colab_demo.ipynb

{
  "cells": [
     {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/chemprop/chemprop/blob/master/colab_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T2XDc8RVq6p6"
      },
      "source": [
        "Please note that this notebook is intended to be run in Google Colab rather than as a Jupyter notebook on your local machine. Please click the \"Open in Colab\" button."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6jZnD-vU9Fd-"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "WGZxehx_44Rj",
        "outputId": "32702633-b1be-4f8c-95d6-44b785b3cd05"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100   163  100   163    0     0   1940      0 --:--:-- --:--:-- --:--:--  1917\n",
            "100   133  100   133    0     0    747      0 --:--:-- --:--:-- --:--:--   747\n",
            "100   651  100   651    0     0   2573      0 --:--:-- --:--:-- --:--:--  2573\n",
            "100 29.6M  100 29.6M    0     0  17.9M      0  0:00:01  0:00:01 --:--:-- 19.7M\n",
            "Cloning into 'chemprop'...\n",
            "remote: Enumerating objects: 15722, done.\u001b[K\n",
            "remote: Counting objects: 100% (17/17), done.\u001b[K\n",
            "remote: Compressing objects: 100% (12/12), done.\u001b[K\n",
            "remote: Total 15722 (delta 7), reused 12 (delta 5), pack-reused 15705\u001b[K\n",
            "Receiving objects: 100% (15722/15722), 360.39 MiB | 20.05 MiB/s, done.\n",
            "Resolving deltas: 100% (7989/7989), done.\n",
            "Obtaining file:///content/chemprop\n",
            "Requirement already satisfied: flask>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (1.1.4)\n",
            "Collecting hyperopt>=0.2.3\n",
            "  Downloading hyperopt-0.2.7-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 5.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib>=3.1.3 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.18.1 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (1.21.6)\n",
            "Requirement already satisfied: pandas>=1.0.3 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (1.3.5)\n",
            "Collecting pandas-flavor>=0.2.0\n",
            "  Downloading pandas_flavor-0.3.0-py3-none-any.whl (6.3 kB)\n",
            "Requirement already satisfied: scikit-learn>=0.22.2.post1 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (1.0.2)\n",
            "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (1.4.1)\n",
            "Collecting sphinx>=3.1.2\n",
            "  Downloading Sphinx-4.5.0-py3-none-any.whl (3.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.1 MB 7.8 MB/s \n",
            "\u001b[?25hCollecting tensorboardX>=2.0\n",
            "  Downloading tensorboardX-2.5-py2.py3-none-any.whl (125 kB)\n",
            "\u001b[K     |████████████████████████████████| 125 kB 48.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.5.1 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (1.11.0+cu113)\n",
            "Requirement already satisfied: tqdm>=4.45.0 in /usr/local/lib/python3.7/dist-packages (from chemprop==1.4.1) (4.64.0)\n",
            "Collecting typed-argument-parser>=1.6.1\n",
            "  Downloading typed-argument-parser-1.7.2.tar.gz (27 kB)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.2->chemprop==1.4.1) (2.11.3)\n",
            "Requirement already satisfied: click<8.0,>=5.1 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.2->chemprop==1.4.1) (7.1.2)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.2->chemprop==1.4.1) (1.0.1)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from flask>=1.1.2->chemprop==1.4.1) (1.1.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.2.3->chemprop==1.4.1) (1.3.0)\n",
            "Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.2.3->chemprop==1.4.1) (2.6.3)\n",
            "Collecting py4j\n",
            "  Downloading py4j-0.10.9.5-py2.py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 63.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.2.3->chemprop==1.4.1) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.2.3->chemprop==1.4.1) (1.15.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->flask>=1.1.2->chemprop==1.4.1) (2.0.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.3->chemprop==1.4.1) (0.11.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.3->chemprop==1.4.1) (1.4.2)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.3->chemprop==1.4.1) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.1.3->chemprop==1.4.1) (3.0.8)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib>=3.1.3->chemprop==1.4.1) (4.2.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.3->chemprop==1.4.1) (2022.1)\n",
            "Requirement already satisfied: xarray in /usr/local/lib/python3.7/dist-packages (from pandas-flavor>=0.2.0->chemprop==1.4.1) (0.18.2)\n",
            "Collecting pandas-flavor>=0.2.0\n",
            "  Downloading pandas_flavor-0.2.0-py2.py3-none-any.whl (6.6 kB)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.22.2.post1->chemprop==1.4.1) (1.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.22.2.post1->chemprop==1.4.1) (3.1.0)\n",
            "Requirement already satisfied: alabaster<0.8,>=0.7 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (0.7.12)\n",
            "Collecting sphinxcontrib-qthelp\n",
            "  Downloading sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl (90 kB)\n",
            "\u001b[K     |████████████████████████████████| 90 kB 9.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: imagesize in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (1.3.0)\n",
            "Collecting sphinxcontrib-devhelp\n",
            "  Downloading sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl (84 kB)\n",
            "\u001b[K     |████████████████████████████████| 84 kB 3.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: docutils<0.18,>=0.14 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (0.17.1)\n",
            "Requirement already satisfied: requests>=2.5.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (21.3)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (4.11.3)\n",
            "Requirement already satisfied: sphinxcontrib-serializinghtml>=1.1.5 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (1.1.5)\n",
            "Collecting sphinxcontrib-applehelp\n",
            "  Downloading sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl (121 kB)\n",
            "\u001b[K     |████████████████████████████████| 121 kB 54.9 MB/s \n",
            "\u001b[?25hCollecting sphinxcontrib-jsmath\n",
            "  Downloading sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl (5.1 kB)\n",
            "Requirement already satisfied: babel>=1.3 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (2.10.1)\n",
            "Collecting sphinxcontrib-htmlhelp>=2.0.0\n",
            "  Downloading sphinxcontrib_htmlhelp-2.0.0-py2.py3-none-any.whl (100 kB)\n",
            "\u001b[K     |████████████████████████████████| 100 kB 9.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: Pygments>=2.0 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (2.6.1)\n",
            "Requirement already satisfied: snowballstemmer>=1.1 in /usr/local/lib/python3.7/dist-packages (from sphinx>=3.1.2->chemprop==1.4.1) (2.2.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->sphinx>=3.1.2->chemprop==1.4.1) (3.8.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->sphinx>=3.1.2->chemprop==1.4.1) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->sphinx>=3.1.2->chemprop==1.4.1) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->sphinx>=3.1.2->chemprop==1.4.1) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.5.0->sphinx>=3.1.2->chemprop==1.4.1) (2021.10.8)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX>=2.0->chemprop==1.4.1) (3.17.3)\n",
            "Collecting typing-inspect>=0.7.1\n",
            "  Downloading typing_inspect-0.7.1-py3-none-any.whl (8.4 kB)\n",
            "Collecting mypy-extensions>=0.3.0\n",
            "  Downloading mypy_extensions-0.4.3-py2.py3-none-any.whl (4.5 kB)\n",
            "Requirement already satisfied: setuptools>=40.4 in /usr/local/lib/python3.7/dist-packages (from xarray->pandas-flavor>=0.2.0->chemprop==1.4.1) (57.4.0)\n",
            "Building wheels for collected packages: typed-argument-parser\n",
            "  Building wheel for typed-argument-parser (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for typed-argument-parser: filename=typed_argument_parser-1.7.2-py3-none-any.whl size=22701 sha256=ffedabdec27500b13dc414601e57e3ac25c00a7211afdcede38db8ff873334f8\n",
            "  Stored in directory: /root/.cache/pip/wheels/ab/9e/09/716b41e56484cac4752e0a6f5a2526ea439c4474bcf5032a7c\n",
            "Successfully built typed-argument-parser\n",
            "Installing collected packages: mypy-extensions, typing-inspect, sphinxcontrib-qthelp, sphinxcontrib-jsmath, sphinxcontrib-htmlhelp, sphinxcontrib-devhelp, sphinxcontrib-applehelp, py4j, typed-argument-parser, tensorboardX, sphinx, pandas-flavor, hyperopt, chemprop\n",
            "  Attempting uninstall: sphinx\n",
            "    Found existing installation: Sphinx 1.8.6\n",
            "    Uninstalling Sphinx-1.8.6:\n",
            "      Successfully uninstalled Sphinx-1.8.6\n",
            "  Attempting uninstall: hyperopt\n",
            "    Found existing installation: hyperopt 0.1.2\n",
            "    Uninstalling hyperopt-0.1.2:\n",
            "      Successfully uninstalled hyperopt-0.1.2\n",
            "  Running setup.py develop for chemprop\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed chemprop-1.4.1 hyperopt-0.2.7 mypy-extensions-0.4.3 pandas-flavor-0.2.0 py4j-0.10.9.5 sphinx-4.5.0 sphinxcontrib-applehelp-1.0.2 sphinxcontrib-devhelp-1.0.2 sphinxcontrib-htmlhelp-2.0.0 sphinxcontrib-jsmath-1.0.1 sphinxcontrib-qthelp-1.0.3 tensorboardX-2.5 typed-argument-parser-1.7.2 typing-inspect-0.7.1\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "sphinxcontrib"
                ]
              }
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "import os\n",
        "\n",
        "!curl -L bit.ly/rdkit-colab | tar xz -C / # shortcut install of rdkit 2020.09.01 for Colab\n",
        "\n",
        "!git clone https://github.com/chemprop/chemprop\n",
        "os.chdir('chemprop')\n",
        "!pip install -e .\n",
        "\n",
        "import chemprop\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib.offsetbox import AnchoredText\n",
        "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
        "from sklearn.decomposition import PCA\n",
        "\n",
        "# You can safely ignore the following errors and do not need to restart the runtime; all subsequent cells should still execute without errors:\n",
        "\n",
        "# ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
        "# datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
        "\n",
        "# WARNING: The following packages were previously imported in this runtime:\n",
        "#   [sphinxcontrib]\n",
        "# You must restart the runtime in order to use newly installed versions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "ZVmFInN_qj4n"
      },
      "outputs": [],
      "source": [
        "def plot_parity(y_true, y_pred, y_pred_unc=None):\n",
        "    \n",
        "    axmin = min(min(y_true), min(y_pred)) - 0.1*(max(y_true)-min(y_true))\n",
        "    axmax = max(max(y_true), max(y_pred)) + 0.1*(max(y_true)-min(y_true))\n",
        "    \n",
        "    mae = mean_absolute_error(y_true, y_pred)\n",
        "    rmse = mean_squared_error(y_true, y_pred, squared=False)\n",
        "    \n",
        "    plt.plot([axmin, axmax], [axmin, axmax], '--k')\n",
        "\n",
        "    plt.errorbar(y_true, y_pred, yerr=y_pred_unc, linewidth=0, marker='o', markeredgecolor='w', alpha=1, elinewidth=1)\n",
        "    \n",
        "    plt.xlim((axmin, axmax))\n",
        "    plt.ylim((axmin, axmax))\n",
        "    \n",
        "    ax = plt.gca()\n",
        "    ax.set_aspect('equal')\n",
        "    \n",
        "    at = AnchoredText(\n",
        "    f\"MAE = {mae:.2f}\\nRMSE = {rmse:.2f}\", prop=dict(size=10), frameon=True, loc='upper left')\n",
        "    at.patch.set_boxstyle(\"round,pad=0.,rounding_size=0.2\")\n",
        "    ax.add_artist(at)\n",
        "    \n",
        "    plt.xlabel('True')\n",
        "    plt.ylabel('Chemprop Predicted')\n",
        "    \n",
        "    plt.show()\n",
        "    \n",
        "    return"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQ67V6hr_6Yz"
      },
      "source": [
        "# Train regression model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pqtogu2T7OTd",
        "outputId": "62b5e76b-5695-4d12-c0d2-b585093cab5f",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/regression.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'regression',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'mse',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'rmse',\n",
            " 'metrics': ['rmse'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': False,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_reg',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['smiles'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'random',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['logSolubility'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "500it [00:00, 76661.50it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 72810.19it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 8698.34it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "0it [00:00, ?it/s]Warning: Repeated SMILES found in data, pickle file of split indices cannot distinguish entries and will not be generated.\n",
            "276it [00:00, 55749.00it/s]\n",
            "Total size = 500 | train size = 400 | val size = 50 | test size = 50\n",
            "Fitting scaler\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 355,201\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:02,  3.39it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:01,  4.16it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:01,  3.87it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  4.49it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:01<00:00,  4.82it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:01<00:00,  3.44it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:01<00:00,  3.63it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:02<00:00,  3.57it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  6.33it/s]\u001b[A\n",
            "                                             \u001b[AValidation rmse = 1.706185\n",
            " 20%|██        | 1/5 [00:02<00:10,  2.53s/it]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:01,  6.11it/s]\u001b[ALoss = 6.5336e-01, PNorm = 34.0268, GNorm = 6.2552, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:01,  4.81it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:01,  4.49it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  4.71it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:01<00:00,  4.82it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:01<00:00,  4.80it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:01<00:00,  5.12it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  4.76it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.387877\n",
            " 40%|████      | 2/5 [00:04<00:06,  2.20s/it]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:01,  4.25it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:01,  4.13it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:01,  4.91it/s]\u001b[ALoss = 4.3137e-01, PNorm = 34.0797, GNorm = 5.1802, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00,  5.14it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:01<00:00,  5.00it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:01<00:00,  5.30it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:01<00:00,  5.22it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  4.86it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.432342\n",
            " 60%|██████    | 3/5 [00:06<00:04,  2.03s/it]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:01,  5.73it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:01,  4.18it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:01,  3.99it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  4.29it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:01<00:00,  4.55it/s]\u001b[ALoss = 4.0926e-01, PNorm = 34.1023, GNorm = 6.6009, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:01<00:00,  4.00it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:01<00:00,  3.90it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  3.87it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.345148\n",
            " 80%|████████  | 4/5 [00:08<00:02,  2.16s/it]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:01,  4.22it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:01,  4.18it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:01,  3.65it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:01<00:01,  3.42it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:01<00:00,  3.55it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:01<00:00,  3.67it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:01<00:00,  4.01it/s]\u001b[ALoss = 3.1846e-01, PNorm = 34.1096, GNorm = 1.1095, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:02<00:00,  4.23it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.262283\n",
            "100%|██████████| 5/5 [00:11<00:00,  2.21s/it]\n",
            "Model 0 best validation rmse = 1.262283 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test rmse = 1.657099\n",
            "Ensemble test rmse = 1.657099\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test rmse = 1.657099\n",
            "Overall test rmse = 1.657099 +/- 0.000000\n",
            "Elapsed time = 0:00:12\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/regression.csv',\n",
        "    '--dataset_type', 'regression',\n",
        "    '--save_dir', 'test_checkpoints_reg',\n",
        "    '--epochs', '5',\n",
        "    '--save_smiles_splits'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WZktZ7DsAAMn"
      },
      "source": [
        "# Predict from file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5a1-7UXcJCss",
        "outputId": "4f36f2cc-196e-4634-f9f8-f5dfbda509ad",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "50it [00:00, 41282.52it/s]\n",
            "100%|██████████| 50/50 [00:00<00:00, 104752.85it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 50\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.06it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  1.84it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_reg.csv\n",
            "Elapsed time = 0:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_reg/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_reg.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_reg'\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "j-4hbBeerlX0",
        "outputId": "e4158008-646b-4102-da0f-d43fa77661e9",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                               smiles  logSolubility     preds\n",
              "0                                      C/C1CCC(\\C)CC1         -4.470 -2.923179\n",
              "1           Cc1ccc(OP(=O)(Oc2cccc(C)c2)Oc3ccccc3C)cc1         -6.010 -2.800287\n",
              "2                                  c1c(Br)ccc2ccccc12         -4.400 -4.282761\n",
              "3          CCOc1ccc(cc1)C(C)(C)COCc3cccc(Oc2ccccc2)c3         -8.600 -3.202824\n",
              "4                     CCC1(C(=O)NC(=O)NC1=O)C2=CCCCC2         -2.170 -2.961018\n",
              "5                                C(Cc1ccccc1)c2ccccc2         -4.620 -2.532373\n",
              "6                                        Cc1cccc(N)c1         -0.850 -1.932497\n",
              "7      CCN(CC(C)=C)c1c(cc(cc1N(=O)=O)C(F)(F)F)N(=O)=O         -6.124 -4.090992\n",
              "8                                              CC(C)O          0.430  0.151779\n",
              "9                                           CCCCCCCCO         -2.390 -3.121081\n",
              "10                           CN(C)C(=O)SCCCCOc1ccccc1         -3.927 -2.256432\n",
              "11       CC2Nc1cc(Cl)c(cc1C(=O)N2c3ccccc3C)S(N)(=O)=O         -3.780 -4.537866\n",
              "12                                          CCCCCCC#C         -3.660 -2.933518\n",
              "13                      COP(=S)(OC)Oc1cc(Cl)c(I)cc1Cl         -6.620 -4.226359\n",
              "14                                     CC(C)CCOC(=O)C         -1.920 -1.432131\n",
              "15                                    CCN(CC)c1ccccc1         -3.030 -3.003822\n",
              "16                            O=N(=O)c1cc(Cl)c(Cl)cc1         -3.200 -3.460910\n",
              "17       ClC1=C(Cl)C2(Cl)C3C4CC(C=C4)C3C1(Cl)C2(Cl)Cl         -6.307 -5.903345\n",
              "18                                 CC(=O)Nc1ccc(F)cc1         -1.780 -2.438263\n",
              "19  CC1(C)C(C=C(Br)Br)C1C(=O)OC(C#N)c2cccc(Oc3cccc...         -8.402 -3.614757\n",
              "20                     O=C1NC(=O)NC(=O)C1(CC)c1ccccc1         -2.322 -3.179535\n",
              "21                                           c1ccncc1          0.760  0.550709\n",
              "22                                         OC1CCCCCC1         -0.880 -2.066501\n",
              "23                                  CCCCCCCCCCCCCCCCO         -7.000 -3.930395\n",
              "24    COC(=O)C1=C(C)NC(=C(C1c2ccccc2N(=O)=O)C(=O)OC)C         -4.760 -3.196916\n",
              "25                                           Nc1nccs1         -0.360  0.473926\n",
              "26                                           CCCC(C)C         -3.740 -2.502151\n",
              "27                                       Cc1cccc(C)c1         -2.820 -2.359880\n",
              "28                 Clc2ccc(Oc1ccc(cc1)N(=O)=O)c(Cl)c2         -5.460 -3.393277\n",
              "29                  O=C1NC(=O)C(N1)(c2ccccc2)c3ccccc3         -4.097 -3.744062\n",
              "30             Oc1ccc(Cl)cc1C(=O)Nc2ccc(cc2Cl)N(=O)=O         -4.700 -3.634063\n",
              "31                                  CC(C)(C)Cc1ccccc1         -4.150 -2.779619\n",
              "32                        CCOC(=O)c1cncn1C(C)c2ccccc2         -4.735 -2.892047\n",
              "33  OCC1OC(CO)(OC2OC(COC3OC(CO)C(O)C(O)C3O)C(O)C(O...         -0.410 -0.263499\n",
              "34                                             ClCC#N         -0.092 -1.213599\n",
              "35                                     CCCCCCCCC(=O)C         -3.300 -3.065029\n",
              "36                                         CCCOC(=O)C         -0.720 -0.813638\n",
              "37                  OC3N=C(c1ccccc1)c2cc(Cl)ccc2NC3=O         -3.952 -3.594748\n",
              "38                       CCOc2ccc1nc(sc1c2)S(N)(=O)=O         -3.810 -2.758790\n",
              "39                                         CC(C)C(C)O         -0.180 -1.060896\n",
              "40                                    Oc2ccc1ncccc1c2         -2.160 -2.433550\n",
              "41             Cc3nnc4CN=C(c1ccccc1Cl)c2cc(Cl)ccc2n34         -4.090 -4.776746\n",
              "42             COc1ccc(cc1)C(c2ccc(OC)cc2)C(Cl)(Cl)Cl         -6.890 -4.065035\n",
              "43                               CCSCc1ccccc1OC(=O)NC         -2.090 -2.406479\n",
              "44                 Clc1ccc(c(Cl)c1)c2c(Cl)ccc(Cl)c2Cl         -7.430 -6.413253\n",
              "45                           Clc1ccc(cc1)c2ccc(Cl)cc2         -6.560 -4.465485\n",
              "46                                         CCCC1CCCC1         -4.740 -3.386112\n",
              "47                                          CCCC(O)CC         -0.800 -1.881690\n",
              "48                                         CCCCCCCC#C         -4.240 -3.152136\n",
              "49  ClC1=C(Cl)C(Cl)(C(=C1Cl)Cl)C2(Cl)C(=C(Cl)C(=C2...         -7.278 -5.784078"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-10e1b467-49c5-4b44-aec0-70a89773f4d7\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>smiles</th>\n",
              "      <th>logSolubility</th>\n",
              "      <th>preds</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>C/C1CCC(\\C)CC1</td>\n",
              "      <td>-4.470</td>\n",
              "      <td>-2.923179</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Cc1ccc(OP(=O)(Oc2cccc(C)c2)Oc3ccccc3C)cc1</td>\n",
              "      <td>-6.010</td>\n",
              "      <td>-2.800287</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>c1c(Br)ccc2ccccc12</td>\n",
              "      <td>-4.400</td>\n",
              "      <td>-4.282761</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CCOc1ccc(cc1)C(C)(C)COCc3cccc(Oc2ccccc2)c3</td>\n",
              "      <td>-8.600</td>\n",
              "      <td>-3.202824</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>CCC1(C(=O)NC(=O)NC1=O)C2=CCCCC2</td>\n",
              "      <td>-2.170</td>\n",
              "      <td>-2.961018</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>C(Cc1ccccc1)c2ccccc2</td>\n",
              "      <td>-4.620</td>\n",
              "      <td>-2.532373</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Cc1cccc(N)c1</td>\n",
              "      <td>-0.850</td>\n",
              "      <td>-1.932497</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>CCN(CC(C)=C)c1c(cc(cc1N(=O)=O)C(F)(F)F)N(=O)=O</td>\n",
              "      <td>-6.124</td>\n",
              "      <td>-4.090992</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>CC(C)O</td>\n",
              "      <td>0.430</td>\n",
              "      <td>0.151779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>CCCCCCCCO</td>\n",
              "      <td>-2.390</td>\n",
              "      <td>-3.121081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>CN(C)C(=O)SCCCCOc1ccccc1</td>\n",
              "      <td>-3.927</td>\n",
              "      <td>-2.256432</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>CC2Nc1cc(Cl)c(cc1C(=O)N2c3ccccc3C)S(N)(=O)=O</td>\n",
              "      <td>-3.780</td>\n",
              "      <td>-4.537866</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>CCCCCCC#C</td>\n",
              "      <td>-3.660</td>\n",
              "      <td>-2.933518</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>COP(=S)(OC)Oc1cc(Cl)c(I)cc1Cl</td>\n",
              "      <td>-6.620</td>\n",
              "      <td>-4.226359</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>CC(C)CCOC(=O)C</td>\n",
              "      <td>-1.920</td>\n",
              "      <td>-1.432131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>CCN(CC)c1ccccc1</td>\n",
              "      <td>-3.030</td>\n",
              "      <td>-3.003822</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>O=N(=O)c1cc(Cl)c(Cl)cc1</td>\n",
              "      <td>-3.200</td>\n",
              "      <td>-3.460910</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>ClC1=C(Cl)C2(Cl)C3C4CC(C=C4)C3C1(Cl)C2(Cl)Cl</td>\n",
              "      <td>-6.307</td>\n",
              "      <td>-5.903345</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>CC(=O)Nc1ccc(F)cc1</td>\n",
              "      <td>-1.780</td>\n",
              "      <td>-2.438263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>CC1(C)C(C=C(Br)Br)C1C(=O)OC(C#N)c2cccc(Oc3cccc...</td>\n",
              "      <td>-8.402</td>\n",
              "      <td>-3.614757</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>O=C1NC(=O)NC(=O)C1(CC)c1ccccc1</td>\n",
              "      <td>-2.322</td>\n",
              "      <td>-3.179535</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>c1ccncc1</td>\n",
              "      <td>0.760</td>\n",
              "      <td>0.550709</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>OC1CCCCCC1</td>\n",
              "      <td>-0.880</td>\n",
              "      <td>-2.066501</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>CCCCCCCCCCCCCCCCO</td>\n",
              "      <td>-7.000</td>\n",
              "      <td>-3.930395</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>COC(=O)C1=C(C)NC(=C(C1c2ccccc2N(=O)=O)C(=O)OC)C</td>\n",
              "      <td>-4.760</td>\n",
              "      <td>-3.196916</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>Nc1nccs1</td>\n",
              "      <td>-0.360</td>\n",
              "      <td>0.473926</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>CCCC(C)C</td>\n",
              "      <td>-3.740</td>\n",
              "      <td>-2.502151</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>Cc1cccc(C)c1</td>\n",
              "      <td>-2.820</td>\n",
              "      <td>-2.359880</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>Clc2ccc(Oc1ccc(cc1)N(=O)=O)c(Cl)c2</td>\n",
              "      <td>-5.460</td>\n",
              "      <td>-3.393277</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>O=C1NC(=O)C(N1)(c2ccccc2)c3ccccc3</td>\n",
              "      <td>-4.097</td>\n",
              "      <td>-3.744062</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>Oc1ccc(Cl)cc1C(=O)Nc2ccc(cc2Cl)N(=O)=O</td>\n",
              "      <td>-4.700</td>\n",
              "      <td>-3.634063</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>CC(C)(C)Cc1ccccc1</td>\n",
              "      <td>-4.150</td>\n",
              "      <td>-2.779619</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>CCOC(=O)c1cncn1C(C)c2ccccc2</td>\n",
              "      <td>-4.735</td>\n",
              "      <td>-2.892047</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>OCC1OC(CO)(OC2OC(COC3OC(CO)C(O)C(O)C3O)C(O)C(O...</td>\n",
              "      <td>-0.410</td>\n",
              "      <td>-0.263499</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>ClCC#N</td>\n",
              "      <td>-0.092</td>\n",
              "      <td>-1.213599</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>CCCCCCCCC(=O)C</td>\n",
              "      <td>-3.300</td>\n",
              "      <td>-3.065029</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>CCCOC(=O)C</td>\n",
              "      <td>-0.720</td>\n",
              "      <td>-0.813638</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>OC3N=C(c1ccccc1)c2cc(Cl)ccc2NC3=O</td>\n",
              "      <td>-3.952</td>\n",
              "      <td>-3.594748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>CCOc2ccc1nc(sc1c2)S(N)(=O)=O</td>\n",
              "      <td>-3.810</td>\n",
              "      <td>-2.758790</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>CC(C)C(C)O</td>\n",
              "      <td>-0.180</td>\n",
              "      <td>-1.060896</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>Oc2ccc1ncccc1c2</td>\n",
              "      <td>-2.160</td>\n",
              "      <td>-2.433550</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>Cc3nnc4CN=C(c1ccccc1Cl)c2cc(Cl)ccc2n34</td>\n",
              "      <td>-4.090</td>\n",
              "      <td>-4.776746</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>COc1ccc(cc1)C(c2ccc(OC)cc2)C(Cl)(Cl)Cl</td>\n",
              "      <td>-6.890</td>\n",
              "      <td>-4.065035</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>CCSCc1ccccc1OC(=O)NC</td>\n",
              "      <td>-2.090</td>\n",
              "      <td>-2.406479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>Clc1ccc(c(Cl)c1)c2c(Cl)ccc(Cl)c2Cl</td>\n",
              "      <td>-7.430</td>\n",
              "      <td>-6.413253</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>Clc1ccc(cc1)c2ccc(Cl)cc2</td>\n",
              "      <td>-6.560</td>\n",
              "      <td>-4.465485</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>CCCC1CCCC1</td>\n",
              "      <td>-4.740</td>\n",
              "      <td>-3.386112</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>CCCC(O)CC</td>\n",
              "      <td>-0.800</td>\n",
              "      <td>-1.881690</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>CCCCCCCC#C</td>\n",
              "      <td>-4.240</td>\n",
              "      <td>-3.152136</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>ClC1=C(Cl)C(Cl)(C(=C1Cl)Cl)C2(Cl)C(=C(Cl)C(=C2...</td>\n",
              "      <td>-7.278</td>\n",
              "      <td>-5.784078</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-10e1b467-49c5-4b44-aec0-70a89773f4d7')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-10e1b467-49c5-4b44-aec0-70a89773f4d7 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-10e1b467-49c5-4b44-aec0-70a89773f4d7');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "df = pd.read_csv('test_checkpoints_reg/fold_0/test_full.csv')\n",
        "df['preds'] = [x[0] for x in preds]\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "hj726FiKqj4q",
        "outputId": "c0f101b0-9c08-4d43-ecf8-84a2ffdb055c"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAEGCAYAAAByy7CMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhU5ZX48e+pXthEiWwKyBIRZW+gQYmJrMoSRJYgEiOOG9kMiiZRQnBJ3MU4MiGTGGNEJ+kGA0YnTojRHzImoRFQQDTs4AgGF1C8vdPd5/dHVbW9VfWt9VZVn8/z1ANV99a9pwvq9Lu/oqoYY0xzfF4HYIxJD5YsjDGuWLIwxrhiycIY44olC2OMK9leBxCJTp06ae/evb0Ow5iMU1NTw549eygpKflYVTs3dU5aJYvevXuzZcsWr8MwJqM4jsOUKVMoLy8HeDfUeVYNMaYFCyaKoqIiCgoKwp5rycKYFuzKK6+sTRRz5swJe64lC2NasLvvvptVq1Y1myggA5LF4cOHGTlyJFlZWYiIPeo8srKyGDlyJIcPH/b6n8mkEMdxeOqppwAYNmwYs2fPdvW+tE8WM2fOZNasWZSVlaGq9qjzKCsrY+bMmcycOdPrfyaTIoJtFNdffz3//Oc/I3qvpNNEsvz8fG3YG5KVlUVZWRm5ubkeRZXaKisradOmDdXV1V6HYjxWtzGzsLCQr33ta43OEZGtqprf1PvTvmRRU1NjiSKM3NxcampqvA7DeMxNomhO2icLY0zzNmzYwObNm6NOFJChyUJE+MY3vlH7vKqqis6dOzNt2rR6582YMYMLLrig3mt33XUX3bt3Jy8vr/bx6aefxhTPs88+y8CBA/H5fCEHlb333nuMGzeOAQMGMHDgQB577LHaY0uXLmXIkCHk5eVxySWX8P7778cUj2k5gs0M06ZNY9++fVEnCsjQZNGuXTt27txJWVkZAH/961/p3r17vXM+/fRTtm7dyokTJzhw4EC9Y4sWLWLbtm21jw4dOsQUz6BBg1i7di0XXXRRyHOys7N55JFHeOeddygqKmLFihW88847APzgBz9gx44dbNu2jWnTpvGTn/wkpnhMy+A4DhdffDHr1q0D4KyzzorpehmZLACmTp3Kiy++CEBBQQHz5s2rd3zt2rVceumlXHHFFRQWFiY0lv79+3PuueeGPefMM89k+PDhALRv357+/ftz5MgRAE499dTa80pKShCRxAVrMkKwjeLVV1+luLg4LtfM2GQRTALl5eXs2LGD888/v97xYAKZN29eo2Gujz76aG0VZNy4cY2u7ThOvWpK3UewNBCLQ4cO8eabb9aLecmSJZx11ln87ne/s5KFCSsejZlN8nosQCSPESNGaEP+H6G+du3aqarqiBEj9Mknn9TFixfr+vXr9atf/aqqqh49elR79eqlNTU1qqo6bNgwfeutt1RV9c4779SHH3640TXjYcyYMbp58+aw5ziOo8OHD9c1a9Y0efy+++7TO+64I6L7NvUZmcxUUlKiF154oWZlZemzzz4b8fuBLRri+5exJQuA6dOn8/3vf79RFWT16tV88skn9OnTh969e3Po0KFmJ9HUlaiSxcmTJ5k9ezZXXnkls2bNavKcK6+8kjVr1kR9D5PZWrduzdChQ+NboghIqynqkbr22mvp0KEDgwcP5tVXX619vaCggHXr1jF69GgADh48yMSJE7n33ntdXbd9+/Zs27YtrrGqKtdddx39+/fnlltuqXds7969nHPOOQA8//zznHfeeXG9t0l/juNw/PhxevXqxYoVKxJzk1BFjlR8RFoNqStYDTl48KB269attgoSNGzYMC0qKtI777xTu3XrpkOHDq19HDx4MEShzZ21a9dq9+7dNTc3V7t06aKXXHKJqqoeOXJEp0yZoqqqr732mgI6ePDg2vu++OKLqqo6a9YsHThwoA4ePFinTZumhw8fjuj+TX1GJnN89tlneuGFF+rZZ5+t5eXlMV2LMNWQtB/uLSKk08/gBfuMMle8GzMzeri3MS1Vwno9QsjoNgtjMtnixYvp1as3a194kU5fOJXiiira5mTh8yVmHI4lC2OSpKZGKT1ZTdvcLEorq2P+Yj/wwAMUV/m4afUONh86zsjep7N83jA6tstNSMLIyGpIVlYWeXl5DBo0iEsvvbR2bsehQ4cQEX784x/Xnvvxxx+Tk5PDjTfeCMDu3bsZO3YseXl59O/fnwULFgDw6quvctppp9XrJn355ZdjinPXrl2MHj2aVq1asWzZspDnqSpLliyhX79+9O/fn+XLl9cee/XVV8nLy2PgwIGMGTMmpnhM4tTUKMdKKrlh5Rb6LfkzN6zcwrGSSmpqImtLchyH2267zT+VIac1i1bvYOOBY1TVKBsPHGNhwZuUnkzMcgQZWbJo06ZNbdfm1VdfzYoVK1iyZAkAffr04cUXX+See+4BPp/kFbRw4UIWLVrEZZddBsBbb71Ve+wrX/kKf/rTn+IW5+mnn87y5cv54x//GPa8p556ivfee49du3bh8/n48MMPAf/8lu985zusW7eOnj171r5uUk/pyWoWFrzJxgPHAGq/2L++Op9TWrn7GjqOw9SpU9m4cSOTJ09mzNixbD50vN45mw8dp21uVtzjhwwtWdQ1evTo2jkWAG3btqV///61sz9XrVrF5ZdfXnv8X//6Fz169Kh9Pnjw4ITF1qVLF0aOHElOTk7Y8/7zP/+TO+64A5/PV/s+gN///vfMmjWLnj171nvdpJ62uVkxfbHrJorCwkLGjRtHaWU1I3ufXu+8kb1Pp7QyMSWLjE4W1dXVvPLKK0yfPr3e68F5I++99x5ZWVl069at9tiiRYsYP348U6ZM4dFHH603Pf21116rVw3Zv39/o3vOnTu3yZGdTz/9dNQ/x/79+1m1ahX5+flMmTKFvXv3ArBnzx4++eQTxo4dy4gRI2K6h0msWL7YDRNFsNejbU4Wy+cNY/QXO5LtE0Z/sSPL5w2jbU5iShYZWQ0pKysjLy+PI0eO0L9/fy6++OJ6xydPnszSpUvp2rUrc+fOrXfsmmuuYdKkSaxbt47nn3+eX/3qV2zfvh1wVw1ZtWpVfH8YoKKigtatW7NlyxbWrl3Ltddey2uvvUZVVRVbt27llVdeoaysjNGjR3PBBRfQr1+/uMdgYhP8Yi8seLNeY6SbL/aRI0fYv39/o+5Rn0/o2C6XX1+dH7dG03AyMlkE2yxKS0uZNGkSK1asYOHChbXHc3NzGTFiRO36ES+88EK993fr1o1rr72Wa6+9lkGDBrFz507X9547dy67d+9u9Pott9zC/Pnzo/p5evToUTtXZObMmVxzzTW1r3fs2JF27drRrl07LrroIrZv327JIgVF88WuqKggNzeX8847j/0HDlAt2dSo1nuvzye1bR5u2z6ilZHJIqht27YsX76cGTNm8J3vfKfesVtvvZUxY8Zw+un1i4br1q1jwoQJ5OTkcPToUY4dO0b37t3ZtWuXq3smomQxY8YM1q9fT58+fdiwYUNtMrjsssu48cYbqaqqorKykk2bNrFo0aK439/ERyRf7GDVY+zYsdx9908oPiksLNiSlC7SUDK6zQL8+yIMGTKk0azSgQMHcvXVVzc6/6WXXmLQoEEMHTqUSZMm8fDDD3PGGWcAjdss/vCHP8QU29GjR+nRowc/+9nPuOeee+jRowefffYZ4F+8J7h83u23386aNWsYPHgwixcv5oknngD8i+pMnjyZIUOGMGrUKK6//noGDRoUU0zGe3XbKIYOHVqvJyUZXaSheDo3REQmA48BWcATqvpAuPNtbkh07DNKH001Ztao0m/Jn6mqMyYj2yfsuXcKvjivmpaSc0NEJAtYAUwBBgDzRGSAV/EY47WamhouvfTSRr0eye4iDcXLasgoYJ+qHlDVSqAQuMzDeIzxlM/n45vf/GajXo9kd5GG4mUDZ3fgvTrPDwPnNzxJRBYAC4DawUd1+Xw+KisrbaOhECorK2sHc5nU5DgOb7zxBmPGjGm0qhskv4s0lJT/X6Sqj6tqvqrmd+7cudHx4cOHs2zZMiorKz2ILrVVVlaybNmy2lXDTeoJTjOfOnVq2OH6wZ4UnwT+THKiAG+TxRGg7kYGPQKvReS5557jueeeo02bNp7vWp5qjzZt2tR+PsZbNTVKcUUVNRr4s0brrUexcuXKlB+u72U1ZDNwjoj0wZ8krgC+HulFevTowebNm+MdmzFxE5xxWnf05mNX5LHiZ48kbeGaePAsWahqlYjcCPwFf9fpk6r6tlfxGJMoTc04valwG0u/fhWDBw1Ki0QBHo/gVNX/Af7HyxiMSbRQM07P63s+A/qd7VFUkUv5Bk5j0l3IcRJJHoEZK0sWxiRY25wsHrsiz/NxErHK6IlkxqSCkpJiVvzsEZZ+/SrO63u+fx1OD8ZJxMqShTEJVLd7dPCgQQzod3bCp5InilVDjEmw7OzstOkeDSc9U5wxKc5xHHw+H+3bt2f9+vVInGeHesGShTFxFqx6tGnThpdeeikjEgVYsjAmrkpLS/ng+Ak2bPhfjp1wUIUMyRXWZmFMvJSWlnKs5CRL/3KYc5eu46Y//DOqjYRSlSULY+Lkg+Mn+OFz//R8+btEsWRhTJz06nZGUncISzZLFsbEwHEcfvGLX6Dq3/Q4FZa/SxRLFsZEKdjrsXDhQnbs2BHR8ndNrW+R6qw3xJgo1B2ZWVBQwNChQwFcLX/X1PoWXuwDEikrWRgToYaJYs6cObXH3Cx/lyr7gETKkoUxEdq0aRNbt25tlCjcCrejeipXR6waYoxLqoqIMHHiRA4cOMCZZ54Z1XWC61sEV84Cf0Po/x0rpV2r7JStjljJwhgXHMdh/PjxrFmzBiDqRAFN7wPy4Owh/Oyve1K6OmIlC2OaUbeNouEG29Go3Qdkfj5tcrPY92Exy17azQvb3yfbJyk7LsOShWlRamr84yHcbtYTrjEzFj6fgMA3ntjUqDpSWlmdkmteWDXEtBjBLssbVm6h35I/c8PKLWHnbpSXlyckUQSlyraEbnm6i3qkmtpF3Ri3iiuquGHllnq/yUd/sSO/vjq/yd/kqsrtt99Ofn5+3BNFUKQlnUQLt4t6yLKOiPwHEDKTqOrCOMRmTNKE67Ksy3Ecjh49yjnnnMODDz6Y0JiC4zKAlKx61BWuGrIF2Aq0BoYDewOPPMB2ITZpJ+SS/HXmbgTbKMaPH09ZWVmyQ0xpIVOZqq4EEJFvA19W1arA818CryUnPGPiJ9hG0HCYdbCNoG5jZmFhIW3atPE44tTiptzzBeBUIFh+OyXwmjFppbbLsom5Gw0TRbovrpsIbpLFA8CbIrIeEOAi4K5EBmVMooRqI7jrrrssUTTDVW+IiJwBnB94uklVjyY0qhCsN8QkSklJCUVFRUyYMMHrUDwVrjek2XEW4l+aeCIwVFWfB3JFZFScYzQm6RzH4aabbsJxHNq1axdTokjH9Ski5WZQ1i+A0cC8wHMHWJGwiIxJgmAbxYoVKygqKorpWtXVNXxcUuF6sFe6cpMszlfV7wLlAKr6CTF2nYrIwyKyS0R2iMhzItIhlusZE4mGjZkXX3xx1NeqqVFKKqu5qWBb2q1PESk3yeKkiGQRGKAlIp2Bmhjv+1dgkKoOAfYAi2O8njGuxLvXo/Skfx5HJi/UG+QmWSwHngO6iMi9wN+A+2O5qaq+FBy3ARQBPWK5njFuffTRRxw+fDhuvR5tcnw45VUZvVBvkNvekPOACfi7Tl9R1X/GLQCR/wZWqep/hTi+AFgA0LNnzxHvvvtuvG5tWpCysjJat26NiFBeXk7r1q3jcl2n/CS//dtBZgzrwW1rdtQO9npsXh6d2rVKyUVswolqbkidNz+jqlcBu5p4Ldz7XgbOaOLQkkCvCiKyBKgCfhfqOqr6OPA4+LtOm4vXmIaCVY+RI0fy6KOPxi1RALRrlc3y/7ePfR+VcNf0gfTtcgr7PixO2dWuYuFmUNbAuk8C7RcjmnuTqk4Md1xE/g2YBkzQdJr6alxJldmUddsobr755rhfPzjf5IXt7/PC9veBujNZM2sFiJA/jYgsFhEHGCIinwUeDvAh8HwsNxWRycAPgemqWhrLtUzqiXTdiERJxhDudFuTIhbNtlmIyP2qGtfeChHZB7QCggsLFKnqt5p7n43gTA+RrhvhViSlFVVl4sSJbNiwIeFDuFOlFBUPMbVZAK+LyGmqeiJwsQ7AWFX9Y7QBqWrfaN9rUp/bdSMiEenGPCLCzTffzD333MvQEfkUl1fRtlUWpRX+L7XPJ3H7kqfTmhSxcFOpujOYKABU9VPgzsSFZNKdm3UjIr6my415HMdh3bp1AHz1q9MYMHQ4x0squeHpQJXo6S0cK6mguromJapK6cRNsmjqnMxNn6aeaOY8NKzH3zLxHH511Qja5mZFPW/CTWnFcRymTp3KjBkzOHLkCKUnq/mk9CQ/eHZHgySzLW13BfOSmy/9FhH5GZ/PB/ku/hW0TIaLdk/OuutGtMnxcaykkm8+szWmfT1DbcwTXAk7mCg2btxIYWEh3bt3p0aVs05v22SSaddCRl3Gk5uSxfeASmBV4FGBP2GYDBfLb99gPb7sZE1c5k2E63VomCiCjZmlldW8d7y0ySpRSUXLGHUZT82WLFS1BLg9CbGYFBOPhsp4NXaGW+WqsLCwUaIAf4L5QtscHp4zhB88u6NOySav2SX2TGPhVvf+d1W9OTAcu1ElU1WnJzQy47nmiv7JukZQqF6H66+/ntGjRzNo0KBG57dvnUNOto9fz89v1BsSKvmYpoUcZyEiI1R1q4iMaeq4qm5IaGRNsHEWfsnq13fbZhEunurqGkoqq2nXKpt9Hxbzl53/Yt75vWIeDu04DvPnz+enP/1poyRhohfVOAtV3Rr4M+lJwYQWbaNjNNz89g0XD8Dx0pP1jj02L4/T2+bEnCiCIzPnz59vySJJwg33fiuwOE2Tj2QGaT6XrC6/YJcpwe+0+ov+Db/k4eJp6thNBdsoq4p+OZSGQ7hnzpwZw09pIhGu0jgt8Gew5+OZwJ/fIMxOZSaxEjE6sqFQpYXT2+ZQVlVTr5TRXDyhjhVXVEVcfbLl+r0VsmShqu+q6rvAxar6Q1V9K/C4DbgkeSGauhIxOrLRPUKUFkoqqxuNeCytCB1PqFj3flDsasRkwwFhObm5nHrqqRElipawkG6yuBlnISJyYZ0nX3L5PpMAyZjlGKq00K5VdqME4vMRMp6mYn1w9hBWrN/XbPWpqZmrTiX86U9/iihR2JDu+HEz63QE8CRwWuClT4FrVfWNBMfWiPWG+CW6NyTUrNG7pg9k0r//b+1r2T5hz71TQAkZT91Y935QzIr1+2rXfQi+3yeNJ3X5BK57KraZq4ma/ZrJYto3RFW3qupQYCj+vUPyvEgU5nPB8QY+kSYbHWPVVIngsXl5/GXnv+qdt3B8X0qaaQQNxlpaWc1dL7xdmyjg8+pKUyWAkooqup7aqt79mmubaVjlaJPjsyHdceRmk6GuIvIboFBVT4jIABG5LgmxxZ3VX91r1yqLX8/PZ8+9U/j1/Hw6ts1l3vm96k0Ou2JUTxY8vdVVET9c9anpNpJt3DyxX71rhGubqZtwblm1jY+dCkSE1344julDu7m6hgnPTVnsKeC3wJLA8z3454j8JkExJUQyxyeks1CfU9vcrHpjLkoqqljw9NbaIn6wDSJUET/cmI1QbSQ9O7Zl9Bc7uhqOHUw4ndu34paLz623eO7Dc4bgE/jgswob0h0DNw2VnVR1NYG9QgJL+KddarYpye6E+5zqVn+imbUZqvoUqtfkU6eMX18dKN1cnR82sQcTznfH9eW2NfWnpP/g2R3cM2Nws9cw4blJFiUi0pHPNxm6ADgR/i2pJxnjEzKB288pnl24oaooHU5p47ptJhhP3y6nNB1/q6yEtO+0JG6qIbcALwBni8jfgc5A2o2GieeEpnQSac9J8HPq3L4V3x3Xl75dTuG946WUn6ymbe7nn1M8Z236fEK7rGp+OqkHfXqM4kRJGR0iLAEE4wlOSW9p/87JELbrNLDs/0LgP4Bz8bd771bVk8kJr75Yuk5bYptFND9zTY3ilJ/EqahqNK27Y4NNc+LVhVt3ZOZrf/s7g4eNiOqaNTVKeVU1JRVVLCzY1mL+neMpXNepm3EWr6vqqIREFqFYx1lk0irMbkQ7zqC4vIobnk7O+ISGiaLvwLyYE3pL+3eOp5jGWQB/F5Gfi8hXRGR48BHnGJMi0eMTUk207TRtWyWvfWf79u1s27aNwsJCBg8bEZdG6Jb275wsbn5N5AX+/Emd1xQYH/9wkivTfwNF204Ta/uOm8+1pqYGn8/Hl7/8ZQ4dOkSnTp2oUbVG6BTmpmQxR1XHNXhkRKLI9HkD0c4jCTWCs0128/9d3HyujuMwbtw4Vq5cCUCnTp2A5EySM9ELt1LWpfjnhJzEP8biclX9RxJjaySec0NayryBaEtP0a5w1dznGm6aeUtshE410e5Idi/wFVXdJSLnAw8BTS6xl45ayriLaHfLKquq4ZvPbK33pd944HjIZFo3KYX6XJtbj8LWxUxt4cqVVaq6C0BVNwHtkxNScliRN7xIkmndqsfeD4qb/lwrqlwtXGONk6krXLLoIiK3BB9NPE9rLWn362hEkkzrDhFfsX4fD84e0vhzzc1mypQptsJVGgvXZhF2P1NVvTshEYUR7/Usktkbkm49L5G0H9So0m/Jn6kKNGJOH9qN747ryzldT6Gk/CSffPwBPc86y4sfw0Qo2tW9E54MRORWYBnQWVU/TvT9GkrW7tfp2HAXSftBw67WF7a/z0dOBY9fNZwpF49n79697N+/n1NOOSXZP4aJI8+WxxORs/Cv5fl/XsWQLOk649Vt+0GTXa1X5LHswfsoKiri5z//uSWKDOBlH+GjwA+B5z2MISnqNhYGi+h9u5xCWWCVqEjmPqRiVaZhKaSk/CTLHryPe++5h4KCAubMmeN1iCYOPEkWInIZcERVt4t4/5890erO5Pz+JfUXZnFbHUn1qkzdKt39P72Lhx56yBJFhnEzkawjcBdwIf5h3n8DfqKqx5p538vAGU0cWgL8CLgksEzfISA/VJuFiCwAFgD07NlzxLvvvhs23lQU/KKXVFSxeO1bUQ0ES6dBZOXl5RQVFTF27FivQzERinUiWSHwITAb/zoWH+FfVi8sVZ2oqoMaPoADQB9geyBR9ADeEJGmEguq+riq5qtqfufOnV2Em3qCxfSeHduG3XQn3NqgqT6IzHEcvvWtb3H8+HFat25tiSIDuUkWZ6rqT1X1YOBxD9A12hsGNirqoqq9VbU3cBgYrqpHo71mOvD5JOTYheLyqmbnqKTKILKmFj0Ojsx84okneP3115Maj0keN8niJRG5QkR8gcflwF8SHVgmCjVB66m/H2y2pyQVBpE1NUns4+IKlj3ySO3IzMmTJyctHpNcbtosHKAdgQV78SeYksDfVVVPTVx49WXCJkMNezTa5Pg498fragc0Qf3Nd8K9N9m9IaHaTZaO7cw7O960kZkZINqJZACoakbNCfFaw4FgxRVVrteOSNYgslBCtZuc1/d8BvQ7O+nxmORy9T9ORKYDFwWevqqqf0pcSC1L3YVvu57aipsn9qNnx7aUVkQ2BiMZQi6Kc9IWw20J3FRDHgBGAr8LvDQP2KKqixMcWyOZUA1pSrosNFtTo3xcXMFNhe5j9LrqZCIT64K9O4A8Va0JPM8C3lTVIXGPtBmZmiwgPcZROI7DskceYe7Xr+K8vl/0J4EwX/5UH0hmGot1nAVAhzp/Py3kWSZq6TCOYsqUKdx7zz28s+PNz+eNhNuDJE3nxJimuUkW9wFvishTIrIS2Ip/FS0TR6kyjqIpqsrs2bObXbimoVRPgCYyYZOFiPjwd5leAKwF1gCjVbXZEZwmMqkwjiIUEeG2226LeOGaVE6AJnJu2iy2hKrDJFsmt1lA6jUGOo7DK6+8wowZM5o9t6nYAWuzSDMxjbMAXhaR7+OfDxIcjIWqHg/9FhMNr8dR1BVso3j99dfZu3cvvXr1CnluuIZMW4A3c7hps5gLfBf4X/ztFVuBzP31buqtwv373/8+bKKA8A2ZtgBv5nAzgrNPMgIxqaG55fqbYg2ZLUOzJQsRaR1Y0XutiKwRkZtFpHUygjPJ9/zzz0fc62ENmS2DmwbO1YAD/Ffgpa8DHVQ16UsgZXoDZ6rYvXs35557ruvzbfBV5oi1gXOQqg6o83y9iLwTn9BMKnAch3nz5nHHHXcwatSoiBIF2E5iLYWbBs43ROSC4JPAVob26z1DBNso1q1bx+HDh6O+jjVkZj43JYsRwD9EJLhkf09gt4i8hX89i6TPETHx0bAxc9asWV6HZFKYm2RhSx9loOLi4oh7PUzL5qbr9F0R+QJwVt3zVfWNRAZmEis3N5czzzzTEoVxrdlkISI/Bf4N2I9/KwACf45PXFgmURzHoby8nM6dO7N69Wpawr4tJj7cVEMuB85W1cpEB2MSy3Ecpk6dSklJCZs3byYrywZNGffcJIud+Nez+DDBsZgECiaKjRs3UlhYaInCRMxNsrgf/3oWO4GK4IuqOj1hUZm4apgorI3CRMNNslgJPAi8xefbAZg08r3vfc8ShYmZm2RRqqrLEx6JSZj77ruPr33ta0ybNs3rUEwaczOC8zURuV9ERovI8OAj4ZGZmDiOw7333ktVVRXdunWzRGFi5qZkMSzw5wV1XrOu0xRWd2Tm+PHjGT16tNchmQzgZlDWuGQEkkm8XB6v4RBuSxQmXtysZ9FVRH4jIn8OPB8gItclPrT01NTmwU3tip4I0SxcY4xbbtosnsK/a3q3wPM9wM2JCijdxXOvjJoapbiiihoN/NlMwtm9ezc7d+60RGESwk2bRSdVXS0iiwFUtUpEbAmkEOK1xFwkC8pUVVWRnZ1Nfn4+hw4dokOHDiGuakz03JQsSkSkI4F5IYG1LU7EemMR+Z6I7BKRt0XkoVivlyritcSc2xKK4ziMGzeOFStWAFiiMAnjJlncArwAnC0ifweeBr4Xy01FZBxwGTBUVZlDJvAAAApnSURBVAcCy2K5XiqJ12ZBbkoowTaKjRs30rVr17jEb0wobnpD3hCRMcC5gAC7VfVkjPf9NvCAqlYE7pEx807itcRcsIRSd6PkYAnllFbZ1phpks7txsijgKHAcGCeiMyP8b79gK+IyCYR2SAiI0OdKCILRGSLiGz56KOPYrxtcsRjiblwJZSqqiqmTp1qicIklZvVvZ8Bzga2AcEKs6rqwmbe9zJwRhOHluDfWHk9sBAYiX+3sy9qM8G0tNW9w43XeOyxx+jevbslChNXsa7unQ8MaO6L3JCqTgwT0LeBtYFrvi4iNUAnID2KDknScDtDx3HYu3cvw4cP56abbvI4OtPSuKmG7KTpEkIs/giMAxCRfkAu8HGc75FRgm0UEydO5MSJmDujjIlYyJKFiPw3/u7S9sA7IvI68VvP4kngycAaGZXA1ZGWXFqSho2Zp512mtchmRYoXDUkYd2ZgSX6vpGo63shUfNBrNfDpIpwyeII0FVV/173RRH5MvCvhEaVZhK5fd+jjz5qicKkhHBtFv8OfNbE6ycCx0xAPOeDNPSjH/2IDRs2WKIwnguXLLqq6lsNXwy81jthEaWheM0HCXIch2uuuYajR4+SnZ3NhRdeGI8wjYlJuGQRbpJBm3gHks7iNR8EPm+jeOaZZ2hJY0pM6guXLLaIyA0NXxSR64GtiQsp/cRrPkjdxsyCggJbCs+klHANnDcDz4nIlXyeHPLxj4mYmejA0kk85oM0TBRz5sxJYMTGRC5kslDVD4AvBWaIDgq8/KKq/r+kRJZmGo62jFRpaSmO41iiMCnLzazT9fjncZgEKC4uplWrVnTt2pWtW7eSnR15ojEmGdzOOjUJ4DgOkydPZv58/yReSxQmlVmy8EjdNopZs2Z5HY4xzbJk4QFrzDTpyJKFB+bOnWuJwqQdSxYeWLp0KYWFhZYoTFqxFrUkcRyHF198kSuuuMJ2CTNpyUoWSRBso7jqqqvYt2+f1+EYExUrWSRYw/Uo+vbt63VIxkTFShYJZAvXmExiySKBXnrpJTZt2mSJwmQEq4YkgKoiIsyePZs9e/bQp08fr0MyJmZWsoizYNVjw4YNAJYoTMawZBFHwUTx8ssv8/HHtrOBySyWLOKkYWPm7NmzvQ7JmLiyZBEHJSUl1uthMp4lizho3bo155xzjiUKk9GsNyQGjuPgOA7dunXjt7/9rdfhGJNQliyiFGyjOH78ONu3bycnJ8frkIxJKEsWUWjYmGmJwrQE1mYRIRvCbVoqSxYRuvXWWy1RmBbJqiERuu+++5g5cyZTpkzxOhRjksqTkoWI5IlIkYhsE5EtIjLKizjcchyHpUuXUllZSadOnSxRmBbJq2rIQ8DdqpoH3BF4npIcx2Hq1Kncf//9FBUVeR2OMZ7xKlkocGrg76cB73sUR1jBRLFx40YKCwu56KKLvA7JGM941WZxM/AXEVmGP2F9KdSJIrIAWADQs2fP5ERH40RhjZmmpUtYshCRl4Ezmji0BJgALFLVNSJyOfAbYGJT11HVx4HHAfLz8zVB4TZy8OBBdu3aZYnCmABRTdr37/ObipwAOqiqiogAJ1T11Obel5+fr1u2bElobJWVleTm5gL+0kX79u0Tej9jUomIbFXV/KaOedVm8T4wJvD38cBej+Kox3EcJkyYwAMPPABgicKYOrxKFjcAj4jIduA+Am0SXqrbRmErcBvTmCcNnKr6N2CEF/duijVmGtO8Fj/cu7q6mmnTplmiMKYZLT5ZZGVlMX/+fEsUxjSjxc4NcRyHnTt3Mnr0aK677jqvwzEm5bXIkkVwmvmkSZM4duyY1+EYkxZaXMmi4XoUHTt29DokY9JCiypZ2MI1xkSvRSWLX/7yl5YojIlSi6qG3HrrrYwZM4ZRo1J6+QxjUlLGlywcx+HKK6/k3XffxefzWaIwJkoZnSyCbRSrVq1i+/btXodjTFrL2GTRsDFz+vTpXodkTFrLyGRhvR7GxF9GJouqqiqqqqosURgTRxnVG+I4Djk5OXzhC1/gH//4Bz5fRuZCYzyRMd+mYNXj8ssvR1UtURgTZxlRsmjYRuFfqc8YE09p/+vXGjONSY60TxZXXXWVJQpjkiDtk8Xdd9/N6tWrLVEYk2BpmSwcx+HJJ58EYOjQocyaNcvjiIzJfGmXLIJtFAsWLODtt9/2OhxjWoy0ShY1NTW1jZkFBQUMHDjQ65CMaTHSqut0z549lJeXU1BQwJw5c7wOx5gWJa2SRWlpKatWrbJEYYwHPNnrNFoi8hHwrge37gR87MF9m5OqcYHFFi2vY+ulqp2bOpBWycIrIrIl1GaxXkrVuMBii1Yqx5ZWDZzGGO9YsjDGuGLJwp3HvQ4ghFSNCyy2aKVsbNZmYYxxxUoWxhhXLFkYY1yxZOGCiOSJSJGIbBORLSKSUpuPiMj3RGSXiLwtIg95HU9DInKriKiIdPI6liAReTjwme0QkedEpIPH8UwWkd0isk9EbvcyllAsWbjzEHC3quYBdwSepwQRGQdcBgxV1YHAMo9DqkdEzgIuAf7P61ga+CswSFWHAHuAxV4FIiJZwApgCjAAmCciA7yKJxRLFu4ocGrg76cB73sYS0PfBh5Q1QoAVf3Q43gaehT4If7PMGWo6kuqWhV4WgT08DCcUcA+VT2gqpVAIf5fACnFkoU7NwMPi8h7+H9ze/ZbqAn9gK+IyCYR2SAiI70OKEhELgOOqGqqbwd3LfBnD+/fHXivzvPDgddSSlpNJEskEXkZOKOJQ0uACcAiVV0jIpcDvwEmpkhs2cDpwAXASGC1iHxRk9Qn3kxsP8JfBfFEuNhU9fnAOUuAKuB3yYwtHdk4CxdE5ATQQVVV/EuHn1DVU5t7XzKIyDrgQVVdH3i+H7hAVT/yOK7BwCtAaeClHvirb6NU9ahngdUhIv8GfBOYoKqlzZyeyDhGA3ep6qTA88UAqnq/VzE1xaoh7rwPjAn8fTyw18NYGvojMA5ARPoBuaTAjEpVfUtVu6hqb1Xtjb9oPTyFEsVk/G0p071MFAGbgXNEpI+I5AJXAC94HFMjVg1x5wbgMRHJBsqBBR7HU9eTwJMishOoBK5OVhUkzf0caAX8NbDPTJGqfsuLQFS1SkRuBP4CZAFPqmrKrRlp1RBjjCtWDTHGuGLJwhjjiiULY4wrliyMMa5YsjDGuGJdpyZiItIR/4Ar8I+QrAaCg8BGBeY3mAxjXacmJiJyF1CsqsvqvJZdZ5KWyRBWsjBxISJP4R+wNgz4u4h8Rp0kEhg0Nk1VD4nIN4CF+EebbgK+o6rV3kRu3LI2CxNPPYAvqeotoU4Qkf7AXODCwPog1cCVSYrPxMBKFiaennVRQpgAjAA2B4ZZtwFSbQ0O0wRLFiaeSur8vYr6JdfWgT8FWKmqqbQmiHHBqiEmUQ4BwwFEZDjQJ/D6K8DXRKRL4NjpItLLkwhNRCxZmERZA5wuIm8DN+Jf5xJVfQf4MfCSiOzAvxbmmZ5FaVyzrlNjjCtWsjDGuGLJwhjjiiULY4wrliyMMa5YsjDGuGLJwhjjiiULY4wr/x/CTfhwHb2e1wAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plot_parity(df.logSolubility, df.preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GE9Tof7UK8cI"
      },
      "source": [
        "# Predict from SMILES list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kswx6y_uJHqW",
        "outputId": "0e6dfa98-c6d6-4d57-ff03-2bb7465f9e9e",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "Validating SMILES\n",
            "Test size = 3\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  6.31it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.33it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to /dev/null\n",
            "Elapsed time = 0:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "smiles = [['CCC'], ['CCCC'], ['OCC']]\n",
        "arguments = [\n",
        "    '--test_path', '/dev/null',\n",
        "    '--preds_path', '/dev/null',\n",
        "    '--checkpoint_dir', 'test_checkpoints_reg'\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args, smiles=smiles)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "54x-eGwxLEZ1"
      },
      "source": [
        "# Load model once, predict multiple times"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ICmfelGD7rcf",
        "outputId": "f0c65fb0-e3b6-4d96-e05b-aa98226beedc",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "Validating SMILES\n",
            "Test size = 3\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 0/1 [00:00<?, ?it/s]\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.25it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.76it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to /dev/null\n",
            "Elapsed time = 0:00:00\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "Validating SMILES\n",
            "Test size = 3\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  0%|          | 0/1 [00:00<?, ?it/s]\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.20it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.66it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to /dev/null\n",
            "Elapsed time = 0:00:00\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', '/dev/null',\n",
        "    '--preds_path', '/dev/null',\n",
        "    '--checkpoint_dir', 'test_checkpoints_reg'\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "\n",
        "model_objects = chemprop.train.load_model(args=args)\n",
        "\n",
        "smiles = [['CCC'], ['CCCC'], ['OCC']]\n",
        "preds = chemprop.train.make_predictions(args=args, smiles=smiles, model_objects=model_objects)\n",
        "\n",
        "smiles = [['CCCC'], ['CCCCC'], ['COCC']]\n",
        "preds = chemprop.train.make_predictions(args=args, smiles=smiles, model_objects=model_objects)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7jCqwGfYqj4w"
      },
      "source": [
        "# Reactions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "6RltQMn8qj4w",
        "outputId": "3fb4a758-4b18-4b62-92ef-aa8553d65b10"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                   AAM         ea\n",
              "0    [O:1]([C:2]([C:3]([C:4](=[O:5])[C:6]([O:7][H:1...   8.898934\n",
              "1    [C:1]1([H:8])([H:9])[O:2][C@@:3]2([H:10])[C@@:...   5.464328\n",
              "2    [C:1]([C@@:2]1([H:11])[C@@:3]2([H:12])[C:4]([H...   5.270552\n",
              "3    [C:1]([O:2][C:3]([C@@:4]([C:5]([H:14])([H:15])...   8.473006\n",
              "4    [C:1]([C:2]#[C:3][C:4]([C:5](=[O:6])[H:12])([H...   5.579037\n",
              "..                                                 ...        ...\n",
              "495  [C:1]([c:2]1[c:3]([H:10])[n:4]([H:11])[n:5][c:...  14.662005\n",
              "496  [C:1]([C@@:2]1([H:11])[N:3]([H:12])[C@:4]1([C:...   5.146017\n",
              "497  [C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C:...   8.362688\n",
              "498  [C:1]([c:2]1[c:3]([H:11])[o:4][c:5]([O:6][H:12...  12.416359\n",
              "499  [N:1]([c:2]1[c:3]([O:4][H:10])[n:5]([H:11])[c:...   7.785556\n",
              "\n",
              "[500 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2d248069-2bb3-4ac5-a85b-64f9892a9355\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AAM</th>\n",
              "      <th>ea</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[O:1]([C:2]([C:3]([C:4](=[O:5])[C:6]([O:7][H:1...</td>\n",
              "      <td>8.898934</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[C:1]1([H:8])([H:9])[O:2][C@@:3]2([H:10])[C@@:...</td>\n",
              "      <td>5.464328</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[C@@:3]2([H:12])[C:4]([H...</td>\n",
              "      <td>5.270552</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[C:1]([O:2][C:3]([C@@:4]([C:5]([H:14])([H:15])...</td>\n",
              "      <td>8.473006</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[C:1]([C:2]#[C:3][C:4]([C:5](=[O:6])[H:12])([H...</td>\n",
              "      <td>5.579037</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>495</th>\n",
              "      <td>[C:1]([c:2]1[c:3]([H:10])[n:4]([H:11])[n:5][c:...</td>\n",
              "      <td>14.662005</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>496</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[N:3]([H:12])[C@:4]1([C:...</td>\n",
              "      <td>5.146017</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>497</th>\n",
              "      <td>[C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C:...</td>\n",
              "      <td>8.362688</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>498</th>\n",
              "      <td>[C:1]([c:2]1[c:3]([H:11])[o:4][c:5]([O:6][H:12...</td>\n",
              "      <td>12.416359</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499</th>\n",
              "      <td>[N:1]([c:2]1[c:3]([O:4][H:10])[n:5]([H:11])[c:...</td>\n",
              "      <td>7.785556</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>500 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2d248069-2bb3-4ac5-a85b-64f9892a9355')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2d248069-2bb3-4ac5-a85b-64f9892a9355 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2d248069-2bb3-4ac5-a85b-64f9892a9355');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "reaction_reg_df = pd.read_csv('tests/data/reaction_regression.csv')\n",
        "reaction_reg_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "faZ5arGiqj4w",
        "outputId": "a0a781ad-1e44-4678-870b-2cf53f9522af"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/reaction_regression.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'regression',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'mse',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'rmse',\n",
            " 'metrics': ['rmse'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': True,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_reaction',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['AAM'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'random',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['ea'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "500it [00:00, 72106.73it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 32351.47it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 3464.18it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "500it [00:00, 69566.51it/s]\n",
            "Total size = 500 | train size = 400 | val size = 50 | test size = 50\n",
            "Fitting scaler\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=193, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=465, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 378,601\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:03,  2.20it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:02,  2.48it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:01<00:02,  2.45it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:01<00:01,  2.38it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:02<00:01,  2.27it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:02<00:00,  2.19it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:03<00:00,  1.90it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:03<00:00,  1.89it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.89it/s]\u001b[A\n",
            "                                             \u001b[AValidation rmse = 2.271719\n",
            " 20%|██        | 1/5 [00:04<00:18,  4.68s/it]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.31it/s]\u001b[ALoss = 7.7459e-01, PNorm = 34.6702, GNorm = 6.2131, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.68it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  6.75it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  7.04it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  7.29it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  7.97it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  7.95it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  8.18it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.165562\n",
            " 40%|████      | 2/5 [00:05<00:08,  2.69s/it]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 10.83it/s]\u001b[ALoss = 8.0665e-01, PNorm = 34.7091, GNorm = 10.4497, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00,  9.40it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  7.63it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  7.55it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  7.57it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  7.62it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.293178\n",
            " 60%|██████    | 3/5 [00:07<00:03,  1.98s/it]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  7.64it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  9.51it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  7.53it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  6.46it/s]\u001b[ALoss = 7.1648e-01, PNorm = 34.7279, GNorm = 2.6229, lr_0 = 2.3714e-04\n",
            "\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  7.92it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  8.29it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.401798\n",
            " 80%|████████  | 4/5 [00:08<00:01,  1.65s/it]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.20it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  7.94it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.68it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  7.77it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  7.52it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  7.13it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  6.87it/s]\u001b[ALoss = 6.5946e-01, PNorm = 34.7336, GNorm = 4.1201, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:01<00:00,  7.33it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.034656\n",
            "100%|██████████| 5/5 [00:09<00:00,  1.93s/it]\n",
            "Model 0 best validation rmse = 2.034656 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test rmse = 1.969818\n",
            "Ensemble test rmse = 1.969818\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test rmse = 1.969818\n",
            "Overall test rmse = 1.969818 +/- 0.000000\n",
            "Elapsed time = 0:00:12\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/reaction_regression.csv',\n",
        "    '--dataset_type', 'regression',\n",
        "    '--save_dir', 'test_checkpoints_reaction',\n",
        "    '--epochs', '5',\n",
        "    '--reaction',\n",
        "    '--save_smiles_splits'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U0uWRx5Zqj4w",
        "outputId": "013b8063-d188-40da-9cb8-5572db99f4be"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "50it [00:00, 19145.08it/s]\n",
            "100%|██████████| 50/50 [00:00<00:00, 40423.13it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 50\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  3.15it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  1.16it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_reaction.csv\n",
            "Elapsed time = 0:00:02\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_reaction/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_reaction.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_reaction'\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "0jAsBzayqj4w",
        "outputId": "f9ed6762-b6fe-4b9c-867c-e658804364c2"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQkAAAEGCAYAAAB2PmCxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZhU1bW331U9AE1QZBDFViFxIrTQYBMlKIZJBoEIiMp1uqjR78sgONwYNd6YT4wmYkw0ehOT6xCTdIOI0cQnXJxQrgFkEBSVSQYBRRACHLob2qbX90dVtdVFVfXprjp1aljv85ynqk6dOufXQ/3O2mvvvbaoKoZhGPEI+C3AMIzMxkzCMIyEmEkYhpEQMwnDMBJiJmEYRkIK/Rbghi5dumiPHj38lmEYOUdDQwPr1q2jurr6c1XtGuuYrDCJHj16sGzZMr9lGEZO4TgOo0eP5uDBgwBb4h1nzQ3DyEPCBrF48WIqKysTHmsmYRh5yOWXX95oEJMnT054rJmEYeQhP/3pT5k1a1azBgFZbBLbtm1jwIABFBQUICK2RWwFBQUMGDCAbdu2+f1nMjIIx3F46qmnAOjXrx+TJk1y9bmsNYkJEyYwceJEamtrUVXbIrba2lomTJjAhAkT/P4zGRlCOAdx3XXX8eGHH7bos5INE7wqKio0unejoKCA2tpaiouLfVKV2dTV1dGuXTsOHz7stxTDZyKTlFVVVVx88cVHHCMiy1W1ItbnszaSaGhoMINIQHFxMQ0NDX7LMHzGjUE0R9aahGEYzfPGG2+wdOnSVhsE5JhJiAhXXHFF4+v6+nq6du3K2LFjmxx30UUXcc455zTZd/fdd3PCCSdQXl7euO3duzcpPc8++yy9e/cmEAgkHAy2d+9eLr74Ys444wx69erFokWLALj00ksbtfTo0YPy8vKk9Bj5QziNMHbsWDZs2NBqg4AcM4n27duzevVqamtrAXj55Zc54YQTmhyzd+9eli9fzr59+9i4cWOT92666SZWrlzZuHXs2DEpPWVlZcydO5fBgwcnPG7atGmMGjWKNWvWsGrVKnr16gXArFmzGrVMmjSJiRMnJqXHyA8cx2HEiBHMmzcPgBNPPDGp8+WUSQCMGTOGl156CYDKykqmTJnS5P25c+cybtw4LrvsMqqqqjzV0qtXL04//fSEx+zbt48333yTa6+9FgjmEqLNSVWZPXv2ET+LYUQTzkEsWLCAAwcOpOScOWcS4S//wYMHeffddzn77LObvB82jilTphwxHPWhhx5qDO+HDBlyxLkdx2nSHIncPvjgg1bp3bRpE127dmXq1Kn069eP6667jurq6ibHLFy4kG7dunHqqae26hpGfpCKJGUssmKCV0vo06cPmzdvprKykjFjxjR577PPPmP9+vWce+65iAhFRUWsXr2asrIyINjcuPXWW+Oeu0OHDqxcuTKleuvr61mxYgWPPPIIZ599NtOmTeP+++/nnnvuaTwmVkRkGJHU1NR4YhCQgyYBMH78eG699VYWLFjA7t27G/fPnj2bf/3rX/Ts2ROA/fv3U1lZyb333uvqvI7jcN5558V87y9/+Qtf//rXW6y1tLSU0tLSxojn4osv5v777298v76+nrlz57J8+fIWn9vIH9q2bUvfvn2ZPn16Sg0CctQkrrnmGjp27MiZZ57JggULGvdXVlYyb948Bg4cCARD/eHDh7s2CS8iieOOO44TTzyRtWvXcvrpp/Pqq682MZtXXnmFM844g9LS0pRe18gNHMdhz549nHzyyTz66KOeXCPnchIQvDvfeOONTfZt3ryZLVu2NOn67NmzJ0cffTRLliwBmuYkysvL2bx5c1I6nn/+eUpLS1m0aBEXXnghI0eOBOCTTz5p0hR65JFHuPzyy+nTpw8rV67kjjvuaHyvqqrKmhpGTMI5iGHDhnHo0CHPrpO1w7JFhGzQ7if2O8pdUp2kzMlh2YaRr3jVixEPMwnDyDJuv/32tBkE5Gji0jBymRkzZjBu3LjGHJfX5FQkUVBQQHl5OWVlZYwbN65x7sXmzZsREX784x83Hvv5559TVFTE97//fQDWrl3Lt771LcrLy+nVqxfXX389AAsWLODoo49uktB85ZVXktK5Zs0aBg4cSJs2bZg5c2bc41577TX69+9PWVkZV199NfX19QA88MADjVrKysooKChgz549SWkyMhvHcbjtttuora2lY8eOaTMIAN8LpLjZzjrrLI0mKL0p7du3b3x+1VVX6YwZM1RVddOmTdqzZ08tLy9vfP+xxx7Tvn376ve+9z1VVb3gggv0r3/9a+P77777rqqqvv7663rhhRceca1k+Oyzz/Ttt9/WO+64Qx944IGYxxw+fFhLS0t17dq1qqp611136R/+8IcjjnvxxRd1yJAhMc8R63dkZB/79+/Xc889VwsKCvS1117z5BrAMo3z/cupSCKSgQMHsn379sbXJSUl9OrVq3E25qxZs7jkkksa3//000+bjEU488wzPdN27LHHMmDAAIqKiuIes3v3boqLiznttNMAGDFiBM8999wRx9lozNzGcRzGjBnDokWLqKqqijldwGty0iQOHz7Mq6++yvjx45vsD8/r2Lp1KwUFBXTv3r3xvZtuuomhQ4cyevRoHnrooSbTxBcuXNikufHRRx8dcc3Iad2R2x//+MdW/QxdunShvr6+0dTmzJnD1q1bmxxTU1PDvHnzXNcqNLKLaINIR5IyFjmVuKytraW8vJzt27fTq1cvRowY0eT9UaNGcdddd9GtWzcuvfTSJu9NnTqVkSNHMm/ePF544QV+97vfsWrVKgDOO+88/v73vye89qxZs1L6s4gIVVVV3HTTTRw6dIgLLriAgoKCJsf87W9/Y9CgQXTq1Cml1zYyg+3bt/PRRx/5ahCQY5FEu3btWLlyJVu2bEFVjximWlxczFlnncWDDz4Y85fevXt3rrnmGl544QUKCwtZvXq162unOpKAYJNp4cKFvP322wwePLix6RHGRmPmJocOHUJVOeOMM1i/fr2vBgE5FkmEKSkp4eGHH+aiiy7iu9/9bpP3brnlFs4///wj7r7z5s1j2LBhFBUVsWPHDnbv3s0JJ5zAmjVrXF0z1ZEEwM6dOzn22GM5dOgQP//5z7nzzjsb39u3bx9vvPEGf/rTn1J+XcM/wk2Mb33rW9xzzz20b9/eb0m5FUlE0q9fP/r06XNEzYjevXtz9dVXH3H8/PnzKSsro2/fvowcOZIHHniA4447DjgyJzFnzpyktO3YsYPS0lJ++ctfMmPGDEpLS9m/fz8QLJrzySefAMGuzl69etGnTx/GjRvH0KFDG8/x/PPPc8EFF2TEP5GRGiJzEH379vVbTiOezd0QkSeAscBOVS0L7XsAGAfUAR8BU1W12UKSNnejddjvKHvwO0np19yNp4BRUfteBspUtQ+wDrjdw+sbRlbQ0NDAuHHjfO/FiIdnJqGqbwJ7ovbNV9X60MvFgBVJMPKeQCDADTfckJEGAf4mLq8BWp3tCwQC1NXV2QI9cairqyMQyNmUU07gOA4rVqzg/PPPz+heKl/+i0TkTqAe+HOCY64XkWUismzXrl1HvN+/f39mzpxJXV2dh0qzk7q6OmbOnEn//v39lmLEITzde8yYMezcudNvOYmJN147FRvQA1gdte/fgUVAidvzxJq7sXXrVq2oqNBAIKCAbRFbIBDQiooK3bp1a1Lj+Q1v2L9/vw4aNEgLCgr02Wef9VuOqiaeu5HW5oaIjAJ+CJyvqjXJnKu0tJSlS5emRphhpIl0F4xJBZ41N0SkkmDEcLqIbBORa4HfAB2Al0VkpYj81qvrG0Ym8uSTT2aVQUAW17g0jGxEVXnnnXcyLl9kNS4Nw0ccx+HSSy9l/fr1iEjGGURzmEkYhoeEcxDPPfcc77//vt9yWoWZhGF4RHSS8qKLLvJbUqswkzAMD8jGXox4mEkYhkcUFhZmvUFAjtaTMAy/cByHQCBAhw4deP311xERvyUljZmEYaSIcBOjXbt2zJ8/PycMAswkDCMlROcgcsUgwHIShpE0uZSkjIWZhGEkydSpU3PWIMBMwjCSZsaMGcyZMycnDQLMJAyPaWhQDhyqp0FDjw2ZP1fIDY7j8NhjjzWWvs/WgVJusMSl4RkNDcru6jpurHyHpZv3MKBHJx6e0o/O7YsJBLI3sReZgxg0aFBGVbb2AoskDM+o+eIwN1a+w6KNu6lvUBZt3M2Nle9Q88Vhv6W1mkiDqKyszHmDADMJw0NKigtYurlJLWSWbt5DSXFBnE9kNtEGMXnyZL8lpQUzCcMzauoOM6BH05XSBvToRE1ddkYSS5YsYfny5XllEGAmYXhISVEBD0/px8CvdqYwIAz8amcentKPkqL0RBKpSpqGCzMNHz6cjRs35pVBgCUuDQ8JBITO7Yv5/dUVlBQXUFN3mJKigrQkLVOVNHUch/Hjx/P973+fSZMmcfzxx3uoOjOxSMLwlEBA+EqbQgISekxTr0YqkqbhHMTChQtpaGjwUG1mYyZh5CTJJk3zNUkZCzMJIydJJml68OBBM4gIzCSMnCSZpGmbNm0YNGiQGUSIuCX1ReQRgitCxURVb/RKVDRWUt9oDQ0NSs0Xh10nTR3HYceOHZx66qlpVJkZtLak/jJgOdAW6A+sD23lgK3Sa2Q8LUmahnMQQ4cOpba2No0qM5+4XaCq+jSAiPxf4FxVrQ+9/i2wMD3yDMN7outBtGvXzm9JGYWbnMQxwFERr78S2mcYWU+uF4xJBW4GU90PvCMirwMCDAbu9lKUYaSLu+++2wyiGVytBSoixwFnh14uUdUdnqqKwhKXhldUV1ezePFihg0b5rcUX0lqLVAJVvQcDvRV1ReAYhH5Roo1GkbacByHadOm4TgO7du3z3uDaA43OYnHgIHAlNBrB3jUM0WG4SHhHMSjjz7K4sWL/ZaTFbjJSZytqv1F5B0AVf2XiFgXqJF1RCcpR4wY4bekrMBNJPGFiBQQGlglIl2B/J3tYmQludaLkc7aoW5M4mHgeeBYEbkX+F/gPs8UGYYH7Nq1i23btuWMQeyuruM7Ty/jtDv/wXeeXsbu6jrPjMJt78YZwDCCXaCvquqHnqiJg/VuGK2ltraWtm3bIiIcPHiQtm3b+i0paQ4cquc7Ty9j0cbdjfsGfrUzv7+6gq+0aV2JmGR7N55R1TWq+qiq/kZVPxSRZ1x87gkR2SkiqyP2dRKRl0VkfejRBmVFkasl6P3AcRxGjBjBzTffDJATBgHprx3qprnRO/JFKD9xlovPPQWMitr3I4KRyKnAq6HXRoh0h5G5THTZ+1wi3bVD45qEiNwuIg7QR0T2hzYH2Am80NyJVfVNYE/U7m8DT4eePw3k7oomrSAXS9Cng+joq6amJqeSlNGku3Zoogle9wH3ich9qnp7iq7XTVU/DT3fAXSLd6CIXA9cD3DSSSel6PKZTa6VoE8HsWpZ/mJCL04+uQfTp09Pi0G0dEp6sqS7dqib5sbbInJ0+IWIdBSRpCMADWZME9WreFxVK1S1omvXrsleLivItRL06SBW9PXD5z/koUceTZtB+NFETGftUDcm8RNV3Rd+oap7gZ+08nqficjxAKHHna08T07idwn6bCRe9NXlmKPifCK15EMT0U1/SSwjaW0p/heBqwnOLL0aF7mNfMLPEvSJSHc43RLC0Vdkd2A4+mptd2BLyIcmoptIYpmI/FJEvhbafkmwYlVCRKQSWAScLiLbRORaguYwQkTWE5w0dn8y4nMRv0rQxyPTe1wCh+v4xYRevkVf+dBEbHYwlYi0B+4i+KUGeBmYoarVHmtrxAZT+YcXA3dSheM4jBkzhpNOOpmHHnmULscclfZIJ1bi9NdTyulcUkxBQaDxmEyNxMIkGkzV7F85ZAY2niFPaUk4ne4vQ1VVFYsWLWLatGkc2ymYW0+3cQUCQqeSIn535Vm0b1PIhp0HqFryMVPOPpnO7YPzIFOxkpifxP2NisivVHW6iPyNGL0QqjreU2VGRuC2zZ+qZfVawnXXXcfAgQMpKyvz5Pxuqa1v4IZnljf5HS3auIffXx28MYcTm8H9wcRmJkRibkmUkwgPvZ4JPBhjM3KIeMPB3fa4pCvL7zgOEyZMYPXq1YiI7wYBiaOtXEhsJhpMtTz0+Eb65Bh+0FwU4KbHJR1fhsih1ldddVVGGAQkjrbCz/3qfUkFiYZlvyci78bb0inS8JbmogA3PS5eZ/mj60FMmDAhJedNBYmirVwY+5LIysaGHr8Xegw3P64gwUhJI/tIRRQQ/jJERyOp+DL4UTCmJUnY5qKtTBz70hISNTe2AIjICFXtF/HWbSKyAuvxyBlSMSDJy4FghYWFHHXUUQkNIpU9K61JwoajLTiyhyXRe9mAm8FUIiKDIl580+XnjCwhVSFxqgeCOY7D3r17adeuHS+99FJCg0jlgK98GGrdEtzY2rXAExGTvPYC13gnyUg3mTgcPNzEUFUWLlxIIBD/vhT5pYbkuxlzoUcilbgZTLUc6Bs2icjJXkbukEkhcXQOIpFBQOq/1H7PB8k03JSv6yYi/w1Uqeo+Efl6aB6GYaSc1iQpU92zEt38unn4qfzuyrMoKS7Iy5KCbuZu/AN4ErhTVfuKSCHwjqqemQ6BYHM38ol/+7d/Y/bs2S3qxfBitGc4EdquKMDu6jqmVa7M2mHVbkg0d8ONSSxV1QEi8k64l0NEVqpquQdaY2ImkT9s2rSJ9957j/HjWzbq36t5I5k8wS2VJFUtG6gWkc58uTjPOYDlJYyU4TgODz74IA0NDfTs2bPFBgHeTbG3JKY7k7iZYLGYr4nIW8AfgR94qsrIG8I5iNtuu40VK1b4LecI8qFeRHMkNIlQ+fzzQ9s3gRuA3qpqw7KNpIlOUlZUxIx2gdSvR+L2fLkwrDpZEjaqVPWwiExR1YeA99OkycgDIg1i4f++xZn9zqJBNWY+IdWJyZacLxPHkKQbN82Nt0TkNyJynoj0D2+eKzOSJvJuWVNXz4GDmbMy2KpVq1i5ciUL//ctTuldnnC0ZKpHQLb0fJlWUjDduEnPhnsx/l/EPgWGpl6OkQoaGkJ35DYFfO4c4o11OxnWqxv/8ey7vnfjNTQ0EAgEOPfcc9m8eTNtO3Rs0nsQa7RkssnDI3o+UpCMzIaSdKnCTSQxWVWHRG1mEBlK4zyGPwbvzLfPfY9RZcczd/k23+ciOI7DkCFDePrp4CJuXbp0cfWFTSZ5GGtex4GD9UklI4PnPBQV/RzyPTrzikT1JMaJyC7g3VC162+mUVdWkgmL/cYKpadXrWRk2fFNjmvNnTOZny2cg3jrrbdo3779l3pdGEAyycNYv4+n3trEr6eUtzoZWVN3mBsrV0aZ7sqc7fFI1Ny4FzhPVdeIyNnALwj2chgx8KPGYyzi3ZlPOfYrTfa5mYsQGVIfOFTPU29t4uHXNrT4Z3Mch5kPPsjjTzzNGad8lZovDtPQoAQC4qoORTLJw1i/j4df28D3hp7S6mRkSZs40U+b3OzxSNTcqFfVNQCqugTokB5J2UmmTC+Od2c+cKi+RXfO6DD9hmeWc1G/UsaceXyLfra6ujpmPvgg1/zfG7lnwS5O+3HT5GSkAay7dzS/v7oibi9Da5KH8X4ftV80tDoZWXMoTvRzKDcjibjDskVkG/DLiF03R75W1V8e8SGPyIZh2Q2qnHbnP6iPCMMLA8K6e0cTkPRFErEjmnI6tCmkviF4F3Rz54w3HPnu8b0Z+as3W/Szbd+5m1v/ut6Xoc1eRHg1dfXsqa5rkgh+YHIfOrUvpqQ4O4dqt3bdjd/TNHqIfm1EkGh6cUlRgWeZ8FhZ9uZCczdfzOaaLc01VxzH4eOPP6Z3794c37WTb0ObvRjn0LawgA5tCrlv4pmc2KmErXtq6NCmkLaFudncSFS+7qfpFJLtxGtbtysMeJarSHSXTLY2RDzT27DzQLPNlXCScv369Xz00UdQ1NbX+gyprpURCAgd2hZRUBBABLp0aJPTXaDNzgLNBLKhuQGx7+o1Xxz2bBahlzMU4y5f1764caxB7RcNR3w5IkdSVlZWMnny5IxJ6mYqmTDmIqll/gz3xLpjeTmL0MtzxwrT2xUG2FP9RdwveyyDiHeuXL7ztoRsMFAraOsxXs4i9HqGYnSPQm19Q8IenPvuu+8Ig4h3rlQ0tfwek5IKMqVXLBFuytd1FpFHRGSFiCwXkV+H6ksYLvByFmG6y6w1F7n853/+J6+88soRBpFqUl0d20+yoV6Fm+ZGFfAmMCn0+nJgFjDcK1HpIF3twEShdrIaIs8dLrN2wzPLPQtb4yUzt326k6+0LaJTp04MHnw+Bw7Ve/p7TXV1bD/JhqK7bpobx6vqPaq6KbTNALp5LcxLWnsnam2IGyvUTtXdMHzu2i8amHbEUOHUhq2xoqJfTOjF7f9xC2+//Xba7vDZcPd1SzbUq3BjEvNF5DIRCYS2S4D/8VqYl7SmHZjpC8Ck44sTPTrynpGl/Gj695gw4SJGjRqVtvZ1LlWLcjvi1E/cmMR3gL8AdaGtCrhBRBwR2e+lOK9ozRcq07/U6fjiRDaPNn68nZ/c8SMmTLiosap1uu7w2XD3bQmZXq/CzeI8KR9lKSI3AdcRrEvxHjBVVQ+m+jphotv+AWn5cvCp/AKEmy2pbIt6uWBvWHN0V91Dj/6OY48qaTwmXe1r61JNL666QEVkvIjMDG1jm/9EwnOdANwIVKhqGVAAXJbMORMRq5lQfaie317Rv0V3olTeqWu+OMxTb23i55P6NNHw6ynlrf5Sex22xoqkbpr9XpNIKp13+Ey/++YSzdq7iNwPDAD+HNo1TUQGqertSV63nYh8AZQAnyRxroSE/7m7dmjDSzeexynHfoWte2po36GwRXeiVN6pS4oLePi1DWzYVc3d43tzyrFfYcPOA0l/qb1cqs9NJGV3+NzEzX/SGKBcVRsARORp4B2gVSahqttFZCbwMVALzFfV+dHHicj1wPUAJ510UmsuBQT/ubsd1YabR5zObc9Flm8rp3P7No13ouZI5RcgHJW8uOoTXlwV9Mcvh1Nn3vg2x3H4bM8+V02JTFpT1EgNbv8jO0Y8PzruUS4QkWOAbwM9ge5AexG5Ivo4VX1cVStUtaJr166tvl5N3WGmDz+N255798hKQi1MOqYqxE134i2Z0YnhodY/ueNHPHTJmTmTLDTc48bqfwa8IyKvAwIMBn6UxDWHA5tUdReAiMwluKbHn5I4Z1xKigo4qXNJRvWrpzMsT2ZugKoyadIkFi9ezPTp0zn2qBJrSuQhzS3OEwAagHOAucBzwEBVnZXENT8GzhGREhERYBjwYRLnS0ggIPErCfnYr56uxFsyXbciwm233da4eK8lC/OThCYRykP8UFU/VdUXQ9uOZC4YKoU3B1hBsPszADyezDmbo6Q4feF9k9D+YD01df5OQmpN163jOPz1r38FYNiwYa5X9zZyEzfNjVdE5FaC8zWqwztVdU/8jyRGVX8C/KS1n28p6QrvY4X2D0zuw8z/Wctn+w+1ai5FsvM7oscujO/bnZtHnAYE61HEqwfx9ttvs379ek4++WTX12oNmVBLwUhMs0VnRGRTjN2qql/1RtKRZEvRmebqQra0IEwqag1EnqPbUW24deTpcRfpiV6b0+sIIhtqKeQLiYrONNu7oao9Y2xpM4hsorm6kC1NlqZiKHhkFDXjojP5j2eje3mC50u3QUB21FIw3NWTaCsiN4vIXBF5TkSmi0jbdIjLNuKNytyw80Dj85YkS1M1FDyccIy7XkRxAS+88EJaDQJyazZnLuNmnMQfgd7AI8BvQs+f8VJUthJr/MMDk/vwXws2tCpZmupJW4nOd8UVV/D++++nNUmZS7M5cxk3OYkPVPXrze3zkmzJSUBUIu7QYQIBaFvUuqRcqtvsMROrE3tRu3cXp55ySovPlyyWk8gcEuUk3JjEn4DfqOri0Ouzge+p6lUpVxqHbDIJN7Qko5/q7H+s6d6TJk1k4sSJrT5nMljvRmaQbLXss4B/isjHodcnAWtF5D2CvRx9UqQzL2jp3dOLNSO0rpbBw75MUvplEGE9Ntcjs3HzVxnluYo8wu/6jAcOHEh7L4aR3bgpOrMlNCnrxMjjVXWFl8JyFb8z+sXFxRx//PFmEIZr3NSTuAf4d+AjgpWkCD0O9U5W7uJXdWTHcTh48CBdu3Zl9uzZSBoXMTayGzf/lZcAX1PVOq/F5ANel5mLheM4jBkzhurqapYuXUpBgY1DMNzjxiRWE6wnsdNjLXlBuqs3hQ1i0aJFVFVVmUEYLcaNSdxHsJ7EauBQeKeqjvdMVY6Trox+tEFkSg7Cuj2zCzf/oU8DPyc4rbvBWzlGKvnBD36QkQZhA6iyCzeDqZaq6oA06YlJrg2mSheffPIJK1asYOzYpAqcp5R4M2WzcYm+XCKpWaDAQhG5T0QGikj/8JZijUaKcByHe++9l/r6erp3755RBgH+dwEbLceNdfcLPZ4Tsc+6QNNAS9vukdO9hw4dysCBA9Oo1h3ZsECu0RQ3g6mGpEOI0ZSWtt2j60FkokGAP13ARnK4yUl0I1gxu7uqjhaRrxMshvvf6RAI+ZmTaEnb3Y+CMclgvRuZR7I5iacIriLePfR6HTA9NdKMeLSk7b527VpWr16dFQYBtkRftuHGJLqo6mxC3Z+qWg9YVRCPcVOQpb6+HoCKigo2b96cFQZhZB9uTKJaRDoTmrchIucA+zxVZTS7ypfjOAwZMoRHH30UgI4dOyY6nWG0Gjfp5JuBF4GvichbQFfAblkek2j4dmQOYtq0aX5LNXIcN70bK0TkfOB0gsv8rVXVLzxXZsQcvp1tSUoj+3HbMf0NoEfo+P4igqr+0TNVRkzq6+sZM2aMGYSRVtzUk3gG+Bqwki8TlkqwiraRRgoLC7n44ouZNm2aGYSRNtxEEhXA17W5ARWGZziOw/r16+nfv7/lIIy046Z3YzVwnNdCjNiEcxDDhw9n3z7rVDLST9xIQkT+RrBZ0QH4QETexupJpJXoJOXRRx/ttyQjD0nU3JiZNhXGEVgvhpEpJDKJ7UA3Ve/BGM8AAAlNSURBVH0rcqeInAt86qkqg4ceesgMwsgIEuUkfgXsj7F/X+g9w0PuuOMO3njjDTMIw3cSmUQ3VX0vemdoXw/PFOUxjuMwdepUduzYQWFhIYMGDfJbkmEkNIlEkwHapVpIvhPOQTzzzDPk27R4I7NJZBLLROQ70TtF5DpgeTIXFZGOIjJHRNaIyIcikpkVUtJEZJKysrIy40rOGflNosTldOB5EbmcL02hAigGJiR53V8D81T1YhEpBkqSPF/WEm0QkydP9luSYTQhrkmo6mfAN0VkCFAW2v2Sqr6WzAVF5GhgMMGlAwmtDJa3q4PV1NTgOI4ZhJGxNFu+LuUXFCkHHgc+APoSjFKmqWp11HHXA9cDnHTSSWdt2bIlrTq95sCBA7Rp04aioiLq6+spLLQisIZ/JFu+LtUUAv2B/1LVfkA18KPog1T1cVWtUNWKrl27plujpziOw6hRo7jqqqsAzCCMjMYPk9gGbFPVJaHXcwiaRl4QmYOYOHGi33IMo1nSbhKqugPYKiKnh3YNI9j0yHksSWlkI37FuT8A/hzq2dgITPVJR1q59NJLzSCMrCPticvWkCvrbixatIjt27fbUGsj40iUuLSMmcc4jsNLL73EZZddlrGrahlGIvxIXOYN4RzElVdeyYYNG/yWYxitwiIJj4iuB3HKKaf4LckwWoVFEh5gBWOMXMJMwgPmz5/PkiVLzCCMnMCaGylEVRERJk2axLp16+jZs6ffkgwjaSySSBHhJsYbb7wBYAZh5AxmEikgbBCvvPIKn3/+ud9yDCOlmEkkSXSSctKkSX5LMoyUYiaRBNXV1daLYeQ8ZhJJ0LZtW0499VQzCCOnsd6NVuA4Do7j0L17d5588km/5RiGp5hJtJBwDmLPnj2sWrWKoqIivyUZhqeYSbSA6CSlGYSRD1hOwiU21NrIV8wkXHLLLbeYQRh5iTU3XPKzn/2MCRMmMHr0aL+lGEZasUgiAY7jcNddd1FXV0eXLl3MIIy8xEwiDo7jMGbMGO677z4WL17stxzD8A0ziRiEDWLRokVUVVUxePBgvyUZhm+YSUQRbRCWpDTyHTOJKDZt2sSaNWvMIAwjhPVuhKirq6O4uJg+ffqwceNGOnTo4Lckw8gILJIg2MQYNmwY999/P4AZhGFEkPcmEZmDsIrWhnEkeW0SlqQ0jObJW5M4fPgwY8eONYMwjGbIW5MoKCjgqquuMoMwjGbIu94Nx3FYvXo1AwcO5Nprr/VbjmFkPHkVSYSne48cOZLdu3f7LccwsoK8iSSi60F07tzZb0mGkRXkRSRhBWMMo/XkhUn89re/NYMwjFaSF82NW265hfPPP59vfOMbfksxjKzDt0hCRApE5B0R+bsX53cch8svv5wtW7YQCATMIAyjlfjZ3JgGfOjFicM5iFmzZrFq1SovLmEYeYMvJiEipcCFwB9Sfe7oJOX48eNTfQnDyCv8iiR+BfwQaIh3gIhcLyLLRGTZrl27XJ3UejEMI/Wk3SREZCywU1WXJzpOVR9X1QpVrejataurc9fX11NfX28GYRgpxI/ejUHAeBEZA7QFjhKRP6nqFa09oeM4FBUVccwxx/DPf/6TQCAvenYNIy2k/dukqreraqmq9gAuA15L1iBGjx7NJZdcgqqaQRhGisnqcRLROQgR8VuSYeQcvpqEqi4AFrTms5akNIz0kLWx+ZVXXmkGYRhpIGtN4qc//SmzZ882gzAMj8kqk3AchyeeeAKAvn37MnHiRJ8VGUbukzUmEc5BXH/99bz//vt+yzGMvCErTKKhoaExSVlZWUnv3r39lmQYeUNWdIGuW7eOgwcPUllZyeTJk/2WYxh5RVaYRE1NDbNmzTKDMAwfEFX1W0OziMguYEsLP9YF+NwDOV5gWlNPtuiEzNB6sqrGnCSVFSbRGkRkmapW+K3DDaY19WSLTsh8rVmRuDQMwz/MJAzDSEgum8TjfgtoAaY19WSLTshwrTmbkzAMIzXkciRhGEYKMJMwDCMhOWkSXq/pkSpEpKOIzBGRNSLyoYgM9FtTPETkJhF5X0RWi0iliLT1W1MYEXlCRHaKyOqIfZ1E5GURWR96PMZPjWHiaH0g9D/wrog8LyId/dQYTU6aBB6u6ZFifg3MU9UzgL5kqGYROQG4EahQ1TKggGDpwUzhKWBU1L4fAa+q6qnAq6HXmcBTHKn1ZaBMVfsA64Db0y0qETlnEl6u6ZFKRORoYDDw3wCqWqeqe/1VlZBCoJ2IFAIlwCc+62lEVd8E9kTt/jbwdOj508BFaRUVh1haVXW+qtaHXi4GStMuLAE5ZxK4WNMjQ+gJ7AKeDDWN/iAi7f0WFQtV3Q7MBD4GPgX2qep8f1U1SzdV/TT0fAfQzU8xLeAa4B9+i4gkp0zC7ZoeGUIh0B/4L1XtB1STOSFxE0Lt+W8TNLbuQHsRaXWF83SjwX7+jO/rF5E7gXrgz35riSSnTIIv1/TYDFQBQ0XkT/5Kiss2YJuqLgm9nkPQNDKR4cAmVd2lql8Ac4Fv+qypOT4TkeMBQo87fdaTEBH5d2AscLlm2OClnDKJVK/p4SWqugPYKiKnh3YNAz7wUVIiPgbOEZESCa5bMIwMTbJG8CJwdej51cALPmpJiIiMIthEHq+qNX7riSYr6knkMD8A/iwixcBGYKrPemKiqktEZA6wgmA4/A4ZNJRYRCqBbwFdRGQb8BPgfmC2iFxLsMzAJf4p/JI4Wm8H2gAvh9aOWayq/8c3kVHYsGzDMBKSU80NwzBSj5mEYRgJMZMwDCMhZhKGYSTETMIwjIRYF6jhGhHpTHCyFMBxwGGCQ8sBvqGqdb4IMzzFukCNViEidwMHVHVmxL7CiIlKRo5gkYSRFCLyFHAQ6Ae8JSL7iTCPUN2Esaq6OTTf40agGFgCfFdVD/uj3HCL5SSMVFAKfFNVb453gIj0Ai4FBqlqOcGmyuVp0mckgUUSRip41kVEMAw4C1gaGnrcjgyfdGUEMZMwUkF1xPN6mkao4TJ3AjytqhlVdcloHmtuGKlmM6Ep7yLSn2ANCgj2ilwsIseG3uskIif7otBoEWYSRqp5DugkIu8D3ydYsxFV/QD4MTBfRN4lWNfxeN9UGq6xLlDDMBJikYRhGAkxkzAMIyFmEoZhJMRMwjCMhJhJGIaREDMJwzASYiZhGEZC/j+rL8pHnEqvvAAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "df = pd.read_csv('test_checkpoints_reaction/fold_0/test_full.csv')\n",
        "df['preds'] = [x[0] for x in preds]\n",
        "\n",
        "plot_parity(df.ea, df.preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ROjlinX0qj4x"
      },
      "source": [
        "# Multiple-Molecule Inputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "1dc99YcRqj4x",
        "outputId": "b34db34b-5fe8-4494-bf58-3790829e17e4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                          mol a smiles  \\\n",
              "0    CC(C)CC1C(=O)NC(C(=O)NC(C(=O)NC(C(=O)NCCC(C(=O...   \n",
              "1    CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...   \n",
              "2    CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...   \n",
              "3    CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...   \n",
              "4    C[C@H]([C@H]1C(=O)NCC[C@@H](C(=O)N[C@H](C(=O)N...   \n",
              "..                                                 ...   \n",
              "254  CC(C)CC1C(=O)NC(C(=O)N2CCCC2C(=O)NC(C(=O)NC(C(...   \n",
              "255  CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...   \n",
              "256  CC(C)CC1C(=O)NC(C(=O)NC(C(=O)NC(C(=O)NCCC(C(=O...   \n",
              "257                              CCOP(=O)(O)OP(=O)(O)O   \n",
              "258  CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...   \n",
              "\n",
              "                                          mol b Smiles  synergy  \n",
              "0    CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=C(C=C3)O)N)C...        1  \n",
              "1    CC(C1CCC(C(O1)OC2C(CC(C(C2O)OC3C(C(C(CO3)(C)O)...        0  \n",
              "2    CC(C)(C(=O)O)ON=C(C1=CSC(=N1)N)C(=O)NC2C3N(C2=...        0  \n",
              "3    CC(C)(C)NCC(=O)NC1=CC(=C2CC3CC4C(C(=O)C(=C(C4(...        1  \n",
              "4    CN1C(=NC(=O)C(=O)N1)SCC2=C(N3C(C(C3=O)NC(=O)C(...        1  \n",
              "..                                                 ...      ...  \n",
              "254                           [N+](=O)([O-])[O-].[Ag+]        0  \n",
              "255                                C1=CC=C2C(=C1)C=CN2        1  \n",
              "256  CC1C2C(C(=O)N2C(=C1SC3CC(NC3)C(=O)N(C)C)C(=O)O...        1  \n",
              "257    C1=CC(=CC=C1C(C(CO)NC(=O)C(Cl)Cl)O)[N+](=O)[O-]        1  \n",
              "258  C1C(C(C(C(C1N)OC2C(C(C(C(O2)CN)O)O)O)O)OC3C(C(...        1  \n",
              "\n",
              "[259 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6348093c-24b1-4442-95f7-e3c9e4a3015c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>mol a smiles</th>\n",
              "      <th>mol b Smiles</th>\n",
              "      <th>synergy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>CC(C)CC1C(=O)NC(C(=O)NC(C(=O)NC(C(=O)NCCC(C(=O...</td>\n",
              "      <td>CC1(C(N2C(S1)C(C2=O)NC(=O)C(C3=CC=C(C=C3)O)N)C...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...</td>\n",
              "      <td>CC(C1CCC(C(O1)OC2C(CC(C(C2O)OC3C(C(C(CO3)(C)O)...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...</td>\n",
              "      <td>CC(C)(C(=O)O)ON=C(C1=CSC(=N1)N)C(=O)NC2C3N(C2=...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...</td>\n",
              "      <td>CC(C)(C)NCC(=O)NC1=CC(=C2CC3CC4C(C(=O)C(=C(C4(...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>C[C@H]([C@H]1C(=O)NCC[C@@H](C(=O)N[C@H](C(=O)N...</td>\n",
              "      <td>CN1C(=NC(=O)C(=O)N1)SCC2=C(N3C(C(C3=O)NC(=O)C(...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>254</th>\n",
              "      <td>CC(C)CC1C(=O)NC(C(=O)N2CCCC2C(=O)NC(C(=O)NC(C(...</td>\n",
              "      <td>[N+](=O)([O-])[O-].[Ag+]</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>255</th>\n",
              "      <td>CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...</td>\n",
              "      <td>C1=CC=C2C(=C1)C=CN2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>256</th>\n",
              "      <td>CC(C)CC1C(=O)NC(C(=O)NC(C(=O)NC(C(=O)NCCC(C(=O...</td>\n",
              "      <td>CC1C2C(C(=O)N2C(=C1SC3CC(NC3)C(=O)N(C)C)C(=O)O...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>257</th>\n",
              "      <td>CCOP(=O)(O)OP(=O)(O)O</td>\n",
              "      <td>C1=CC(=CC=C1C(C(CO)NC(=O)C(Cl)Cl)O)[N+](=O)[O-]</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>258</th>\n",
              "      <td>CCC(C)CCCC(=O)NC(CCN)C(=O)NC(C(C)O)C(=O)NC(CCN...</td>\n",
              "      <td>C1C(C(C(C(C1N)OC2C(C(C(C(O2)CN)O)O)O)O)OC3C(C(...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>259 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6348093c-24b1-4442-95f7-e3c9e4a3015c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6348093c-24b1-4442-95f7-e3c9e4a3015c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6348093c-24b1-4442-95f7-e3c9e4a3015c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "multimolecule_df = pd.read_csv('tests/data/classification_multimolecule.csv')\n",
        "multimolecule_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xPRnEIzpqj4x",
        "outputId": "265d990d-17c2-4e5a-88e0-6e156f4aa566"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/classification_multimolecule.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'classification',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'binary_cross_entropy',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'auc',\n",
            " 'metrics': ['auc'],\n",
            " 'minimize_score': False,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 2,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': False,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_multimolecule',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['mol a smiles', 'mol b Smiles'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 1,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'random',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['synergy'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "259it [00:00, 59711.14it/s]\n",
            "100%|██████████| 259/259 [00:00<00:00, 66070.11it/s]\n",
            "100%|██████████| 259/259 [00:00<00:00, 4817.51it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "Class sizes\n",
            "synergy 0: 40.93%, 1: 59.07%\n",
            "0it [00:00, ?it/s]Warning: Repeated SMILES found in data, pickle file of split indices cannot distinguish entries and will not be generated.\n",
            "15it [00:00, 3535.72it/s]\n",
            "Total size = 259 | train size = 207 | val size = 26 | test size = 26\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (sigmoid): Sigmoid()\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "      (1): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=600, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 709,801\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
            " 20%|██        | 1/5 [00:01<00:05,  1.44s/it]\u001b[A\n",
            " 40%|████      | 2/5 [00:03<00:04,  1.61s/it]\u001b[A\n",
            " 60%|██████    | 3/5 [00:05<00:03,  1.83s/it]\u001b[A\n",
            " 80%|████████  | 4/5 [00:07<00:01,  1.82s/it]\u001b[A\n",
            "100%|██████████| 5/5 [00:07<00:00,  1.24s/it]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.36it/s]\u001b[A\n",
            "                                             \u001b[AValidation auc = 0.612500\n",
            " 20%|██        | 1/5 [00:07<00:31,  7.95s/it]Epoch 1\n",
            "\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
            " 20%|██        | 1/5 [00:01<00:04,  1.17s/it]\u001b[A\n",
            " 40%|████      | 2/5 [00:02<00:03,  1.18s/it]\u001b[A\n",
            " 60%|██████    | 3/5 [00:04<00:02,  1.40s/it]\u001b[A\n",
            " 80%|████████  | 4/5 [00:04<00:01,  1.17s/it]\u001b[A\n",
            "100%|██████████| 5/5 [00:04<00:00,  1.27it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  7.06it/s]\u001b[A\n",
            "                                             \u001b[AValidation auc = 0.525000\n",
            " 40%|████      | 2/5 [00:13<00:18,  6.32s/it]Epoch 2\n",
            "\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.40it/s]\u001b[ALoss = 6.6435e-01, PNorm = 45.8618, GNorm = 1.1497, lr_0 = 3.8312e-04\n",
            "\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.32it/s]\u001b[A\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.33it/s]\u001b[A\n",
            " 80%|████████  | 4/5 [00:03<00:00,  1.22it/s]\u001b[A\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.76it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  6.45it/s]\u001b[A\n",
            "                                             \u001b[AValidation auc = 0.525000\n",
            " 60%|██████    | 3/5 [00:16<00:10,  5.03s/it]Epoch 3\n",
            "\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
            " 20%|██        | 1/5 [00:00<00:03,  1.30it/s]\u001b[A\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.34it/s]\u001b[A\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.35it/s]\u001b[A\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.35it/s]\u001b[A\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.93it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  6.50it/s]\u001b[A\n",
            "                                             \u001b[AValidation auc = 0.543750\n",
            " 80%|████████  | 4/5 [00:19<00:04,  4.35s/it]Epoch 4\n",
            "\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.39it/s]\u001b[A\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.38it/s]\u001b[A\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.39it/s]\u001b[ALoss = 6.1639e-01, PNorm = 45.8724, GNorm = 0.7997, lr_0 = 1.0000e-04\n",
            "\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.35it/s]\u001b[ALoss = 5.9861e-01, PNorm = 45.8728, GNorm = 1.6585, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.94it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  7.07it/s]\u001b[A\n",
            "                                             \u001b[AValidation auc = 0.543750\n",
            "100%|██████████| 5/5 [00:23<00:00,  4.64s/it]\n",
            "Model 0 best validation auc = 0.612500 on epoch 0\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test auc = 0.709091\n",
            "Ensemble test auc = 0.709091\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test auc = 0.709091\n",
            "Overall test auc = 0.709091 +/- 0.000000\n",
            "Elapsed time = 0:00:25\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/classification_multimolecule.csv',\n",
        "    '--dataset_type', 'classification',\n",
        "    '--save_dir', 'test_checkpoints_multimolecule',\n",
        "    '--epochs', '5',\n",
        "    '--save_smiles_splits',\n",
        "    '--number_of_molecules', '2',\n",
        "    '--split_key_molecule', '1' # defaults to 0 (1st column) if not specified\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d252c71a-fe9d-4c80-d131-3980c1aa92ce",
        "scrolled": true,
        "id": "ug8U-ZdIqj4x"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "26it [00:00, 59689.06it/s]\n",
            "100%|██████████| 26/26 [00:00<00:00, 50627.62it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 26\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.1.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.66it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  1.51it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_multimolecule.csv\n",
            "Elapsed time = 0:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_multimolecule/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_multimolecule.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_multimolecule',\n",
        "    '--number_of_molecules', '2',\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ra4xqKLHqj4x"
      },
      "source": [
        "# Split Type"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VTlgNjW1qj4x",
        "outputId": "a76314d5-9cab-4b88-d9af-a4e3a47fb19d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/regression.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'regression',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'mse',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'rmse',\n",
            " 'metrics': ['rmse'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': False,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_splits',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['smiles'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'scaffold_balanced',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['logSolubility'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "500it [00:00, 140183.96it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 37730.10it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 60734.20it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "100%|██████████| 500/500 [00:00<00:00, 10684.16it/s]\n",
            "Total scaffolds = 148 | train scaffolds = 85 | val scaffolds = 37 | test scaffolds = 26\n",
            "Label averages per scaffold, in decreasing order of scaffold frequency,capped at 10 scaffolds and 20 labels:\n",
            "Scaffold 0\n",
            "Task 0: count = 134 | target average = -1.964269\n",
            "\n",
            "\n",
            "Scaffold 1\n",
            "Task 0: count = 121 | target average = -2.929471\n",
            "\n",
            "\n",
            "Scaffold 2\n",
            "Task 0: count = 17 | target average = -6.661176\n",
            "\n",
            "\n",
            "Scaffold 3\n",
            "Task 0: count = 12 | target average = -3.795000\n",
            "\n",
            "\n",
            "Scaffold 4\n",
            "Task 0: count = 9 | target average = -2.160778\n",
            "\n",
            "\n",
            "Scaffold 5\n",
            "Task 0: count = 8 | target average = -4.232875\n",
            "\n",
            "\n",
            "Scaffold 6\n",
            "Task 0: count = 6 | target average = -3.096667\n",
            "\n",
            "\n",
            "Scaffold 7\n",
            "Task 0: count = 5 | target average = -3.716800\n",
            "\n",
            "\n",
            "Scaffold 8\n",
            "Task 0: count = 5 | target average = -5.497000\n",
            "\n",
            "\n",
            "Scaffold 9\n",
            "Task 0: count = 4 | target average = -5.857500\n",
            "\n",
            "\n",
            "0it [00:00, ?it/s]Warning: Repeated SMILES found in data, pickle file of split indices cannot distinguish entries and will not be generated.\n",
            "276it [00:00, 61958.25it/s]\n",
            "Total size = 500 | train size = 400 | val size = 50 | test size = 50\n",
            "Fitting scaler\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 355,201\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.62it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.49it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.62it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.66it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  9.07it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.93it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.70it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.63it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.382019\n",
            " 20%|██        | 1/5 [00:01<00:04,  1.12s/it]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  9.24it/s]\u001b[ALoss = 7.9258e-01, PNorm = 34.0214, GNorm = 5.8171, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.79it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.44it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.81it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.43it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.94it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  9.10it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.269503\n",
            " 40%|████      | 2/5 [00:02<00:03,  1.10s/it]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.97it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.74it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  9.03it/s]\u001b[ALoss = 6.3368e-01, PNorm = 34.0696, GNorm = 2.3688, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.60it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.25it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  7.99it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.03it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.28it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.918212\n",
            " 60%|██████    | 3/5 [00:03<00:02,  1.13s/it]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.58it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.80it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.67it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.68it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.68it/s]\u001b[ALoss = 4.3541e-01, PNorm = 34.0939, GNorm = 3.4748, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.69it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.58it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.60it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.776913\n",
            " 80%|████████  | 4/5 [00:04<00:01,  1.13s/it]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  9.34it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  9.18it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  9.06it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  9.32it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.73it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.66it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.86it/s]\u001b[ALoss = 3.7779e-01, PNorm = 34.1042, GNorm = 2.0183, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00,  9.01it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.774567\n",
            "100%|██████████| 5/5 [00:05<00:00,  1.12s/it]\n",
            "Model 0 best validation rmse = 1.774567 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test rmse = 1.813285\n",
            "Ensemble test rmse = 1.813285\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test rmse = 1.813285\n",
            "Overall test rmse = 1.813285 +/- 0.000000\n",
            "Elapsed time = 0:00:07\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/regression.csv',\n",
        "    '--dataset_type', 'regression',\n",
        "    '--save_dir', 'test_checkpoints_splits',\n",
        "    '--epochs', '5',\n",
        "    '--split_type', 'scaffold_balanced',\n",
        "    '--save_smiles_splits'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V6d_mNCkqj4x",
        "outputId": "594f518b-fd1d-463c-dd17-aa5fe6937515"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "50it [00:00, 64686.98it/s]\n",
            "100%|██████████| 50/50 [00:00<00:00, 109683.68it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 50\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.35it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  1.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_splits.csv\n",
            "Elapsed time = 0:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_splits/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_splits.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_splits',\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "wyzQ_o50qj4x",
        "outputId": "0065779f-0869-4f6a-83b7-3ca58208a023"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ0AAAEJCAYAAACOg3IeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXhV5bX48e/KxBBBZKyAAipYZAoYVMRKmRRygQKKwFWxYou3tQVEf7WKWnzUqlesV1qsonXsfRIcaPVWpU5orQIymCAi8yDgwKSwyUBIsn5/nHNiEk5OTs60z7A+z7Of5Ex7r5wk67x7vcMWVcUYY4KV5nYAxpjEYknDGNMoljSMMY1iScMY0yiWNIwxjWJJwxjTKK4lDRE5TUSWicgGEflMRGa5FYsxJnji1jgNETkVOFVV14pIC2ANMF5VN7gSkDEmKBluHVhVvwK+8n7viMjnQCeg3qTRtm1b7dq1a2wCNCaFVFVVsXnzZoqLiw+oartAz3UtadQkIl2B/sBKP4/NAGYAnH766axevTqmsRmT7BzHYfTo0ZSVlQHsauj5rhdCReQk4GVgtqoeqfu4qi5S1VxVzW3XLmACNMY0ki9hrFixgvz8/KBe42rSEJFMPAnjf1V1iZuxGJOKrrzyyuqEMWnSpKBe42YhVIBngUOqOjuY1+Tm5qqdnhgTOZ988gnbt2/nsssuA0BE1qhqbqDXuNnSGAxcDQwTkULvltfYnezZs4eBAweSnp6OiNhWY0tPT2fgwIHs2bMn8r89k7Acx+GZZ54BoH///tUJI1iuJQ1V/beqiqr2VdUc7/Z6Y/czYcIEJk6cSGlpKapqW42ttLSUCRMmMGHChGj8Ck0C8tUwfvazn/H555+HtA/XTk9C4e/0JD09ndLSUrKyslyKKr6Vl5fTrFkzKisr3Q7FuKxm0bOgoIDLL7/8hOfE++lJRFRVVVnCCCArK4uqqiq3wzAuCyZhBCvhk4YxpmHvv/8+q1atCjthQJImDRHhqquuqr5dUVFBu3btGDNmTK3njR8/ngsuuKDWffPmzaNTp07k5ORUb999911Y8bz44ov06tWLtLS0egenlZWVcd5559GvXz969erF7373u+rHVJW5c+fSo0cPevbsyYIFC8KKx6QOX/lhzJgxbN26NeyEAUmaNLKzs1m/fj2lpaUAvPXWW3Tq1KnWc7777jvWrFnD4cOH2b59e63HbrzxRgoLC6u3Vq1ahRVP7969WbJkCRdffHG9z2nSpAnvvvsuRUVFFBYWsnTpUlasWAHAM888w+7du9m4cSOff/45U6ZMCSsekxocx2HkyJEsXboUgNNOOy0i+03KpAGQl5fHa6+9BkB+fj5Tp06t9fiSJUsYO3YsU6ZMoaCgIKqx9OzZk7PPPjvgc0SEk046CYDjx49z/PhxPENZ4M9//jN33nknaWmeX1f79u2jGq9JfL4axnvvvcfRo0cjuu+kTRq+ZFBWVsa6des4//zzaz3uSyRTp049Yfjsww8/XH1qMnTo0BP27ThOrdOXmtuGDaFP0q2srCQnJ4f27dszcuTI6pi3bdvG4sWLyc3NZfTo0WzZsiXkY5jkF8mipz9xMWEtGvr27cvOnTvJz88nL6/2mLFvvvmGLVu2cNFFFyEiZGZmsn79enr37g14Tk9uvvnmevfdokULCgsLIx5zeno6hYWFfPfdd0yYMKE6pmPHjtG0aVNWr17NkiVLmD59Oh988EHEj28SX0lJSVQTBiRxSwNg3Lhx3HzzzSecmrzwwgt8++23dOvWja5du1Ynl2BFq6Xh06pVK4YOHVp9Ltq5c2cmTpwIeAazrVu3LuxjmOTUtGlT+vXrF7WEAUnc0gCYPn06rVq1ok+fPrz33nvV9+fn57N06VIGDRoEwI4dOxgxYgT33ntvUPuNRktj//79ZGZm0qpVK0pLS3nrrbe45ZZbAE8vz7Jly+jWrRvvv/8+PXr0iOixTeJzHIdDhw7RpUsXFi5cGN2DuT3UuTHbueeeq3V5foTasrOzT7hv2bJl+h//8R+6Y8cO7dixo1ZVVdV6vH///rpixQr93e9+px07dtR+/fpVbzt27Dhhf42xZMkS7dSpk2ZlZWn79u31kksuUVXVvXv36ujRo1VVtaioSHNycrRPnz7aq1cvveuuu6pf/+2332peXp727t1bL7jgAi0sLGzU8f29RyZ5HDlyRAcPHqxnnnmmlpWVhbUvYLU28H+Y8MPIRYRE+hncYO9R8op00TMlhpEbk6qi3UtSH0saxiSoW2+9NeYJA5K8EGpMMrvnnnsYO3Ysl156aUyPm5QtjfT0dHJycujduzdjx46tnjuyc+dORITbb7+9+rkHDhwgMzOTX/3qVwBs2rSJH//4x+Tk5NCzZ09mzJgBwHvvvcfJJ59cq3v17bffDivOjRs3MmjQIJo0acL8+fPrfd4777zDgAEDyMnJ4aKLLmLr1q0A/Otf/2LAgAFkZGTw0ksvhRWLSQyO43DLLbdQWlpKq1atYp4wgOTvPZk2bZrec889qqq6Y8cO7datm+bk5FQ//uijj2q/fv30hhtuUFXVSy65RP/+979XP75u3TpV/b73JZK++eYb/fjjj/W2227TBx98sN7nde/eXTds2KCqqgsXLtRrrrmm+ucpKirSq6++Wl988cV6X+/vPTKJ58iRI3rRRRdpenq6vvvuu1E5BkH0niRlS6OmQYMGsXfv3urbzZs3p2fPntWzTRcvXswVV1xR/fhXX31F586dq2/36dMnarG1b9+egQMHkpmZGfB5IsKRI56F2g8fPkzHjh0B6Nq1K3379q2ek2KSl+M45OXlsXz5cgoKCvxOb4iVpK5pVFZW8s4773DdddfVut83L6VDhw6kp6fTsWNHvvzyS8AzhHzYsGFceOGFXHLJJVx77bXVs1w/+OADcnJyqvfz8ssvc+aZZ9ba9+TJk9m0adMJscyZM4dp06aF9HM8+eST5OXl0axZM1q2bFk9+9WkhroJI5ZFT3+SMmmUlpaSk5PD3r176dmzJyNHjqz1+KhRo7jjjjvo0KEDkydPrvXYtddey6WXXsrSpUt55ZVXePzxxykqKgLgRz/6Ef/4xz8CHnvx4sWR/WHwTKB7/fXXOf/883nwwQeZM2cOTz75ZMSPY+LT3r172bZtW1wkDEjSQmizZs0oLCxk165dqOoJw2qzsrI499xzeeihh/z+Ejp27Mj06dN55ZVXyMjIYP369UEfe/LkyX7npDz33HMh/Sz79++nqKioesbr5MmT+eijj0Lal0ksx44dQ1X54Q9/yJYtW+IiYUCStjR8mjdvzoIFCxg/fjy//OUvaz120003MWTIEFq3bl3r/qVLlzJ8+HAyMzP5+uuvOXjwIJ06dWLjxo1BHTPSLY1TTjmFw4cPs3nzZnr06MFbb71Fz549I3oME398pyQ//vGPufvuu8nOznY7pO81VCmNpy3UuSdjxozR5557Tnfs2KG9evU64flPP/10de/JjTfeqD169NC+fftq37599fnnn1dVT+9Jy5Yta81JCdRjEYyvvvpKO3XqpC1atNCTTz5ZO3XqpIcPH1ZV1dGjR+vevXtV1TN3pXfv3tq3b18dMmSIbtu2TVVVP/74Y+3UqZM2b95cW7dureecc47f4/h7j0z8qtlLEu7fWGNhc08M2HuUSNwuetrcE2MSSFVVFWPHjo2bXpL6WNIwJk6kpaVx/fXXx3XCgCQohKalpVFeXm4XTKpHeXm5Df6Kc47jsHbtWoYMGXLCKnPxKOH/mgYMGMD8+fMpLy93O5S4U15ezvz58xkwYIDboaSkqirl6LEKqtT7terEupJventeXh779u1zIcoQNFQpjafNX+/J7t27NTc3V9PS0hSwrcaWlpamubm5unv37nAK6iYElZVVuu9ImU55fLmeeetrOuXx5brvSJlWVn6/YpxvxS03eknqQ7z3nojIKOARIB14UlXvD/R8f70nxsSjo8cq+Pmzq1m+/WD1fYPOaMMT1+RyUpMM1xbQaUhc956ISDqwEBgNnANMFZFz3IrHmEhqnpXOqp2Hat23auchmmelA/D000/HXcIIlps1jfOAraq6XVXLgQLgJy7GY0zElJRXMrBr7dHGA7u2pqS8EoBf//rXfPzxxwmXMMDdpNEJ2F3j9h7vfbWIyAwRWS0iq/fv3x+z4IwJR/PMdBZM7c+gM9qQkSYMOqMNj0zJ4S+PLWTLli2ISMIWqOO+y1VVFwGLwFPTcDkcY4KSlia0yc7iiWtyaZ6VTnHZceY/8HvuveceunTpQvfu3d0OMWRuJo29QM3LWHf23mdMUkhLE79Fz/Hjx7sdWljcPD1ZBXQXkW4ikgVMAV51MR5jIi5ee0nC4VpLQ1UrRORXwD/xdLk+paqfuRWPMdGSkZGRNAkDXK5pqOrrwOtuxmBMNDiOQ1paGi1atGDZsmWIiNshRUzcF0KNSTS+U5JmzZrx5ptvJlXCAEsaxkRU3RpGsiUMSIIJa8bEi2QsevpjScOYCLn22muTPmGAJQ1jIuaee+7hpZdeSuqEAZY0jAmL4zg8+uij1ZcaSPSBW8GwpGFMiHw1jJkzZ7Ju3Tq3w4kZSxrGhKBm0TM/P59+/fq5HVLMWNIwppHqJoxJkya5HVJMWdIwppFWrlzJmjVr4iJhBLMOaaTZ4C5jgqSqiAgjRoxg+/btnHrqqa7GU1WlHCwuZ2b+J6zaeYiBXVuzYGp/2mRnkZYWvUFl1tIwJgiO4zBs2DBefvllANcTBkDJ8Upm5n/C8u0HqahSlm8/yMz8Tyg5XhnV41rSMKYBvhrGBx98QFVVldvhVGtoHdJosaRhTADxXPRsaB3SaLGkYUw9ysrK4jZhgP91SBdM7U/zzOi2NKwQakw9mjRpwuDBg5k1a1bcJQw4cR3SkvJKmmemR7UICgGShoj8Ec+VuvxS1ZlRicgYlzmOw9dff0337t154IEH3A4nIN86pED116gfM8Bjq4E1QFNgALDFu+UAdrVlk5R8NYxhw4ZRWlrqdjhxqd7UpKrPAojIL4CLVLXCe/sx4IPYhGdM7NRdD6NZs2ZuhxSXgimEngK0rHH7JO99xiSNVFlAJxKCOQm6H/hERJYBAlwMzItmUMbE2rx58yxhBCmoq8aLyA+A8703V6rq11GNqh521XgTLcXFxaxYsYLhw4eHtZ+qKqXkeGVMezMiKSJXjRfPyqgjgH6q+gqQJSLnRShGY1zjOA6zZs3CcRyys7MjkjAOFpfz82dX02PuG/z82dUcLC6PySSyWAqmpvEoMAiY6r3tAAujFpExMeCrYSxcuJAVK1ZEZJ9uzQWJtWBqGuer6gAR+QRAVb/1XkbRmIRUt+g5cuTIiOzXrbkgsRZMS+O4iKTjHeglIu2A+Jm1Y0wjRLOXxK25ILEWTNJYAPwNaC8i9wL/Bu6LalTGRMn+/fvZs2dPVHpJ3JoLEmvB9p78EBiOp8v1HVX9PNqB+WO9JyZUpaWlNG3aFBGhrKyMpk2bRuU41nvi2cnzqrpRVReq6p9U9XMReT5yYRoTXY7jMHLkSObMmQMQtYQB388FSRPv1wRKGMEK5vSkV80b3vrGueEcVEQeFJGNIrJORP4mIq3C2Z8x9alZwxg8eLDb4SSFepOGiNwqIg7QV0SOeDcH2Ae8EuZx3wJ6q2pfYDNwa5j7M+YENjQ8OupNGqp6n6q2AB5U1ZberYWqtlHVsP7JVfVN3wQ4YAXQOZz9GVOXqjJ+/PikTxhurEYezOnJxyJysu+GiLQSkUhee2468EYE92cMIsLs2bPJz89P6oThxgjUBntPRKRQVXPq3PeJqvZv4HVvAz/w89Bc73B0RGQukAtM1HoCEZEZwAyA008//dxdu3YFjNekNsdx+PDDDxk1apTboUTd0WMV/PzZ1SzffrD6vkFntOGJa3JDXpAnmN6TYPbsrzXS4OtUdUSgx0Xkp8AYYHh9CcO7n0XAIvB0uTZ0XJO6HMchLy+PVatWsW3bNjp16uR2SFEVz6uRrxaRP4jImd7tD3hW9AqZiIwCfgOMU9WScPZlDHyfMJYvX85f//rXpE8YEN+rkf8aKAcWe7djwA1hHvdPQAvgLREp9K4GZkxIaiaMZC561hW3q5GrajHw20geVFXPiuT+TGorKChIuYQB8bka+f+o6mwR+T/8rEququOiGpkxQfrZz37GoEGD6N27t9uhxJwbq5EHOopvqPj8WARiTGM4jsO0adO4++676d27d0omDLcEWo18jffr+7ELx5iG1RzpOW3aNEsYMRbo9ORTAl8sqW9UIjKmHlVVSvGx4zTPzmbRU89y+NABBl1wfsMvDPFYiTxbNZoCnZ6M8X719ZT4TleuIkAyMZFnf8Ce9+DA0WPMKihk1c5DDOzamgVT+1NVpRF/L3wjLWfmf1LrWG2ys1Luffcn0NyTXaq6Cxipqr9R1U+92y3AJbELMbWlymK1DSk5XsmsgsKYrL+ZKmt9hiqYcRoiIoNr3LgwyNeZCLA/YE8NI5ajH1Nlrc9QBfPPfx3wqIjsFJGdeFYnnx7VqEy1VP8D9hU9t3+xN2ajH1Nlrc9QNZg0VHWNqvYD+uG59kmOqq6NfmgGUvsPuGYvyf6v9sZs9GOqrPUZqmBmuXYAfg90VNXRInIOMEhV/xKLAGtKxTVCU7Uo528BnVgWhFO1+BypWa7PAE8Dc723N+OZgxLzpJGKYjlUOJ7+Ua6//voTFtCJ5ehHN0ZaJopgahptVfUFvNc68a64lfxt4zgSi8Vq462X5t5772XJkiUpNZckUQSTNIpFpA3fXyzpAuBwVKMyMRcPvTSO4/DQQw9RVVVFt27dGDeu/ulNbixzF89xxFIw7a45wKvAmSLyIdAOsPSfZNzupalZwxgyZAi5ufWfVsdLnSde4oi1gC0N7+UKhni3C4HrgV6qui4GsZkYcrOXpm7RM1DCgPhoFcVTHLEWMGmoaiUwVVUrVPUzVV2vqsdjFJuJIbe6GUO5zECoraJIn0q43TpzSzCnJx+KyJ/w9JgU++60sRrJxa0FXYqKiigsLGzUAjol5ZXMHHYWl/Y+lbPan8TWfUf55/qvKCmvrLenIxqnEr7WWc2FfX2ts2TucQlmnMYyP3erqg6LTkj1C3ecRjx1KSaCaL5fVVVVpKV5GroHDhygbdu2Qb+2srKKg8XltSavPTIlhzbZWaSn+288R2Pl7mSsaURqnMYkVT0QoZhc09Av2BJKbdH8h3AchzFjxjB9+nSuueaaRiUMgNKKqurJawDLtx9kVkGhJwHUkzSicSrhVuvMbYEuyzhWRPYD60Rkj3eiWsIKVLSKtzEK8SBaRT5fDePDDz8kOzs7pH34EsC8sedQeOdItt+Xx5+vGkDTjLR66xXRKvSmwgWf6wpUCL0X+JGqdgQuA+6LTUjREeiTJlWr4IFE45M5UtdWLSmv5I9Tc8jrcyq/+Otaesx9g1/8dS2Hist5fvlOv0nf5pNETqCkUaGqGwFUdSWeSw4krECfNKlaBffx16sQiU/mWvstO878hx6KyLVVm2emc1H3diesrzGroJCf5HTym/Rrnkpsvnc0T1yTm9C1BzcFqmm0F5E59d1W1T9EL6zI833S1D1Hb56ZnrJVcKi/dtG6eWa971eo+31o5s2MGjU65CX6atadsptk+E30LZtlVn9fN+nbfJLICPTOPUHt1kXd2wklUNEqUEJJdjVPzYDqT2nfJ3GoRT5/+73ppfU8cU3ggVv1qZuEVt8+wm+iP1J6vPr7VEj6bgi0GvldsQwkFur7pAm1Cp4MPS6BTs18xT1o/CdzpE/56iahv3+yl0em5FDw8RfV4zWKj1Ww9otDVq+IMlu2z6uxVfBk6XGJVq9CcdnxiO63bhKa938b2PDVYaacfzrzXv2Ms29/g+ufX0Ovjifzl59avSKarO0WokDNereaxKG0fJpnpvPYVQP4tuQ4p7Vuzu5DJZzSPDOsT2nHcZj/0EM8MvsmSirSIrJff3Wnrm1OYlZ+YZ3fgWe8hiWM6LGWRojircclnJZPeWUVty75lLNvf4Nbl3xKeWVVWLHcd999bN2yFZX0iO3XX5fp6W2ax9XvIFU0mDREpI2I/FFE1orIGhF5xLu+RkqLt7U7Qx1r4nld3UsDFIY1RuXOO+/kkUcfY/bidRHbr78u05Jj8fU7SBXBtDQKgH14BnhdDuzHM3ktpcXbYKFQWz7htphqjsX44stvOOI4NG3alNYts6MybLtm3al5Vnz9DlJFMCffp6rq3TVu3yMikyNxcBG5Cc8Fptsl2vyWYHtcYtXDEupYk3DGqPgbi/HwFX05yfszR3vsS6rO/XBbMC2NN0VkioikebcrgH+Ge2AROQ3Pldq+CHdfbvGN8fCNKvXNY/GJZQ9LqC2fcFpM/k6JbnxhnSdJxqgllopzP9wWzNR4B8jGu7AwnkTjW1dDVbVlSAcWeQm4G3gFyA2mpRFvlzBoaCZoNKZjNxRPKK2akF+nSo+5b1DhTYLj+nXkhqFn0b2DZ8xEs4x0yiqrrBWQQIKZGh/MxZJaqGqaqmZ4tzTvfS3CSBg/AfaqalEQz50hIqtFZPX+/ftDOVzUNFR8jHUPS6ifuqG+zik5Vl2IHNevIzdfcjbzXv2MHnPfYMZzazhUUk6zjDRrBSSZoD7uRGQccLH35nuq+o8gXvM28AM/D80FbiPIi0ir6iJgEXhaGsG8JlYaSgrJOqeluLiYpk2b0qJZExZMzWFmfiE3DD2Lv3+yh3njelWvplXw8Rdce1E3WtSzxoVJTA3+5YrI/cBA4H+9d80SkcGqemug16nqiHr21wfoBhSJCEBnYK2InKeqXzcmeLfVlxSKj1WQnZWRlHNafNPbe/bsyRNPPEGb7CY8cU0uzTLTaNa/M7e8vO77n3VKDoLnNMaN05NkGOYfj4L5uMsDclS1CkBEngU+AQImjfqo6qdAe99t70Wlg6ppxBt/SeGBy/ry9L93MPX8LrTJzkqq6n7N9TBmz54NfH9q45Qd55aX11Un0HYtmnCssoqZzxX6rfdEW2VlFQdLypmV787xk1mw7cZWNb4/ORqBJCJfl9+iaeey6Z7RzBvXi/lvbuIPb2+prm0kS3W/oQV06k5Vv2HoWfy/F+sO7orNwkZVVUpxeWX1EHNbWCmygmlp/B74xLvAsOCpbfw2UgGoatdI7csNaWlCdpOMWr0IkBzDmWs27/d/e4QuXboye/Zsvwvo1D1VO6v9Sa4N8S45XlnvehuJ/juJBw1dLCkNT1frBcAS4GU8V4xP+RGhNcXbkHKfcK7zUXeMydylu3lo4eNMnHiZ3+fXHZex+1CJa+9J86x0tu47Gpe/k2QQzDiN1Q3128ZKvI3T8InHpezDjaneMSbTckHwW6Op2TIpO15J8bEKZrpQUzh6rIKnPtjO+DqF2Uem5tA2u0nCniLGQjDjNIJJGvcDBzjxYkmH6n1RlMRr0oD4q9SHO7Cs7sAtgIw0YfO9o7nyiZU1EkEOber5Rwz2PYn0e+dLmPkrd9VaoCc7K73e66IYj4gM7gImAzcA/wLWeLf4/M91UbwVPMMZWOY4Dtu/2Ou3ef/FwZITZ67W0+QP5j2JxlB7X4F6+o/OoHuHkyg97hkXYwkjMoIZEdrNz3ZGLIIzoQu1zuLrJfndbb/l4Sv61Jo78sjUHN7fvK/W81ftPETzJqEXF6N1+Yh4S+LJJJjBXU2BXwIXAQp8ADymqmVRjs2EIdSBZa+88gpdunTl4T8upE3LZiyadi7ZWRls2XeUgpVfMPHczqzZ9R2vFn0JeBPRsUpOahraCNd4W8zINCyY3/RzgAP80Xv7P4HngUnRCsqEL9Rp4//5n1cycuxlzFpcVGvA2sJlW3m16EuWbz/EfRP78PqnXzGwa2senNSXNAl91GeyDrVPZsEUQjeo6jkN3RcL8VwIdUOkCoiO4zB16lTuvPNOzuk34IQC6pwR3Zl2YVdaNM1k676jnNU+G/B0q7ZqnslzH+1kwbtbQ+ohiceep1QWqULoWhG5oMZOz8cKoa6LVAHRV8NYunQpe/bsOeF0YVy/jozv35lf/HUtZ9/+BvNe/YyDxeWA0uakLJ77aCd/eHtLo+sRvjEkCGQ3SeeJaXbls0QRTNI4F/hIRHZ654ksBwaKyKcisi6q0Zl6RaKAWHdo+MSJE08ooN4w9KzqOSXVlz/ML6T0eBXZTTJY8O7WWvsMph5RN+Fd98xqSo9XgmJFywQQTNIYhWdW6hDv1s173xhgbPRCM4GEW0A8evSo37kkdUd2BhoOXnysgpnDzqr1WDA9NHbB7cQWTJfrLuAInolqbXybqu7yPmZcEO7Q9aysLE499dQTJp/VXfW7pLzC73G2fHOUGc+tYcp5pzNnRPfGLTFoPSYJLZhC6N3AT4FteLpcwbPM37DohnYiK4R+L9QCouM4lJWV0a5dO1QV75omtfZbs7jaLCONQyXHT5j+P//NTbxa9CWDzmjj6ZZtkhF0MTbWyyCa4EVqGPkmoI+qlkcyuFBY0qitsb0njuOQl5dHcXExq1atIj299id7oCvIl1Z41vrc8s3R6u5X+H5oeZoEX4ewHpP4FUzSCCatr8eznsa+hp5oYqu+C1r740sYy5cvp6Cg4ISEAQ1favLosQrmvfpZ2GMq7NIDiS2YQuh9eNbT+KeIvOrboh2YCV/NqfH7Dh3m9NO7+F1Ax6ehWkMkL0tgw7wTVzAfD88CDwCf8v1lDEyc83sho4WP075l83pf09DoTGshGAiupVGiqgtUdZmqvu/boh6ZCYv/Cxl9GrBbM5iWhLUQTDAtjQ9E5D7gVeCY705VXRu1qEzYQunWtJaECUYwSaO/9+sFNe5TIOZdriY4juPwzaHDIU0Ea0xx1aSmBv8qVHVoLAIxkeEbGt6lS1ceXvg4N77wadJcc8XEh2DW0+iAZ0Xyjqo6WkTOwbO48F+iHp1plLrXJWnfsrmdapiIC6YQ+gyeq8R39N7eDMyOVkAmdJs2bWL9+vXV3apWtDTREMxJa1tVfUFEbgVQ1QoRsZlFcaSiooKMjAxyc3PZuXMnrVq1avhFxoQomJZGsYi0wTvvxLu2xgapSeEAAArhSURBVOGoRmWC5jgOQ4cOZeHChQCWMEzUBZM05uDpbj1TRD7Es/zfr6MalQmKr4axfPlyOnTo4HY4JkUE03uyVkSGAGfjuSzjJlU9HvXITEANXVvVmGgJtiP+PKCr9/kDRARVfS5qUZmAKioqyMvLs4RhXBFMl+vzwJlAIeArgCqe0xTjgoyMDC6//HJmzZplCcPEXDAtjVzgHG1o4Y1GEpFf47lyWyXwmqr+JpL7T0aO47BlyxYGDBjArFmz3A7HpKhg19P4AfBVpA4qIkOBnwD9VPWYiLSP1L6Tla+GsWHDBnbs2MHJJ5/sdkgmRdWbNETk//CchrQANojIx9SesDYujOP+ArhfVY9592UL/ARQt+hpCcO4KVBLY34Uj9sD+JGI3AuUATer6ip/TxSRGcAMgNNPPz2KIcUn6yUx8SZQ0tgLdFDVD2veKSIXEcSpioi8jee0pq653uO2xjNzdiDwgoic4a9uoqqLgEXgWSO0oeMmm4cfftgShokrgZLG/wC3+rn/sPexgNc8UdUR9T0mIr8AlniTxMciUgW0BfY3GHGKue222xg+fDiDBw92OxRjgMAjQjuo6qd17/Te1zXM4/4dGAogIj2ALOBAmPtMGo7jcO211/L111+TkZFhCcPElUBJI9AkhmZhHvcp4AwRWQ8UANdEuks3UflqGM8//zx2uQYTjwKdnqwWkZ+r6hM17xSRnwFrwjmo9xoqV4Wzj2RUs+iZn5/PmDFj3A7JmBMEShqzgb+JyJV8nyRy8ZxKTIh2YKmmbsKYNGmS2yEZ41e9SUNVvwEu9A7E6u29+zVVfTcmkaWYkpISHMexhGHiXjCzXJcBy2IQS0o6evQoTZo0oUOHDqxZs4aMDFvM18S3YNbTMFHiOA6jRo1i2rRpAJYwTEKwpOGSmjWMiRMnuh2OMUGzpOECK3qaRGZJwwWTJ0+2hGESliUNF9xxxx0UFBRYwjAJySpvMeI4Dq+99hpTpkxh0KBBbodjTMispREDvhrG1VdfzdatW90Ox5iwWEsjyuquh3HWWWe5HZIxYbGWRhTZAjomGVnSiKI333yTlStXWsIwScVOT6JAVRERLrvsMjZv3ky3bt3cDsmYiLGWRoT5Tknef/99AEsYJulY0oggX8J4++23OXDAFiIzycmSRoTULXpedtllbodkTFRY0oiA4uJi6yUxKcOSRgQ0bdqU7t27W8IwKcF6T8LgOA6O49CxY0eefvppt8MxJiYsaYTIV8M4dOgQRUVFZGZmuh2SMTFhSSMEdYueljBMKrGaRiPZ0HCT6ixpNNJNN91kCcOkNDs9aaTf//73TJgwgdGjR7sdijGusJZGEBzH4Y477qC8vJy2bdtawjApzZJGAxzHIS8vj/vuu48VK1a4HY4xrrOkEYAvYSxfvpyCggIuvvhit0MyxnWWNOpRN2FY0dMYD1eShojkiMgKESkUkdUicp4bcQSyY8cONm7caAnDmDrc6j35b+AuVX1DRPK8t3/sUiy1lJeXk5WVRd++fdm+fTstWrRwOyRj4opbpycKtPR+fzLwpUtx1OI4DsOHD+f+++8HsIRhjB9uJY3ZwIMishuYD9xa3xNFZIb3FGb1/v37oxZQzRqGrRhuTP2idnoiIm8DP/Dz0FxgOHCjqr4sIlcAfwFG+NuPqi4CFgHk5uZqNGK1oqcxwYta0lBVv0kAQESeA2Z5b74IPBmtOBpSWVnJmDFjLGEYEyS3Tk++BIZ4vx8GbHEpDtLT05k2bZolDGOC5Fbvyc+BR0QkAygDZsQ6AMdxWL9+PYMGDeK6666L9eGNSViuJA1V/TdwrhvHhu+nt69bt44dO3bQpk0bt0IxJuGk3CzXuuthWMIwpnFSahi5LaBjTPhSKmk89thjljCMCVNKnZ7cdNNNDBkyhPPOi7upLsYkjKRvaTiOw5VXXsmuXbtIS0uzhGFMmJI6afhqGIsXL6aoqMjtcIxJCkmbNOoWPceNG+d2SMYkhaRMGtZLYkz0JGXSqKiooKKiwhKGMVGQVL0njuOQmZnJKaecwkcffURaWlLmRGNclTT/Vb5TkiuuuAJVtYRhTJQkRUujbg1DRNwOyZiklfAfx1b0NCa2Ej5pXH311ZYwjImhhE8ad911Fy+88IIlDGNiJCGThuM4PPXUUwD069ePiRMnuhyRMakj4ZKGr4YxY8YMPvvsM7fDMSblJFTSqKqqqi565ufn06tXL7dDMiblJFSX6+bNmykrKyM/P59Jkya5HY4xKSmhkkZJSQmLFy+2hGGMi0Q1KtcfigoR2Q/sisGh2gIHYnCcxrK4Gsfiapy2QLaqtgv0pIRKGrEiIqtVNdftOOqyuBrH4mqcYONKqEKoMcZ9ljSMMY1iScO/RW4HUA+Lq3EsrsYJKi6raRhjGsVaGsaYRrGkYYxpFEsa9RCRHBFZISKFIrJaROLmgiki8msR2Sgin4nIf7sdT00icpOIqIi0dTsWABF50PterRORv4lIK5fjGSUim0Rkq4j81s1YfETkNBFZJiIbvH9TswK+QFVt87MBbwKjvd/nAe+5HZM3lqHA20AT7+32bsdUI7bTgH/iGYDX1u14vDFdAmR4v38AeMDFWNKBbcAZQBZQBJwTB+/RqcAA7/ctgM2B4rKWRv0UaOn9/mTgSxdjqekXwP2qegxAVfe5HE9NDwO/wfPexQVVfVNVK7w3VwCdXQznPGCrqm5X1XKgAPiJi/EAoKpfqepa7/cO8DnQqb7nW9Ko32zgQRHZDcwHbnU5Hp8ewI9EZKWIvC8iA90OCEBEfgLsVdV4vpTddOANF4/fCdhd4/YeAvxzukFEugL9gZX1PSehJqxFmoi8DfzAz0NzgeHAjar6sohcAfwFGBEHcWUArYELgIHACyJyhnrbli7GdRueU4GYCxSXqr7ifc5coAL431jGlkhE5CTgZWC2qh6p93kx+FtLSCJyGGilqiqe5c0Pq2rLhl4Xg7iW4jkvX+a9vQ24QFX3uxhTH+AdoMR7V2c8p3PnqerXbsXlIyI/Ba4HhqtqSQNPj2Ycg4B5qnqp9/atAKp6n1sx+YhIJvAP4J+q+odAz7XTk/p9CQzxfj8M2OJiLDX9HU8xFBHpgaeg5uqMSVX9VFXbq2pXVe2Kp9k9IE4Sxig8dZZxbiYMr1VAdxHpJiJZwBTgVZdjwvuh+Bfg84YSBqT46UkDfg48IiIZQBkww+V4fJ4CnhKR9UA5cE0sTk0S2J+AJsBb3uvhrFDV/3IjEFWtEJFf4elhSgeeUtV4WLNyMHA18KmIFHrvu01VX/f3ZDs9McY0ip2eGGMaxZKGMaZRLGkYYxrFkoYxplEsaRhjGsW6XE2jiUgbPIO5wDMSsxLwDS47zzuvwiQp63I1YRGRecBRVZ1f476MGpPETJKxloaJCBF5Bs8guP7AhyJyhBrJxDsYbYyq7hSRq4CZeEazrgR+qaqV7kRuGstqGiaSOgMXquqc+p4gIj2BycBgVc3Bc2pzZYziMxFgLQ0TSS8G0WIYDpwLrPIO624GxNOaIKYBljRMJBXX+L6C2i3Zpt6vAjyrqvGyPolpJDs9MdGyExgAICIDgG7e+98BLheR9t7HWotIF1ciNCGxpGGi5WWgtYh8BvwKz7qTqOoG4HbgTRFZB7yFZ41KkyCsy9UY0yjW0jDGNIolDWNMo1jSMMY0iiUNY0yjWNIwxjSKJQ1jTKNY0jDGNMr/BylM2pp2GTgHAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "df = pd.read_csv('test_checkpoints_splits/fold_0/test_full.csv')\n",
        "df['preds'] = [x[0] for x in preds]\n",
        "\n",
        "plot_parity(df.logSolubility, df.preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WNacITwVqj4x"
      },
      "source": [
        "# Ensembling and Uncertainty"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RpXC83NMqj4x",
        "outputId": "58085646-7157-42d2-d012-5b132b3c02bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/reaction_regression.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'regression',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 5,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'mse',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'rmse',\n",
            " 'metrics': ['rmse'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': True,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_ensemble',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['AAM'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'scaffold_balanced',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['ea'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "500it [00:00, 58463.72it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 82506.57it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 40976.01it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "100%|██████████| 500/500 [00:00<00:00, 10191.78it/s]\n",
            "Total scaffolds = 203 | train scaffolds = 147 | val scaffolds = 29 | test scaffolds = 27\n",
            "Label averages per scaffold, in decreasing order of scaffold frequency,capped at 10 scaffolds and 20 labels:\n",
            "Scaffold 0\n",
            "Task 0: count = 141 | target average = 8.010435\n",
            "\n",
            "\n",
            "Scaffold 1\n",
            "Task 0: count = 13 | target average = 9.241358\n",
            "\n",
            "\n",
            "Scaffold 2\n",
            "Task 0: count = 7 | target average = 7.232327\n",
            "\n",
            "\n",
            "Scaffold 3\n",
            "Task 0: count = 7 | target average = 7.289696\n",
            "\n",
            "\n",
            "Scaffold 4\n",
            "Task 0: count = 7 | target average = 6.381221\n",
            "\n",
            "\n",
            "Scaffold 5\n",
            "Task 0: count = 6 | target average = 6.927956\n",
            "\n",
            "\n",
            "Scaffold 6\n",
            "Task 0: count = 6 | target average = 8.997394\n",
            "\n",
            "\n",
            "Scaffold 7\n",
            "Task 0: count = 5 | target average = 7.520033\n",
            "\n",
            "\n",
            "Scaffold 8\n",
            "Task 0: count = 5 | target average = 8.068549\n",
            "\n",
            "\n",
            "Scaffold 9\n",
            "Task 0: count = 4 | target average = 7.458364\n",
            "\n",
            "\n",
            "500it [00:00, 33611.98it/s]\n",
            "Total size = 500 | train size = 400 | val size = 50 | test size = 50\n",
            "Fitting scaler\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=193, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=465, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 378,601\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.23it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.67it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.28it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.25it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.732327\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.37it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[ALoss = 9.6142e-01, PNorm = 34.6711, GNorm = 2.1700, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 13.80it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.44it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.71it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.69it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.342255\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.38it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.77it/s]\u001b[ALoss = 9.8149e-01, PNorm = 34.7060, GNorm = 1.2760, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.13it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.00it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.06it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.459109\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.45it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.33it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.05it/s]\u001b[ALoss = 7.0097e-01, PNorm = 34.7278, GNorm = 3.2558, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.54it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.76it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.278640\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.42it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 16.01it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.23it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.89it/s]\u001b[ALoss = 6.5832e-01, PNorm = 34.7368, GNorm = 1.5438, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.61it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.200886\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.41it/s]\n",
            "Model 0 best validation rmse = 2.200886 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test rmse = 2.063817\n",
            "Building model 1\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=193, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=465, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 378,601\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.93it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.94it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.96it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.73it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.382831\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.41it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[ALoss = 6.9498e-01, PNorm = 34.7264, GNorm = 1.8911, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.06it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.37it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.79it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.60it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.378330\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.40it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.05it/s]\u001b[ALoss = 9.4992e-01, PNorm = 34.7562, GNorm = 6.6177, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.00it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.93it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.61it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.331827\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.40it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.56it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.99it/s]\u001b[ALoss = 6.8234e-01, PNorm = 34.7773, GNorm = 1.9858, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.72it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.43it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.327268\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.40it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 16.28it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.16it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 15.22it/s]\u001b[ALoss = 6.7731e-01, PNorm = 34.7855, GNorm = 2.4887, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.43it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.261885\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.40it/s]\n",
            "Model 1 best validation rmse = 2.261885 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 1 test rmse = 2.100716\n",
            "Building model 2\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=193, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=465, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 378,601\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.98it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.67it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.79it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.99it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.236768\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.41it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[ALoss = 7.8392e-01, PNorm = 34.6743, GNorm = 2.7400, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.99it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.29it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.41it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 13.93it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.621285\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.46it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.33it/s]\u001b[ALoss = 8.8711e-01, PNorm = 34.7103, GNorm = 7.5940, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00, 13.74it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.08it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.32it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.178063\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.41it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 13.27it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.66it/s]\u001b[ALoss = 6.6988e-01, PNorm = 34.7295, GNorm = 2.6633, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.68it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.51it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.198801\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.48it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.20it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.05it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.54it/s]\u001b[ALoss = 6.1597e-01, PNorm = 34.7375, GNorm = 3.5315, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.22it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.209044\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.47it/s]\n",
            "Model 2 best validation rmse = 2.178063 on epoch 2\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 2 test rmse = 2.126568\n",
            "Building model 3\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=193, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=465, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 378,601\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.78it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.35it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.29it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 13.90it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.507891\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.35it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[ALoss = 1.0369e+00, PNorm = 34.6045, GNorm = 1.8233, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.78it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.40it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.60it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.78it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.420495\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.36it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.53it/s]\u001b[ALoss = 8.9386e-01, PNorm = 34.6317, GNorm = 5.4623, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.17it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 15.15it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.78it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.388413\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.39it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.10it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.42it/s]\u001b[ALoss = 7.7755e-01, PNorm = 34.6499, GNorm = 5.8817, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.67it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.49it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.239898\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.39it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.27it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.59it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.91it/s]\u001b[ALoss = 6.7354e-01, PNorm = 34.6578, GNorm = 2.4950, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00, 15.13it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.189930\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.39it/s]\n",
            "Model 3 best validation rmse = 2.189930 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 3 test rmse = 2.075962\n",
            "Building model 4\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=193, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=465, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 378,601\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.87it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.55it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 15.28it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 15.21it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.780604\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.41it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[ALoss = 9.4477e-01, PNorm = 34.6045, GNorm = 0.9831, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.10it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 13.97it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.19it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.46it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.337362\n",
            " 40%|████      | 2/5 [00:01<00:02,  1.41it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.08it/s]\u001b[ALoss = 7.8064e-01, PNorm = 34.6381, GNorm = 1.7224, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.41it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.30it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.26it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.411323\n",
            " 60%|██████    | 3/5 [00:02<00:01,  1.46it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 14.75it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 14.59it/s]\u001b[ALoss = 7.3826e-01, PNorm = 34.6611, GNorm = 4.0007, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.46it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.22it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.280935\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.42it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 15.93it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00, 15.13it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 14.99it/s]\u001b[ALoss = 6.4870e-01, PNorm = 34.6699, GNorm = 5.0482, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00, 14.78it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 2.204269\n",
            "100%|██████████| 5/5 [00:03<00:00,  1.42it/s]\n",
            "Model 4 best validation rmse = 2.204269 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 4 test rmse = 2.108382\n",
            "Ensemble test rmse = 2.067441\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test rmse = 2.067441\n",
            "Overall test rmse = 2.067441 +/- 0.000000\n",
            "Elapsed time = 0:00:20\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/reaction_regression.csv',\n",
        "    '--dataset_type', 'regression',\n",
        "    '--save_dir', 'test_checkpoints_ensemble',\n",
        "    '--epochs', '5',\n",
        "    '--reaction',\n",
        "    '--save_smiles_splits',\n",
        "    '--ensemble_size', '5',\n",
        "    '--split_type', 'scaffold_balanced'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wt8ItRG3qj4x",
        "outputId": "3026c834-4ff7-4c2f-b141-17d5a4950a58"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "50it [00:00, 58514.29it/s]\n",
            "100%|██████████| 50/50 [00:00<00:00, 104232.21it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 50\n",
            "Predicting with an ensemble of 5 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/5 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.58it/s]\u001b[A\n",
            " 20%|██        | 1/5 [00:00<00:01,  2.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.60it/s]\u001b[A\n",
            " 40%|████      | 2/5 [00:00<00:01,  2.02it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.70it/s]\u001b[A\n",
            " 60%|██████    | 3/5 [00:01<00:00,  2.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.58it/s]\u001b[A\n",
            " 80%|████████  | 4/5 [00:01<00:00,  2.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.49it/s]\u001b[A\n",
            "100%|██████████| 5/5 [00:02<00:00,  2.05it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_ensemble.csv\n",
            "Elapsed time = 0:00:03\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_ensemble/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_ensemble.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_ensemble',\n",
        "    '--ensemble_variance'\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "5WaSuowCqj4x",
        "outputId": "0d6e3b38-b835-4d32-dae9-8939cf5c23fb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  AAM         ea  ea_epi_unc\n",
              "0   [C:1]([C@@:2]1([H:10])[C@:3]([C:4]([H:12])([H:...   7.437500    0.051191\n",
              "1   [C:1]([C@@:2]1([H:10])[C@:3]([C:4]([H:12])([H:...   8.662633    0.362828\n",
              "2   [O:1]([C@@:2]1([H:9])[C:3]([H:10])([H:11])[N:4...   8.302808    0.073829\n",
              "3   [O:1]([C:2]1([H:7])[C:3]([H:8])([H:9])[O:4][C:...  10.508875    0.356629\n",
              "4   [O:1]([C:2]1([C:6](=[O:7])[H:13])[C:3]([H:9])(...   9.142585    0.174254\n",
              "5   [O:1]([C:2]1([C:6](=[O:7])[H:13])[C:3]([H:9])(...   8.655645    0.116326\n",
              "6   [N:1](=[C:2]1\\[O:3][C@@:4]2([H:9])[C:5]([H:10]...   7.597202    0.044275\n",
              "7   [N:1](=[C:2]1\\[O:3][C@@:4]2([H:9])[C:5]([H:10]...   8.631742    0.175392\n",
              "8   [O:1]=[C:2]1[C:3]([H:8])([H:9])[N:4]2[C:5]([H:...   9.267970    0.310400\n",
              "9   [C:1]([c:2]1[c:3]([H:11])[n:4]([H:12])[c:5]([O...   8.703622    0.050446\n",
              "10  [C:1]([O:2][C@@:3]1([H:11])[C@@:4]2([H:12])[C:...   7.670585    0.076409\n",
              "11  [C:1]([C@@:2]12[C:3]([H:11])([H:12])[C@:4]1([O...   7.286619    0.107827\n",
              "12  [C:1]([C@:2]12[C@@:3]3([H:11])[C@:4]4([H:12])[...   8.877699    0.374892\n",
              "13  [C:1]([C@:2]12[C@@:3]3([H:11])[C@:4]4([H:12])[...   5.327089    0.162265\n",
              "14  [C:1](#[C:2][C:3]#[C:4][C@@:5]1([H:9])[C:6]([H...   7.464116    0.278382\n",
              "15  [C:1](/[C:2](=[N:3]\\[O:4][H:11])[C@@:5]1([H:12...   8.581939    0.084502\n",
              "16  [C:1]1([H:7])([H:8])[C@@:2]2([H:9])[C:3]([H:10...   7.991354    0.048565\n",
              "17  [O:1]([c:2]1[n:3][n:4]([H:8])[n:5][c:6]1[H:9])...   6.699617    0.014819\n",
              "18  [C:1]([c:2]1[n:3][n:4]([H:10])[n:5][c:6]1[H:11...   7.758332    0.024714\n",
              "19  [N:1]([c:2]1[c:3]([H:10])[o:4][n:5][c:6]1[N:7]...   7.714368    0.011509\n",
              "20  [N:1]([c:2]1[c:3]([H:10])[c:4]([N:5]([H:11])[H...   7.620089    0.049530\n",
              "21  [C:1]([c:2]1[c:3]([H:11])[c:4]([N:5]([H:12])[H...   7.897841    0.023132\n",
              "22  [C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C@...   8.073214    0.064748\n",
              "23  [O:1]([C@@:2]1([H:9])[C:3]([H:10])([H:11])[C@:...   7.238061    0.030620\n",
              "24  [C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C@...   8.160344    0.162666\n",
              "25  [C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C@...   7.405831    0.057869\n",
              "26  [C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C@...   7.736525    0.051980\n",
              "27  [O:1]([C@@:2]1([H:9])[C:3]([H:10])([H:11])[C@:...   7.024516    0.036442\n",
              "28  [O:1]([C@@:2]1([H:8])[C:3]([H:9])([H:10])[C@:4...   8.845977    0.183325\n",
              "29  [C:1]([C@:2]12[C:3]([H:10])([H:11])[C:4]([H:12...   8.116722    0.182136\n",
              "30  [C:1]([C@@:2]1([H:11])[O:3][C:4]([H:12])([H:13...   8.908027    0.265048\n",
              "31  [C:1]([C@@:2]1([H:11])[O:3][C:4]([H:12])([H:13...   7.425850    0.088779\n",
              "32  [O:1]=[C:2]1[C:3]([H:8])([H:9])[O:4][C@@:5]2([...   8.702798    0.237794\n",
              "33  [C:1]([C@@:2]1([C:3]([O:4][H:12])([H:10])[H:11...   7.963273    0.081806\n",
              "34  [C:1]1([H:6])([H:7])[C:2]([H:8])([H:9])[C@@:3]...   8.077995    0.145824\n",
              "35  [N:1]#[C:2][C@@:3]1([H:7])[C:4]([H:8])([H:9])[...   7.693681    0.044768\n",
              "36  [C:1]1([H:8])([H:9])[O:2][C@@:3]2([H:10])[C:4]...   7.850417    0.090569\n",
              "37  [C:1]1([H:8])([H:9])[O:2][C@@:3]2([H:10])[C:4]...   7.768584    0.075748\n",
              "38  [C:1]([C:2]#[C:3][C:4]([N:5]1[C:6]([H:13])([H:...   8.004091    0.077657\n",
              "39  [N:1]([C:2](=[O:3])[C:4]([N:5]1[C:6]([H:12])([...   7.967761    0.065627\n",
              "40  [C:1]([C:2]#[C:3][C:4]([N:5]1[C:6]([H:13])([H:...   9.012047    0.067033\n",
              "41  [N:1]([C:2](=[O:3])[C:4]([N:5]1[C:6]([H:12])([...   7.712333    0.052295\n",
              "42  [C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C:...   7.955681    0.023752\n",
              "43  [C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C:...   9.677749    0.084125\n",
              "44  [C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C:...   8.062699    0.030905\n",
              "45  [C:1]([C:2]1([C:3]([O:4][H:12])([H:10])[H:11])...  10.491706    0.295304\n",
              "46  [C:1]1([H:7])([H:8])[C:2]([H:9])([H:10])[C:3](...   9.052074    0.064273\n",
              "47  [C:1](#[C:2][C:3]1([H:8])[C:4]([H:9])([H:10])[...   8.012644    0.064509\n",
              "48  [C:1]([O:2][C@@:3]1([H:11])[C:4]([H:12])([H:13...   7.975821    0.075249\n",
              "49  [O:1]([C:2]([C:3]1([H:10])[C:4]([H:11])([H:12]...   9.594182    0.106705"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e8059f09-4dac-4e08-9849-6e56be6902eb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AAM</th>\n",
              "      <th>ea</th>\n",
              "      <th>ea_epi_unc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[C:1]([C@@:2]1([H:10])[C@:3]([C:4]([H:12])([H:...</td>\n",
              "      <td>7.437500</td>\n",
              "      <td>0.051191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[C:1]([C@@:2]1([H:10])[C@:3]([C:4]([H:12])([H:...</td>\n",
              "      <td>8.662633</td>\n",
              "      <td>0.362828</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[O:1]([C@@:2]1([H:9])[C:3]([H:10])([H:11])[N:4...</td>\n",
              "      <td>8.302808</td>\n",
              "      <td>0.073829</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[O:1]([C:2]1([H:7])[C:3]([H:8])([H:9])[O:4][C:...</td>\n",
              "      <td>10.508875</td>\n",
              "      <td>0.356629</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[O:1]([C:2]1([C:6](=[O:7])[H:13])[C:3]([H:9])(...</td>\n",
              "      <td>9.142585</td>\n",
              "      <td>0.174254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>[O:1]([C:2]1([C:6](=[O:7])[H:13])[C:3]([H:9])(...</td>\n",
              "      <td>8.655645</td>\n",
              "      <td>0.116326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>[N:1](=[C:2]1\\[O:3][C@@:4]2([H:9])[C:5]([H:10]...</td>\n",
              "      <td>7.597202</td>\n",
              "      <td>0.044275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>[N:1](=[C:2]1\\[O:3][C@@:4]2([H:9])[C:5]([H:10]...</td>\n",
              "      <td>8.631742</td>\n",
              "      <td>0.175392</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>[O:1]=[C:2]1[C:3]([H:8])([H:9])[N:4]2[C:5]([H:...</td>\n",
              "      <td>9.267970</td>\n",
              "      <td>0.310400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>[C:1]([c:2]1[c:3]([H:11])[n:4]([H:12])[c:5]([O...</td>\n",
              "      <td>8.703622</td>\n",
              "      <td>0.050446</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>[C:1]([O:2][C@@:3]1([H:11])[C@@:4]2([H:12])[C:...</td>\n",
              "      <td>7.670585</td>\n",
              "      <td>0.076409</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>[C:1]([C@@:2]12[C:3]([H:11])([H:12])[C@:4]1([O...</td>\n",
              "      <td>7.286619</td>\n",
              "      <td>0.107827</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>[C:1]([C@:2]12[C@@:3]3([H:11])[C@:4]4([H:12])[...</td>\n",
              "      <td>8.877699</td>\n",
              "      <td>0.374892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>[C:1]([C@:2]12[C@@:3]3([H:11])[C@:4]4([H:12])[...</td>\n",
              "      <td>5.327089</td>\n",
              "      <td>0.162265</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>[C:1](#[C:2][C:3]#[C:4][C@@:5]1([H:9])[C:6]([H...</td>\n",
              "      <td>7.464116</td>\n",
              "      <td>0.278382</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>[C:1](/[C:2](=[N:3]\\[O:4][H:11])[C@@:5]1([H:12...</td>\n",
              "      <td>8.581939</td>\n",
              "      <td>0.084502</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>[C:1]1([H:7])([H:8])[C@@:2]2([H:9])[C:3]([H:10...</td>\n",
              "      <td>7.991354</td>\n",
              "      <td>0.048565</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>[O:1]([c:2]1[n:3][n:4]([H:8])[n:5][c:6]1[H:9])...</td>\n",
              "      <td>6.699617</td>\n",
              "      <td>0.014819</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>[C:1]([c:2]1[n:3][n:4]([H:10])[n:5][c:6]1[H:11...</td>\n",
              "      <td>7.758332</td>\n",
              "      <td>0.024714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>[N:1]([c:2]1[c:3]([H:10])[o:4][n:5][c:6]1[N:7]...</td>\n",
              "      <td>7.714368</td>\n",
              "      <td>0.011509</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>[N:1]([c:2]1[c:3]([H:10])[c:4]([N:5]([H:11])[H...</td>\n",
              "      <td>7.620089</td>\n",
              "      <td>0.049530</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>[C:1]([c:2]1[c:3]([H:11])[c:4]([N:5]([H:12])[H...</td>\n",
              "      <td>7.897841</td>\n",
              "      <td>0.023132</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C@...</td>\n",
              "      <td>8.073214</td>\n",
              "      <td>0.064748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>[O:1]([C@@:2]1([H:9])[C:3]([H:10])([H:11])[C@:...</td>\n",
              "      <td>7.238061</td>\n",
              "      <td>0.030620</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C@...</td>\n",
              "      <td>8.160344</td>\n",
              "      <td>0.162666</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>[C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C@...</td>\n",
              "      <td>7.405831</td>\n",
              "      <td>0.057869</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C@...</td>\n",
              "      <td>7.736525</td>\n",
              "      <td>0.051980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>[O:1]([C@@:2]1([H:9])[C:3]([H:10])([H:11])[C@:...</td>\n",
              "      <td>7.024516</td>\n",
              "      <td>0.036442</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>[O:1]([C@@:2]1([H:8])[C:3]([H:9])([H:10])[C@:4...</td>\n",
              "      <td>8.845977</td>\n",
              "      <td>0.183325</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>[C:1]([C@:2]12[C:3]([H:10])([H:11])[C:4]([H:12...</td>\n",
              "      <td>8.116722</td>\n",
              "      <td>0.182136</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[O:3][C:4]([H:12])([H:13...</td>\n",
              "      <td>8.908027</td>\n",
              "      <td>0.265048</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[O:3][C:4]([H:12])([H:13...</td>\n",
              "      <td>7.425850</td>\n",
              "      <td>0.088779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>[O:1]=[C:2]1[C:3]([H:8])([H:9])[O:4][C@@:5]2([...</td>\n",
              "      <td>8.702798</td>\n",
              "      <td>0.237794</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>[C:1]([C@@:2]1([C:3]([O:4][H:12])([H:10])[H:11...</td>\n",
              "      <td>7.963273</td>\n",
              "      <td>0.081806</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>[C:1]1([H:6])([H:7])[C:2]([H:8])([H:9])[C@@:3]...</td>\n",
              "      <td>8.077995</td>\n",
              "      <td>0.145824</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>[N:1]#[C:2][C@@:3]1([H:7])[C:4]([H:8])([H:9])[...</td>\n",
              "      <td>7.693681</td>\n",
              "      <td>0.044768</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>[C:1]1([H:8])([H:9])[O:2][C@@:3]2([H:10])[C:4]...</td>\n",
              "      <td>7.850417</td>\n",
              "      <td>0.090569</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>[C:1]1([H:8])([H:9])[O:2][C@@:3]2([H:10])[C:4]...</td>\n",
              "      <td>7.768584</td>\n",
              "      <td>0.075748</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>[C:1]([C:2]#[C:3][C:4]([N:5]1[C:6]([H:13])([H:...</td>\n",
              "      <td>8.004091</td>\n",
              "      <td>0.077657</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>[N:1]([C:2](=[O:3])[C:4]([N:5]1[C:6]([H:12])([...</td>\n",
              "      <td>7.967761</td>\n",
              "      <td>0.065627</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>[C:1]([C:2]#[C:3][C:4]([N:5]1[C:6]([H:13])([H:...</td>\n",
              "      <td>9.012047</td>\n",
              "      <td>0.067033</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>[N:1]([C:2](=[O:3])[C:4]([N:5]1[C:6]([H:12])([...</td>\n",
              "      <td>7.712333</td>\n",
              "      <td>0.052295</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>[C:1]([C@@:2]1([H:11])[C:3]([H:12])([H:13])[C:...</td>\n",
              "      <td>7.955681</td>\n",
              "      <td>0.023752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>[C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C:...</td>\n",
              "      <td>9.677749</td>\n",
              "      <td>0.084125</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>[C:1]([C@@:2]1([H:10])[C:3]([H:11])([H:12])[C:...</td>\n",
              "      <td>8.062699</td>\n",
              "      <td>0.030905</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>[C:1]([C:2]1([C:3]([O:4][H:12])([H:10])[H:11])...</td>\n",
              "      <td>10.491706</td>\n",
              "      <td>0.295304</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>[C:1]1([H:7])([H:8])[C:2]([H:9])([H:10])[C:3](...</td>\n",
              "      <td>9.052074</td>\n",
              "      <td>0.064273</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>[C:1](#[C:2][C:3]1([H:8])[C:4]([H:9])([H:10])[...</td>\n",
              "      <td>8.012644</td>\n",
              "      <td>0.064509</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>[C:1]([O:2][C@@:3]1([H:11])[C:4]([H:12])([H:13...</td>\n",
              "      <td>7.975821</td>\n",
              "      <td>0.075249</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>[O:1]([C:2]([C:3]1([H:10])[C:4]([H:11])([H:12]...</td>\n",
              "      <td>9.594182</td>\n",
              "      <td>0.106705</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e8059f09-4dac-4e08-9849-6e56be6902eb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e8059f09-4dac-4e08-9849-6e56be6902eb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e8059f09-4dac-4e08-9849-6e56be6902eb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ],
      "source": [
        "preds_df = pd.read_csv('test_preds_ensemble.csv')\n",
        "preds_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "IDoQXtURqj4y",
        "outputId": "da11e8ff-9738-419f-d269-fd18215ff400"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQkAAAEGCAYAAAB2PmCxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZxU1bXvv6u7aUYVBUQBESIECVODrYioiSABWuQKgsg1QkCC98UImuuLwWiu3ud4nSIGX5IbFU183ThFvDESFNEY0xAGAduBSUBAkEn0QDc03bXeHzVY3VRVV1fVqVNVvb6fz/lUn6H2XnW66nf2XmvvtUVVMQzDiEae1wYYhpHZmEgYhhETEwnDMGJiImEYRkxMJAzDiEmB1wbEQ/v27bVbt25em2EYOYfP52PDhg0cPnx4n6p2iHRNVohEt27dWLlypddmGEZO4TgOo0eP5siRIwDbol1n3Q3DaIIEBWLZsmWUlpbGvNZEwjCaINdcc01IICZOnBjzWhMJw2iC3HXXXSxYsKBBgYAsFokdO3Zw7rnnkp+fj4jYFrbl5+dz7rnnsmPHDq//TUYG4TgO8+fPB2DgwIFceeWVcb0va0Vi3LhxjB8/nqqqKlTVtrCtqqqKcePGMW7cOK//TUaGEPRBzJgxg48//rhR75VsmOBVXFys9aMb+fn5VFVVUVhY6JFVmU11dTUtW7aktrbWa1MMjwl3UpaVlTFhwoTjrhGRVapaHOn9WduS8Pl8JhAxKCwsxOfzeW2G4THxCERDZK1IGIbRMO+88w4rVqxIWCAgx0RCRPjBD34Q2q+pqaFDhw6MGTOmznVXXHEF559/fp1jd955J507d6aoqCi0HTx4MCl7XnjhBfr06UNeXl7MwWAHDx5kwoQJnH322fTu3Zvy8vI65x9++GFEhH379iVlj9F0CLoRxowZw6ZNmxIWCMgxkWjdujUVFRVUVVUB8MYbb9C5c+c61xw8eJBVq1bx1Vdf8emnn9Y5d/PNN7NmzZrQ1rZt26Ts6du3Ly+//DIXX3xxzOtmz57NqFGj+OSTT1i7di29e/cOndu+fTuLFy+ma9euSdliNB0cx2HEiBEsWrQIgDPOOCOp8nJKJABKSkp47bXXACgtLWXy5Ml1zr/88stcfvnlXH311ZSVlblqS+/evenVq1fMa7766iv+9re/cd111wF+X0K4ON18883813/9FyLiqq1GbhD0Qbz99tscOnQoJWXmnEgEf/xHjhxh3bp1DB48uM75oHBMnjz5uOGojz76aKircckllxxXtuM4dboj4dtHH32UkL1btmyhQ4cOTJs2jYEDBzJjxgwOHz4MwMKFC+ncuTMDBgxIqGyjaZEKJ2UksmKCV2Po378/W7dupbS0lJKSkjrnvvjiCzZu3MiFF16IiNCsWTMqKiro27cv4H9q33LLLVHLPuGEE1izZk1K7a2pqWH16tU8/vjjDB48mNmzZ3P//fczZ84c7r33XhYvXpzS+ozcpLKy0hWBgBxsSQCMHTuWW2655biuxvPPP8+XX35J9+7d6datW0hM4sWNlkSXLl3o0qVLqMUzYcIEVq9ezebNm9myZQsDBgygW7du7Nixg0GDBrF79+6E6jFymxYtWjBgwICUCwTkYEsCYPr06bRt25Z+/frx9ttvh46XlpayaNEihgwZAvib+pdeein33HNPXOW60ZI47bTTOOOMM1i/fj29evViyZIlfOc736Ffv37s2bMndF1wunz79u1TWr+R3TiOw4EDBzjzzDOZN2+eK3XkZEuiS5cuzJo1q86xrVu3sm3btjqhz+7du3PSSSexfPlyoK5PoqioiK1btyZlx5/+9Ce6dOlCeXk5l112GSNHjgTg888/r9MVevzxx7nmmmvo378/a9as4bbbbkuqXqNpEPRBDB8+nKNHj7pWT9YOyxYRssF2L7F7lLuk2kmZk8OyDaOp4lYUIxomEoaRZcyZMydtAgEuioSIPCUie0SkIuzYgyLyiYisE5E/iUhyQxoNowly991389prr6VFIMDdlsR8YFS9Y28AfVW1P7ABmJPKCvPz8ykqKqJv375cfvnlobkXW7duRUS4/fbbQ9fu27ePZs2a8ZOf/ASA9evX873vfY+ioiJ69+7NzJkzAXj77bc56aST6jg033zzzaTsfO655+jfvz/9+vXjggsuYO3atRGv27JlC4MHD6ZHjx5MmjSJ6upqwD+eI2jLt7/97aSHjxuZj+M43HrrrVRVVdG2bduQEzwtuJn8BOgGVEQ5Nw54Lp5yzjnnHK2P3/S6tG7dOvT3lClT9O6771ZV1S1btmj37t21qKgodP6JJ57QAQMG6A033KCqqt///vf1lVdeCZ1ft26dqqouXbpUL7vssuPqSob33ntPDxw4oKqqf/nLX/S8886LeN3EiRO1tLRUVVWvv/56feKJJ467Zu7cuTpt2rSI7490j4zs4+uvv9YLL7xQ8/Pz9a233nKlDmClRvn9eemTmA68Hu2kiMwUkZUisnLv3r2NLnzIkCHs3LkztN+qVSt69+4dmo25YMECrrrqqtD5Xbt20aVLl9B+v379Gl1nvFxwwQWcfPLJAJx//vkR08ypKm+99VaoSTl16lReeeWV466LND/FyB0cx6GkpITy8nLKysoiThdwG09EQkR+AdQAz0W7RlV/p6rFqlrcoUPENUOiUltby5IlSxg7dmyd48F5Hdu3byc/P59OnTqFzt18880MGzaM0aNH8+ijj9aZJv7uu+/W6W5s3rz5uDonTZoUcSTms88+G9PWJ598ktGjRx93fP/+/bRt25aCAv94ty5dutQRPYBt27axZcsWhg0b1vBNMbKO+gKRLh9EfdI+4lJEfgiMAYYHmjkpo6qqiqKiInbu3Env3r0ZMWJEnfOjRo3ijjvuoGPHjkyaNKnOuWnTpjFy5EgWLVrEwoUL+e1vfxvyFVx00UX8+c9/jln3ggULGm3v0qVLefLJJ/n73//e6PcCoS9Ofn5+Qu83MpudO3eyefNmTwUC0tySEJFRwM+AsapameryW7ZsyZo1a9i2bRuqetww1cLCQs455xwefvjhiDe9U6dOTJ8+nYULF1JQUEBFRcVx10SjsS2JdevWMWPGDBYuXEi7du2OO9+uXTsOHjxITU0N4M8OXj83RllZmXU1cpCjR4+iqpx99tls3LjRU4EA3HNcAqXALuAYsAO4DtgEbAfWBLbfxFNWIo7L1atXa9euXfXYsWO6ZcsW7dOnj6qqVlRU6Pz581VV9emnnw45Ll9//XWtrq5WVdVdu3bpaaedprt27XLFcblt2zY966yz9L333ot53YQJE+o4LufNmxc69/HHH+uZZ56pPp8v6vsj3SMjswk6KW+//fa01osXjktVnayqp6tqM1XtoqpPqmoPVT1DVYsC27+5Vf/AgQPp37//cbM8+/Tpw9SpU4+7fvHixfTt25cBAwYwcuRIHnzwQU477TTgeJ/Eiy++mJRt//mf/8n+/fv58Y9/TFFREcXF34yGLSkp4fPPPwfggQce4JFHHqFHjx7s378/lJgG/K2Iq6++2pLR5BDhPohMyiFiczdyGLtH2YPXTkqbu2EYGYzP5+Pyyy/3PIoRDRMJw/CYvLw8rr/++owUCMjipDN5eXlUV1fbAj1RqK6uJi/PngGZjOM4rF69mu9+97sZHaXK2m/RoEGDeOihh0LzGYxvqK6u5qGHHmLQoEFem2JEITjdu6SkpE4GsowkWtgjk7ZIIdDt27drcXGx5uXlKWBb2JaXl6fFxcW6ffv2xOJhhqt8/fXXOnToUM3Pz9cXXnjBa3NUNXYINGu7G126dGHFihVem2EYjSLdCWNSQdZ2NwwjG3n66aezSiAgix2XhpGN3HjjjVx44YVZ5S+yloRhuIzjOEyaNImNGzciIlklEGAiYRiuEvRBvPTSS3z44Ydem5MQJhKG4RL1nZRXXHGF1yYlhImEYbhANkYxomEiYRguUVBQkPUCARbdMIyU4jgOeXl5nHDCCSxdujQnpvKbSBhGigh2MVq2bMnixYtzQiDARMIwUkJ9H0SuCASYT8IwkiaXnJSRMJEwjEbw6Bsbjjs2bdq0nBUIMJEwjEbx2JKNxx27++67efHFF3NSIMBEwjBiEqnlAP4uxhNPPBFKfZ+tA6XiwUTCMGIQqeUQ9EHMmjWLdevWeWBVejGRMIxG4DtaGXJSlpaWZlTqe7fI2pT6huEmPp9SeayWls3yqTpWS8uCPCqra2hVWMD6zVv4av9ehgw532szU0aslPo2TsIw6uHzKfsPVzOr9H1WbD3Aud1O4bGriyj752fMfWsT53Y7hbmTB+LzKXl5uTMeIhrW3TCMelQeq2VW6fuUf7qfGp9S/ul+ZpetYWTf00P7s0rfp/JYrdempgUTCcOoR6vCfFZsPVDn2IqtB+hxaps6+60Km8Zq7iYShlGPyupazu12Sp1j53Y7hU17DtXZr6y2loRhNElaNcvnsauLGPKtdhTkCUO+1Y7Hri7irxW7QvtzJw+kVbOm0ZIwx6Vh1KO6+ijzHrmfO/71Wnr1OI+qYz5aFuQx/aJvccOwnlQdq6VVs/wm4bQEa0kYxnE0b96cI1VVfLh2NWfd9jptmheQn59Hm+YFnHXbX2jTvKDJCATEaEmIyOP4V4SKiKrOcsUiw/AIx3HYvXs3PXv25IEHHgDgf696zWOrvCdWS2IlsApoAQwCNga2IsBW6TVyiuBQ62HDhlFVVeW1ORlF1JaEqj4DICL/C7hQVWsC+78B3k2PeYbhPvXzQbRs2TJ0bvbwnh5alhnE45M4GTgxbL9N4FhMROQpEdkjIhVhx04RkTdEZGPgtcFyDMNNGkoYc/OIb9fZb4qiEY9I3A+8LyLzReQZYDVwbxzvmw+Mqnfs58ASVe0JLAnsG4Zn3HnnnY1KGFNfNJoCcU3wEpHTgMGB3eWqujuuwkW6AX9W1b6B/fXA91R1l4icDrytqr0aKscmeBlucfjwYZYtW8bw4cO9NsVTYk3warAlIf6MnpcCA1R1IVAoIuclaEtHVd0V+Hs30DFGvTNFZKWIrNy7d2+C1RnG8TiOw+zZs3Ech9atWzd5gWiIeLobTwBDgMmBfQeYl2zF6m/CxAqx/k5Vi1W1uEOHDslWZxjANz6IefPmsWzZMq/NyQriEYnBqnoDcARAVb8k8RDoF4FuBoHXPQmWYxiNpr6TcsSIEV6blBXEIxLHRCSfwFNfRDoAvgTrexWYGvh7KrAwwXIMo1Hketp7N4lHJOYCfwJOFZF7gL8D9zX0JhEpBcqBXiKyQ0Suwx8pGSEiG/H7Oe5P2HLDaAR79+5lx44dSQuEz6ccOlqDTwOvvszP7JYs8UY3zgaGA4I/hPmx24aFY9ENI1Gqqqpo0aIFIsKRI0do0aJFwmVFylg1d/JA2rUuzPq5HMlGN/6gqp+o6jxV/bWqfiwif0i9mYYRH6GnuS/209xxHEaMGMFPf/pTgKQEAiJnrGoKGari6W70Cd8J+CfOccccw4hN8Gn+o2dW8u3bX+dHz6xk/+Hq44Qi3AcxdOjQlNQdLWNVrmeoiioSIjJHRBygv4h8Hdgc/BEJczganhDP09wtJ2W0jFW5nqEqqkio6n2qegLwoKqeGNhOUNV2qjonjTYaOUC0lbAaS0NPc1XliiuucCWK0apZPnMnD6yTsaopZKiKp7vxTxE5KbgjIm1FJHfXNDNcIdJKWInQ0NNcRLjpppsoLS1NeZgzL09o17qQ/55azIZ7RvPfU4tzwmnZEPGIxH+o6lfBHVU9CPyHeyYZRnSiPc3zaqtZtGgRAJdffjkTJ050pf68PPFnphJpMhmq4slxGUlILDem4QnhT/NWhflUVteSV1vNyJEjWbFiBZs3b6Zz585em5lTxNOSWCkij4jIWYHtEfwZqwyjQYJhys33lqRs8FH401yrqxg5ciTl5eX88Y9/NIFwgXhE4kagGlgQ2I4CN7hplJEbhIcre8UIVyaK4ziUlJRQXl5uQ61dpEGRUNXDqvrz4IxMVZ2jqofTYZyRGuIdfJRq3B58VFZWZgKRBmJly/6Vqt4kIv9DhCndqjrWVctyhEff2OBpNiMvhxK7PfhoxowZDBkyhL59+6akPCMysVoSwaHXDwEPR9iMOEhV6C9RvBxK7MbgI8dxGDduHBUVFYhITglEpk4ei5Ute1Xg9Z30mWOkGq+GEvt8CgrP/Wgwn+2v5FdvbuCLr48mNfgofCTllClTck4gEmnxpaOlGqu78QGxM0f1d8UiI6UEn+bln+4PHQs+zds0dyeSHfkLX0Tr5gW0KEhsebz6Q63HjRvnguXeEd7iA0Itvv+eWhzz//TYko2ui0Ss7sYY4HJgUWC7JrC9DvzFVauMlOHFUOLIXZw1+JSUCEQuOikb2+JzI7QcjVjdjW0AIjJCVQeGnbpVRFaThenw0+lE9PmUymO1oX+iVwvMhg8+atksPy2L3aa6i1NQUMCJJ56YkQKRqu9UY1p86XZGxzNOQkRkaNjOBXG+L2MIqu6s4T3T4hBye3xAYwkOPpq7ZGNahhKnymHpOA4HDx6kZcuWvPbaaxknEJA6x3RjWnzpdkbH82O/DnhCRLaKyFb82bOnu2KNC3jxg83U5CTpakWloosT7GJcdtll+Hw+/Cs75BbhM2MbM3ks3c7oBj1XgSjHgOBM0PDJXtlAog6hZGiqyUmCRJpf0ZguTn0fRF5eVjVc46a+0zHY4gNifjfT7YyOJ31dRxF5EihT1a9E5DuBpLZZgRc/2KaanCScRGdL5rKTMtjtrfX5cI4cS9jpmG5ndDwSPR/4K9ApsL8BuMkVa1zAix9sU01Okgquv/76rBCIxkYXgt3ep979lJ1fHmHms6vCur9HqayuW0asofThLbX1d7uf16LBbNkiskJVzxWR94NRDhFZo6pFrlgUgWSyZXs1LDkY3UhXRMEtgp8jkW5DImzZsoUPPviAsWMzd9R/It+pQ0dr+NEzK7lzbB/ufPXDOl2FId9qx33j+9G6eQHtWvvXvYq3/G4/f42t91+W9GdKKls2cFhE2vHN4jznA1njl0i36obX26Z5AWfd9pesTU5SJ+nsL9xz+jqOw8MPP4zP56N79+4ZLRCQmGM62O3tcWqbiN3fM05pFSoj0xzf8YjET/GvvHWWiLwHPIt/+njWkO4QYK6Qji9r0Adx6623snr16pSV6yaJ+LmC3d5New5F7P5u2nMoVEZjyp89vGcSnyQ+YopEIH3+dwPbBcD1QB9VXee6ZS7g5WzMbMRtp299J2VxccTWbsaRiJ8r6Kf6a8UuHriyfx1/1QNX9mfe0k2hMhpTfjq+0zFFQlVrgcmqWqOqH6pqhaoec90qIyNw0+mbzVGMRBzTwW7v9Iu+ReeTW/C7Keew4Z7R3De+H4+8sZ69zjeT3zLO8a2qMTfgUeDXwEXAoODW0PtSuZ1zzjmarTyyeL3XJiRMba1P93x9RK/+bbmeNec1vfq35brn6yNaW+tLuux3331XW7durS+88EIKLE0/tbU+dY4c05rAayL35Mb/tzpqGcHya32Jl98YgJUa5fcXT3RjaWRt0WGplavo2Fqg3pHq6IbP5wsNjtq3bx/t27dPlameEIwuJHKfgvM+vE5MBLGjG/EMz5qoqvtSbJORJcQ7CjAeHMdhzJgxTJ8+nalTp2a9QARJNMweFAavBaIhYi3zd7mI7AXWiciOwMQuw0iIoA/ivffeo3Xr1l6bk1IyLWSZamI5Lu8BLlLVTsCVwH3pMcnINhpawi+bnZTxkOtzdWKJRI2qfgKgqsuBE9JjUvbgVRbqTCGeKfjV1dU5LRCzh/fM+bk6sUTiVBH5aXCLsN+kqTMaMQNyRqSbmhofh47W0LJZPpv2HOKpdz+N+PkLCwsZPXp0TgoE+P0JGReyTDFRoxsiEnO9T1W9yxWLIpCJ0Y3gWPz6Y/DdnIKeKdTW+thfWc3s0jUhR90DV/bnlfd3MP2ib9GmeQGO4/DZZ5/Rp08fr80N4eY8lHTPcUk1CUU33BQBEbkZmIF/PsgHwDRVPeJWfW6Qrf3QaF/m8OOHj9bQqjCfqmO+iOcrj9VStvyzOjk6bn1pHXeO7UOrwvyQD2Ljxo1s3ryZNm3aePyp3Z/ol8ooUKaR9mweItIZmAUUq2pfIB+4Ot12JEs29kOjTdiqrfXVOT7z2VXs/PJIqAsR6fwVA7swdkCnUNkrth6gZ8c2HKmuCfkgfv3rX0cViHSvMZHtEQgv1+TwKuVPAdBSRAqAVsDnHtkRorH/hGzsh8b6odQ/futL6xjZ9/SY52+4pEeo7HO7ncJn+ys5ePgIZ57ZjdLSUiZOnBjRjnTNLg0nW1t+4M39Cift7SJV3SkiDwGfAVXAYlVdXP86EZkJzATo2rWrqzYl0hT1Igt1skT7obRuXhDxeI9T29DxxOYxzxfkScgn8dBi/xyExx6fx6mnnBTVDi9SCnqx/kiq8OJ+hRNP+rp2IvK4iKwWkVUi8lggv0RCiMjJwL8A3fFnu2otIj+of52q/k4DixR36NAh0eriItGmaLZNQY/WRTp8tCbi8c8PVnHLyF58tr8ySteqhvV3j+bOsX14aPF6Xl37OSu2HqD9ySfGtMOLp3o2tvyCeN0Kiqe7UQbswT+gagKwF1iQRJ2XAltUda/6Z5S+jH8aumck+0/I9GG1QWL9UOoff+DK/hQW5PG/X1jHI29sOG5689zJA8nPE37w++WM/NXfeHWtv8cYj18mlli51YRuTDbqWHjhG/Da/xXPBK+KgIMx/NgHqtovoQpFBgNPAefi727Mxz8D7fFo73E7BJrKcGamh8IaE90QEb79i9ep8SljB3Tihkt60OPUNlRV15KXB80L8th36Cg3la1tVMQgUvcuGEKdPLirfznAZpl3/7xMheh2vbFCoPGIxCPAP4HnA4cmAOep6i1JGHQXMAmoAd4HZqjq0WjXuy0SqfonePUlaizxClk08fzttedw/R9WsWLrAWYN68HUod04oUWzuH/UPp9ypKaWWp/SqrCATXsOMW/pJl5d+3ko3+Olj7yTcffPy7Exbj98khUJB2gN+AKH8oDDgb9VVWN3QFNAqkQi1o1OxT8hGwZYNUbIIl372OQiypZ/xiNvfrNyVWM+Y3iZf5wxmF63+1sqQQryhPV3j+as2/7S6LLdxqcaalkFKcgTNtwzmrwsXzwoqaniqpoTczYa+nGkYjCM1w6meGiMpzzSIjstm+Ux961Nda6L5zMGRRglVH8w32P9iMOmPYcaVXa6yOYISTLENU5CRMaKyEOBbYzbRrlBOgbTeO1giofGCln4IjtSc5SqBD5jeJy/ZVj985ZuOs4h+uBEf77HeMtOJ9kcIUmGBuVPRO7H72R8LnBotogMVdU5rlqWYtLxlA9+ieq3VjLpS5To0zA41HrK1KnMnTylUZ8xXKDDWw/BiMh94/vRtV0rKo/WUuvzsdc5Ghp/MXdyEXniFxqv/RLJLl+YrcTTRioBilTVByAiz+B3NmaVSKSjqRj6Ek0pplXzfCqP+qMAmUQiQhaeD+Kmm25q9A+lVYTWw60vrWPF1gPsdY7SunkBKLRpUYDPpzw5tRifQqvm+ez8sorS5Z8yefCZGeHAzOU5GtGI91O2BYKP4ehD6TKYdD7lq47V8qNnVx7n+wA8D4829mkYLWFMY34o4QJ9XOshQv2HqmuOm2FaunxbaIapkV7iiW5cDTwALAUEuBj4uaomM6CqUaQjupEqokU4nvxhMYeP1mZ8eDQcVWXkyJG89dZbSeWDaExEJdr9u3NsH3p2bJP1UYRMJeEQqIjk4R8X8S5+vwTAP1V1d8qtjEEm5pOIRrQw2Zpffp8fPeteeNQtAVyyZAlffvll0glj4rUv2v1bf/doqo7ldhTBSxJeCzTgh/iZqu5S1VcDW1oFItuIFuFo1Ty24zSZ4b6pniXoOA6vvPIKAMOHD09JRqnwKEmseS6xhmxnkgO4KRGPW+1NEblFRM4QkVOCm+uWZSnRwmSVR6OHDpP9kacyvBv0QVx11VVs27at0e9Plkj377HJRbQuzP0oQqYSj09iS4TDqqrfcsek48mm7gZEblpD9OXkK4/VJjVSM1UjATMlq3X4/dv4xSF6ntrGBMJlkh1x2T31JuU20cJk0aIKyY7hSEV4N1MEAurev5G/+htb77/MM1uM+PJJtAhkyH5ZRF4SkZtEpEU6jMs1ovXLkx2pmYqRgAsXLswIgajP7OE9vTahyRNPd+N5wAH+GDj0r0BbVY2cm8wFsq27UZ+GPPupmD0aqqNZfui1sU309evX06tXr0a9x8gNkp0F+pGqfqehY26SzSIRrwB4kYfCcRwmT57ML3/5S8477zxX6zIym4RDoAFWi8j5YYUNBrLzF+sB8UYe4g0RpoqgD2LRokXs2LHD1bqM7CYer9Y5wD9E5LPAfldgvYh8gD/K0d8163KATJw+Xt9JOX78eM9sMTKfeERilOtW5DCZloPg0KFDGRPFMLKDBrsbqroN+Br/xK52wU1VtwXOGTHItBwEhYWFnH766SYQRtzE47j8P8APgc34l+UDfzdjmLumfUM2Oy4hM5LjOo7DkSNH6NChA6qK2EQpI4ykBlMBVwFnqWp1as3KbeqvndmqWX7IKZluHMehpKSEw4cPs2LFCvLzbQ6EET/xfGMr8OeT2OOyLTlDJmXNDgpEeXk5ZWVlJhBGo4knBHof8L6I/FVEXg1ubhuWzWTK4rT1BcJ8EEYixNOSeAZ/0pkP+CatvhGDTAl73njjjSYQRtLEIxKVqjrXdUtyiEwJe957771MmDCBMWOyMsG5kSHE0914V0TuE5EhIjIouLluWRbjZdjTcRzuueceampq6NSpkwmEkTTxPNYGBl7PDzumQNpCoNmGV6nXw0dSDhs2jCFDhrhan9E0iCefxCXpMCTXSHfq9fpDrU0gjFQRTz6JjiLypIi8Htj/johc575pRrxkUsIYI/eIxycxH/gr0CmwvwG4yS2DjMazfv16KioqTCAMV4inHdxeVZ8XkTkAqlojIpmxOGMTp6amhoKCAoqLi9m6dStt27b12iQjB4mnJT7z2vMAAArOSURBVHFYRNoRmLcRyC3xlatWGQ3iOA6XXHIJ8+bNAzCBMFwjHpH4KfAqcJaIvAc8C9zoqlVGTII+iPLycjp27Oi1OUaOE090Y7WIfBfohX+Zv/Wqesx1y4yImJPSSDfxxubOA7oFrh8kIqjqs65ZZUSkpqaGkpISEwgjrTQoEiLyB+AsYA0QdFgq/m5HQohIW+D3QN9AWdNVtTzR8poKBQUFTJgwgdmzZ5tAGGkjnpZEMfAdbSg7TeN4DFikqhNEpBBolcKycw7Hcdi4cSODBg1i9uzZXptjNDHicVxWAKelqkIROQm4GHgSQFWrVfVgqsrPNYI+iEsvvZSvvrKgkpF+orYkROR/8HcFTgA+EpF/AkeD51V1bIJ1dgf2Ak+LyABgFTBbVQ/Xq38mMBOga9euCVaV3dR3Up500klem2Q0QaLmuAxENKKiqu8kVKFIMbAMGKqqy0XkMeBrVb0j2nuyPcdlIlgUw0gniea43Al0VNX36hV2IbArCXt2ADtUdXlg/0Xg50mUl5M8+uijJhBGRhDLJ/Er/Kn06/NV4FxCqOpuYLuIBBedHA58lGh5ucptt93GO++8YwJheE4skeioqh/UPxg41i3Jem8EnhORdUARcG+S5eUEjuMwbdo0du/eTUFBAUOHDvXaJMOIKRKxJgO0TKZSVV2jqsWq2l9Vr1DVL5MpLxcI+iD+8Ic/0NT8L0ZmE0skVorIj+ofFJEZ+CMSRooId1KWlpZayjkjo4jluLwJ+JOIXMM3olAMFALj3DasqVBfICZOnOi1SYZRh6gioapfABeIyCX4h08DvKaqb6XFsiZCZWUljuOYQBgZSzyzQJcCS9NgS5Pi0KFDNG/enI4dO7Jq1SoKCtK//J9hxEM8w7KNFOM4DqNGjWLKlCkAJhBGRmMikWbCfRDjx48PHff5lENHa+q8GkYmYI+wNBLNSZlJCwwbRn2sJZFGJk2aFDGKkSkLDBtGJEwk0sgdd9xBWVnZcVGMTFlg2DAiYSLhMo7jUFZWBsCQIUMizsUILjAcTnCBYcPwGhMJFwn6IK699lo2bdoU9TovFxg2jIYwx6VL1M8H0aNHj6jXhi8w3LJZPlXH0rPAsGHEg7UkXCCRhDHBBYbnLtlIm+YFJhBGxmAi4QKLFy9m+fLlCSWMuXnEt12yyjASw7obKURVERGuvPJKNmzYQPfu3b02yTCSxloSKSLYxXjnHX/qTxMII1cwkUgBQYF488032bdvn9fmGEZKMZFIkvpOyiuvvNJrkwwjpZhIJMHhw4ct7b2R85hIJEGLFi3o2bOnCYSR01h0IwEcx8FxHDp16sTTTz/ttTmG4SomEo0k6IM4cOAAa9eupVmzZl6bZBiuYiLRCOo7KU0gjKaA+STixNbmNJoqJhJx8u///u8mEEaTxLobcXLvvfcybtw4Ro8e7bUphpFWrCURA8dxuOOOO6iurqZ9+/YmEEaTxEQiCo7jUFJSwn333ceyZcu8NscwPMNEIgJBgSgvL6esrIyLL77Ya5MMwzNMJOpRXyDMSWk0dUwk6rFlyxY++eQTEwjDCGDRjQDV1dUUFhbSv39/Pv30U0444QSvTTKMjMBaEvi7GMOHD+f+++8HMIEwjDCavEiE+yBiZbQ2jKaKZyIhIvki8r6I/NkrG8xJaRgN42VLYjbwsVeV19bWMmbMGBMIw2gAT0RCRLoAlwG/96J+gPz8fKZMmWICYRgN4FV041fAz4CoHkIRmQnMBOjatWvKKnYch4qKCoYMGcJ1112XsnINI1dJe0tCRMYAe1R1VazrVPV3qlqsqsUdOnRISd3B6d4jR45k//79KSnTMHIdL1oSQ4GxIlICtABOFJE/quoP3Ky0fj6Idu3auVmdYeQMaW9JqOocVe2iqt2Aq4G30i0Q5oMwjPhpEuMkfvOb35hAGEaCiKp6bUODFBcX68qVKxN+v8/nY+XKlZx33nkptMowcgcRWaWqxZHO5WxLwnEcrrnmGrZt20ZeXp4JhGEkSE6KRNAHsWDBAtauXeu1OYaR1eScSNR3Uo4dO9Zrkwwjq8kpkbAohmGknpwSiZqaGmpqakwgDCOF5ETSGcdxaNasGSeffDL/+Mc/yMvLKe0zDE/J+l9TsItx1VVXoaomEIaRYrK6JVHfByEiXptkGDlH1j52zUlpGOkha0Xi2muvNYEwjDSQtSJx11138fzzz5tAGIbLZJVIOI7DU089BcCAAQMYP368xxYZRu6TNSIR9EHMnDmTDz/80GtzDKPJkBUi4fP5Qk7K0tJS+vTp47VJhtFkyIoQ6IYNGzhy5AilpaVMnDjRa3MMo0mRFSJRWVnJggULTCAMwwOyIumMiOwFtrlQdHtgnwvluoHZ6g5mq58zVTVixumsEAm3EJGV0bLxZBpmqzuYrQ2TFY5LwzC8w0TCMIyYNHWR+J3XBjQCs9UdzNYGaNI+CcMwGqaptyQMw2gAEwnDMGLSZEVCRPJF5H0R+bPXtsRCRNqKyIsi8omIfCwiQ7y2KRoicrOIfCgiFSJSKiItvLYpHBF5SkT2iEhF2LFTROQNEdkYeD3ZSxsDNkWy88HAd2CdiPxJRNqmy54mKxLAbOBjr42Ig8eARap6NjCADLVZRDoDs4BiVe0L5ONf6zWTmA+Mqnfs58ASVe0JLAnse818jrfzDaCvqvYHNgBz0mVMkxQJEekCXAb83mtbYiEiJwEXA08CqGq1qh701qqYFAAtRaQAaAV87rE9dVDVvwEH6h3+F+CZwN/PAFek1agIRLJTVRerak1gdxnQJV32NEmRAH4F/AzweW1IA3QH9gJPB7pGvxeR1l4bFQlV3Qk8BHwG7AK+UtXF3loVFx1VdVfg791ARy+NiZPpwOvpqqzJiYSIjAH2qOoqr22JgwJgEPB/VXUgcJjMaA4fR6Av/y/4ha0T0FpEfuCtVY1D/eMBMnpMgIj8AqgBnktXnU1OJIChwFgR2QqUAcNE5I/emhSVHcAOVV0e2H8Rv2hkIpcCW1R1r6oeA14GLvDYpnj4QkROBwi87vHYnqiIyA+BMcA1msYBTk1OJFR1jqp2UdVu+B1rb6lqRj7xVHU3sF1EegUODQc+8tCkWHwGnC8ircS/tsFwMtTJWo9XgamBv6cCCz20JSoiMgp/F3msqlams+6syCfRxLkReE5ECoFPgWke2xMRVV0uIi8Cq/E3h98nw4Y8i0gp8D2gvYjsAP4DuB94XkSuw5+O4CrvLPQTxc45QHPgjcD6MstU9d/SYo8NyzYMIxZNrrthGEbjMJEwDCMmJhKGYcTERMIwjJiYSBiGERMLgRpxIyLt8E+CAjgNqMU/bBzgPFWt9sQww1UsBGokhIjcCRxS1YfCjhWETUIycgRrSRhJISLzgSPAQOA9EfmaMPEI5EQYo6pbA3M5ZgGFwHLgx6pa643lRryYT8JIBV2AC1T1p9EuEJHewCRgqKoW4e+qXJMm+4wksJaEkQpeiKNFMBw4B1gRGFbckgyeTGV8g4mEkQoOh/1dQ90WajCFnQDPqGraMioZqcG6G0aq2UpgOruIDMKfXwL8UZEJInJq4NwpInKmJxYajcJEwkg1LwGniMiHwE/w52NEVT8CbgcWi8g6/DkbT/fMSiNuLARqGEZMrCVhGEZMTCQMw4iJiYRhGDExkTAMIyYmEoZhxMREwjCMmJhIGIYRk/8PDK4Jd1BX8r0AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "df = pd.read_csv('test_checkpoints_ensemble/fold_0/test_full.csv')\n",
        "plot_parity(df.ea, preds_df.ea, preds_df.ea_ensemble_uncal_var)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b9IskYn_qj4y"
      },
      "source": [
        "# Fingerprint"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1JIABGusqj4y",
        "outputId": "fc4f5f14-394d-41f3-de6c-de79c5b1eb58"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "500it [00:00, 78074.23it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 99405.22it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 500\n",
            "Encoding smiles into a fingerprint vector from 1 models.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/10 [00:00<?, ?it/s]\u001b[A\n",
            " 10%|█         | 1/10 [00:00<00:03,  2.72it/s]\u001b[A\n",
            " 40%|████      | 4/10 [00:00<00:00,  9.70it/s]\u001b[A\n",
            " 70%|███████   | 7/10 [00:00<00:00, 14.14it/s]\u001b[A\n",
            "100%|██████████| 10/10 [00:00<00:00, 18.05it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:01<00:00,  1.00s/it]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_fingerprint.csv\n",
            "Elapsed time = 0:00:02\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'tests/data/regression.csv',\n",
        "    '--preds_path', 'test_preds_fingerprint.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_reg',\n",
        "    '--fingerprint_type', 'MPN'\n",
        "]\n",
        "\n",
        "args = chemprop.args.FingerprintArgs().parse_args(arguments)\n",
        "preds = chemprop.train.molecule_fingerprint.molecule_fingerprint(args=args)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q2zDA4Hxqj4y",
        "outputId": "ff1ff2de-ad2d-4595-ecdb-b8266b6edd7c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(500, 300)"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "preds.squeeze().shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "ucy2Bn5Kqj4y",
        "outputId": "892efe4c-5df3-42d5-e51b-a256065ba3f3"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEGCAYAAACJnEVTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3hUxfrA8e+csy2bQnooIfTeIYAggiAg9oaIgoq9l6tX/anXq97r1Xvt5VYbdhRRLICoSBfpvRNCCSmEFFK3nXPm98eGQCQkAUISyHyeZx9yzp4yGzbvzs68MyOklCiKoiiNg1bfBVAURVHqjgr6iqIojYgK+oqiKI2ICvqKoiiNiAr6iqIojYitvgtwPGJjY2Xr1q3ruxiKopwGVq1alSOljDuZa5w/PFTm5pnV32u970cp5ZiTuVddOa2CfuvWrVm5cmV9F0NRlNOAEGLPyV4jN89k+Y9J1R6nN9sRe7L3qiunVdBXFEWpSxKwsOq7GLVKBX1FUZRjkEgCsvrmndOJCvqKoihVUDV9RVGURkIiMc+wqWpU0FcURamChQr6iqIodabE6yfEYcMTMAix29C0uhteJAFTBX1FUZS64fUH+OuXc5izPoWuLRN4ddLFxEaE1mkZzrSavhqRqyhKg2SYFl8t3cis1dvwGyZrd2Xw3LRfKPb66qwMEghIWe3jdKJq+oqiNEiGabI7O7/Cvoy8wjotg0Secc07qqavKEqD5HLYGT+kF3ZdL993zdm9cNj0Ks6qZRLMGjxOJ6qmryhKg9U8OoKZf7qJOet20LtNc9o1jcFhq7uwFRyRe2ZRQV9RlAYrxGEnxGFn4rC+9VQCgYmop3ufGiroK4qiHEOwI1cFfUVRlEYhmKevgr6iKEqjYamavqIoSuOgavqKoiiNiERgnmGZ7SroK4qiVEE17yiKojQSEoFf1uFgsDqggr6iKMoxBAdnqeYdRVGURkN15CqKojQSUgpMqWr6iqIojYalavqKoiiNQ7Aj98wKk2fWq1EURalFqiNXURSlkTFVnr6iKErjcCaOyD2zXo2iKEots6RW7aMmhBBjhBDbhBApQoj/O8XFPiZV01cURTmG4IRrJ183FkLowL+AUcA+YIUQ4jsp5eaTvvhxUkFfURTlGCSCQO1MwzAASJFSpgIIIT4HLgNU0FcURWkopKSmg7NihRArj9h+W0r59hHbLYC0I7b3AQNroYjHTQV9RVGUYxI1HZyVI6VMPtWlqQ312pErhIgUQkwTQmwVQmwRQgyqz/IoiqIcSRKs6Vf3qIF0oOUR24ll++pcfdf03wBmSynHCiEcgLuey6MoilJBLaVsrgA6CCHaEAz244HrauPCx6vegr4QogkwFJgEIKX0A/76Ko+iKMrvSUStLKIipTSEEPcCPwI68L6UctNJX/gE1GdNvw1wAJgshOgFrAIekFKWHHmQEOJ24HaApKSkOi+koiiNlwQCtTT3jpRyFjCrVi52EuqzTd8G9AX+I6XsA5QARw1YkFK+LaVMllImx8XF1XUZFUVp1ARmDR6nk/oM+vuAfVLKZWXb0wh+CCiKojQIktobkdtQ1FtppZRZQJoQolPZrvOoh4EKiqIoVTnTavr1nb1zH/BpWeZOKnBTPZdHURSlnJTitKvJV6deg76Uci1wWgxoUBSl8Ql25NbKNAwNRn3X9BVFURowtUauoihKoxHsyD292uyro4K+oihKFc60RVRU0FcURTmG2hqR25CooK8oilIFtTC6oihKIyElBCwV9BVFURqFYPOOCvqKoiiNxuk24rY6KugriqIcg0rZVBRFaVRU846iKEqjUsM1ck8bKugriqIcQzB7R829oyiK0iiowVmKoiiNjGreURRFaSRU9o6iKEojo7J3FEVRGgkpBYYK+oqiKI2Hat5RFEVpJFSbvqIoSiOjgr6iKEojofL0FUVRGhmVp68oitJISAmGWkRFURSl8VDNO4qiKI2EatNXFEVpZKQK+oqiKI3HmdaRe2b1UCiKotQiKYNt+tU9TpYQ4iUhxFYhxHohxHQhRGQtFL9SKugrSh0wDJMSj58Sjx/DNOu7OEqNCUxLq/ZRC34GukspewLbgcdr46KVUc07inKKeX0BNu7I5J+fLADgnglD6dGxOS6nvZ5LptREXbTpSyl/OmJzKTD2VN1L1fQV5RSTUvLIP6azfVc223dl88g/vkFKWd/FUmrg0Nw7NWjeiRVCrDzicftJ3PZm4IdaeQGVUDV9RTnFDuSX4PMb5dv+gMGBvGKSmkfXY6mUGpHBdv0ayJFSJld1gBBiDtC0kqeelFJ+W3bMk4ABfHqcJa0xFfQV5RRLiAknqVkUezPzAWjZLIqE2Ih6LpVSU7WVvSOlHFnV80KIScDFwHnyFH4VVEFfUU4xm03jvecn8NVPawG4cnRvbDbVsno6kGUduaeaEGIM8CgwTEpZeirvVe9BXwihAyuBdCnlxfVdHkWpbbqmEep2ct0l/QGw6Srgn07qqPvln4AT+FkIAbBUSnnnqbhRvQd94AFgC6C+7ypnNBXsT091lL3T/pTfpEy9vguFEInARcC79VkORVGUykgZDPrVPU4n9V3Tf51gO1b4sQ4oS326HSApKamOiqUoihJ0pk24Vm81fSHExUC2lHJVVcdJKd+WUiZLKZPj4uLqqHSKcnz8foPiUh+madV3UZRaJmX1j7pW1hd6Quqzpn82cKkQ4kLABUQIIT6RUk6sxzIpynHz+gJMmbGSran7Of+cLgzq3ZYQlxpteyaQCKyGuYjKDiHEV8BkKeXm4zmx3l6NlPJxKWWilLI1MB6YqwK+crop9fr516cLeWfqEhat3MmfXpvBhu0ZasTtGUTW4FEPehGco+ddIcRSIcTtQogaJcM0yI8wRTltSFi+fk+FXQtX7KgwAlc5jTXQjlwpZZGU8h0p5WDgMeBpIFMI8aEQospMoAYR9KWU81WOvnK66ts1scL24D5tcdjrO0dCqTUNsKovhNCFEJcKIaYTTIh5BWgLfA/Mqupc9c5UlJPgDnHwwI3DiQgLYWvqfkYP6UKfbi3RtDMr46Mxa6ApmTuAecBLUsolR+yfJoQYWtWJKugryklyOe3cdNVZBAyTEKcdm+2EEyuUBkYCltUgg/4NUsrFR+4QQpwtpfxVSnl/VSc2iOYdRTnduZx2wkNdKuCfaSQgRfWPuvdmJfveqsmJqqavKIpShYaUiCWEGAQMBuKEEA8d8VQEUKMahwr6iqIoVWlAQR9wAGEEY/eRMxkUUsPVtlTQVxRFOaaGNbeOlHIBsEAI8YGUck+1J1RCBX1FUZSqNKCavhDidSnlg8A/hRBHlUxKeWl111BBX1EU5VgkyIaVvfNx2b8vn+gFVNBXFEWpUsMJ+ocmqCxr5jkhKugriqJUpWE172ygihJJKXtWdw0V9BVFUarSgII+wYXTT0qVQb9s1rY4KeXO3+3vKaVcf7I3VxRFadAODc5qIE40Y+dIxxyRK4QYB2wFvhJCbBJC9D/i6Q9O9saKoiing4a0iIoQYnHZv0VCiMLf/1uTa1RV038C6CelzBRCDAA+FkI8LqWcTkPq2VAURTmVGlD2jpRySNm/x1xitjpVBX1dSplZdoPlQojhwAwhREsaWiuXoijKKXJ0NnzDIIToCwwhGI8XSynX1OS8qiZcKxJCtDu0UfYBcC5wGdDtxIuqKA1PaYmP0hIfXm+gvouiNCQ1mUu/fubT/zPwIRADxAIfCCH+VJNzq6rp38XvmnGklEVCiDHAuBMsq6IcN9OyMKTJ1sJ0mroiibCHEGJz1tr1S0t8vPXaj+zYlsmw4V0Yd90gnGqNWwWAeptFszoTgF5SSi+AEOLvwFrguepOPGbQl1KuO8b+APDpiZVTUY6fzwowcckbZHjyAfhjl0u5qEU/QnTHSV+7tMTHW6/OZs6PGwH4aNcimidGc+55XdF1NfO4QkNtzM4AXIC3bNsJpNfkRPWuVhq8n7PWlwd8gHdTfsEmam/e+u3bsipsb9qwj0DArLXrK6c5qwaPOiKEeEsI8SZQAGwSQnwghJgMbAQO1uQaanCW0uCF21w4NTtum4N8fwlumwNLWtRw+vAqabrGsOFd+HjyovJ9I0d3x6WadxRocHn6wMqyf1cB04/YP7+mFzjuoF+WvTNeSvnS8Z6rKDVRUuJDEwIEhIQ4GBjTkVnnPolhWRzwFeK3/Ihayhp2uexcM2EQzROj2LRhHyPP70Gb9vG1cm3lzNCQsneklB+e7DVqFPSFEHHA1cC1QHMqfsIoyknx+APYdR1TWgQCJm+98gNbNqYzfGQ3Jt40hAKfj5eXL+ablC2Mat2e10dciEOvvS+pTped4SO7MWRYJ1yuk+8nUM4wDSjoHyKE2EUlJZNStq3u3GP+5QghwoErgeuAjsDXQBspZeKJF1VRKir1+/lqw2beWPwbboedp0ecS+8BbVi3ajejLunJkr1prM7K4KZufbmkXWdu++kbFqbt5vw2HWq1HLquoddCx7Ci1JHkI352EayUR9fkxKqqS9nAcuBPBBP/pRDiihMuoqJUosDr49k58wA46PXy8KwfWXTHLUhL8sPenfx9cbCt/T/LlvPZNeO4qG0ncjyl9VlkpZFpSM07h0gpc3+363UhxCrgz9WdW1X2zuME04D+DTx+5EAtRaktWUXFFbaL/X5KfQG6905i+tYt5ftNKfluy1aGt2zDZe271HUxlcZKEpyGobpHLRFCPCyEkEKI2GqO63vEI1kIcSc1bK6vKk//dYKfHm2B8cA3QHMhxGPAdCnl9pq/FEWpXNeEOFpFReLQdS7q0pHOsbFEhDg5UOKjXXQ023Jyyo/tGBvDmDYdEKJBZVMoZ7o6qumXJcmMBvbW4PBXjvjZAHZTw0Gz1X4ySClTgeeB54UQ3Ql25s4C2tfkBopyLCU+P+v2ZfLNDddhCUla0UFah0eR6/EQn9CEF84fhWVZrMjIYEyH9lzSuTO3zJjOuxerVkal7tRh885rwKPAt9UdKKUcfqI3qaojtz2QIKX89YgbbRRC/ABMPtEbKmcuv2VgSYvU4gxahSagCw2XfuzpEgzL4p7PvuOXh29hxp6ttHZEccN70yjxB+gUH8vHN17NMyNG4LLb8ZkG106fypacA2zI3s+AFiqfQKkjNQv6sUKIlUdsvy2lfLumtxBCXAakSynXVfVNVgjxUFXXkVK+Wt29qqrpv06wXf/3Cgh+Il1S3cWVxiXfX8RdK1+myCjFrtl4sdfddI1ohSYq7zoq8vpIio4k11vKOc1a84cvZ1HiD054ti07hy9Xb+SGAb0p9Pq4eeZ0tuQcwKZpdIiJqcuXpTR2NQv6OVLK5KoOEELMAZpW8tSTBKeyH12D+5zwlMqHVBX0E6SUG36/U0q5QQjR+mRvrJxZvKafL/b+QpERzKwJWAYf7Z7N090mEWoLqfScuLBQ7h8xiBZhEeR5PVhWxb8uwzJBCCwpSQgNw9ZU4w8DB+OsxRx9RamKkLXXvCOlHFnpPYToAbQBDtXyE4HVQogBUsoKc4RIKZ892XJU9dcTWcVzlf8VK42WAFxaxakLnNqxpzLw+ALsyytg9fZ0/B6DYd3a8uioc7jz8+8ImCaJkU24pm9PVqXsI6+olFfPuwA0CLHbsGm1N++OolTrFC+iUla5Lh8GLoTYDSRLKXOOdY4QIhF4Czi7bNci4AEp5b7q7ldV0F8phLhNSvnO7252K8F5HxSlnFN3ML7VSJblbWZvaTbRjghub3fpMWv56XkFjHvlE8yy2v0t5/XnphHJ/PbQHaQXFNI6OpL0/EIemzyLEp+fBS/cSZiz9qZTVpSaaoh5+gT7VT8jOCgLYGLZvlHVnVhV0H8QmC6EmMDhIJ8MOACVPqEcJVQP4d/9/kix4SHcFgK/65DyGyZSSgTw3crN5QEfYPaabdxy3gCEkMS7Q7n1zWls2HP4m+3BYi9hLhX0lXpQx0FfStm6BofFSSmPTKj5QAjxYE2uX1We/n5gcNkyid3Lds+UUs6tyYWVxkfXNHQ0nPrRzTo+w2BLZjaPfj2bUV3a061FxUnNOjSLZduBA5QaAQYkJlb4QOjRuimxEaGnvPyKcpRabNOvZblCiInAlLLta4Hfj9KtVFUpmy7gToL5+BuA96SUxkkW9MjrtwQ+AhIIfpa+LaV8o7aur9Qejz+A1zDYsC+L3i2bYdd1QhzHP/Xwg1NnklVYzLTVG7n+ruu5Y9RAZq7eSvumMTx+1QiemTuXEr+fXs2aMvmBq5m/MRUBDOveDpdDdd4q9aRhBv2bCbbpv1a2/StwU01OrOov6UMgQLCD4AKgC8Emn9piAA9LKVeXTe62Sgjxs5Rycy3eQzlJlmWxJSubSZOnETAtXHYbU24bT9u4qOOa6VIXGjnFwcyeAo+P6yd/yTsTr2DisL7syMnl6V/mMndnKhd26ogQApfDzpi+nU7Vy1KUGhN1uEhKTUkp9wCXnsi5Vc2901VKOVFK+T9gLDD0RG5wLFLKTCnl6rKfi4AtQIvavIdy8kr8Af63YDkBM/jO9wYMJv+6ik37syn2+2p8HZ9hcEWfruXb3oBBfEQoui54cdFi5u5MpX1MNE+MGEa46rBVlEoJIW4TQnQo+1kIId4XQhQIIdYLIfrW5BpVVdUCh36QUhqncr6Tsrz/PsCySp67HbgdICkp6ZSVQamcEILw33Wghrsc7DqYT8+mlY0zqVyo08GTFwxnTNeOZBYUMqZ7J+y6jk3T+ODqK9CEQAJ2XaVjKg1Mw2reeQD4oOzna4FeQFuC8fMN4JzqLlBVTb+XEKKw7FEE9Dz0sxCi8OTKfZgQIgz4CnhQSnnUdaWUb0spk6WUyXFxcbV1W6WGwpwOHj1/KK1jogDo3DSOW87pz7y9uwhYx/e912W3cXb7Vozt14MwpwO7riOEwO1w4LLbCbHbsWlq2WalAZGHB2hV9ahDhpTyUIX8YuAjKWWulHIOUKNsh6qyd055lUsIYScY8D+VUn59qu+nnJjo0BCm3z0BT8AgYJlMXrea2/okB5c0VJQzXcOq6VtCiGZAPnAe8LcjnqvRoNl6S4kQwfai94AtNZkkSKk/Nl3HputYUhJhc/LAwEHoQlNNMUrj0LCC/p8JLo6uA99JKTcBCCGGAak1uUB95sGdDVwPbBBCrC3b94SUclY9lkmpgtsRXE5QV00wSiMhaFjZO1LKGUKIVkC4lDL/iKdWAtfU5Br1FvSllIsJ/k6VelRq+JBIbELHQhJyCteJNS0LQ1qkFuXQLKQJDl3HbVPr0ioNWAMcnFWWWDO8kuSaAiHEBilldlXnqxEvjZjPDPDilq+Ys38tIbqTJ7pezaDYzriOI/BblsQXMJi9ahuaJji/b0ecdlulq1t5TYPLfn6X3cV56ELwysDLGNmiE65KRvAqSoPRwIJ+mVuAQcC8su1zCU6X00YI8Rcp5cfHOlF9T2/EthSmMWd/sGXNY/p4eevX6MeY+/5YAqbJDa9+zrNTfubpT3/i5te/xBcwKz12+p717C7OA4Jr3r60fl6lxylKgyJr8Kh7NqCLlPIqKeVVQNeykgwEHqvqRBX0G7ESw1th22P6EcfZ4paRV8iOjMMzwG5O209uUUmlxzp+NyWyQ9eR1fzBlPr8FJZ62ZqejS9gYB5nmqiinKwGlrJ5SMuy+dEOyS7bl8cRY6wqo5p3GrHk6A60CU1gV0nwvXNdq2H4LeO45quPjQjFpmkYZcHYYdOJCq08c+ySpO5MTV3L2rx03DY7T/U5H72KtE+/YbBk+17++MlMDMsiMboJ0/4wQc22qdSthtm8M18IMQP4smx7bNm+UOBgVSeqoH8GC1h+0ko3YkmLVqE9sGsVg6Vd03lvwP1sK0onyhFGjDPiuDtydU3w3A1jeP3bRWhC8Mcrh1Xang/g0m18fO5ECvwemjhC8JjBTmRLWkctqWgYJj5PgCinkyGdWzN/cyr78gr4cukGbhjaV2UQKXVDNqzsnSPcA1wJDCnb/hD4SkopgSoXTVdB/wxgWgYSE0saCKFjEw4MGWDq3r+wu2QdAC1COjOh9d8qBH5NaDh0jR6RrU/43kIXdGgXw9d/ugFNCEzLQrcdDvo+048mNAQCv2Vg12xk+3K5e9UU9nsLaBOawNsD7sFtO1wuj8fPqmWpTPtiGbFx4Txx93Ai3S6+WbkZh029ZZU61gBr+lJKKYRYDPgJlnB5WcCvlvoLOgOUGDl8nfYoBYEMElyduDTxBQxplAd8gHTPVgoDOcQ4T35Ou5KAHykluib4Yuc6NAEdo2J4feuPeM0Ad3ccwZD4DmhCsCZ/Oy9v+5xSw8tlLYZwQ+sLeHHLdPZ7CwDYVbKf79OXc3XS2eW1/dwDRTz75LTy+2Wm5/PkP8aSkpXLZcldOVBYwrwNO+nbtgVJsZGEOFX2j3LqNLSUTQAhxDjgJWA+wdT3t4QQj0gpp1V5Iiron/Z8ZgkLs/9DQSADgP3ebazOm0rf6PFo6FgEM2kEGiF6+FHne00vlrRw6a7yoFvs8aEJgdtVsanHZxhklRbz77VLcdls3NdnECC5vHVPLpr3KoUBDwCPrZ7K3FGPEWZ38tLWKeWLpX+1bwETW51/VBl+/ze1cUNahe3tWzNJiArno3vGsTv7INe+8ilG2ayfz004nzF9OmG3qdHByinSAIM+8CTQ/1BOvhAiDpgDVBv0VcPoaU/itYor7PFZRZjSYnSzu3BqoTi0EIYn3Iw44jPekhZFgSI+2v0Z/0z5H1sKt1Li9bFzXw5//2AOr09ZQF5hKaZ5uEHTlJJLpn/EF9s28OGmNVz9/RSuatOLwoCnPOADWEj2lAQX8SkxK2YIHQwUcU/HC3FowbIkueO4pEX/Cm36/Qe2w24/HMT7n9UOny+AaUq+WLy2POADfL5oLb5Ara3toygV1SRds34+FLTfDcLKpYbxXNX0T1MefwCbrqHhoH/0tWSUbkBiYddC6B01lhA9lDxvc+7r+DECWJS9jZ8ydnBZUi8AAlaAl7a9wa6S3QDk+HL5c8enuOnZz/D6g0F07bZ0PvrLBBDBqRe25R+g8Ig59HcV5FMaCBBmd9IjMpENB/cBEO+KoHOTZhiWyQVNBzIz8zcAEkPiiHNGEeOIZOawP5PjKyTBFYn9d9lCoWFO/vneLUyfupz4hAjGjj+LELeDgGGSGNOkwrHNoiLUuG7llBE0zOYdYLYQ4kcOL5d4DVCjKWxU0D/N+A2TgGniMwymr9zC+OQugODGth+y37uNFu5eWFJQavh5bfMCHln5LQgIWCbvDLq+/Dq6sJUHfIAWIc3ZmJpRHvABdmfm4QsYFFk+QnQ7XWPieW7ISJ75dS6GtEhwh+G22/GZft4+axLT01ZTYvi4vGVfkBBic3JX+8sZ02wgB/3F9InqiF3Ty2v1SbbKp8p2uRy0bRfP3Q+MQtd1HM7g29Ru0xl/Tm827t3P/A076ZaUwONjR6gUTuWUaohBX0r5iBDiKoJzmEFwudnpNTlXBf3TSKk/wIpd+3jlx0XomuDBUWdjWJKv0x4n3tWBKEcLfj3wIec1vZ+mrh68lHwVf1gxlWxPEeNa9aNvzOFFaAwZoFN4B5yakz5RPdGFTtfQZjjttvLmkm5tm+Jw2nh85XQW7N9OpMPNa8njeOXcMfywK4VH+w/BlBZuu5OfMzZQavhw63ZSi7LpFRW8l1N30Dmi1Qm93hD30cE8xGHnL9eOwj3pIjz+AC67egsrp1gDDPoAUsqvCE5Nf1zUX8xpwJKyPB3y/s++J2AGO2cf+nwmix+/kzZhA9lVvIxs7w5cejgJrs5sOLCfbrEJzBxxH5om8BqBCnPcODUnD3W8jyKjiLnZc4m0R6LrgrefvIb3v1uK2+XggQlDmZO5mQX7twNw0F/Ks+u/54uhtxMT5uI/KfMZ17of/WNbM6p5d5yaHQuJKa1TOp9OaFnN3u1Uk7UpdaABBf2yBa0qK5EgmMkZUd01VNBvwHyGQcr+XNbvy+T87h3JLS4tD/gAnoBBxsFCRjf9PzYVzMBrFdMt4hLeXb+al1YsoV9CCz69aBx2oRNqr1hrPjSA6plNz+C1gp2tWwq3cG/7e3n6tjFIJCsL9nDQ76lwXnEgOKDq9qXB+ZwSXBH0jm5ZnmevA3ZqnkljWRZ+y+JAaQmxIW4QEGJTKZhKA9HAZtmUUh6dgnecVNBvoIq9Pg4UlZB6II8LenTC4/eQFB1JUnQT9uYFc9w7JMTQtEk4OhqydBDd46O5++cZ/JYZTHlctT+dtKICOkTFVHqPzYWbywM+wIbCDQghCHM7MC2LzjSlV3QiH+78jUxP8J63dhjCL5lby88Z3bzLUR2xx6PECHDlV5+xIz+XULudyRddRZ+EZmqBFqXhaEBBvzaooN/AeAwPmrSxcPtu/jg12Bkf6nTwzb3j8JspTLt7Al+s2IBNE4xN7kGIPdik0jk+Hilg5f70Ctdr4jx2J2eSu+JC8wnOBGTZO1zXNKKdbnymwYzz7mVd3j4SQyOJtLv5YvdKRjXvwlWt+tI+Iv6EX6thWXyycS078svSOwMBnlsyn88uHaeCvtJgNNBpGE6YCvoNiM/0sSp/Fd3C+vDuwhXl+0t8fj5dupkJg/YS5sji+kFnYyJx6XaEEGgIIlwuvEaAvw0ZzbO//YJhSe7rMwi3zYGUstL5cCLsEdzR9g5mZc4iwh7BDa1uQDsi1dep23FoNnyWn77RidjL2ukntB3A1VY/3LrjJOfAkeUTtR1yqmbRLPX5sekaUgY/bEJVf4BSQw2peac2qKDfAPhNH0KALgT9ovuSVxI4aqbKuHAXDq0Jhb4FbCqO5tPda3m65+XEOsOQ0k+xfyOWNLiqY28u79AVw7JYvjeNP0yfyZtXXoxN17BrOl7TA+gYloVEkhyVTK/IXiDBpbsqfDj4TB+mtNhbmkbr0CSkKXHoDpy6HedJdtR6/AEkcEOPPnyxZSP7igpwaDqPnnUOzlqu5Zf6/Hy/cguvfLsQ05LcfF4yN41IVh3BSvXqb/DVKaOCfj0yLZOA38SnFfDF3qfI9afRyt2Ly1s8xaMXDuWBT79nb14Bg9olcU3/PuwpziApYjQ5BVC6EooAACAASURBVPv57UAKj62eyjtnXcf2nP8jxzMXKf1EOPvSIfodzvv3J+R7gu31MzZvpUvLWNqER+O1PMzO2MSM9DW0dMfwePfLibS7j/omIKUkw5vFXza/iN/y49Sc/LXbE7RwN6/yNXm8fmxlUyIYhknI76ZysCxJsd/Pv+cvJa/Ew4MjBzPn2pvYU3CQFuHhaELUetOOBP4xfX75SN7//bSMsYN6qqCv1IwK+kptCfgtSv3FzD34X3L9wc7XPaXrWJH3NW5xNp/dNR63zU7ANHlk0Wxm7tqG22ZnysXjuKbVQLI8B/H6Je2jX6IdkqyiH9hb+CylgX20j4thxd5g+75pSb7du4FJHfqT5jnA61t/ACClKIsSw8uLfSYQandVKJvH9PJN+kz8lh8An+Xjm4xZ3Nx6AiG2yufLL/Z7cTnsBCyT/KJScrKLad8qvsIcPl7D4LaPv2bdviwAZm/ezsKHb6NTTGzt/nKP4AsYFaZugGBHeQJhp+yeypmhAY/IPWFq7p16UGr4CRgGS5bswOG0UWoWVHjeYx0k3ZvH+XPewhKSN9ctZeaubWXnBnhr9VJuaTeMJ7pdytvzVjPgmf8x/Pn32ZLWntZNniPMEY23bIBVl4Q4zu/Sgdn7tpLtKWFD/r4K99pamIFdP/qzXwBOrWJN2Kk5jjlXvsfw8036Ui5c8Bcm/PYKu4wsQuNdmL+rJglBecDXNUGziHBW782o+S/vBLjsNkb16lC+3adN86Omc1CUYxGWrPZxOlE1/TrmMwwKPT5smp+h57bBkoLk6CtJT38ekNiFk16RlzBzbzqfDp1EoT+P0oC/wjVsmo4pLUqKA7y3cCUABR6TP3+9kEVP3oamBZh87ZXsKyykaUQYf141iwOeYlqGRdLE1ZX3ds7DlMGa74iEbgQss3wCtENCbCFc0/JKthfv5IAvhwRnPFcnXo5Lr/iN4JAcfyH/SQl+gygxvDy36QumDn6Mgx4P4b+bJqFjfAyJUU146sIRWKZFTHgoXr+By3F8b8eAYWBYEsMwsdt07LqOrh9dj3E7HfxtwhhuGpGM3zDp1jIBpxrJq9SEatNXTlRxwEeut4TFWbu4vHV7sounseHAWySGTyQx9AHuaTuNYvMAEY4mCGyMaxNPsZHFD1kfc3+fB1mQtot9xYXEhYRyf7+z2HBwH5G+ioPvSv1+TAuc9lDsOrS16XyUspJQh4OvRt5MiG4nxBbJh4Pv4tu0VbQNi+fixH7HHD3bxB7Biz2fxWN6CdFd6FUMutrvqbhCW2GgFEOabMzNpmWTyPL9Dl1n8o1jsWka9//nG9btyiTC7eTNOy+ne6sEbMfRnl9U6ueO16Zh13US45rw6PjhxDYJrfRYl91G96SmVV4v4DeCHeo2/ZjfaJTGp66ad4QQ9xFcEcsEZkopHz0V91FBvw6UBPxke4q4acHn6JrGpa1as6vgdeLdF9HEcRtPfP0zaXkFXNO/B5f1bspBn4crPvuML8ePZWTT6xCal9ljJ5FeXEDT0HCm7FqOYRnc1r4j7eJj2JkdzHO/tE+XCpUSl83OpA79CVhmhRG5nSJacH/nOOzCVmXKpa2s9u/Qqu/w7BHZiqauKLK8+QCcG9+DA6Ul9G/aAq8vgN2moes6uqYR6Xbx5aL1rNuVCUBhqY8Xps5l8oPjahz0C0u9fD5/LS/cfhG6TWBaEgtJiddPqOv4Omg9JV7St2cyd8piup7Vkd4jumNz6LjclX+rURqZOgj6QojhwGVALymlTwhx4gNgqqGC/inkN3wELJBIQmwa00ffxN/WzMGQPsCiWei9/OHzX1ixO9jh+tcZ82gTG02nZrH8Y8woUvMLGJTYKjh3jmmwIHsLc7I207VJCx7tPoZs30G+vPc6lqemEe5y0rl5/FETkDl0G45K2uxNEyxhYhoBNAShjuPPZLEsi5KAv6y3Cz4b/BBzstYTqrvoG92OA8UedqRmk9Q8ioOilLTSfM5JaI8uNbokxtG9VQIb9wQXZff6jeOqXUspuXhwVyavXM2nq4MrhF3ctRN/vXDkcb+O3Rv28sDZf+LQanO3/mMiAy/oQ2Kn5thUM1CjV0c1/buAv0spfQC/myu/Vql39CkgpcRr+PEYFv9ZuQyPYXBXvwGYmExs349SQyfefQERruZsyqj4f7sxYz89WiYwO/83EkNiGChaYpqCa7//gvv7DeLlvj3ZnpdDSv4B2kRG47TbOKdTm2OWxWcEO3QLfT7CHU7suobXNPhow1peXfYrlpTc0KM3jww6B7f9cDNPqT9AwDRJOZBLt2YJgGT7gVwkkk5xcQgJfmny3vZlzN63lW5RTXmm7xguapFMwDQo8fr5+ItlnDOgHdOz1vCfHQsAiHGG8u2wezByffzr/ivQNY0ir5e8PA+6XnnQL/X7KfYHkEjCHA5CHQ7CQpwYyPKADzBj8zYeOvfs4/oA8xR7mT15HkcuLzpvymKGXD4Ab4mPsEj1J9Lo1SzoxwohVh6x/baU8u3juEtH4BwhxN8AL/BHKeWKas45IeodXcsCZlnNF42rv/qM1Pw8AGambGPJpNvpFRPOWxsXcU+3Fyj1eRnVtQPfrt0MgF3XuKB7R2ZmrGJBdnDf3tIcnu4xjtSCfO746dvy+ywcfxsOUXVTiJSStIMFXD9lGjklpXSIjWHKxHHYdY1Xli7GLAt0H6xfw629k8uDvmlZbMzcz82ffIUpJXPuu5lnfprL/J27ABjSphX/vvISftq3lbc2LwZgR2EOAE/3OZ8Ip4udezKZ/esWHrt1JG8s/IlnelxMrBbGkoJU1uXtZWDftry06UeWHEihX0wrnuxxcaWLnpf6/by0aDGrMzIY1607HWNj6dG8KVjBmUcFFf8mj7ct3uGy075vxQ/N1t1a4i31EWNXU0E0erLG0zDkSCmTqzpACDEHqKxj6UmCsTgaOAvoD0wVQrSt6WLnx0MF/VoUMAN4LA86dgq9ZnnABzjo9bI5J5tusfHc2XUgr29egM8y+L+LhtOjRTy7cg4yfkBP7A7Ba9u+Lz9veW4KLt3Oy8NG86fFc/GaBvf2OYuYkBCctqprtMV+P3/7ZQE5JcE1anfk5PLu0pXcdlZyecAHiHS6cNo0Sg0fgmD7+DtLVhCwLLo1jafUHygP+ACLd+2hJOBnbV7FVMtN+VkEDJMSy0/39s2IjQwlYBhMPfsONq1M5dcfNzFqVFe6dmrNRzt/48s9wYpReulBImwh3NN5OGG/Hy9gGGQVFfPe5Vfw2c+rmbUxm6iRLjJzComODGVS/758sGI1EhjbsxvRIZWPITgW3aYz+vphpG1JZ/H05XTq3447X7mRkPAQnCFq8FZjV5t5+lLKY7Y9CiHuAr4uC/LLhRAWEAscqJ27H6aCfi3xmF6K/EW4bWH8d9ti7u50LnHuUA6UlgDBrJWO0bFM+XUtFw9oR3JsIvcsm8q8rG082Pk87u8zCJsmKDI86EIrT6nsF90WnxXgrLitLJt4KxoaBZ6F2KzNQB8AvKafXcVZ7CrO4pz44Lz2Dt0O8nDzTnk5DQNNCMZ27sZXWzfx6KBzmNijJzuK9/OPVd/jMQPc2WEED44YxMKU3XgCAcKcDjQhsMo+KARg0wSXt+7BZztXl1/7/MTOOG02dKGha4Ip/5iEpkN2aRF661AmPDyKlx74nBad49lamFmhXDuK9mPIo6tUUsLjQ4fy2H9nsiYl2Pcxe/lWZjx/C69+uZCJo/tyy1n9EEIQ6rAT4jj+6SGcbic3PXctNz13LaZhYrPbVMBXDqv9ynZlvgGGA/OEEB0BB5BzKm6kBmedJFMaeMwShLQIsYWwMncP7+5Ywn5PIZ9cPpbzWrfl7JZJfHL5WHKLSnhl5mK8Pkn3yBYAZHoKWJG3m90luTy95gfyfF4+P/sPDIrtyNiWg/hrr/G4dRueoofZn9WBzKwOlBZcjyWDKZJe08+0tEXctfJNXtw6lWuXPF++GLnbYef+IYNwlGXERIW4uHVgPxy6zkMDz2bB9bcyvGUbJPCHVZ+wvSiLtNJc/rRuGglNQrl9cDKvXXUR8WGhLLn3dsb17I5D13lw6GByfQW0CYvmg6HXcl27vvyt34VMaj8AJCxPSSOnqBSpSf63ZSnn/vBfrlz0Efet+o7H3rqOHavSuDSxd4Xf44UtelaaOhpitxEb6i4P+AC+gMmKbWk8OfE8/vn1r6zeso/YUDfuGrblGwGT4oJSfJ7D4x9CwlyEhLkIiwzFFaqWX1QOE7L6Ry14H2grhNgIfA7ceCqadkDV9E+Kz/SypXAZBYE8Wrk7kujuxKHBeVcteJtXkq/i7+eNxqZpbNidxRUfBBceMU1Jvi/Y5NIjqgV3dTqXR5d/y9IDu/lh32bmX3gf93c6B0sa2IVASi/h4fdSWPQyAHZ7N1zOQQBoCKan/VpepmLDy8yM5VzXagS6ptGzWVN+u+92UnLy6JIQhyAYNCMcTkoCft5YtpI/DB9EYeDwYikSSY6viLvOGciDX81iYcouujdL4L/jL+OZ80dQ6Pfw1Kb/8nyPu5mdmkL32OZYhqSo1Md1r35GYWlwArl/XH8hEc7DzTVLs/ewNX8/4WEhdIxJ4v3Bk5ibtZXBce3pE9WSomIfrib2CrOChjoceHwB2jWPYWdGMDVVE4KOSXG8+Nsinr5pNGGOykcKVza7qLfUx+Lv1zBv2jLa90zi2ocvxFXJsoyKAtTZ4CwppR+YeOrvpGr6J8y0TPaUbCbakUCr0DbsLvmVvSVrGBzfjq5NmlJq+Ll76RS+T1/H7gN53PXeNwRMi5Hd2xMfEUETp5tVFz/J+4Nv5OUNv7D0wG4A/JbJb9m7WXvwJ5bnzcDCRNPCCA+7i2YJy4iP+4GEuBlAMFAZ0iLWWXFKgRYhsWhlwc7tsBPhctE3sTk6gm17DzDp+Slc+n/vYQQs9hUUIqSgd9ThdWwTXE1IcscyeelqVu5NZ3jHtoQ6HTz/0wJ8hkGow8HopgNYnb+VS9p1YnrKZpq7w8nMK+DWUQPo2aopUsI7Py9nZPMOFcoW6nLStX9rrv3vF6TtLeGimN70i2xFTn4xQoMCn5d3163i622b8RoBABw2nX/dfyWj+nWkX8dEXrzzYjI8xXy0fi3oHDWStzTgZ92BTF5f+yub87LLRzRblsWGJTt45d4PWD1/C1Pf/JG3//QlnmIvinIswqr+cTpRNf0T4Df9+C0vXrMUTfiZnfEy3SLPpyCQT7g/nU+H3cye4jxCNAclngCRzhC++eP1aEIjoUkouqZhGgYgkVKyo+BwX42GoF9sS37I3M8FzW/EpbuD+zU3mpaEjYoLnzg1O090G89T6z8k3ZPDqKb9GBzb5agyW9JCCHjrq8XszgoOoPpmwUbuGTyQ535awCsXTWB25jr8lsEVLZPJLiwhwuVk7v23sDFzP01cTqLdbgQCGzqj48/CEiYlhp9/jbgYt93B13uX4m/h5+X+FzJl3jrW787CodmwCQ1DWlzaqisdouN4YMoMeiY2ZcXudCYvXsVnd1xDQnQTvIZBnsfDrJTtrNmfyfc7tvLP0RcT6nAQHxXG4xNGsCs/n1/2pPL+klW4bDbCHU60IwaYGZbJ/PRd3D0vmOn05tolfDT6as5u1oqAz2DFnI0Vfi/rft1eK+8J5cx1ugX16ohT1GxUs5sLMQZ4g+DSqu9KKf9e1fHJycly5cqVVR1ySnlNH6Y0MaRBeul+Wrpj2XhwNh0jzmNl7h5W5+9laHwnekS2RZc2ftmWwhPf/YyUkvvPHcTYft248te/c2u7kYxtOQgdjYC0KDH8vLjhF3K8xdzacRDdo+IRwsSuOXDb3NWWy7BMTGlh12x4TR9u2+EmFdOy8Fp+ZmUsQdc0hsck8/InC5m9fBsOm860524kzO0kraCA9rHR7Mk9SNrBQoa0TUIC1300lU1ZwbEE7193Of1atmBD/l7ahzdFIPgsdSmTOg7hnhXvsLUw2O4e6wxnyqCHyMgvIDQcwu0RmJbEoelIaaGhs3DnbjQhGNw2iTmpO3ly7hy8hsGY9h14dvgIhn3yHqWBAOtvuQdhSnKyirBssPRgBq8tX4JD13lm2AiGJLWuML6g0O/lnnnfsShjd/m+Ma068tKQCwizO1izYAtPXv1m+XMX3zyMm5+6gpAwNfr2TCOEWFVdGmV1wqJayl7nPVDtcUu+euSk71VX6q2mL4TQgX8Bo4B9wAohxHdSys31Vaaq+Ew/WwpT6RjeiqU5azgrth8pxRl0jbyEPF8xf9v0DQFp8l36Cl7oNZEWIdEMbNuM5Y/cRUnAT36gmAfWvIPfMthRlMlvOdt5besMXuxzPa1DY/hjjyHYhQOJ5F/bfmZdwV5GNe3JpHZDsWtV/zfZNB1b2bw4RwZ8ABOTe1e9TIY3mAjwc9Zynp9wDynpuVwzshdhYXYcQiOnqIS7pnxLdnEw2+iDiVcihCgP+Oe2b0OLmDAuX/gChYYHm9D5R88buLnDOZQYvvKAD5DjK2KPJxvDUcQLm/7L450fIMneEtNrYXPoPPzdD8zZkYpd11n58F08M38e3rIso9kpOxjbtRsjWrVlaXoamhD4/Qb33jYZp9PGbQ+MZPGNtwVHKVsGJhaGZWErq+3bhEbbJlEsytiNU9cZntiWMa06UFjixR6q02VgO5547zZ+mbqMDr2TGHvPaJWpo1TpTJtauT6bdwYAKVLKVAAhxOcE555okEHflBY2YWNXcRqdIjowadlzlJpebELnme63Mq7VYD7dvQiAOVnreaDTRUTYbQQsD+sKd/PUhinl1xrZtBcrc3dywFfIC5u+5r8D7iDK0YR3Un7h012/YkgTAJ/lP+lssQxPTnnAB0gtycAjPUx+chxfpy/izZT1XNB0AM0iI8kvPdyZuzotk3Paty7fHt2lHV+nL6HQCB5jSJNP9i6gU3hzwp0uIuxuCgPBzmldaCS6Y3Bocbyb/BqGz2LauwvZtGo3g0d146kx57IqPROPP4BA4DfNCmX2GQatmjTh5l59kcCWTen4/QZ+v8Grz83gYH4Jvc5vx19X/4zHDPCHHkPpG5uIS7Ph0m081m8YF7XqTI+YpkgpCRgmDl3nxWnzePDycxh8YW/6ntsFu8OG4zjn6VEaoTMs6NdnR24LIO2I7X1l+xocS0pKDA9O3U68K5Zv0xdTWpYWaUiTaWlzGRbfrfz4rk0SCbM52Vk0n6l7bqRXVAyv9JnIda3O4X/97yTaEcZ36cER1gWB0mBeu9DoH9Ou/BoxznCuaz2k0nlzjkecMxLbESN3XZqDaFc4k3fP5r3UH5i7fw2PrPsfht3LyM7ty48LddhJimrCVb26IgiuKxuiVwyQIbodu03Hb5q83OdGujdJon14U/7a41rsQkcTNvYXFvDZv3/h03/OYe1vKfz7L9+yZt42ru3dA69hsHRPGvcMGFB+zW5x8Qxv05bbeyUTbnNy07SvaduxafmUyW63g0su78uN86awLHsv63MzuW3Bl3gCAQLS4pudW1iQtos4WyiXvfUxyX/9Fw98PgNvwOChy4by/bItmJYkNMKtAr5SrUODs+ogZbPONPiOXCHE7cDtAElJSdUcfapI8v0lxDib4NJduPWKKX4hZfvahMZzdlxnrmx5FhoBVuZ+iNcsYPre2+gQPpKrky4h2pHI2EUv4zWDmSk3tDkXicSm6fSMbMVP5z3B3pIc2oc3Lc/AORm60Hiq2028nzoDXWjc0e5yTEvyy/41FY5bkruBif17kV1UzOC2rRjXtwezd+zghgF9eHTEUNwOOz7ZgSU529hZnEWcM4L7Ol7Euqxsvt+xlXuSB/Jav0lICXZNRxcar2+dwSUx/Vi/NLXCvTb8tpM+N/Yl3Okkv7iUG3v24YrOXcn1lNIxJpbFKbtpERnBc/Pn0zUhHo9u8vant/PJu4uIiQ3Di0luWcorBDOYdhXnsfdgAR9tWcvzg0bx3Ix57MsPLk6zfNc+pq7cwAUd2jO0R5szruamnELy9FskpTr1GfTTgZZHbCeW7augbNKityHYkVs3RQsKmF4QBgJB69AYTAkCwfikUazO30ZqSQZxzkhub3cZLpHDP/tdBcDOwu9oFTYUmxZsXzekly2FM2ju7k6csx1fnPMQa/JSSQqNI8YRjqusBh1SNq1Ct8iWlRfoBDiEnR4RHXi9z4MIRFlnr58eTdqw8MD68uP6R3emc1gCr111Edtyc5i2ZROXdAyOsJWWhSlNLEvwet9b8ZkBop2h/LpvL7fODGbJfLl1E99dPYEOUTH8sjcFt93Jre1HsenAHvoM6cC29Ye/1J01oitndW7Lb13a4fEH2Lx+H3Fx4XRJjOXFHxfy8fJ1TLvzOp4ddR57Cg7y9pqV9G/eggefuJCd+3OxCY1uUQlsyg/O0JnobkLXqATaN4mlW1w8bSKi6dg0lt927i2/50GPB5tdJ620iOhwN6qOr9TYmRXz6zXorwA6CCHaEAz244Hr6rE85QJWAJAYmBhGEXsK/kOsewgxrsEUGTmUBHJ4tfd9+KWJS3PgM4txiBj2FPyHVlH3k567Cp9ZzNlxdzMn82/4rGIS3X1pFz4MXdMJQWdwXOdT/jq8hkFKbi7PL1hAwDR5eMgQejdrRpg9hEc6X8Pd7S9ja9FeTGnSOaIlhiGZvzOVfokt6NusOZaUeAMBIlwuCn1eNuRmcevcr2geGsG0Cybwp/m/lN9LAjNTtjOpVx8eXvwD57ZoS0mgM4ObJ9Htlpa4Qu1sWbWXIaO7M2B4F3y+AHa7DU+RHwGsXrGbULeTB4efzZb9B/D4AuT5Pdz2ffBDZeqmjWQUFRHrdPPe2jVMHjmeKTvXUBLwc1vngSzMTOXh377Db5n0jG7GJ+ddx08bt5NZUEyY08G1A3oT4nJw86fTeX/sFQxMqr0PVuXMdro131Sn3oK+lNIQQtwL/EgwZfN9KeWm+irPIaZlcjCQxxvbXyAvkEOX8B7c2uZhlmVcyaAWU4lytCJEj0JKLy7NwcLsz8gP7OesmEtoGfkg+d5UOkVcRJ5vB25bJNe3/RxD+rFrrvKaf1264auvKPB6CXc6CXHYyPIWsPlgBgNi27AiJ408n4exrfsEv23ocGGXTizZvZdbvvwGS1o8cu4Qhrdry4NzZvHW6IuJdIawp+ggqYX59E5oxuzUHeX36pXQlGKfj/lX3sbBUi9SSqSlIXRJ7ys6csG4gaSU5DFsxv+4pkMvbm2bzJyfNvDBewsB+PD9hbz/8e28M/FK/JbJcwvnV3gtP6em8PfzRvPUT7+QkpPLe2OvIMRuw6ZrPL9mDn4r2CG8Pi+Tn9K2MfWOa1mWksbZndogkUycOg2fYTI/dTd9WzTHfsSCLcU+PyF2G96AQYjdViH3X2nEJKCad2qPlHIWMKs+y3BIieHFZwbYVZKBJQ3u7/AEr+94ji1FG1iVv5xmoRdQEtiNZYvAkE4My8uM9BfZWbwWgK2FS7mv43+xaU7iQzoQynYC3q+RthtxiEhs2vHN/lgb8kpLKfAGO5z/ct4Idvv288TSbwBw6w6mDL2VFzfMoX14HFFaOK0jm6ALjYdm/IDPCAbQR2b8yPL77ySzpIj/rVnOjEsm8fn2deR5S3l15Bj+UHQWhmVR4PHRPCKCv/46j5eGjyEuNJRXlv7K2qxMRrVtz53J/UFAuOXi//oNZ3iLdtikxozvDvcteDx+Zs9az4TrB6NZgn7Nm/PVlsPJXD3iE9h7MNhOv6+gELfDjtCC0zL8PsvJFzBY9NMWtm3JwNxTSlqCwdYDwSymMZ06lAd8nz+ABTw74xd+2ryDbs0TeGPcxcSFV77sotIInVkxX03DAFBq+Hh/5y+8sPlrCvw+wuxhLMhezwVNrwg+b3pw6DE0cXTDZwYYOfufRDkjSS0+vICH9f/t3Xd4VFX6wPHvuXd6Jj2kkJAEEgKB0LtUBREQ7LrWn3UR26q79t52dXV3Leu6667ouoq6NnQVxIY0NShSpEMCSQikkV6m3nt+f0wIRKkqpHA+zzPPk7lzZ+6b9s6Zc899X0y21H1DtD2VHZXXU1JzH7vrnyOvdCJCBA906KMqxuUiIyaGsWlpnJjRnZfzv255rMnw827RKsYn9mRN1U4+ztvKW+vXU+PxtCR8AENKyuobODWjF8+t+obX1q1hevcsurjtFHsq+du2j3hu+0dEhGvEOp38Y8rpWHWdao+Hu8aM448TJ7Ohopx3Nq5nzsbV3LH0EwbFJTNn1Wp21daRkNC6hES3bjGYUlLR2MhJ3TO4ftgIesfFcV7fHO4aO55AMMhVw4Yw/4pLEAJ+9fHrrKzYxa0DJ2ARoT/nPtEJTO3Wm7dez2X1ykImTOxDjMvFlKyezD7nTHrGxgDg8QdYVVjCm9+t5YPvN+ELGqws2sUDH35Og9d3DH5DSkegVu90Ej7Dg1Wz0xT08diGd/m8bC0Ayyo28tywmVT66xkXP5oISxQnxI3DLjRA4+2ClVT7PRQ1lJMW1peCxtBl/QJBWlhfBFbqvZ+1HMeUDdR7FhEVdvox/x6tmsbb559PfaCJan8VsXY3UNbyeKzdTUPQx4mJWfxh4zK2VVdxbt8cBiQlsqakFICecbGkRkdxReRgZvTsjUPXSQhzk2BE4DUCXNp9PK8WLGXWNy/y4fhbuWrhOzw75nT+tehbFm/axsDUJB4582Tmbt5AQqSbFWU7ufqz93j5lHN45btV3HLnqTxy/1x2FFUxaXIOI07IZHV5Cf9esYoNFRVcO3w4T50yDU8wSMmuGsakpjKxZwYV/ipqA34md+vJzEXv8vTYGSw743rq/F6SXZEYPoO7HzyTjJ4J6LrGmTnZnJqdRZjViqZpNHr9PDb3C/qlJbGj+dPDHsXVtc2NcA6P1xugsclHmMuGlOA8wou9mhq8WKyhZuyGYeJQF4u1K2r1TifgN30sLHuTJcZdOAAAIABJREFU3MqPuD37Rb6tym/1+IrKfLIj0rAKKw/l/Ll51UvoHzE5LAqAP61bzMODb2VJxRxqA7sZGnMKAbOegNmAzdIdf3DvMkWHrS9tQdc0IhwOXDYLbxR9wB39JnPrigayI5OYlJjNiC7p7GiooaCqli+LChmanEyVv4EXzjudhVu3EzAMTuuTzfvb1/Pypu+YnNqTS3sNocLTyFvbvuf97evJiIzlgWGn8ZdNH5JXX845PfoxJ3cVH6zeCMCSLQU89cmXXDNxBLM3hEpobKutIiHMzbRevcirqeGBJ86jS6SbkqpattXX4DGC+A2Dwpoabv/kEwDO7tuXO8aO5bZV77GmqpgJSVnc0/9kZvYZxsi4bnxTWUymM5qawjoWLP2Oq64+iT45KS0/Cwtg36czl6YJcrcW0eQLcNmkofx3xVoCzReJXTBsAFZdJxg0EEK0XCOwPz5fgHvue5uVqwqx2Szcfus0Ro3qicP+4zLRPn8ACTiaa/77PH6CQYOKXTX8+0/zWbc8nxsfPY9hJ/VRib+9OEZVNo+l4zLpl3oKWLY7tCpkl6eIoTEZLGwe6QMMj80kMzyRMMuP5+FPTOrJad36Ma94HbF2N7f3u4wKXz4FDbl8XvM+4+JnkhX3b3ZW30HQqCAu/Gpsetdj9r3tT0AGaDQaWFWzlJdPuJy8sirmrdpMIF1jeI9u/PbrjxmUlMQfJk1kdv5CvqnM46be0xgSncnzG3J5+vtQ6eYN1eUMj0+lPujlye9DVx9vq6/CbxrcNuhEkl3RVNQXUljZeuRcuLuaCJudGIcLh25hRo/elDTU47baOOeN/7XsZ9U1Fv/u13ywbSMzhw/j6x07aAoEiLDbuWbYMObtWseSsjwAlpRupSpjLDFWjRRLBH16DOaNV79i8RcbeObvl+FwHLyZimlKhmd2Y953m5g8MIv3Zl3EkrwChqQmkxkfg88b4N15qwgLszN9Uj8cdut+yzevXFXAylWFAPj9QZ559lPGjunVap+gYdDQ5OM/877FME0uPXU4LquF/724iM/fXE63nolc9/C5/OPB93j6zjd5fcVDR/gbVo6W0MVZnSvrH5dJv8a/t6rl4vLXuC37FhIdURQ2VnB6ynAywxNx6ftfaePQrTw4aCqPDT0NQ5oIDOLsaZR41jI2/mp6R07GImykxc0GBEJY0UTb1mt36k4uTb+SDbXrWberhCteCp3MnbN8NbdMHstb512AzwxQE2ikiz2CuoCHUk8NMhoW7Wx9YVVj0M/aytJW2zZWl5PujuMf63Ipb2rkjMF9+WD1xpZOW2cN6cuSvAJywhJYePYVRDud5FdV4dJaJ+aAYeILGnyxYxup4VEsueoqimpqyIyJpcHjY17x3gqZ9+dM49tvCnnm34swTMmoQd159LencdkV4xDaoedmwhw27jlnIrHhLt77Zh2XnziUi4cPxGrR8fkCXHjDS1RWh+oQLVy6iScfPG+/byR+f+sSEsGg+aM3B8OQXPbg6+yqCL0ZWnSdkXGxvPRIaOBRtKUUvzfAhbecSu7n6wn4gwT8AVzuY3/yX9mPTlZl87g6keszfATNIL0ihhBuiQZgW+M6VlV/woVpw7mv37mYrGa3N++gDbbDrHYsmoZdt2DT7TgtkQyNPZ+cqOlYNQdCaOhaOLrmbvOEv4dDd5Dp6sOHq7e22j5/7WYChsGfN37KI2s/ooe7Ky+fcB1npY7Apmucktq6Hn6s3cX0tOyWk6YA01OzCZomvaPjGZmQSt+kBN667kKuP2kUsy8/m/6pSfz+40Vc/+YHGIakNtDEW5vWkxwVweDUvZ+CJvXOINxu44nxU7HqoZaL2fHxWBC4nTbOTR8MhK74nZCcxfOvLcMwJVPH9eH+G6exqbaCOtOPTxo0Bvw0Bvw0BA58QtZps3LNKaN4/JJTGdwjGasltKJnc35ZS8IHWLd5Fz7//k/GjxyRQUZGPABCwJWXjyMQaL3v7pqGloQPEOFysOGb1lOKeWt3kNgtllN+NYKCDTtxuFTVz/ZCSHnIW0dyXIz0/aafan81n5Z9TJQ1ipMTJnNzr2f5vmYZ4dYoujp7sLkulxhbIpvrvmR03DlHfAxtn/o2+/IG/QSkwaa6nfSOSMYqdByHaGh+tLisdvqnJPLuqr2XQ/RKjGNLfRlzi0JLJ5dXbGfBpBtYV72TrIg4fpWVQ4zDycqKEqamZZEVHUeNz8P/pl7O29u+p090AiclZ3Br7ofcOmAC8Q43T61exkXZ/bh83EBW7Shh5mtzqfGElo6u2VnChF7duWX4aBr8fl64+CzW7ipFQ9ArIS5UjqJLIv27JO4NXNexA1NT+tA9PJbciu0tbzrJCZFcf9l4zlzwCvl1VWhC8McTptLNHcHtufMZHt+NB4dNxmXd/898fz11U7pGo2kCs/kEXlSEE+cPRvkeb4Cqmkbytpfx16cuYeeuamKiw3A4rDh+UNMnNspNuMtOfVPoDWhzUTmzzh7Gm89+gmmEhpFjZwwm4A/yq1kn8eL9b9H9mctUuef2oBPO6bdpPf0jdbj19KXZBEIDGQB0mkyTW7//LX4z1EGpd3g2v8m8GV1oeM0GQCIQVPrySXT0RBd2NK31CH1/rfcOx5a6XVyZ+w98ZhC7ZmH2yFlkRbTdHL83EOCvC7/m84359EtJ5IEZE7n2mzmsqCrEqVv564jzyQyPRyCJtYfTGPSwumYjhgSHppPi6kp5Q4Dckh0MS0yhb2wCNk3HYwTQpAABNYFaXip4jd2+KiYljKOPYwAznpuDrml8cdOVOK1W/v5ZLgs35jM4LZnbpo9H1wQCgcN68HGIlJLvt+7CMEy2bCunsrqBiMHh3Pvtpy37xDvdfHbGlQx480kArsoezs39xx4w8f+Qx+snd+V2Xnz9S1wOGzfNnEiPtC7Y9+nQlbe9nKtueSW02sZu5cFbZjBsYBrW5vhDUzRBJGCx6RSWVvPX/y7FNE2uOWcMPRKiKSvazYI5X9EjJ4VxMwbz4QsL2bq6kGsev5DIuHB1gdjP9EvU048IT5bDB193yP0+X3K3qqffVqTZBN55yLqHAD9E/J6N3tSWhA+wqX5jaFWGZsFhlOOp/g1CjydWuPHULcWdsIQ97QgN00u1ZzFNgW0khp+DLiLQtcObsmkIeHkxfxE+M/Rx32cGeTF/EffknIXb2jajOIfVynUnjuLaCSORSDR9b82fKzJHh0pIa4Int7zBN5UbyApP5cG+V2LVbARkkLpAE9mxXVhdUUKyO4JZX73BuMQMTkvti8tiQ0jB3/Jmk9dYAMB/Ct/igex07p4ygSGpXbHrOv9buZHZS0Jv3tsrqjFMk9unTyDCeeifq2lK0rvGcOXDb/D7a6aRmhjNRztbd7+y6ToBc+9E7IbqMoLy8CdmnQ4bY4ZlMGJQd6SUOOzWVit4PF4/7y1YjdE8Svf6Asx+fRkD+qaESks0+fl64QbeenEpUTFh3HDf6fToGsuj100HwOUILRvt3ieFK+45A4tVxzRNpl1xIgBhEWouv13pQAPjw9Hpkj5CR9Y/AjTP5Xrmku5+EoFANn9OS3J0bflas6QBPoK+RQBY7BMQe/rPmg1sr36c0vrXACiufZ5hKQvRWx5vwpQBNGEDJBatdZcrTQjCLK0TWZjF/otUz/w5XD+Y0vjLsHOp9Tdh1y3saKziw13LWF4ZmgLaXF/IDk852xrKeWbzXFwWO1f2mMpZmYN5Yt3nXJ41HF2v5/cbn8IqLFze/TyuybiMz8uX8nn5MnymjwJPEZP6DiLK7iYQMFlfXNbq+JtKKloti6/3h353Nl3H3lxa2usLoOsagtDP9T8PXUhQwuOLlnLThNEMi0/h2/JinBYr9w+byLvb9q7GOrN7Dk7LwVfz/JDVasF6gKdYLTppKTGttnVNjGr5unp3PY/f/lbL/Xtnvcxz796Aez8N2Pc0cNHRse5nqklpY7LztUvsfEkf2Tyt0yzwLW7dwjUZ17OgdD4R1ggu6HYxOnvm4K2Exc0n4P0YoYVhsY9FiNAoXBdOyhvebXkpQzZQ0TiPrhGXYJhe1lQ8yK7Gj9CEjZzYO0l2T22V+F0WO9dmTWZNTSFFjbtJDYvj2qzJuCzt4+QuQFPQz9cVW3kxfymRNie39zmVVGdCy+O60Ogdnsadq/+NXbfy1ODrWV+5m6KGGjShkROVyPWrnsFoHkk/tuk5/j74UVL1QdybPYJHNz3FyNjBOIQD05RommDG4GzeWbF3Jc4p/bKaSylIqn0e7v76EwrqqjivZ38u7DUQMyD58rt8/vTC5/j8QS46bRjnnzaEWq+HOSu/p7i2jmemnYZfGnRxumjw+4mw2bms11AmJGcwLD4Fq3aAcy7+AFJCcXkNqQnRCCGwWfe/7x6aJphx8gA255ezJHcL2T2T+N3VJxPWnNTzNuxqtf/Owkq0w1hRpLRTaqTfzkkDXP8HTS+G7lv7Y9fDGBA1kOzwUMNw5z59Z4XQQDixuc740UuZ0ofTkk5jYFPLtjBbaA12nX8ruxo/at7Pz/rKx0kJn/Gj14iyuphzwg14jQB23YpVtK952rpAE7es/G/LJ5/rv32FN8deh12z4jNDCVEIgRCCyQlD2VZTw125C/jjyGlc03sMZb6KloQP4DG8VPrqePTrZTw4ZgJPDnoQp2bHkIKHly/im9JiXpx0Fq/O+hXzVm9iSHoyE7J74LRZafD7uHHJByzdVYAuBBlR0fh9Aay6hUf//jFeX2ia7KW3v2bKhD7Ex7o5u38f3vl+Ayc9O5uJWRnceuIY8quqGZOeysAuXbFp+n7PxQQCQfwBg0DAYFX+ThbkbuL7/F28dNcFJMZGHPRn5mn089xD7zH93GHcOutkfP4gbvfeN/LBJ2TidNnwNIWmFIeOySIYMLDaOt+/23Ghc+X8zpf0hRYG7hvBdQHIOrBkIYQdndbJ/rBeS9jIjv8rmytuwWvsJMl9IW5bPwwzQMCsb7WvKQOhEcEP8ouu6ejo2PX29dHdb/poCjaxobakJeED7GiqAgEvDb+H3Mr19A5Px2cEuDrjVKyahR01dXwy/Uq2NpSwsGw156aNIt4eS7mvEoAeYam4dCfba6r4eHs+gxJSMKXk+TXf8MrGUHG6k96dzTMTpnPzlNE4rFb05hOWVl1vuQZgSlov+oTH8+Fn6zh5bHZLwt9jZ1UNL5Z+ya0TJzC0WzJuu43EiAjiw910cbuxWQ78p+3x+PkqN4/X/5tLZKSLG66bxDXnjMbvN3jt05XMOuMEXAfoquX3BXnnxcUsfH8VC98PrXgaMjaLO5+6iLDw0CdEm93C8+/fyPtzviYuIYKp5w5TV9h2YMLsXPM7nS7pAwjNCVraz34dTVhwWFLJSfwPmrAgpYGuOTFML9GOAUTZ+1PjCzUi6RH5f5jS3zLf3575DB//KZjNDk8RN/W8izCLncZgaB59aEw6UoJbD6ObrTt//m45dqFz34iTcFptVEU1EmlzsrmkmCmJg/EGA/yh/x18UfYlurAwJm4E9yxeSMA0GZuSjk3XafD7WLO7dJ/jG7y2eQ0ndE1tSfgAAdNgSmov3ti6hhNTuhNms/OvV5YxcmB3ThyZxRe5oRO2GalxZKcn8MiX8xC9BFFuJ9/t2kX3uBiklLhs+0+wppQ0+v0Y0qRwRyVFO6oIbKvg7vve4dm/XsJnBXmMzEk76CotKSVN9a3X/nub/K3u2+xW4rtGcelvTkbTBRbLwaeLlHZM0ukuzuqUSf+XJISORTSX2W3OBbrmwDTqGJH4LPX+PKxaBA5LArp2ZJ8k2orH8PBt9XIAVlYv59XRV/F24UqibWFc0H0kDl3HZxhoQvLE6FNoCDbQYPg4Y9E/qPDW09UVxdvjZmLVdc5Y8gTxjgjOTzuBk+L78cLq76j3+XnnzPPJjIklKE1cVitnZfZl4Y69V/eemdkH1w/OlLqtdu4fMZG+sQnE2cNC00qa4NFnF/DYXWdy1pQBVDc2MbRfGg+s/ZBLM0bx+pq1/PnLUJmIf61YwfsXXUSf+Pgffc++YJDC6hoe+XQRnkCAWSOGcW/Wadx331x27arBZrEwf+1mnrpgOs791M3Zw+6wcs5V41n4v1XU1zShWzQuumESDuePn2Ozq3+vjk7Q8S6+OhT1V3mEgmY9AgsCDU04cOiJ2C1xCOQBL9Bqbyya3rKa6c3iV8mJ7M9VmTPJrdyMkJKAabCptoSUMCePb/4tZ6dczb+3bqXCG5rSqvI14rLayC3PozbQRG2giYfXvcvbRct5ZsgVXDlwCLnl27l/1XIGx3bjvPRBTErN4JUp5/BpYT4TUzMYnpiCZT8nV50WKxdkDSBoGgT9Jr++aAz/eHkx51/zAs89dgHuFDtnLP47Zd46bsqexMwle2v3mFLy5rp13D52LM79LL259PV32N0Y6q17/fvz+GLm5aSnxxEX66astp7oMBfaYZxziYgJ4z+L7yRv/U7SMhOw2nR0NZrvvFTSP34FjBq27L6NBv964lxTSI++lTBbyqGf2M5YhJVpSTP4qORDJJIkRzJVgVr+uPFNRnfpw5Prv2BcQjqfly+gLliNhk7A3FtjRgBIyAhPQENgNp8TKPfVYdctfF1RwLW5oSWL84s3UOltZFbvMYxNTmdYYiK60LAd5ByHPxBECMHyoh2k94nn7RdnIU2Jw2Vle20V7544iwpvPVFWJ5kxMWytrGx5bnaXLq06YjUF/ZR7GnAKW0vCBwiaJturarj8/8bQb0A3/rVsBbdPG4/jME622pr3yRna/bB+3koHp5L+8SloNpBXeR9VnoUA7Kp/GZc1i9iwM7Dp7X8ef18O3cEpiadySuI0pBT4TA91gSYeH3AFutA4O20gSc4wiktDf+zr6r7i11lT+Lp8G02GH6umU+/3EQhIHu53Pq8ULsZlsfO73jMorqnlk50bWx3vi9KtXJU1inpZy5cV72ARNkZ3OQeH7mr16cgfDDWh13WN0tp6Vu4oYfZXK0iOiuCfF53J7z5awLKiQlIiIujqDufpU07loYkT8RkGG8rLmdKzJ6f17o2l+TyB3zDYVF1GkiuScJudPgld2FAWKrYX7XTQv1sidTFe3C47Zw/ty7cFxUzuk3mMfgtKh6Dm9I9vTYHWxcoaAxuprBtMj/DUdrX2/nA4dAcBw0+jWc0npc/TGKxlVOzZ6CKVj0pyqfDVcl3PGWyoW4ldc9LdHc3Xp95Gjb8Bt8WJIU1qAh6GxfZgWFwGmhDYNQuVwsMJ8T14p3BvV7Ehsd0QAp7Pu4kmow6ALfXfMDPzaTSh0xj0stvbwMqqAoZG92D9jgrsuoWrxg5hUp8ebCnZTffYaL4sCpUwLq6ro7iujvl5W5iakcWfp0zBousEDaPVtE5QGiQ4Irjyf++RFRPL7F+dyeur1tLo83PFiMGU1TVw+nOvEjAMurjDeO+aiw+66kc5PqnVO8cpgYX4sLMoqPlj8xaNSOfpvL5tJbdlp7dlaD+dkLxReD+V/mIA3il+lGt7vsC2xlLW1GyjV0RX7uz9NNX+Kp7Y/CSV/iomdBnPifET+E/Bf7ko7Vzu3/AkTouDXu7udBG9uTf3cz487VLu7H8yH+/cSL/ortzc90Qag2UtCR+g0r8Tj1GPYUbwVUUet696AwhV0Hx3zE1U+73cuuI9gtLkxj7jqfV7GJ6cwvKdoVgFMLxrCrVeD3PLvkJKk8t7TGz17QWCBn/J/YqNuyvYuLuCrVVV3DV6HAMSEnHb7bhtAb667WoKKqvpGR+H3sZXSivtkex00zvt60qhdkzXHHSNuJjs+H+REjmL7IR3Kfba+bJi408qxNYeCERLwg+RlHi2kOKMBWD2tk/xGD7+lv88hU1FNAQb+LBkHsVNxdg0K6Y0QQh2ecqZmjSBN7eup9rn4ex5c4jVI/jbqHM5pVsSFiGIsMZiEXuXUrr0SML0CDyGnzcK9vbudeo2ouwurvxyDkvK8vmqfDuXLn0VIQTPTD2V8/v2Y3xaOrNPO5PUqCjWebbx6vZF5ESltZRsaPn+hEalZ+88/obd5by5cV3LKiyXzUq4w06/5EQcVktLaWVFaSEJJf1D3X4mIcRAIUSuEGK1EGKFEGL4zw9+/9RI/wjompMI+xi2NSXywpYVfFuVx+96n4Fhmh3y7dOQQdLC+lPYGLrWwCrsdA8bSIytoWWfKFsE5d7yVs8r95UTbYsiyhbBdRkXkhqWzOqKEpbs2g5AaVM9L6z/lhFJSfQO7wGEEvDF6Q+zqHwOFmHh5MQrAAhIgy6O8JbXTnREUu5toNrvadnmNYJsri1lSfkmJvROZWRcBg6LhbeLvuKbqq3c2fdshsVm/mjljdtqZdbg4SwrKsSQEpum8+tBQ3HbOtZUnNLGjs3szuPAg1LKj4QQ05rvTzgaB1JJ/wjZdCs5UT3oF50RWrkiZUuVyo7Grrs4t9u9fFv1Pg2BKgZET6PUW83F6ScxI3kkLosDn+FjbJcxLCz/AgCX7mRozBC21m8DKcmKCK1gyY5OIMUdSXFDLXbdwu1DxlMX8PCXTR8wpetgRsb2JMXVi/NS7wwdW3MhhCDaqnFz76mc1W0osXYXpZ46klwRdAuLZkdjNQCx9jD6R6cwKCaVrbVl2HQNm27hrNRRzEgZhlO37Xf5p6Zp9E9I4MvLZrKiZCcjU7odceE1RTlG6/QlsKf+RySw6yD7/iydsp6+cvi8QR8b6/Op9FWzvGoDyyvXck3meYyPzaHI4+Xh1fN4ZuS5bKnfTHWgkjFxJ2DX7BjSxLVPD+GgaRI0TYrqa+jqDmdp+Qb+sOFtgtLArln4aMJ92HXrjyqMSinxmz52+8qp8u+mZ3gfpITagI+Xti4naJr8OusEvti+nQeWLmJKRiaPTZi833X4irKvX6KefqQzSZ6Qftkh91uw6bFCYPc+m/4ppfzn4R5HCJENfExo8lEDTpBSFh5ZtIdHjfSPcwYmLxd8SH7DjpZtlb4afNLK0xvms6p6B9M+e46pyX05MbEXxU3bmVP0F+7v2/rv2aJpWDSNrOg4ChrKeWj9f1se85lBKn0NVNR76RUb16qZic/08EX5fBaUvgNArC2eW3s/SlPAy7W9x+AzAry2dh1PfRua939/yyYGJXTl4pwBLUszFeWokRKMw5rf2X2oNxghxGdA4n4euhuYCNwspXxHCHEeMBuYdKThHg71X3Mc8BsefEYjXqMBv+Fp9ZhFWBgbN7jVttFx/fAbwdCJWqA+4OXNgu9YXLYVr+FjYvyZyIM0JUlyRpPh3lueeVB0D3QsnPPe62ypqmy1r02zs7D8w5b7lf5yvq9ZQbTDwR3f/wVP0MfH2/JaPWd7TTUBo3VDckU5an6hE7lSyklSypz93N4HLgX21HF/C1AncpWfJmCESifUBUr4avdcwizRjIu/BJsWmpqx61amdx1HvCOGdbV5jI0bQII9Ak3YmZk1ju+qimgK+omzu7k6ayxRNhvQH7t+4M5fVk3nhRHXsbhsPaaEUXG9+M2n8zGkZF7+ZvrExWNrvmrWlCYOzYXf3FvELNwSQX5DMcWeMlbXbuKinP7cs/hzAGyaziU5A9T0jnLsHJsp8F3AeGARcBKw9aB7/wwq6XdihumlsPZ5ShvfJ8yawaSER/i07FV8RlNL0gew6zZGxw1keEwOuvDiDeSB1p0e4bF8Mfm3FDRU0jMyHqSJ/TCuPtaEhkO3MSE+h/e2bmTSp/9md/PSybEp6Vj3mZaRSC5Im8krBc/iMZoYFD2KzPBsSjxVXJI2naExvYhKiKZflwRWlOxiakZPoh2qnaByjEjAPCZJ/9fA00IIC+AFZh6tA6mk30lJKan0LKao7l8A+I1ytlbex6jY2zDk3tr0pjQwpB9d2NBEEF2E4bJmIwl9CvAZQXpGxOP4Cf0A7BYrp2b0Zk1ZKZurd3N2Vl+GJiW3uq7BqtnIDMvmoZy/Y0oDicSm2UkLSyItLKllvwEJSQxISNrfYRTlKJJwBP2Vf/JRpFwGDDnqB0Il/U5LYlDvX99qW2NgG5HWRHQtNDVjSoOGQClflNxLrX8HmRFTGN7lOiz7fApwaT/vT8Rts3H36AkYponDYm2Z1tnX3tpFaspGaWckh3sit8NQJ3I7KU1YSHKfjdgnkSa6z0AXFqzNST8ofSwufYhK3xaC0sOm2rkUNy7/xWMJs9qIsDv2m/AVpd07BlfkHktqpN+J2fV4RibPp6RhLuG2HGKco1tG+QAaOg2BklbPqfEXIOXYUO9gRVE6XFI/FPWf3YnpmhOnNZXuUdfTJWxiq4QPIDHpFXl6y32LcJIVeapK+IrS4jBG+R3sTUGN9DsR06wH9kyhmASljiasWA4wL2/VnPSPuYgk1yCqfPn0CJ+IVQs7ZvEqSrsnAVVa+ecTQjwBzAD8QD5wuZSypi1i6SxM00NV9c14vPPRtDhio5/FZhtMYyAfmxaJRY/dbw9fi+YgyTWYJNfg/byqoigdbSR/KG31Of5TIEdK2R/YAtzZRnF0Gl7fQjzeeYDENCuorr0LMKhpepv1JaMpr/sXhtnY1mEqSgfTXIbhULcOpE2SvpTyEylbFovnAh2v0Ww7YxgVre6bZjVCODDMWgBK6/+GJjpmNVBFaTMSpDQPeetI2sMZuyuAjw70oBBiZnNTgRUVFRUH2u24F+Y6C13bW8sp3H0tptlITVPoR6sJB53rQ6qiHCOmPPStAzlqc/oHqyjXXGAIIcTdQBCYc6DXaS5P+k8IlVY+CqF2CkI4SUz4Cp/vKyyWdHQ9gZLaf2LKJjThpFv070EGQagLoBTliHSyOf2jlvSllActCyqEuAyYDkyUHamofzslhBUhrDidoT6xhmmQGHk1CZFXomFHYqJpqmaNohwRKdXqnV+CEGIKcBswXkrZdKj9lSOnazqgll8qys/WycakbbVO/1nADnzaXHwrV0opB3dbAAAFMElEQVQ5q41iURRFOQCJ7GS9G9ok6UspM9viuIqiKEfk2JVWPmbUFbmKoigH08GWZB6KSvqKoigHIAGpRvqKoijHCXlsmqgcSyrpK4qiHERnO5ErOtISeSFEBVC4z6Y4YHcbhfNTdbSYO1q80PFi7mjxQseIOU1K2eXnvIAQYgGh7/VQdkspp/ycYx0rHSrp/5AQYoWUcmhbx3EkOlrMHS1e6Hgxd7R4oWPGrIS0h9o7iqIoyjGikr6iKMpxpKMn/X+2dQA/QUeLuaPFCx0v5o4WL3TMmBU6+Jy+oiiKcmQ6+khfURRFOQIq6SuKohxHOk3SF0L8TgghhRCHs6a2zQghnhBCbBJCfC+EmCuEiGrrmA5ECDFFCLFZCJEnhLijreM5GCFENyHEF0KIDUKI9UKIG9s6psMlhNCFEKuEEB+2dSyHIoSIEkK83fw3vFEIMaqtY1KOTKdI+kKIbsBkoKitYzkMHaIpvBBCB/4GTAX6ABcIIfq0bVQHFQR+J6XsA4wErmvn8e7rRmBjWwdxmJ4GFkgpewMD6DhxK806RdIHniTUlKXdn5XuQE3hhwN5UsptUko/8AZwehvHdEBSyhIp5crmr+sJJaPkto3q0IQQKcCpwAttHcuhCCEigXHAbAAppV9KWdO2USlHqsMnfSHE6cBOKeWato7lJzhoU/g2lgzs2Od+MR0giQIIIdKBQcDyto3ksDxFaMDSEap6dQcqgJeap6NeEEKo9mwdTIcouHawJuvAXYSmdtqNX6opvHLkhBBu4B3gJillXVvHczBCiOlAuZTyOyHEhLaO5zBYgMHADVLK5UKIp4E7gHvbNizlSHSIpH+gJutCiH6ERh9rmtsupgArhRDDpZSlxzDEVjpJU/idQLd97qc0b2u3hBBWQgl/jpTy3baO5zCMBk4TQkwDHECEEOJVKeXFbRzXgRQDxVLKPZ+g3iaU9JUOpFNdnCWEKACGSinbbfW/5qbwfyHUFL6ireM5ECGEhdCJ5omEkv23wIVSyvVtGtgBiNC7/stAlZTypraO50g1j/RvkVJOb+tYDkYIsRS4Skq5WQjxABAmpby1jcNSjkCHGOl3Mh2iKbyUMiiEuB74GNCBF9trwm82GrgEWCuEWN287S4p5fw2jKkzugGYI4SwAduAy9s4HuUIdaqRvqIoinJwHX71jqIoinL4VNJXFEU5jqikryiKchxRSV9RFOU4opK+oijKcUQlfaXdEEIYQojVQoh1Qoi3hBCu5u2JQog3hBD5QojvhBDzhRBZ+zzvJiGEt7k2zIFee4EQoqYjVLJUlKNJJX2lPfFIKQdKKXMAPzCr+aKrucAiKWWGlHIIocqkCfs87wJCF4+ddZDXfoLQOn5FOa6ppK+0V0uBTOBEICCl/MeeB6SUa6SUSwGEEBmAG7iHUPLfLynl50D9UY1YUToAlfSVdqe5BMRUYC2QA3x3kN3PJ1T2eSnQSwiRcJB9FeW4p5K+0p44m0sorCDUEGf2YTznAuANKaVJqNjauUcxPkXp8FTtHaU98UgpB+67QQixHjhnfzs3V1ntyd46RjZgO6H6Roqi7Ica6Svt3ULALoSYuWeDEKK/EGIsoVH+A1LK9OZbV6CrECKtrYJVlPZOJX2lXWvuN3AmMKl5yeZ64FGglNB8/twfPGVu8/ZWmksCvwVMFEIUCyFOObqRK0r7pKpsKoqiHEfUSF9RFOU4opK+oijKcUQlfUVRlOOISvqKoijHEZX0FUVRjiMq6SuKohxHVNJXFEU5jvw/r3+LhkEs68EAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "df = pd.read_csv('tests/data/regression.csv')\n",
        "\n",
        "pca = PCA(n_components=2)\n",
        "pca_xy = pca.fit_transform(preds.squeeze())\n",
        "plt.scatter(pca_xy[:,0], pca_xy[:,1], s=30, c=df.logSolubility, edgecolor='w')\n",
        "plt.colorbar(label='logSolubility')\n",
        "plt.xlabel('PCA 1')\n",
        "plt.ylabel('PCA 2')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oGUPCknvqj4y"
      },
      "source": [
        "# Spectra"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bwF0PzS9qj4y",
        "outputId": "19a410d9-09dd-49f0-877c-0143444be71c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/spectra.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'spectra',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': ['tests/data/spectra_features.csv'],\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'sid',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'sid',\n",
            " 'metrics': ['sid'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 6,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': False,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_spectra',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['smiles'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'random_with_repeated_smiles',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['400', '402', '404', '406', '408', '410'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': True,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "200it [00:00, 41173.10it/s]\n",
            "100%|██████████| 200/200 [00:00<00:00, 70623.07it/s]\n",
            "100%|██████████| 200/200 [00:00<00:00, 12587.57it/s]\n",
            "Number of tasks = 6\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "0it [00:00, ?it/s]Warning: Repeated SMILES found in data, pickle file of split indices cannot distinguish entries and will not be generated.\n",
            "1it [00:00, 431.65it/s]\n",
            "Total size = 200 | train size = 160 | val size = 20 | test size = 20\n",
            "Normalizing spectra and excluding spectra regions based on phase\n",
            "100%|██████████| 4/4 [00:00<00:00, 1356.39it/s]\n",
            "100%|██████████| 1/1 [00:00<00:00, 3226.39it/s]\n",
            "100%|██████████| 1/1 [00:00<00:00, 3084.05it/s]\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=305, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=6, bias=True)\n",
            "    (5): nn_exp()\n",
            "  )\n",
            ")\n",
            "Number of parameters = 358,206\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/4 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 1/4 [00:00<00:00,  5.98it/s]\u001b[A\n",
            " 50%|█████     | 2/4 [00:00<00:00,  6.22it/s]\u001b[A\n",
            " 75%|███████▌  | 3/4 [00:00<00:00,  6.46it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00, 2031.14it/s]\n",
            "Validation sid = 0.038611\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.46it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/4 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 1/4 [00:00<00:00,  8.71it/s]\u001b[A\n",
            " 50%|█████     | 2/4 [00:00<00:00,  8.68it/s]\u001b[A\n",
            " 75%|███████▌  | 3/4 [00:00<00:00,  8.37it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00, 2200.58it/s]\n",
            "Validation sid = 0.008126\n",
            " 40%|████      | 2/5 [00:01<00:01,  1.61it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/4 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 1/4 [00:00<00:00,  8.40it/s]\u001b[A\n",
            " 50%|█████     | 2/4 [00:00<00:00,  7.55it/s]\u001b[A\n",
            " 75%|███████▌  | 3/4 [00:00<00:00,  7.94it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00, 2185.67it/s]\n",
            "Validation sid = 0.005895\n",
            " 60%|██████    | 3/5 [00:01<00:01,  1.67it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/4 [00:00<?, ?it/s]\u001b[ALoss = 1.3642e-03, PNorm = 34.2049, GNorm = 0.0428, lr_0 = 1.2915e-04\n",
            "\n",
            " 25%|██▌       | 1/4 [00:00<00:00,  7.79it/s]\u001b[A\n",
            " 50%|█████     | 2/4 [00:00<00:00,  7.99it/s]\u001b[A\n",
            " 75%|███████▌  | 3/4 [00:00<00:00,  8.26it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00, 1776.49it/s]\n",
            "Validation sid = 0.003364\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.71it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/4 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 1/4 [00:00<00:00,  8.34it/s]\u001b[A\n",
            " 50%|█████     | 2/4 [00:00<00:00,  8.52it/s]\u001b[A\n",
            " 75%|███████▌  | 3/4 [00:00<00:00,  8.32it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00, 2113.00it/s]\n",
            "Validation sid = 0.002056\n",
            "100%|██████████| 5/5 [00:02<00:00,  1.69it/s]\n",
            "Model 0 best validation sid = 0.002056 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "100%|██████████| 1/1 [00:00<00:00, 2075.36it/s]\n",
            "Model 0 test sid = 0.003194\n",
            "100%|██████████| 1/1 [00:00<00:00, 1853.43it/s]\n",
            "Ensemble test sid = 0.003194\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test sid = 0.003194\n",
            "Overall test sid = 0.003194 +/- 0.000000\n",
            "Elapsed time = 0:00:04\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/spectra.csv',\n",
        "    '--dataset_type', 'spectra',\n",
        "    '--save_dir', 'test_checkpoints_spectra',\n",
        "    '--epochs', '5',\n",
        "    '--features_path', 'tests/data/spectra_features.csv',\n",
        "    '--split_type', 'random_with_repeated_smiles',\n",
        "    '--save_smiles_splits'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)       "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e2tFb6TKqj4y",
        "outputId": "2b2727ef-4c08-41f0-f20f-d0af4657895d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "20it [00:00, 54436.13it/s]\n",
            "100%|██████████| 20/20 [00:00<00:00, 24535.27it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 20\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  5.01it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00, 2023.30it/s]\n",
            "100%|██████████| 1/1 [00:00<00:00,  1.99it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_spectra.csv\n",
            "Elapsed time = 0:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_spectra/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_spectra.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_spectra',\n",
        "    '--features_path', 'tests/data/spectra_features.csv'\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0kinNKidqj4y"
      },
      "source": [
        "# Pretraining / Transfer Learning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dM8pI9p6qj4y",
        "outputId": "85c7135a-a819-4c1d-d930-36f0f35cc16f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': None,\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/regression.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'regression',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'mse',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'rmse',\n",
            " 'metrics': ['rmse'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': False,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_transfer',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': True,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['smiles'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'random',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['logSolubility'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "500it [00:00, 87964.10it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 139327.13it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 73659.23it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "0it [00:00, ?it/s]Warning: Repeated SMILES found in data, pickle file of split indices cannot distinguish entries and will not be generated.\n",
            "276it [00:00, 59113.92it/s]\n",
            "Total size = 500 | train size = 400 | val size = 50 | test size = 50\n",
            "Fitting scaler\n",
            "Building model 0\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of parameters = 355,201\n",
            "  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.31it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.47it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.36it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.39it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.26it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.10it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.36it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.30it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.706185\n",
            " 20%|██        | 1/5 [00:01<00:04,  1.16s/it]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.90it/s]\u001b[ALoss = 6.5336e-01, PNorm = 34.0268, GNorm = 6.2552, lr_0 = 7.1875e-04\n",
            "\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.24it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  7.47it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  7.47it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  7.92it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.14it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.23it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:01<00:00,  8.04it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.387877\n",
            " 40%|████      | 2/5 [00:02<00:03,  1.17s/it]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.08it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  7.63it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.07it/s]\u001b[ALoss = 4.3137e-01, PNorm = 34.0797, GNorm = 5.1802, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.47it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.28it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.47it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.78it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.70it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.432342\n",
            " 60%|██████    | 3/5 [00:03<00:02,  1.11s/it]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.47it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  7.91it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.39it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.76it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.67it/s]\u001b[ALoss = 4.0926e-01, PNorm = 34.1023, GNorm = 6.6009, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.47it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.25it/s]\u001b[A\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.67it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.345148\n",
            " 80%|████████  | 4/5 [00:04<00:01,  1.12s/it]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 12%|█▎        | 1/8 [00:00<00:00,  8.48it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00,  8.25it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00,  8.33it/s]\u001b[A\n",
            " 50%|█████     | 4/8 [00:00<00:00,  8.36it/s]\u001b[A\n",
            " 62%|██████▎   | 5/8 [00:00<00:00,  8.53it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00,  8.58it/s]\u001b[A\n",
            " 88%|████████▊ | 7/8 [00:00<00:00,  8.51it/s]\u001b[ALoss = 3.1846e-01, PNorm = 34.1096, GNorm = 1.1095, lr_0 = 1.0000e-04\n",
            "\n",
            "100%|██████████| 8/8 [00:00<00:00,  8.53it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.262283\n",
            "100%|██████████| 5/5 [00:05<00:00,  1.13s/it]\n",
            "Model 0 best validation rmse = 1.262283 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test rmse = 1.657099\n",
            "Ensemble test rmse = 1.657099\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test rmse = 1.657099\n",
            "Overall test rmse = 1.657099 +/- 0.000000\n",
            "Elapsed time = 0:00:06\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/regression.csv',\n",
        "    '--dataset_type', 'regression',\n",
        "    '--save_dir', 'test_checkpoints_transfer',\n",
        "    '--epochs', '5',\n",
        "    '--save_smiles_splits'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)       "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3LVDRXsCqj4z",
        "outputId": "a56b6210-41e1-4a5f-da3d-768162de42cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Command line\n",
            "python /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py -f /root/.local/share/jupyter/runtime/kernel-bfa3852f-0b16-4879-895c-2ecd119e032c.json\n",
            "Args\n",
            "{'activation': 'ReLU',\n",
            " 'adding_h': False,\n",
            " 'aggregation': 'mean',\n",
            " 'aggregation_norm': 100,\n",
            " 'atom_descriptor_scaling': True,\n",
            " 'atom_descriptors': None,\n",
            " 'atom_descriptors_path': None,\n",
            " 'atom_descriptors_size': 0,\n",
            " 'atom_features_size': 0,\n",
            " 'atom_messages': False,\n",
            " 'batch_size': 50,\n",
            " 'bias': False,\n",
            " 'bias_solvent': False,\n",
            " 'bond_feature_scaling': True,\n",
            " 'bond_features_path': None,\n",
            " 'bond_features_size': 0,\n",
            " 'cache_cutoff': 10000,\n",
            " 'checkpoint_dir': None,\n",
            " 'checkpoint_frzn': 'test_checkpoints_transfer/fold_0/model_0/model.pt',\n",
            " 'checkpoint_path': None,\n",
            " 'checkpoint_paths': None,\n",
            " 'class_balance': False,\n",
            " 'config_path': None,\n",
            " 'crossval_index_dir': None,\n",
            " 'crossval_index_file': None,\n",
            " 'crossval_index_sets': None,\n",
            " 'cuda': False,\n",
            " 'data_path': 'tests/data/regression.csv',\n",
            " 'data_weights_path': None,\n",
            " 'dataset_type': 'regression',\n",
            " 'depth': 3,\n",
            " 'depth_solvent': 3,\n",
            " 'device': device(type='cpu'),\n",
            " 'dropout': 0.0,\n",
            " 'empty_cache': False,\n",
            " 'ensemble_size': 1,\n",
            " 'epochs': 5,\n",
            " 'explicit_h': False,\n",
            " 'extra_metrics': [],\n",
            " 'features_generator': None,\n",
            " 'features_only': False,\n",
            " 'features_path': None,\n",
            " 'features_scaling': True,\n",
            " 'features_size': None,\n",
            " 'ffn_hidden_size': 300,\n",
            " 'ffn_num_layers': 2,\n",
            " 'final_lr': 0.0001,\n",
            " 'folds_file': None,\n",
            " 'freeze_first_only': False,\n",
            " 'frzn_ffn_layers': 0,\n",
            " 'gpu': None,\n",
            " 'grad_clip': None,\n",
            " 'hidden_size': 300,\n",
            " 'hidden_size_solvent': 300,\n",
            " 'ignore_columns': None,\n",
            " 'init_lr': 0.0001,\n",
            " 'log_frequency': 10,\n",
            " 'loss_function': 'mse',\n",
            " 'max_data_size': None,\n",
            " 'max_lr': 0.001,\n",
            " 'metric': 'rmse',\n",
            " 'metrics': ['rmse'],\n",
            " 'minimize_score': True,\n",
            " 'mpn_shared': False,\n",
            " 'multiclass_num_classes': 3,\n",
            " 'no_atom_descriptor_scaling': False,\n",
            " 'no_bond_features_scaling': False,\n",
            " 'no_cache_mol': False,\n",
            " 'no_cuda': False,\n",
            " 'no_features_scaling': False,\n",
            " 'num_folds': 1,\n",
            " 'num_lrs': 1,\n",
            " 'num_tasks': 1,\n",
            " 'num_workers': 8,\n",
            " 'number_of_molecules': 1,\n",
            " 'overwrite_default_atom_features': False,\n",
            " 'overwrite_default_bond_features': False,\n",
            " 'phase_features_path': None,\n",
            " 'pytorch_seed': 0,\n",
            " 'quiet': False,\n",
            " 'reaction': False,\n",
            " 'reaction_mode': 'reac_diff',\n",
            " 'reaction_solvent': False,\n",
            " 'resume_experiment': False,\n",
            " 'save_dir': 'test_checkpoints_transfer',\n",
            " 'save_preds': False,\n",
            " 'save_smiles_splits': False,\n",
            " 'seed': 0,\n",
            " 'separate_test_atom_descriptors_path': None,\n",
            " 'separate_test_bond_features_path': None,\n",
            " 'separate_test_features_path': None,\n",
            " 'separate_test_path': None,\n",
            " 'separate_test_phase_features_path': None,\n",
            " 'separate_val_atom_descriptors_path': None,\n",
            " 'separate_val_bond_features_path': None,\n",
            " 'separate_val_features_path': None,\n",
            " 'separate_val_path': None,\n",
            " 'separate_val_phase_features_path': None,\n",
            " 'show_individual_scores': False,\n",
            " 'smiles_columns': ['smiles'],\n",
            " 'spectra_activation': 'exp',\n",
            " 'spectra_phase_mask_path': None,\n",
            " 'spectra_target_floor': 1e-08,\n",
            " 'split_key_molecule': 0,\n",
            " 'split_sizes': (0.8, 0.1, 0.1),\n",
            " 'split_type': 'random',\n",
            " 'target_columns': None,\n",
            " 'target_weights': None,\n",
            " 'task_names': ['logSolubility'],\n",
            " 'test': False,\n",
            " 'test_fold_index': None,\n",
            " 'train_data_size': None,\n",
            " 'undirected': False,\n",
            " 'use_input_features': False,\n",
            " 'val_fold_index': None,\n",
            " 'warmup_epochs': 2.0}\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n",
            "500it [00:00, 84545.54it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 94034.26it/s]\n",
            "100%|██████████| 500/500 [00:00<00:00, 62988.89it/s]\n",
            "Number of tasks = 1\n",
            "Fold 0\n",
            "Splitting data with seed 0\n",
            "Total size = 500 | train size = 400 | val size = 50 | test size = 50\n",
            "Fitting scaler\n",
            "Building model 0\n",
            "Loading and freezing parameters from test_checkpoints_transfer/fold_0/model_0/model.pt.\n",
            "MoleculeModel(\n",
            "  (encoder): MPN(\n",
            "    (encoder): ModuleList(\n",
            "      (0): MPNEncoder(\n",
            "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
            "        (act_func): ReLU()\n",
            "        (W_i): Linear(in_features=147, out_features=300, bias=False)\n",
            "        (W_h): Linear(in_features=300, out_features=300, bias=False)\n",
            "        (W_o): Linear(in_features=433, out_features=300, bias=True)\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (ffn): Sequential(\n",
            "    (0): Dropout(p=0.0, inplace=False)\n",
            "    (1): Linear(in_features=300, out_features=300, bias=True)\n",
            "    (2): ReLU()\n",
            "    (3): Dropout(p=0.0, inplace=False)\n",
            "    (4): Linear(in_features=300, out_features=1, bias=True)\n",
            "  )\n",
            ")\n",
            "Number of unfrozen parameters = 90,601\n",
            "Total number of parameters = 355,201\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/5 [00:00<?, ?it/s]Epoch 0\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00, 22.10it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 20.92it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.297117\n",
            " 20%|██        | 1/5 [00:00<00:02,  1.82it/s]Epoch 1\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[ALoss = 3.4231e-01, PNorm = 34.0884, GNorm = 2.9127, lr_0 = 7.1875e-04\n",
            "\n",
            " 38%|███▊      | 3/8 [00:00<00:00, 20.64it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 21.47it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.315881\n",
            " 40%|████      | 2/5 [00:01<00:01,  1.98it/s]Epoch 2\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 25%|██▌       | 2/8 [00:00<00:00, 18.60it/s]\u001b[ALoss = 3.2340e-01, PNorm = 34.1108, GNorm = 2.3179, lr_0 = 6.1897e-04\n",
            "\n",
            " 50%|█████     | 4/8 [00:00<00:00, 19.06it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 19.28it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.287416\n",
            " 60%|██████    | 3/5 [00:01<00:01,  1.83it/s]Epoch 3\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00, 20.79it/s]\u001b[ALoss = 3.2418e-01, PNorm = 34.1170, GNorm = 1.4311, lr_0 = 2.3714e-04\n",
            "\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 20.52it/s]\u001b[A\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.276664\n",
            " 80%|████████  | 4/5 [00:02<00:00,  1.82it/s]Epoch 4\n",
            "\n",
            "  0%|          | 0/8 [00:00<?, ?it/s]\u001b[A\n",
            " 38%|███▊      | 3/8 [00:00<00:00, 21.26it/s]\u001b[A\n",
            " 75%|███████▌  | 6/8 [00:00<00:00, 21.53it/s]\u001b[ALoss = 2.9644e-01, PNorm = 34.1188, GNorm = 0.9773, lr_0 = 1.0000e-04\n",
            "\n",
            "                                             \u001b[A\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "                                     \u001b[AValidation rmse = 1.261734\n",
            "100%|██████████| 5/5 [00:02<00:00,  1.83it/s]\n",
            "Model 0 best validation rmse = 1.261734 on epoch 4\n",
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n",
            "Model 0 test rmse = 1.561036\n",
            "Ensemble test rmse = 1.561036\n",
            "1-fold cross validation\n",
            "\tSeed 0 ==> test rmse = 1.561036\n",
            "Overall test rmse = 1.561036 +/- 0.000000\n",
            "Elapsed time = 0:00:03\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--data_path', 'tests/data/regression.csv',\n",
        "    '--dataset_type', 'regression',\n",
        "    '--save_dir', 'test_checkpoints_transfer',\n",
        "    '--epochs', '5',\n",
        "    '--checkpoint_frzn', 'test_checkpoints_transfer/fold_0/model_0/model.pt'\n",
        "]\n",
        "\n",
        "args = chemprop.args.TrainArgs().parse_args(arguments)\n",
        "mean_score, std_score = chemprop.train.cross_validate(args=args, train_func=chemprop.train.run_training)       "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3EKNdFKKqj4z",
        "outputId": "ae294d3c-347c-41e0-d930-967fac6c2d29"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading training args\n",
            "Setting molecule featurization parameters to default.\n",
            "Loading data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "50it [00:00, 16168.01it/s]\n",
            "100%|██████████| 50/50 [00:00<00:00, 97723.77it/s]\n",
            "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:490: UserWarning: This DataLoader will create 8 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n",
            "  cpuset_checked))\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Validating SMILES\n",
            "Test size = 50\n",
            "Predicting with an ensemble of 1 models\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r  0%|          | 0/1 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
            "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
            "Loading pretrained parameter \"ffn.1.weight\".\n",
            "Loading pretrained parameter \"ffn.1.bias\".\n",
            "Loading pretrained parameter \"ffn.4.weight\".\n",
            "Loading pretrained parameter \"ffn.4.bias\".\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n",
            "  0%|          | 0/1 [00:00<?, ?it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  4.35it/s]\u001b[A\n",
            "100%|██████████| 1/1 [00:00<00:00,  2.06it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving predictions to test_preds_transfer.csv\n",
            "Elapsed time = 0:00:01\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "arguments = [\n",
        "    '--test_path', 'test_checkpoints_transfer/fold_0/test_smiles.csv',\n",
        "    '--preds_path', 'test_preds_transfer.csv',\n",
        "    '--checkpoint_dir', 'test_checkpoints_transfer',\n",
        "]\n",
        "\n",
        "args = chemprop.args.PredictArgs().parse_args(arguments)\n",
        "preds = chemprop.train.make_predictions(args=args)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "chemprop_colab_demo.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}