Tutorials/Data_Preparation_Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b46621c5",
   "metadata": {},
   "source": [
    "# Data Preparation Tutorial"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07750fa8",
   "metadata": {},
   "source": [
    "The Data preparation contains 3 steps: Descriptor Generation Feature Engineering and Pair Generation.    \n",
    "If you have finished your own data curation, please check the output file format by the end of each step and make yours fit for next step.    \n",
    "Please keep the dataframe column names as ('smiles','value','assay')."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0c423278",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Smiles</th>\n",
       "      <th>pChEMBL Value</th>\n",
       "      <th>Assay ChEMBL ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.33</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.39</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.19</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.22</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>7.56</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Smiles  pChEMBL Value  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...           7.33   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...           7.39   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...           7.19   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...           7.22   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...           7.56   \n",
       "\n",
       "  Assay ChEMBL ID  \n",
       "0   CHEMBL3885768  \n",
       "1   CHEMBL3885772  \n",
       "2   CHEMBL3885774  \n",
       "3   CHEMBL3885775  \n",
       "4   CHEMBL3885768  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "#load data\n",
    "df = pd.read_csv('Data/chembl_fax.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f303aa7",
   "metadata": {},
   "source": [
    "## Descriptor Generation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e32a2cf",
   "metadata": {},
   "source": [
    "Generate RDKIT_1D descriptors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a623616f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from DPDI.utilities.descriptor_generator import DescriptorGenerator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3e0b1ced",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>value</th>\n",
       "      <th>assay</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MaxAbsEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>HeavyAtomMolWt</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfonamd</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_tetrazole</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiocyan</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.33</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.39</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.19</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.22</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>7.56</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>-3.790320</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>0.167192</td>\n",
       "      <td>0.507890</td>\n",
       "      <td>427.511</td>\n",
       "      <td>410.375</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 211 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles  value          assay  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.33  CHEMBL3885768   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.39  CHEMBL3885772   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.19  CHEMBL3885774   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.22  CHEMBL3885775   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...   7.56  CHEMBL3885768   \n",
       "\n",
       "   MaxEStateIndex  MinEStateIndex  MaxAbsEStateIndex  MinAbsEStateIndex  \\\n",
       "0       12.841695       -3.817283          12.841695           0.039520   \n",
       "1       12.841695       -3.817283          12.841695           0.039520   \n",
       "2       12.841695       -3.817283          12.841695           0.039520   \n",
       "3       12.841695       -3.817283          12.841695           0.039520   \n",
       "4       12.773431       -3.790320          12.773431           0.167192   \n",
       "\n",
       "        qed    MolWt  HeavyAtomMolWt  ...  fr_sulfide  fr_sulfonamd  \\\n",
       "0  0.371398  455.565         434.397  ...           0             1   \n",
       "1  0.371398  455.565         434.397  ...           0             1   \n",
       "2  0.371398  455.565         434.397  ...           0             1   \n",
       "3  0.371398  455.565         434.397  ...           0             1   \n",
       "4  0.507890  427.511         410.375  ...           0             1   \n",
       "\n",
       "   fr_sulfone  fr_term_acetylene  fr_tetrazole  fr_thiazole  fr_thiocyan  \\\n",
       "0           0                  0             0            0            0   \n",
       "1           0                  0             0            0            0   \n",
       "2           0                  0             0            0            0   \n",
       "3           0                  0             0            0            0   \n",
       "4           0                  0             0            0            0   \n",
       "\n",
       "   fr_thiophene  fr_unbrch_alkane  fr_urea  \n",
       "0             1                 0        0  \n",
       "1             1                 0        0  \n",
       "2             1                 0        0  \n",
       "3             1                 0        0  \n",
       "4             1                 0        0  \n",
       "\n",
       "[5 rows x 211 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DG = DescriptorGenerator(df)\n",
    "df = DG.rdkit_1d()\n",
    "df.to_csv('Data/chembl_fax_rdkit1d.csv',index=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "645b33a8",
   "metadata": {},
   "source": [
    "## Feature Engineering (with scaler) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "755b5a94",
   "metadata": {},
   "source": [
    "Exclude columns with low variance and high correlation. It contains a sklearn StandardScaler."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ca7d32bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>value</th>\n",
       "      <th>assay</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MaxAbsEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>HeavyAtomMolWt</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfonamd</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_tetrazole</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiocyan</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.33</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.39</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.19</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.22</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>7.56</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>-3.790320</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>0.167192</td>\n",
       "      <td>0.507890</td>\n",
       "      <td>427.511</td>\n",
       "      <td>410.375</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 211 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles  value          assay  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.33  CHEMBL3885768   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.39  CHEMBL3885772   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.19  CHEMBL3885774   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.22  CHEMBL3885775   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...   7.56  CHEMBL3885768   \n",
       "\n",
       "   MaxEStateIndex  MinEStateIndex  MaxAbsEStateIndex  MinAbsEStateIndex  \\\n",
       "0       12.841695       -3.817283          12.841695           0.039520   \n",
       "1       12.841695       -3.817283          12.841695           0.039520   \n",
       "2       12.841695       -3.817283          12.841695           0.039520   \n",
       "3       12.841695       -3.817283          12.841695           0.039520   \n",
       "4       12.773431       -3.790320          12.773431           0.167192   \n",
       "\n",
       "        qed    MolWt  HeavyAtomMolWt  ...  fr_sulfide  fr_sulfonamd  \\\n",
       "0  0.371398  455.565         434.397  ...           0             1   \n",
       "1  0.371398  455.565         434.397  ...           0             1   \n",
       "2  0.371398  455.565         434.397  ...           0             1   \n",
       "3  0.371398  455.565         434.397  ...           0             1   \n",
       "4  0.507890  427.511         410.375  ...           0             1   \n",
       "\n",
       "   fr_sulfone  fr_term_acetylene  fr_tetrazole  fr_thiazole  fr_thiocyan  \\\n",
       "0           0                  0             0            0            0   \n",
       "1           0                  0             0            0            0   \n",
       "2           0                  0             0            0            0   \n",
       "3           0                  0             0            0            0   \n",
       "4           0                  0             0            0            0   \n",
       "\n",
       "   fr_thiophene  fr_unbrch_alkane  fr_urea  \n",
       "0             1                 0        0  \n",
       "1             1                 0        0  \n",
       "2             1                 0        0  \n",
       "3             1                 0        0  \n",
       "4             1                 0        0  \n",
       "\n",
       "[5 rows x 211 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from DPDI.utilities.feature_engineering import FeatureEngineer\n",
    "df = pd.read_csv('Data/chembl_fax_rdkit1d.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9e90bca1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>value</th>\n",
       "      <th>assay</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>MaxPartialCharge</th>\n",
       "      <th>MinPartialCharge</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_piperzine</th>\n",
       "      <th>fr_priamide</th>\n",
       "      <th>fr_pyridine</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>-0.448360</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>-0.621792</td>\n",
       "      <td>-0.820514</td>\n",
       "      <td>-0.492365</td>\n",
       "      <td>-0.450592</td>\n",
       "      <td>-0.829236</td>\n",
       "      <td>-0.916730</td>\n",
       "      <td>0.740195</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.426563</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>0.657483</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>-0.080236</td>\n",
       "      <td>-0.092748</td>\n",
       "      <td>-0.386665</td>\n",
       "      <td>1.32011</td>\n",
       "      <td>-0.139876</td>\n",
       "      <td>-0.187936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>-0.392466</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "      <td>-0.621792</td>\n",
       "      <td>-0.820514</td>\n",
       "      <td>-0.492365</td>\n",
       "      <td>-0.450592</td>\n",
       "      <td>-0.829236</td>\n",
       "      <td>-0.916730</td>\n",
       "      <td>0.740195</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.426563</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>0.657483</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>-0.080236</td>\n",
       "      <td>-0.092748</td>\n",
       "      <td>-0.386665</td>\n",
       "      <td>1.32011</td>\n",
       "      <td>-0.139876</td>\n",
       "      <td>-0.187936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>-0.578778</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "      <td>-0.621792</td>\n",
       "      <td>-0.820514</td>\n",
       "      <td>-0.492365</td>\n",
       "      <td>-0.450592</td>\n",
       "      <td>-0.829236</td>\n",
       "      <td>-0.916730</td>\n",
       "      <td>0.740195</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.426563</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>0.657483</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>-0.080236</td>\n",
       "      <td>-0.092748</td>\n",
       "      <td>-0.386665</td>\n",
       "      <td>1.32011</td>\n",
       "      <td>-0.139876</td>\n",
       "      <td>-0.187936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>-0.550831</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "      <td>-0.621792</td>\n",
       "      <td>-0.820514</td>\n",
       "      <td>-0.492365</td>\n",
       "      <td>-0.450592</td>\n",
       "      <td>-0.829236</td>\n",
       "      <td>-0.916730</td>\n",
       "      <td>0.740195</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.426563</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>0.657483</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>-0.080236</td>\n",
       "      <td>-0.092748</td>\n",
       "      <td>-0.386665</td>\n",
       "      <td>1.32011</td>\n",
       "      <td>-0.139876</td>\n",
       "      <td>-0.187936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>-0.234101</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>-0.711091</td>\n",
       "      <td>-0.805257</td>\n",
       "      <td>1.444990</td>\n",
       "      <td>0.277246</td>\n",
       "      <td>-1.288409</td>\n",
       "      <td>-0.915685</td>\n",
       "      <td>1.229818</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.426563</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>2.169050</td>\n",
       "      <td>-0.103807</td>\n",
       "      <td>-0.080236</td>\n",
       "      <td>-0.092748</td>\n",
       "      <td>-0.386665</td>\n",
       "      <td>1.32011</td>\n",
       "      <td>-0.139876</td>\n",
       "      <td>-0.187936</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 132 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles     value          assay  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn... -0.448360  CHEMBL3885768   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn... -0.392466  CHEMBL3885772   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn... -0.578778  CHEMBL3885774   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn... -0.550831  CHEMBL3885775   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c... -0.234101  CHEMBL3885768   \n",
       "\n",
       "   MaxEStateIndex  MinEStateIndex  MinAbsEStateIndex       qed     MolWt  \\\n",
       "0       -0.621792       -0.820514          -0.492365 -0.450592 -0.829236   \n",
       "1       -0.621792       -0.820514          -0.492365 -0.450592 -0.829236   \n",
       "2       -0.621792       -0.820514          -0.492365 -0.450592 -0.829236   \n",
       "3       -0.621792       -0.820514          -0.492365 -0.450592 -0.829236   \n",
       "4       -0.711091       -0.805257           1.444990  0.277246 -1.288409   \n",
       "\n",
       "   MaxPartialCharge  MinPartialCharge  ...  fr_piperzine  fr_priamide  \\\n",
       "0         -0.916730          0.740195  ...     -0.426563    -0.103807   \n",
       "1         -0.916730          0.740195  ...     -0.426563    -0.103807   \n",
       "2         -0.916730          0.740195  ...     -0.426563    -0.103807   \n",
       "3         -0.916730          0.740195  ...     -0.426563    -0.103807   \n",
       "4         -0.915685          1.229818  ...     -0.426563    -0.103807   \n",
       "\n",
       "   fr_pyridine  fr_sulfide  fr_sulfone  fr_term_acetylene  fr_thiazole  \\\n",
       "0     0.657483   -0.103807   -0.080236          -0.092748    -0.386665   \n",
       "1     0.657483   -0.103807   -0.080236          -0.092748    -0.386665   \n",
       "2     0.657483   -0.103807   -0.080236          -0.092748    -0.386665   \n",
       "3     0.657483   -0.103807   -0.080236          -0.092748    -0.386665   \n",
       "4     2.169050   -0.103807   -0.080236          -0.092748    -0.386665   \n",
       "\n",
       "   fr_thiophene  fr_unbrch_alkane   fr_urea  \n",
       "0       1.32011         -0.139876 -0.187936  \n",
       "1       1.32011         -0.139876 -0.187936  \n",
       "2       1.32011         -0.139876 -0.187936  \n",
       "3       1.32011         -0.139876 -0.187936  \n",
       "4       1.32011         -0.139876 -0.187936  \n",
       "\n",
       "[5 rows x 132 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FE = FeatureEngineer(df)\n",
    "df = FE.filter(std_thd=0.1,corr_thd=0.9,returnfeatures=True)\n",
    "df.head()\n",
    "#df.to_csv('Data/GLP1R/glp1r_chembl_rdkit1d_filtered.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75f39a49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardScaler()\n",
      "129\n"
     ]
    }
   ],
   "source": [
    "#Scaler and featurelist are saved for further prediction\n",
    "print(FE.scaler) #run scaler.transform() for new data to predict\n",
    "print(len(FE.featurelist)) #remained features name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2325ef4f",
   "metadata": {},
   "source": [
    "## Feature Engineering (without scaler) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "027e80f0",
   "metadata": {},
   "source": [
    "The scaling step is done before model training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6992ee4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>value</th>\n",
       "      <th>assay</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MaxAbsEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>HeavyAtomMolWt</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfonamd</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_tetrazole</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiocyan</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.33</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.39</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.19</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.22</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>434.397</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>7.56</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>-3.790320</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>0.167192</td>\n",
       "      <td>0.507890</td>\n",
       "      <td>427.511</td>\n",
       "      <td>410.375</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 211 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles  value          assay  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.33  CHEMBL3885768   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.39  CHEMBL3885772   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.19  CHEMBL3885774   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.22  CHEMBL3885775   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...   7.56  CHEMBL3885768   \n",
       "\n",
       "   MaxEStateIndex  MinEStateIndex  MaxAbsEStateIndex  MinAbsEStateIndex  \\\n",
       "0       12.841695       -3.817283          12.841695           0.039520   \n",
       "1       12.841695       -3.817283          12.841695           0.039520   \n",
       "2       12.841695       -3.817283          12.841695           0.039520   \n",
       "3       12.841695       -3.817283          12.841695           0.039520   \n",
       "4       12.773431       -3.790320          12.773431           0.167192   \n",
       "\n",
       "        qed    MolWt  HeavyAtomMolWt  ...  fr_sulfide  fr_sulfonamd  \\\n",
       "0  0.371398  455.565         434.397  ...           0             1   \n",
       "1  0.371398  455.565         434.397  ...           0             1   \n",
       "2  0.371398  455.565         434.397  ...           0             1   \n",
       "3  0.371398  455.565         434.397  ...           0             1   \n",
       "4  0.507890  427.511         410.375  ...           0             1   \n",
       "\n",
       "   fr_sulfone  fr_term_acetylene  fr_tetrazole  fr_thiazole  fr_thiocyan  \\\n",
       "0           0                  0             0            0            0   \n",
       "1           0                  0             0            0            0   \n",
       "2           0                  0             0            0            0   \n",
       "3           0                  0             0            0            0   \n",
       "4           0                  0             0            0            0   \n",
       "\n",
       "   fr_thiophene  fr_unbrch_alkane  fr_urea  \n",
       "0             1                 0        0  \n",
       "1             1                 0        0  \n",
       "2             1                 0        0  \n",
       "3             1                 0        0  \n",
       "4             1                 0        0  \n",
       "\n",
       "[5 rows x 211 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from DPDI.utilities.feature_engineering import FeatureEngineer\n",
    "\n",
    "df = pd.read_csv('Data/chembl_fax_rdkit1d.csv')\n",
    "df_ = df.copy()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fca801ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>value</th>\n",
       "      <th>assay</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>MaxPartialCharge</th>\n",
       "      <th>MinPartialCharge</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_piperzine</th>\n",
       "      <th>fr_priamide</th>\n",
       "      <th>fr_pyridine</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.33</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>0.250295</td>\n",
       "      <td>-0.383682</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.39</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>0.250295</td>\n",
       "      <td>-0.383682</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.19</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>0.250295</td>\n",
       "      <td>-0.383682</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>7.22</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "      <td>12.841695</td>\n",
       "      <td>-3.817283</td>\n",
       "      <td>0.039520</td>\n",
       "      <td>0.371398</td>\n",
       "      <td>455.565</td>\n",
       "      <td>0.250295</td>\n",
       "      <td>-0.383682</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>7.56</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>12.773431</td>\n",
       "      <td>-3.790320</td>\n",
       "      <td>0.167192</td>\n",
       "      <td>0.507890</td>\n",
       "      <td>427.511</td>\n",
       "      <td>0.250377</td>\n",
       "      <td>-0.355654</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 132 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles  value          assay  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.33  CHEMBL3885768   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.39  CHEMBL3885772   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.19  CHEMBL3885774   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...   7.22  CHEMBL3885775   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...   7.56  CHEMBL3885768   \n",
       "\n",
       "   MaxEStateIndex  MinEStateIndex  MinAbsEStateIndex       qed    MolWt  \\\n",
       "0       12.841695       -3.817283           0.039520  0.371398  455.565   \n",
       "1       12.841695       -3.817283           0.039520  0.371398  455.565   \n",
       "2       12.841695       -3.817283           0.039520  0.371398  455.565   \n",
       "3       12.841695       -3.817283           0.039520  0.371398  455.565   \n",
       "4       12.773431       -3.790320           0.167192  0.507890  427.511   \n",
       "\n",
       "   MaxPartialCharge  MinPartialCharge  ...  fr_piperzine  fr_priamide  \\\n",
       "0          0.250295         -0.383682  ...             0            0   \n",
       "1          0.250295         -0.383682  ...             0            0   \n",
       "2          0.250295         -0.383682  ...             0            0   \n",
       "3          0.250295         -0.383682  ...             0            0   \n",
       "4          0.250377         -0.355654  ...             0            0   \n",
       "\n",
       "   fr_pyridine  fr_sulfide  fr_sulfone  fr_term_acetylene  fr_thiazole  \\\n",
       "0            1           0           0                  0            0   \n",
       "1            1           0           0                  0            0   \n",
       "2            1           0           0                  0            0   \n",
       "3            1           0           0                  0            0   \n",
       "4            2           0           0                  0            0   \n",
       "\n",
       "   fr_thiophene  fr_unbrch_alkane  fr_urea  \n",
       "0             1                 0        0  \n",
       "1             1                 0        0  \n",
       "2             1                 0        0  \n",
       "3             1                 0        0  \n",
       "4             1                 0        0  \n",
       "\n",
       "[5 rows x 132 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FE = FeatureEngineer(df)\n",
    "df = FE.filter(std_thd=0.1,corr_thd=0.9,returnfeatures=True)\n",
    "columns = ['smiles','value','assay']\n",
    "columns.extend(FE.featurelist)\n",
    "df = df_[columns]\n",
    "df.to_csv('Data/chembl_fax_rdkit1d_filtered.csv',index=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd026c58",
   "metadata": {},
   "source": [
    "## Normalize Data with Minmax Scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0d491023",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('Data/chembl_fax_rdkit1d_filtered.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0d926742",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>value</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>MaxPartialCharge</th>\n",
       "      <th>MinPartialCharge</th>\n",
       "      <th>FpDensityMorgan1</th>\n",
       "      <th>BCUT2D_MWHI</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_piperzine</th>\n",
       "      <th>fr_priamide</th>\n",
       "      <th>fr_pyridine</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>5.15</td>\n",
       "      <td>12.011251</td>\n",
       "      <td>-5.127858</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.112995</td>\n",
       "      <td>379.869</td>\n",
       "      <td>0.240814</td>\n",
       "      <td>-0.618693</td>\n",
       "      <td>0.722222</td>\n",
       "      <td>16.149901</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>9.72</td>\n",
       "      <td>15.468666</td>\n",
       "      <td>-0.034857</td>\n",
       "      <td>0.342591</td>\n",
       "      <td>0.865972</td>\n",
       "      <td>706.294</td>\n",
       "      <td>0.572606</td>\n",
       "      <td>-0.319324</td>\n",
       "      <td>1.400000</td>\n",
       "      <td>79.918732</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 130 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     value  MaxEStateIndex  MinEStateIndex  MinAbsEStateIndex       qed  \\\n",
       "min   5.15       12.011251       -5.127858           0.000000  0.112995   \n",
       "max   9.72       15.468666       -0.034857           0.342591  0.865972   \n",
       "\n",
       "       MolWt  MaxPartialCharge  MinPartialCharge  FpDensityMorgan1  \\\n",
       "min  379.869          0.240814         -0.618693          0.722222   \n",
       "max  706.294          0.572606         -0.319324          1.400000   \n",
       "\n",
       "     BCUT2D_MWHI  ...  fr_piperzine  fr_priamide  fr_pyridine  fr_sulfide  \\\n",
       "min    16.149901  ...           0.0          0.0          0.0         0.0   \n",
       "max    79.918732  ...           2.0          1.0          2.0         1.0   \n",
       "\n",
       "     fr_sulfone  fr_term_acetylene  fr_thiazole  fr_thiophene  \\\n",
       "min         0.0                0.0          0.0           0.0   \n",
       "max         1.0                1.0          1.0           1.0   \n",
       "\n",
       "     fr_unbrch_alkane  fr_urea  \n",
       "min               0.0      0.0  \n",
       "max               1.0      1.0  \n",
       "\n",
       "[2 rows x 130 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Normalize Data regardless of group\n",
    "FE = FeatureEngineer(df)\n",
    "#save min max value of each column for reverse transform in further prediction\n",
    "df = FE.normalize('Data/minmax.csv')\n",
    "FE.minmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "93693106",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>smiles</th>\n",
       "      <th>value</th>\n",
       "      <th>assay</th>\n",
       "      <th>MaxEStateIndex</th>\n",
       "      <th>MinEStateIndex</th>\n",
       "      <th>MinAbsEStateIndex</th>\n",
       "      <th>qed</th>\n",
       "      <th>MolWt</th>\n",
       "      <th>MaxPartialCharge</th>\n",
       "      <th>MinPartialCharge</th>\n",
       "      <th>...</th>\n",
       "      <th>fr_piperzine</th>\n",
       "      <th>fr_priamide</th>\n",
       "      <th>fr_pyridine</th>\n",
       "      <th>fr_sulfide</th>\n",
       "      <th>fr_sulfone</th>\n",
       "      <th>fr_term_acetylene</th>\n",
       "      <th>fr_thiazole</th>\n",
       "      <th>fr_thiophene</th>\n",
       "      <th>fr_unbrch_alkane</th>\n",
       "      <th>fr_urea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>0.477024</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>0.240192</td>\n",
       "      <td>0.257329</td>\n",
       "      <td>0.115358</td>\n",
       "      <td>0.343176</td>\n",
       "      <td>0.231894</td>\n",
       "      <td>0.028575</td>\n",
       "      <td>0.785021</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>0.490153</td>\n",
       "      <td>CHEMBL3885772</td>\n",
       "      <td>0.240192</td>\n",
       "      <td>0.257329</td>\n",
       "      <td>0.115358</td>\n",
       "      <td>0.343176</td>\n",
       "      <td>0.231894</td>\n",
       "      <td>0.028575</td>\n",
       "      <td>0.785021</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>0.446389</td>\n",
       "      <td>CHEMBL3885774</td>\n",
       "      <td>0.240192</td>\n",
       "      <td>0.257329</td>\n",
       "      <td>0.115358</td>\n",
       "      <td>0.343176</td>\n",
       "      <td>0.231894</td>\n",
       "      <td>0.028575</td>\n",
       "      <td>0.785021</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...</td>\n",
       "      <td>0.452954</td>\n",
       "      <td>CHEMBL3885775</td>\n",
       "      <td>0.240192</td>\n",
       "      <td>0.257329</td>\n",
       "      <td>0.115358</td>\n",
       "      <td>0.343176</td>\n",
       "      <td>0.231894</td>\n",
       "      <td>0.028575</td>\n",
       "      <td>0.785021</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...</td>\n",
       "      <td>0.527352</td>\n",
       "      <td>CHEMBL3885768</td>\n",
       "      <td>0.220448</td>\n",
       "      <td>0.262623</td>\n",
       "      <td>0.488022</td>\n",
       "      <td>0.524445</td>\n",
       "      <td>0.145951</td>\n",
       "      <td>0.028822</td>\n",
       "      <td>0.878645</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 132 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              smiles     value          assay  \\\n",
       "0  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...  0.477024  CHEMBL3885768   \n",
       "1  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...  0.490153  CHEMBL3885772   \n",
       "2  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...  0.446389  CHEMBL3885774   \n",
       "3  N=C(N)c1cccc(CN2CC[C@H](NS(=O)(=O)c3ccc(-c4ccn...  0.452954  CHEMBL3885775   \n",
       "4  O=C1[C@@H](NS(=O)(=O)c2cc3ncccc3s2)CCN1Cc1cc2c...  0.527352  CHEMBL3885768   \n",
       "\n",
       "   MaxEStateIndex  MinEStateIndex  MinAbsEStateIndex       qed     MolWt  \\\n",
       "0        0.240192        0.257329           0.115358  0.343176  0.231894   \n",
       "1        0.240192        0.257329           0.115358  0.343176  0.231894   \n",
       "2        0.240192        0.257329           0.115358  0.343176  0.231894   \n",
       "3        0.240192        0.257329           0.115358  0.343176  0.231894   \n",
       "4        0.220448        0.262623           0.488022  0.524445  0.145951   \n",
       "\n",
       "   MaxPartialCharge  MinPartialCharge  ...  fr_piperzine  fr_priamide  \\\n",
       "0          0.028575          0.785021  ...           0.0          0.0   \n",
       "1          0.028575          0.785021  ...           0.0          0.0   \n",
       "2          0.028575          0.785021  ...           0.0          0.0   \n",
       "3          0.028575          0.785021  ...           0.0          0.0   \n",
       "4          0.028822          0.878645  ...           0.0          0.0   \n",
       "\n",
       "   fr_pyridine  fr_sulfide  fr_sulfone  fr_term_acetylene  fr_thiazole  \\\n",
       "0          0.5         0.0         0.0                0.0          0.0   \n",
       "1          0.5         0.0         0.0                0.0          0.0   \n",
       "2          0.5         0.0         0.0                0.0          0.0   \n",
       "3          0.5         0.0         0.0                0.0          0.0   \n",
       "4          1.0         0.0         0.0                0.0          0.0   \n",
       "\n",
       "   fr_thiophene  fr_unbrch_alkane  fr_urea  \n",
       "0           1.0               0.0      0.0  \n",
       "1           1.0               0.0      0.0  \n",
       "2           1.0               0.0      0.0  \n",
       "3           1.0               0.0      0.0  \n",
       "4           1.0               0.0      0.0  \n",
       "\n",
       "[5 rows x 132 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.to_csv('Data/chembl_fax_rdkit1d_normalized.csv',index=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d02843e4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}