create_dateset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "premier-closing",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84bab4fc",
   "metadata": {},
   "source": [
    "# Create dataset for training and test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "chief-orchestra",
   "metadata": {},
   "source": [
    "## Preprocess for train, test, validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "sublime-spokesman",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"data/train.csv\")\n",
    "val_data = pd.read_csv(\"data/val.csv\")\n",
    "test_data = pd.read_csv(\"data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "interesting-venture",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug</th>\n",
       "      <th>Cell</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>682298</td>\n",
       "      <td>PC_3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>141549</td>\n",
       "      <td>OVCAR_8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>56410</td>\n",
       "      <td>MDA_MB_231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>603077</td>\n",
       "      <td>U251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>757804</td>\n",
       "      <td>IGROV1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Drug        Cell\n",
       "0  682298        PC_3\n",
       "1  141549     OVCAR_8\n",
       "2   56410  MDA_MB_231\n",
       "3  603077        U251\n",
       "4  757804      IGROV1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cb1d365",
   "metadata": {},
   "source": [
    "## Convert drug and cell name to graph nodes' index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "victorian-clerk",
   "metadata": {},
   "outputs": [],
   "source": [
    "converter = dict(\n",
    "    pd.DataFrame(np.load(\"data/idxs.npy\", allow_pickle=True)).T[[1, 0]].values\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "controversial-season",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_idx(X):\n",
    "    X[\"Drug\"] = [converter[(i)] for i in X[\"Drug\"]]\n",
    "    X[\"Cell\"] = [converter[(i)] for i in X[\"Cell\"]]\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "overhead-relay",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = get_idx(train_data)\n",
    "val_data = get_idx(val_data)\n",
    "test_data = get_idx(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "narrow-prediction",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Drug</th>\n",
       "      <th>Cell</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>194</td>\n",
       "      <td>319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>67</td>\n",
       "      <td>316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>146</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>242</td>\n",
       "      <td>312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Drug  Cell\n",
       "0   194   319\n",
       "1    67   316\n",
       "2    23   270\n",
       "3   146   279\n",
       "4   242   312"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a89dcc8f",
   "metadata": {},
   "source": [
    "# Get edge index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "potential-morgan",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500627</th>\n",
       "      <td>3046</td>\n",
       "      <td>324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500628</th>\n",
       "      <td>3046</td>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500629</th>\n",
       "      <td>3046</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500630</th>\n",
       "      <td>3046</td>\n",
       "      <td>327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500631</th>\n",
       "      <td>3046</td>\n",
       "      <td>328</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1500632 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0    1\n",
       "0           0  269\n",
       "1           0  274\n",
       "2           0  275\n",
       "3           0  276\n",
       "4           0  279\n",
       "...       ...  ...\n",
       "1500627  3046  324\n",
       "1500628  3046  325\n",
       "1500629  3046  326\n",
       "1500630  3046  327\n",
       "1500631  3046  328\n",
       "\n",
       "[1500632 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edge_index = pd.DataFrame(np.load(\"data/edges.npy\")).T\n",
    "edge_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ancient-finding",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0</td>\n",
       "      <td>281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0</td>\n",
       "      <td>282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0</td>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502351</th>\n",
       "      <td>3046</td>\n",
       "      <td>324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502352</th>\n",
       "      <td>3046</td>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502353</th>\n",
       "      <td>3046</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502354</th>\n",
       "      <td>3046</td>\n",
       "      <td>327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502355</th>\n",
       "      <td>3046</td>\n",
       "      <td>328</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1499128 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0    1\n",
       "5           0  279\n",
       "7           0  281\n",
       "8           0  282\n",
       "9           0  283\n",
       "10          0  284\n",
       "...       ...  ...\n",
       "1502351  3046  324\n",
       "1502352  3046  325\n",
       "1502353  3046  326\n",
       "1502354  3046  327\n",
       "1502355  3046  328\n",
       "\n",
       "[1499128 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edge_index = (\n",
    "    edge_index.merge(\n",
    "        test_data.rename(columns={\"Drug\": 0, \"Cell\": 1}),\n",
    "        on=[0, 1],\n",
    "        how=\"outer\",\n",
    "        indicator=True,\n",
    "    )\n",
    "    .query('_merge == \"left_only\"')\n",
    "    .drop(\"_merge\", axis=1)\n",
    ")\n",
    "edge_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "clear-catholic",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[   0,    0,    0,  ..., 3046, 3046, 3046],\n",
       "        [ 279,  281,  282,  ...,  326,  327,  328]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edge_index = torch.tensor(edge_index.values.T).int()\n",
    "edge_index = edge_index.type(torch.int64)\n",
    "edge_index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "baa00d86",
   "metadata": {},
   "source": [
    "## Process for torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "meaningful-physics",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_drug = train_data.values[:, 0]\n",
    "train_cell = train_data.values[:, 1]\n",
    "val_drug = val_data.values[:, 0]\n",
    "val_cell = val_data.values[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "northern-belize",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = np.load(\"data/train_labels.npy\")\n",
    "val_labels = np.load(\"data/val_labels.npy\")\n",
    "\n",
    "train_labels = torch.tensor(train_labels).float()\n",
    "val_labels = torch.tensor(val_labels).float()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b6fab92",
   "metadata": {},
   "source": [
    "## Get feature matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "organizational-exploration",
   "metadata": {},
   "outputs": [],
   "source": [
    "drug = pd.read_csv(\"data/drug_sim.csv\", index_col=0)\n",
    "cell = pd.read_csv(\"data/cell_sim.csv\", index_col=0)\n",
    "gene = pd.read_csv(\"data/gene_sim.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cultural-transparency",
   "metadata": {},
   "outputs": [],
   "source": [
    "drug = torch.tensor(drug.values).float()\n",
    "cell = torch.tensor(cell.values).float()\n",
    "gene = torch.tensor(gene.values).float()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c663fb2b",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "# Create the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "comparable-institution",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([[1.0000, 0.9683, 0.9683,  ..., 0.9394, 0.9603, 0.9575],\n",
       "         [0.9683, 1.0000, 0.9932,  ..., 0.9477, 0.9659, 0.9650],\n",
       "         [0.9683, 0.9932, 1.0000,  ..., 0.9514, 0.9697, 0.9688],\n",
       "         ...,\n",
       "         [0.9394, 0.9477, 0.9514,  ..., 1.0000, 0.9472, 0.9500],\n",
       "         [0.9603, 0.9659, 0.9697,  ..., 0.9472, 1.0000, 0.9598],\n",
       "         [0.9575, 0.9650, 0.9688,  ..., 0.9500, 0.9598, 1.0000]]),\n",
       " tensor([[1.0000, 0.0170, 0.0069,  ..., 0.4732, 0.0140, 0.2338],\n",
       "         [0.0170, 1.0000, 0.4351,  ..., 0.0421, 0.4509, 0.0916],\n",
       "         [0.0069, 0.4351, 1.0000,  ..., 0.0196, 0.2926, 0.0337],\n",
       "         ...,\n",
       "         [0.4732, 0.0421, 0.0196,  ..., 1.0000, 0.0267, 0.3622],\n",
       "         [0.0140, 0.4509, 0.2926,  ..., 0.0267, 1.0000, 0.0919],\n",
       "         [0.2338, 0.0916, 0.0337,  ..., 0.3622, 0.0919, 1.0000]]),\n",
       " tensor([[1., 0., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 1., 0.,  ..., 0., 0., 0.],\n",
       "         [0., 0., 1.,  ..., 0., 0., 0.],\n",
       "         ...,\n",
       "         [0., 0., 0.,  ..., 1., 0., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 1., 0.],\n",
       "         [0., 0., 0.,  ..., 0., 0., 1.]]),\n",
       " tensor([[   0,    0,    0,  ..., 3046, 3046, 3046],\n",
       "         [ 279,  281,  282,  ...,  326,  327,  328]]),\n",
       " array([194,  67,  23, ...,  53, 193,   7]),\n",
       " array([319, 316, 270, ..., 272, 327, 296]),\n",
       " array([217, 257,  75, ..., 202, 205,   8]),\n",
       " array([285, 283, 308, ..., 273, 270, 325]),\n",
       " tensor([0., 0., 0.,  ..., 0., 0., 0.]),\n",
       " tensor([0., 0., 0.,  ..., 0., 0., 0.])]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = [\n",
    "    drug,\n",
    "    cell,\n",
    "    gene,\n",
    "    edge_index,\n",
    "    train_drug,\n",
    "    train_cell,\n",
    "    val_drug,\n",
    "    val_cell,\n",
    "    train_labels,\n",
    "    val_labels,\n",
    "]\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "26c93448",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(data, 'train.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46e5e1a8",
   "metadata": {},
   "source": [
    "## Create test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "funky-columbus",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_drug = test_data.values[:, 0]\n",
    "test_cell = test_data.values[:, 1]\n",
    "\n",
    "test_labels = np.load(\"data/test_labels.npy\")\n",
    "test_labels = torch.tensor(test_labels).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "earlier-criticism",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = [drug, cell, gene, edge_index, test_drug, test_cell, test_labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "korean-recipe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(test, 'test.pt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}