SampleSizeScaleUp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"../../Chapter_3/\")\n",
    "\n",
    "import pickle\n",
    "\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "import fastai\n",
    "from fastai.basics import Learner, DataLoaders\n",
    "from fastai.vision.all import *\n",
    "from fastai.callback.all import *\n",
    "from fastai.torch_core import trainable_params\n",
    "\n",
    "# from Utils.model_datasets import RPIDataset\n",
    "from Utils.train import set_seed, MainModelDenseInputSizeCalculator, main_model_splitter\n",
    "from Utils.model_utils import LossLogger\n",
    "from Utils.model_datasets import EncodedRPIDataset\n",
    "from Utils.model_blocks import *\n",
    "from Utils.train import BlockInputShapeReformatter, MainModelDenseInputSizeCalculator\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.path import Path\n",
    "import matplotlib.patches as patches\n",
    "import seaborn as sbn\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import accuracy_score, balanced_accuracy_score, f1_score, roc_auc_score, matthews_corrcoef"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.3.1'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fastai.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of interactions: 142\n"
     ]
    }
   ],
   "source": [
    "# The number of Tat samples we have to refine model\n",
    "with open(\"../Data/TrainingData/FullEmbeddedHIVTrainingData.pickle\",'rb') as infile:\n",
    "    TAT_TRAINDATA_COUNT = len(pickle.load(infile)[0])\n",
    "    print(f\"Number of interactions: {TAT_TRAINDATA_COUNT}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "NNI_DIR = \"/data/Chapter_3/nni\"\n",
    "res_params = pd.read_csv(f\"{NNI_DIR}/experiment_results/Main_StackedRes.csv\", index_col = 0).dropna().sort_values('final_metric')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def determine_block_name_from_params(params):\n",
    "    \"\"\"\n",
    "    Searches for a parameter exclusive to a different block type\n",
    "    within the given parameters, and returns the corresponding\n",
    "    block name. \n",
    "    \"\"\"\n",
    "    \n",
    "    param_names = params.keys()\n",
    "    if 'pooling_type' in param_names:\n",
    "        return 'ConvPool'\n",
    "    elif 'reccurent_type' in param_names:\n",
    "        return 'Reccurent'\n",
    "    elif 'layer_count' in param_names:\n",
    "        return 'Attention'\n",
    "    elif 'hidden_sizes' in param_names:\n",
    "        return 'StackedRes'\n",
    "    elif 'hidden_layer_sizes' in param_names:\n",
    "        return 'Dense'\n",
    "    elif 'sae_type' in param_names:\n",
    "        return 'SAE'\n",
    "    \n",
    "class BlockInializer:\n",
    "    \"\"\"\n",
    "    Used to initialize MainModel blocks from the parameters and \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, seed):\n",
    "        self.seed = seed\n",
    "    \n",
    "    def _reformat_block_act(self, block_params):\n",
    "        preprocessed_block_str = re.sub(r\"'act': 'nn.(.+?)\\(\\)'\", r\"'act': nn.\\1()\", block_params)\n",
    "        preprocessed_block_str = re.sub(r\"'act_cls': 'nn.(.+?)\\'\", r\"'act_cls': nn.\\1\", preprocessed_block_str)\n",
    "        return preprocessed_block_str\n",
    "    \n",
    "    def _reformat_list_str(self, block_params):\n",
    "        preprocessed_block_str = re.sub(r\"'\\[(.*)\\]'\", r\"[\\1]\", block_params)\n",
    "        return preprocessed_block_str\n",
    "    \n",
    "    def generate_block_str(self, block_name, input_size, block_params):\n",
    "        return f\"{block_name}Block({input_size}, **{block_params})\"\n",
    "    \n",
    "    def _reformat_sae_block(self, input_size, preprocessed_block_str):\n",
    "        return preprocessed_block_str.replace(f\"{input_size}, \",\"\")\n",
    "    \n",
    "    def initialize_block(self, block_name, input_size, block_params):\n",
    "        block_str = self.generate_block_str(block_name, input_size, block_params)\n",
    "        preprocessed_block_str = self._reformat_block_act(block_str)\n",
    "        preprocessed_block_str = self._reformat_list_str(preprocessed_block_str)\n",
    "        \n",
    "        if block_name == 'SAE':\n",
    "            preprocessed_block_str = self._reformat_sae_block(input_size, preprocessed_block_str)\n",
    "        \n",
    "        set_seed(self.seed)\n",
    "        print(preprocessed_block_str)\n",
    "        return eval(preprocessed_block_str)\n",
    "\n",
    "def swap_seq_input_order(data : list) -> None:\n",
    "    seq1, seq2 = data[0], data[1]\n",
    "    data[0] = seq2\n",
    "    data[1] = seq1\n",
    "\n",
    "def initiate_lr_option(lr_option, lr_min, lr_steep):\n",
    "    \"having min first is intentional. It results in better performance\"\n",
    "    options = {'slice' : slice(lr_min, lr_steep), 'min' : lr_min}\n",
    "    return options[lr_option]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'StackedRes'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SEED = 255\n",
    "\n",
    "PARAMS = res_params.iloc[0,:].to_dict()\n",
    "\n",
    "_ = PARAMS.pop(\"final_metric\")\n",
    "LR_OPTION = PARAMS.pop(\"lr_max_arg\")\n",
    "BATCHSIZE = PARAMS.pop(\"batchsize\")\n",
    "MAIN_ACT = eval(PARAMS.pop(\"act\"))\n",
    "\n",
    "main_model_params = {key : value for key,value in PARAMS.items() if 'rna' not in key and 'prot' not in key}\n",
    "rna_block_params = {key.replace(\"rna_\",\"\") : value for key,value in PARAMS.items() if 'rna' in key}\n",
    "prot_block_params = {key.replace(\"prot_\",\"\") : value for key,value in PARAMS.items() if 'prot' in key}\n",
    "\n",
    "rna_block_name = determine_block_name_from_params(rna_block_params)\n",
    "prot_block_name = determine_block_name_from_params(prot_block_params)\n",
    "\n",
    "# TEMPORARY: ASSUMES THAT PROT AND RNA BLOCKS ARE IDENTICAL\n",
    "block_name = prot_block_name\n",
    "block_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################\n",
    "# Load params\n",
    "######################################################\n",
    "\n",
    "with open(\"../Data/TrainingData/FullEmbeddedTrainingData.pickle\",'rb') as handle:\n",
    "    data = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def perform_fold_evaluation()\n",
    "\n",
    "def stratify_sample_n_entries_from_data_list(data_list : list, n : int) -> list:\n",
    "    percent = n / len(data_list[0])\n",
    "    \n",
    "    if percent != 1:\n",
    "\n",
    "        _, sample_idx = train_test_split(range(len(data_list[0])), random_state = 255, test_size = percent, \n",
    "                                             stratify = data_list[-1])\n",
    "\n",
    "        stratified_sample =[X[sample_idx] for X in data_list]\n",
    "        return stratified_sample\n",
    "    else:\n",
    "        return data_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive sample count: 500.0\n",
      "Negative sample count: 500.0\n"
     ]
    }
   ],
   "source": [
    "t = stratify_sample_n_entries_from_data_list(data, 1000)\n",
    "print(f\"\"\"Positive sample count: {len(t[-1]) - t[-1].sum()}\n",
    "Negative sample count: {len(t[-1]) - t[-1].sum()}\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "def specificity_score(y_true, y_pred):\n",
    "    tn, fp, _, _ = confusion_matrix(y_true, y_pred).ravel()\n",
    "    specificity = tn / (tn+fp)\n",
    "    return specificity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score, average_precision_score, recall_score\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import os\n",
    "\n",
    "shape_reformatter = BlockInputShapeReformatter()\n",
    "BLOCKNAME = \"StackedRes\"\n",
    "PATIENCE = 10\n",
    "\n",
    "SAMPLE_COUNTS = [100,200,5000,10000,15000,20000,len(data[0])]\n",
    "METRICS = ['acc','f1','rocauc','avg_precision','sensitivity', 'specificity']\n",
    "\n",
    "performance_dict = {n : {metric : list() for metric in METRICS} for n in SAMPLE_COUNTS}\n",
    "\n",
    "if os.path.exists(\"5_fold_validation_performance.csv\"):\n",
    "    fold_eval_performance = pd.read_csv(\"5_fold_validation_performance.csv\", index_col = 0)\n",
    "\n",
    "else:\n",
    "\n",
    "    for n in tqdm(SAMPLE_COUNTS):\n",
    "\n",
    "        sampled_data = stratify_sample_n_entries_from_data_list(data, n)\n",
    "\n",
    "        skf = StratifiedKFold(n_splits = 5, random_state = 255, shuffle = True)\n",
    "        skf.get_n_splits(sampled_data[0], sampled_data[-1])\n",
    "\n",
    "        for train_index, test_index in tqdm(skf.split(sampled_data[0], sampled_data[-1]), total = 5):   \n",
    "\n",
    "            #####################################################\n",
    "            # Generate train, test, val data\n",
    "            #####################################################\n",
    "\n",
    "            train_data, test_data = zip(*[(X[train_index], X[test_index]) for X in sampled_data])\n",
    "\n",
    "            test_idx, val_idx = train_test_split(range(len(test_data[0])), random_state = 255, test_size = 0.5, \n",
    "                                                 stratify = test_data[-1])\n",
    "\n",
    "            test_data, val_data = zip(*[(X[test_idx], X[val_idx]) for X in test_data])\n",
    "\n",
    "    #         print(f\"\"\"Train data: {len(train_data[0])} | + samples: {sum(train_data[-1])} | - samples: {len(train_data[-1]) - sum(train_data[-1])}    \n",
    "    #     Test data: {len(test_data[0])} | + samples: {sum(test_data[-1])} | - samples: {len(test_data[-1]) - sum(test_data[-1])}\n",
    "    #     Val data: {len(val_data[0])} | + samples: {sum(val_data[-1])} | - samples: {len(val_data[-1]) - sum(val_data[-1])}\n",
    "    #     ---\"\"\")\n",
    "\n",
    "            #####################################################\n",
    "            # Preprocess train, test, val data\n",
    "            #####################################################\n",
    "\n",
    "            refine_train_data = [shape_reformatter.reformat_input(BLOCKNAME, train_data[i].float()) for i in range(2)] + [train_data[-1].float()]\n",
    "            swap_seq_input_order(refine_train_data)\n",
    "            refine_train_dataset = EncodedRPIDataset(refine_train_data)\n",
    "\n",
    "            refine_val_data = [shape_reformatter.reformat_input(BLOCKNAME, val_data[i].float()) for i in range(2)] + [val_data[-1].float()]\n",
    "            swap_seq_input_order(refine_val_data)\n",
    "            refine_val_dataset = EncodedRPIDataset(refine_val_data)\n",
    "\n",
    "            refine_test_data = [shape_reformatter.reformat_input(BLOCKNAME, test_data[i].float()) for i in range(2)] + [test_data[-1].float()]\n",
    "            proteins, rnas, y_true = refine_test_data\n",
    "            y_true = y_true.reshape(-1,1)\n",
    "\n",
    "            #####################################################\n",
    "            # Generate model\n",
    "            #####################################################\n",
    "\n",
    "            block_initializer = BlockInializer(SEED)\n",
    "\n",
    "            # Temporarily initializes blocks to calculate input size\n",
    "            rna_block = block_initializer.initialize_block(rna_block_name, 1024, rna_block_params)\n",
    "            prot_block = block_initializer.initialize_block(prot_block_name, 1024, prot_block_params)\n",
    "\n",
    "            dense_input_calculator = MainModelDenseInputSizeCalculator(refine_train_data[0], refine_train_data[1])\n",
    "            dense_input_size = dense_input_calculator.calcualte_input_shape(rna_blocks  = [rna_block.cpu()], \n",
    "                                                                            prot_blocks = [prot_block.cpu()])\n",
    "\n",
    "            set_seed(255)\n",
    "            rna_block = block_initializer.initialize_block(rna_block_name, 1024, rna_block_params)\n",
    "            prot_block = block_initializer.initialize_block(prot_block_name, 1024, prot_block_params)\n",
    "\n",
    "            main_model_params = eval(block_initializer._reformat_list_str(str(main_model_params)))\n",
    "\n",
    "            set_seed(255)\n",
    "            main_model = MainModel(rna_blocks  = [rna_block],\n",
    "                                   prot_blocks = [prot_block],\n",
    "                                   dense_input_size = dense_input_size,\n",
    "                                   act = MAIN_ACT,\n",
    "                                   **main_model_params).cuda()\n",
    "\n",
    "            #####################################################\n",
    "            # Train model\n",
    "            #####################################################\n",
    "\n",
    "            set_seed(255)\n",
    "            loaders = DataLoaders.from_dsets(refine_train_dataset, refine_val_dataset,\n",
    "                                             bs = BATCHSIZE,\n",
    "                                             device = 0)\n",
    "\n",
    "            learner = Learner(loaders, main_model,\n",
    "                              splitter = main_model_splitter,\n",
    "                              cbs = [EarlyStoppingCallback(patience = PATIENCE)],\n",
    "                              loss_func = nn.BCELoss())\n",
    "\n",
    "            lr_min, lr_steep = learner.lr_find()\n",
    "\n",
    "            max_lr_arg = initiate_lr_option(LR_OPTION, lr_min, lr_steep)\n",
    "            learner.fit_one_cycle(1000, lr_max = max_lr_arg)\n",
    "\n",
    "            #####################################################\n",
    "            # Evaluate performance on HIV-1 data\n",
    "            #####################################################\n",
    "\n",
    "            learner.model.eval()\n",
    "\n",
    "            y_pred = learner.model(rnas.cuda(), proteins.cuda()).cpu().detach()\n",
    "            y_pred_rounded = y_pred.round()\n",
    "\n",
    "    #         print(f\"ACCURACY: {round(accuracy_score(y_true, y_pred),3)}\")\n",
    "    #         print(f\"F1 SCORE: {round(f1_score(y_true, y_pred),3)}\")\n",
    "    #         print(f\"ROC AUC: {round(roc_auc_score(y_true, y_pred),3)}\")\n",
    "    #         print(f\"AVERAGE PRECISION: {round(average_precision_score(y_true,y_pred),3)}\")\n",
    "\n",
    "            # METRICS = ['acc','f1','rocauc','avg_precision','sensitivity', 'specificity']\n",
    "\n",
    "            performance_dict[n]['acc'].append(round(accuracy_score(y_true, y_pred_rounded),3))\n",
    "            performance_dict[n]['f1'].append(round(f1_score(y_true, y_pred_rounded),3))\n",
    "            performance_dict[n]['rocauc'].append(round(roc_auc_score(y_true, y_pred), 3))\n",
    "            performance_dict[n]['avg_precision'].append(round(average_precision_score(y_true, y_pred),3))\n",
    "            performance_dict[n]['sensitivity'].append(round(recall_score(y_true, y_pred_rounded),3))\n",
    "            performance_dict[n]['specificity'].append(round(specificity_score(y_true, y_pred_rounded),3))\n",
    "\n",
    "            print(\"---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>100.0</th>\n",
       "      <th>200.0</th>\n",
       "      <th>5000.0</th>\n",
       "      <th>10000.0</th>\n",
       "      <th>15000.0</th>\n",
       "      <th>20000.0</th>\n",
       "      <th>24294.0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>acc</th>\n",
       "      <td>[0.8, 0.6, 0.8, 0.9, 0.5]</td>\n",
       "      <td>[0.85, 0.65, 0.75, 0.8, 0.95]</td>\n",
       "      <td>[0.952, 0.95, 0.948, 0.972, 0.96]</td>\n",
       "      <td>[0.974, 0.973, 0.965, 0.971, 0.981]</td>\n",
       "      <td>[0.969, 0.976, 0.974, 0.975, 0.985]</td>\n",
       "      <td>[0.976, 0.979, 0.982, 0.98, 0.974]</td>\n",
       "      <td>[0.986, 0.986, 0.98, 0.978, 0.981]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>[0.833, 0.667, 0.8, 0.909, 0.667]</td>\n",
       "      <td>[0.857, 0.741, 0.737, 0.818, 0.952]</td>\n",
       "      <td>[0.953, 0.951, 0.949, 0.973, 0.961]</td>\n",
       "      <td>[0.975, 0.974, 0.966, 0.972, 0.981]</td>\n",
       "      <td>[0.969, 0.976, 0.975, 0.976, 0.985]</td>\n",
       "      <td>[0.977, 0.979, 0.982, 0.98, 0.974]</td>\n",
       "      <td>[0.986, 0.986, 0.981, 0.978, 0.981]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rocauc</th>\n",
       "      <td>[0.92, 0.6, 0.84, 0.88, 0.76]</td>\n",
       "      <td>[0.93, 0.84, 0.85, 0.97, 1.0]</td>\n",
       "      <td>[0.985, 0.992, 0.983, 0.993, 0.989]</td>\n",
       "      <td>[0.992, 0.992, 0.992, 0.996, 0.997]</td>\n",
       "      <td>[0.991, 0.994, 0.996, 0.991, 0.995]</td>\n",
       "      <td>[0.993, 0.995, 0.995, 0.996, 0.994]</td>\n",
       "      <td>[0.996, 0.994, 0.995, 0.995, 0.994]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>avg_precision</th>\n",
       "      <td>[0.927, 0.578, 0.853, 0.877, 0.821]</td>\n",
       "      <td>[0.953, 0.852, 0.899, 0.977, 1.0]</td>\n",
       "      <td>[0.982, 0.991, 0.981, 0.987, 0.988]</td>\n",
       "      <td>[0.99, 0.987, 0.975, 0.994, 0.995]</td>\n",
       "      <td>[0.988, 0.992, 0.986, 0.989, 0.993]</td>\n",
       "      <td>[0.988, 0.994, 0.994, 0.995, 0.99]</td>\n",
       "      <td>[0.995, 0.992, 0.993, 0.994, 0.989]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sensitivity</th>\n",
       "      <td>[1.0, 0.8, 0.8, 1.0, 1.0]</td>\n",
       "      <td>[0.9, 1.0, 0.7, 0.9, 1.0]</td>\n",
       "      <td>[0.984, 0.964, 0.976, 0.992, 0.996]</td>\n",
       "      <td>[0.998, 1.0, 0.994, 0.996, 0.996]</td>\n",
       "      <td>[0.995, 0.992, 1.0, 0.996, 0.993]</td>\n",
       "      <td>[0.989, 0.997, 0.997, 0.996, 0.995]</td>\n",
       "      <td>[0.995, 0.998, 0.999, 0.996, 0.996]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specificity</th>\n",
       "      <td>[0.6, 0.4, 0.8, 0.8, 0.0]</td>\n",
       "      <td>[0.8, 0.3, 0.8, 0.7, 0.9]</td>\n",
       "      <td>[0.92, 0.936, 0.92, 0.952, 0.924]</td>\n",
       "      <td>[0.95, 0.946, 0.936, 0.946, 0.966]</td>\n",
       "      <td>[0.943, 0.96, 0.948, 0.955, 0.977]</td>\n",
       "      <td>[0.964, 0.961, 0.966, 0.963, 0.952]</td>\n",
       "      <td>[0.976, 0.973, 0.961, 0.96, 0.965]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           100.0    \\\n",
       "acc                      [0.8, 0.6, 0.8, 0.9, 0.5]   \n",
       "f1               [0.833, 0.667, 0.8, 0.909, 0.667]   \n",
       "rocauc               [0.92, 0.6, 0.84, 0.88, 0.76]   \n",
       "avg_precision  [0.927, 0.578, 0.853, 0.877, 0.821]   \n",
       "sensitivity              [1.0, 0.8, 0.8, 1.0, 1.0]   \n",
       "specificity              [0.6, 0.4, 0.8, 0.8, 0.0]   \n",
       "\n",
       "                                           200.0    \\\n",
       "acc                  [0.85, 0.65, 0.75, 0.8, 0.95]   \n",
       "f1             [0.857, 0.741, 0.737, 0.818, 0.952]   \n",
       "rocauc               [0.93, 0.84, 0.85, 0.97, 1.0]   \n",
       "avg_precision    [0.953, 0.852, 0.899, 0.977, 1.0]   \n",
       "sensitivity              [0.9, 1.0, 0.7, 0.9, 1.0]   \n",
       "specificity              [0.8, 0.3, 0.8, 0.7, 0.9]   \n",
       "\n",
       "                                           5000.0   \\\n",
       "acc              [0.952, 0.95, 0.948, 0.972, 0.96]   \n",
       "f1             [0.953, 0.951, 0.949, 0.973, 0.961]   \n",
       "rocauc         [0.985, 0.992, 0.983, 0.993, 0.989]   \n",
       "avg_precision  [0.982, 0.991, 0.981, 0.987, 0.988]   \n",
       "sensitivity    [0.984, 0.964, 0.976, 0.992, 0.996]   \n",
       "specificity      [0.92, 0.936, 0.92, 0.952, 0.924]   \n",
       "\n",
       "                                           10000.0  \\\n",
       "acc            [0.974, 0.973, 0.965, 0.971, 0.981]   \n",
       "f1             [0.975, 0.974, 0.966, 0.972, 0.981]   \n",
       "rocauc         [0.992, 0.992, 0.992, 0.996, 0.997]   \n",
       "avg_precision   [0.99, 0.987, 0.975, 0.994, 0.995]   \n",
       "sensitivity      [0.998, 1.0, 0.994, 0.996, 0.996]   \n",
       "specificity     [0.95, 0.946, 0.936, 0.946, 0.966]   \n",
       "\n",
       "                                           15000.0  \\\n",
       "acc            [0.969, 0.976, 0.974, 0.975, 0.985]   \n",
       "f1             [0.969, 0.976, 0.975, 0.976, 0.985]   \n",
       "rocauc         [0.991, 0.994, 0.996, 0.991, 0.995]   \n",
       "avg_precision  [0.988, 0.992, 0.986, 0.989, 0.993]   \n",
       "sensitivity      [0.995, 0.992, 1.0, 0.996, 0.993]   \n",
       "specificity     [0.943, 0.96, 0.948, 0.955, 0.977]   \n",
       "\n",
       "                                           20000.0  \\\n",
       "acc             [0.976, 0.979, 0.982, 0.98, 0.974]   \n",
       "f1              [0.977, 0.979, 0.982, 0.98, 0.974]   \n",
       "rocauc         [0.993, 0.995, 0.995, 0.996, 0.994]   \n",
       "avg_precision   [0.988, 0.994, 0.994, 0.995, 0.99]   \n",
       "sensitivity    [0.989, 0.997, 0.997, 0.996, 0.995]   \n",
       "specificity    [0.964, 0.961, 0.966, 0.963, 0.952]   \n",
       "\n",
       "                                           24294.0  \n",
       "acc             [0.986, 0.986, 0.98, 0.978, 0.981]  \n",
       "f1             [0.986, 0.986, 0.981, 0.978, 0.981]  \n",
       "rocauc         [0.996, 0.994, 0.995, 0.995, 0.994]  \n",
       "avg_precision  [0.995, 0.992, 0.993, 0.994, 0.989]  \n",
       "sensitivity    [0.995, 0.998, 0.999, 0.996, 0.996]  \n",
       "specificity     [0.976, 0.973, 0.961, 0.96, 0.965]  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fold_eval_performance = fold_eval_performance.applymap(lambda x: eval(x))\n",
    "fold_eval_performance.columns = fold_eval_performance.columns.astype(float)\n",
    "fold_eval_performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_frame(df,TAT_TRAINDATA_COUNT, axes):\n",
    "    \n",
    "    sbn.lineplot(data = df.iloc[:,:2].melt(), x = 'variable', y = 'value', marker = 'o', ax = axes[0])\n",
    "    sbn.lineplot(data = df.iloc[:,2:].melt(), x = 'variable', y = 'value', marker = 'o', ax = axes[1])\n",
    "    \n",
    "    y_min, y_max = axes[0].get_ylim()\n",
    "        \n",
    "    axes[0].set_xlabel(\"RPI samples\")\n",
    "    axes[1].set_xlabel(\"RPI samples\")\n",
    "    \n",
    "    axes[1].set_ylabel(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_between_to_available_data(plt_data, avilable_data_count : int, ax, **kwargs):\n",
    "    x = plt_data.lines[0].get_xdata().reshape(-1,1)\n",
    "    y = plt_data.lines[0].get_ydata().reshape(-1,1)\n",
    "\n",
    "    y_min, _ = ax.get_ylim()\n",
    "\n",
    "    bottom = [y_min]*2\n",
    "\n",
    "    linear = LinearRegression().fit(x,y)\n",
    "    available_data_count_y = linear.predict(np.array([avilable_data_count]).reshape(-1,1))\n",
    "    \n",
    "    ax.fill_between([int(x[0][0]), avilable_data_count],[y[0][0], available_data_count_y[0][0]], [y_min]*2, **kwargs)\n",
    "\n",
    "def remove_axis(ax, left = False, right = False, top = False, bottom = False):\n",
    "    \n",
    "    ax.tick_params(axis = 'y',    \n",
    "                   which = 'both', \n",
    "                   left = not left, \n",
    "                   right = not right, \n",
    "                   labelleft = not left,\n",
    "                   labelright = not right,\n",
    "    )\n",
    "    \n",
    "    ax.tick_params(axis = 'x',         \n",
    "                   which = 'both',  \n",
    "                   top = not top,\n",
    "                   bottom = not bottom,\n",
    "                   labeltop = not top,\n",
    "                   labelbottom = not bottom,\n",
    "    )\n",
    "    \n",
    "    ax.spines['left'].set_visible(not left)\n",
    "    ax.spines['right'].set_visible(not right)\n",
    "    ax.spines['top'].set_visible(not top)\n",
    "    ax.spines['bottom'].set_visible(not bottom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_sample_size_upscale_graph(df, metric, axes, lineplot_kwgs = dict(), fill_between_kwgs = dict()):\n",
    "    assert axes.shape == (2,2), f\"axes shape needs to be 2x2, gave shape: {axes.shape}\"\n",
    "    \n",
    "    \n",
    "    # marker = 'o', err_style=\"bars\",\n",
    "    smll_plt_data = sbn.lineplot(data = df.iloc[:,:2].melt(), x = 'variable', y = 'value', ax = axes[1,0], **lineplot_kwgs)\n",
    "\n",
    "    sbn.lineplot(data = df.iloc[:,2:].melt(), x = 'variable', y = 'value', ax = axes[0,1], **lineplot_kwgs)\n",
    "\n",
    "    small_data_plt = df.iloc[:,:2].melt()\n",
    "    \n",
    "    # alpha = 0.3\n",
    "    fill_between_to_available_data(smll_plt_data, TAT_TRAINDATA_COUNT, axes[1,0], **fill_between_kwgs)\n",
    "\n",
    "    axes[1,1].set_xlim(axes[0,1].get_xlim())\n",
    "    axes[0,0].set_ylim(axes[0,1].get_ylim())\n",
    "\n",
    "    axes[1,1].set_xlabel(\"RPI count\")\n",
    "    axes[1,0].set_xlabel(\"RPI count\")\n",
    "\n",
    "    axes[0,1].set_xlabel(\"\")\n",
    "    axes[0,1].set_ylabel(\"\")\n",
    "\n",
    "    axes[0,0].set_ylabel(metric)\n",
    "    axes[1,0].set_ylabel(metric)\n",
    "\n",
    "    remove_axis(axes[0,0], bottom = True, top = True, right = True)\n",
    "    remove_axis(axes[0,1], top = True, left = True, right = True, bottom = True)\n",
    "    remove_axis(axes[1,0], top = True, right = True)\n",
    "    remove_axis(axes[1,1], left = True, top = True, right = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-33-bb2842e61461>:14: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.\n",
      "  plt.tight_layout()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAADQCAYAAADRcMzjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjzklEQVR4nO3de3xU9Z3/8dcnk3sIEBPkFvAGomiRaryCvait6K5rrbZqtVrrT+yu1l93ax/atdVqt7V2/XV/66PuumhtxdpVV2vF/am0pbqKd1BArhJRuWMSEgiQ+3x+f5wzYQiBTHBOZpK8n4/HPObM91zmc2ZO5pPzPd/z/Zq7IyIiEqWcTAcgIiIDn5KNiIhETslGREQip2QjIiKRU7IREZHIKdmIiEjkcjMdQF+YMWOGP//885kOY7CxiLevNvsi0Uvb3/GgOLOpra3NdAgiIoPaoEg2IiKSWUo2Iv3A7c8s4/ZnlmU6DJEDNiiu2Yj0d8s3bs90CCKfiM5sREQkcpEmGzObYWarzKzazG7uZv4hZjbPzJaY2YtmVpk07y4zWxo+Lk4q/42ZfWBmi8LH1Cj3QURkf/pbFWem4o2sGs3MYsC9wBeA9cBbZjbH3ZcnLXY3MNvdHzKzM4A7ga+b2V8BxwNTgQLgRTN7zt0TdQnfc/cnoopdRKKT+KG77bxjMhxJevS3Ks5MxRvlmc1JQLW7r3H3VuBR4Pwuy0wG/hJOv5A0fzLwkru3u/tOYAkwI8JYRaSPLN+4vd/9QMsnF2WyGQusS3q9PixLthj4cjh9AVBqZuVh+QwzKzazCuDzwLik9X4SVr39i5kVRBO+iMjAEo87N5w5kZvPOYqaxhbi8b67NzrTrdFuBH5pZt8AXgI2AB3u/kczOxF4FagBXgM6wnW+D2wG8oFZwE3AHV03bGYzgZkA48ePj3YvRGRQSvx4F+fHqGlsobwkn5ycqDvPSE1bR5zmtg6a24Jnx9nW1M5NTy5hfX0TlWVF3H9FFZNGlvZJzFEmmw3seTZSGZZ1cveNhGc2ZjYEuNDdG8J5PwF+Es77HfBeWL4pXL3FzH5NkLD24u6zCJIRVVVV6tpERNIqHndWbWlM6ce7I+7hD38Hze3x3dNtcVraOmhJlLXvTg6dz+0dtITTLV3WTSzfkrTtlvC5o8tZy398/QR+/N/LWV/fBMD6+iaumb2Ap/5uGiNKo68gijLZvAVMNLPDCJLMJcDXkhcIq8i2unuc4IzlwbA8Bgx39zozmwJMAf4Yzhvt7pvMzIAvAUsj3AcRkT20d8T5sG4X8bhzzewFe/14/9OXjuWWp5bSkkgS7R20dRz4/7v5sRwK8nIozItRmJdDYW6sc7okP5fykhwK8mJheffLFeTGOLyipDPWhPX1TbS2d+zjndMrsmTj7u1mdj0wF4gBD7r7MjO7A1jg7nOAzwF3mpkTVKNdF66eB7wc5BO2A5e7e3s47xEzG0HQQdwi4FtR7YOIDG51O1pYubmRFZu2s3JzIys3b2f1lh20tMd5bOYp3f54H1SSz8mHHRQkgMSPf24sSBi5iWQQJoG9kkSMgtycPZJELE1VXDWNLVSWFe0Rc2VZEfm5sbRsvyeRXrNx92eBZ7uU3Zo0/QSwVxNmd28maJHW3TbPSHOYIjLItbR3UP3xDlZuChJKkFgaqWls6VymYkgBR48u5YpTD2HSqKGMGlbY7Y/36GFF/OLiqRnYi/0rL8nn/iuqOs/GEtV+5SX5ffL+mW4gICLSZ9ydTduaWbl5Oys2BQll1ebtvF+zs/MaR35uDkeOHMJnjxzBUaNKOWrUUCaNKt3rukY87hn98e6tnBxj0shS7rpwCsX5MSrLivu0QYOSjUiWy+YWT9lsZ0s7q7Y07nm2smk725vbO5cZO7yIo0eX8sXJo5g0qpSjR5dyaHkJubGe7wrJ9I/3gcjJMf68YgvQ9zfVKtmIZLHetHgarDriztqtu1i5aTsrwjOVlZsb+ahuV+cyJfkxjho9lPOOGxOcrYweypEjSxlWlPeJ3jsnx7hn3moAHrv21E+0rb6SqZ4blGxEsljdztZuWzz96soTWV+/i9LCPIYW5VJamEdpYS5D8nMHdBKq39naeaF+1eZGVmxu5L3NjTS1BS2qcgwOrSjh2DHDuOj4yvBsZShjhxcN6M+lP1CyEclizW0d3bZ4atjVytUPLdhreTMYUpDL0DD5lBYmTwfPQ4t2vx7a9bkol6K8GGFL0LRLtUqwtT3OmtrEBfuwGmxTI5u3N3cuU1acx9Gjh3LpSePDs5VSJh5cSlF+37Sukt5RshHJUmvrdvFh3c5uWzyNGV7E09dNo7G5ncbmNrY3t9HY3M72xOum4LmxuZ3N25tZ/fHu1+09dFESy7FuE1ViOjkxlRbunbhKC3MpzNv7B39fVYIjSvNZuiE4U0k0M36/ZkfnvSl5MWPCwaWcdkQ5R40uZdKooRwdXrCPKilK+inZiGShl1fXcP3v3uFTY4fyb5cdz9898vYeP9Bjhxcx7qDiXm/X3Wlq6+hMUtuSklIiYXW+bkq8bmfd1l2dy+xoacd7uEcxPzdnj+QztDCPb58xge/+1+K9qgR/+NeTufbhhQCMHlbIUaNK+fxRB3e2BDt8RAl5KVywz5TJY4ZmOoR+QclGJIu4Ow+8/AF3PreCiQeX8pMLPsW4suK0tXgyM4rzcynOz2Xk0MID2kY87uxobd99VtVNwtq+V8JqIzeW022V4KHlxTw68xSOGlXK8OLsbDa8PwNlqISoKdmIZInmtg5ufnIJf1i0kXOOHcXdXzmOkoLgTzSbWjzl5BhDw2o1KEp5vX3dwX5QSQGTRunsYKDL3nNTkUFkQ0MTF933Kk8v3sj3zp7Ev112fGeiGSgSd7BXlgUJKttvgpT0GlhHs0g/9PqaOq575G1a2+P86soqzjhqZKZDikR/vAlS0kfJRiRD3J3Zr33Ej/97OYeUFzPriiqOGDEk02FFqj/eBCnpoWQjkgHNbR3c+vRSHl+wnrOOPphfXDw1vAYiMjAp2Yj0sS3bm7n24YUsWtfADWdO5DtnTlRVkgx4SjYifWjhR/V867cL2dXSzn2Xn8CMY0dlOiSRPqFkI9JHHn1zLT98eiljhhfxyP86mSNHlmY6JJE+k1LTZzP7qZkNT3pdZmb/FFlUIgNIa3ucH/zhXW7+/bucekQFc66brkQjg06q99mc4+4NiRfuXg+cG0lEImly+zPLuP2ZZRmNoaaxhcseeJ3fvr6Wb332CH79jRMZVqyGADL4pFqNFjOzAndvATCzIqCgh3VEMmr5xu0Zff8l6xuYOXshDU2t3HPpp/mb48Yc8LbU/5b0d6kmm0eAeWb26/D1VcBD0YQk0v89uXA933/qXUYMKeDJvz2NY8YM+0TbU/9b0t+llGzc/S4zWwKcGRb92N3nRheWSP/U3hHnp8+u5MFXPuDUw8u597LjOUjdsYik3hrN3Z8DnoswFpF+bevOVq7/3du8+n4dV007lH889+is7ho/U1QlODillGzMrBFIjGCRD+QBO91dR40IsGzjNmbOXkjNjhbu/spxXHRCZaZDylqqEhycUq1G62ynacHQeOcDp0QVlEh/8szijXzvicUML8rnv649lePGDc90SCJZp9fn+B74A3B2+sMR6T864s7PnlvJt//zHY4dM4xnvj1diUZkH1KtRvty0sscoApojiQikX5g2642vv3oO7z0Xg2XnTye2847hvxcXZ8R2ZdUGwiclzTdDnxIUJUmMui8t6WRa2YvYGNDEz+94FN87eTxmQ5JJOules3mqqgDEekPnl+6me8+vojiglwenXkKJxxyUKZDEukXUq1GKwSuBo4BChPl7v7NiOISySrxuPN/563mnnmrOW7ccP7j8hMYNayw5xVFBEi9gcDDwCiCRgH/A1QCjVEFJZJNGpvbmPnwQu6Zt5qvnFDJYzNPUaIR6aVUk80Ed/8hwb01DwF/BZzc00pmNsPMVplZtZnd3M38Q8xsnpktMbMXzawyad5dZrY0fFycVH6Ymb0RbvMxM9Pt2RKZNTU7+NK9r/DCqo+5/W+O4ecXTaEwL5bpsET6nVSTTVv43GBmxwLDgIP3t4KZxYB7gXOAycClZja5y2J3A7PdfQpwB3BnuO5fAccDUwmS2o1mlriB9C7gX9x9AlBPUL0nknZ/WbmF83/5CvW72vjt1Sdz5WmHEtxmJiK9lWqymWVmZcAPgDnAcuDnPaxzElDt7mvcvRV4lL1bsE0G/hJOv5A0fzLwkru3u/tOYAkwI7yh9AzgiXC5h4AvpbgPIilxd+59oZqrH1rA+PJi5lw/jVOPKM90WCL9WkrJxt0fcPd6d3/J3Q9394Pd/b4eVhsLrEt6vT4sS7YYSNzDcwFQamblYfkMMys2swrg88A4oBxocPf2/WwTADObaWYLzGxBTU1NKrspws6Wdq773dv889xVnDdlDE986zQqy4ozHZZIv5fpkTpvBD5rZu8AnwU2AB3u/kfgWeBV4D+B14CO3mzY3We5e5W7V40YMSINocpAt7ZuFxf++6s8v3Qzt5x7NP96yVSK8nV9RiQdohypcwPB2UhCZVjWyd03uvuX3f3TwC1hWUP4/BN3n+ruXwAMeA+oA4abWe6+tilyIF5eXcN5v5zPpm3N/Oaqk7jmM4fr+oxIGqWabGJm1jkyZ4ojdb4FTAxbj+UDlxBc7+lkZhVmlojh+8CDYXksrE7DzKYAU4A/ursTXNu5KFznSuDpFPdBZC/uzv0vreHKB99k1NBC5lw/jc8cqTNhkXSLbKROd283s+uBuUAMeNDdl5nZHcACd58DfA6408wceAm4Llw9D3g5/M9yO3B50nWam4BHw2q8d4BfpbgPIntoau3g5t8v4elFGznn2FHc/ZXjKClIeYgnEemF3ozUuRg4KyxKaaROd3+W4NpLctmtSdNPsLtlWfIyzQQt0rrb5hqClm4iB2xDQxMzZy9g+abt3PjFI7nu8xNUbSYSod78G/cOwRmHh9Mi/dLra+q47pG3aW2P88AVVZx59MhMhyQy4KXaGu2rwJsE10q+CrxhZhftfy2R7OLuPPTqh1z+wBsMK87jD9dPU6IR6SOpntncApzo7h8DmNkI4M90UwUmko2a2zq49emlPL5gPWcdfTC/uHgqQwvzMh2WyKCRarLJSSSaUB0HMMqnSCZs3tbMt367kEXrGrjhzIl858yJ5OTo+oxIX+ox2YRdxLxlZnMJbrAEuJguF/5Fskk87txw5kSK8mIs27iNwtwc7rv8eGYcOzrToYkMSj0mG3d3MzsJuBWYHhbPcvenIo1M5ADF486qLY3c9OQS1tc3UVlWxH2Xn8Dk0UN7XllEIpFqVdhCYJ27/0P4UKKRrFW3s5VrZi9gfX0TAOvrm/jWbxdSt7M1w5GJDF6pXrM5GbjMzD4CdiYKw6EBRLJKa3tHZ6JJWF/fRGt7r7rXE5E0SjXZnB1pFCJplJ8bo7KsaI+EU1lWRH6uOtUUyZRUhxj4qLtH1MGJHIjyknzuv6KKyrIiIEg0919RRXmJBnUVyRR1BCUDTk6OMWlkKXddOIXi/BiVZcWUl+SrubNIBinZyICUk2PcM281AI9de2qGoxER3ZgpIiKRU7IREZHIKdmIiEjklGxERCRySjYiIhI5JRsREYmcko2IiEROyUZERCKnZCMiIpFTshERkcgp2YiISOSUbEREJHJKNiIiEjklGxERiZySjYiIRE7JRkREIqdkIyIikYs02ZjZDDNbZWbVZnZzN/MPMbN5ZrbEzF40s8qkeT83s2VmtsLM7jEzC8tfDLe5KHwcHOU+iIjIJxdZsjGzGHAvcA4wGbjUzCZ3WexuYLa7TwHuAO4M1z0NmAZMAY4FTgQ+m7TeZe4+NXx8HNU+iIhIekR5ZnMSUO3ua9y9FXgUOL/LMpOBv4TTLyTNd6AQyAcKgDxgS4SxiohIhKJMNmOBdUmv14dlyRYDXw6nLwBKzazc3V8jSD6bwsdcd1+RtN6vwyq0Hyaq10REJHtluoHAjcBnzewdgmqyDUCHmU0AjgYqCRLUGWZ2erjOZe7+KeD08PH17jZsZjPNbIGZLaipqYl6PyQLTR4zlMljhmY6DBEBciPc9gZgXNLryrCsk7tvJDyzMbMhwIXu3mBm1wCvu/uOcN5zwKnAy+6+IVy30cx+R1BdN7vrm7v7LGAWQFVVlad536QfuO28YzIdgoiEojyzeQuYaGaHmVk+cAkwJ3kBM6sws0QM3wceDKfXEpzx5JpZHsFZz4rwdUW4bh7w18DSCPdBRETSILJk4+7twPXAXGAF8Li7LzOzO8zsb8LFPgesMrP3gJHAT8LyJ4D3gXcJrussdvdnCBoLzDWzJcAigjOl+6PaBxERSQ9zH/g1TGb2vLvPyHQcIiKD1aBINiIiklmZbo0mIiKDgJKNiIhETslGREQip2QjIiKRU7IREZHIKdmIiEjklGxERCRySjYiIhI5JRsREYlclL0+Z40ZM2b4888/n+kwBpuoxxlS1xci0Uvb3/GgOLOpra3NdAgiIoPaoEg2Eq3bn1nG7c8sy3QYIpLFBkU1mkRr+cbtmQ5BRLKczmxERCRySjYiIhI5JRsREYmcko2IiEROyUZERCKnZCMiIpFT02f5ROJx54YzJ1KcH6OmsYXyknxycqLuPEBE+hslGzlg8bizaksjNz25hPX1TVSWFXH/FVVMGlmqhCMie8i6ajQzm2Fmq8ys2sxu7mb+IWY2z8yWmNmLZlaZiTgHs8bmNv60fAuL1jdwzewFrK9vAmB9fRPXzF5A3c7WDEcoItkmq85szCwG3At8AVgPvGVmc9x9edJidwOz3f0hMzsDuBP4et9HO3i0dcRZvK6Bl1fX8kp1Le+sa6Aj7jx+7SmdiSZhfX0Tre0dGYpURLJVViUb4CSg2t3XAJjZo8D5QHKymQz8Qzj9AvCHvgxwMHB33q/ZyfzVNcyvruP1NXXsaGnHDKaMHca1nzmc6RMrOKS8hMqyoj0STmVZEfm5sQxGLyLZKNuSzVhgXdLr9cDJXZZZDHwZ+FfgAqDUzMrdva5vQhyYane08Ep1befZy6ZtzQCMP6iY844bw+kTKzjtiHKGF+d3rhOPO/dfUdVZlZa4ZlNekr+vtxGRQSrbkk0qbgR+aWbfAF4CNgB71duY2UxgJsD48eP7Mr5+oam1gzc/3Mr81TW8vLqWlZsbARhWlMe0CeVcP6GC0yeMYHx58T63kZNjTBpZyl0XTqE4P0ZlWbFao4lItyJLNmb2U+Dn7t4Qvi4DvuvuP9jPahuAcUmvK8OyTu6+keDMBjMbAlyYeI8uy80CZgFUVVUN+oG2OuLOso3beHl1LfNX17Lwo3paO+Lkx3I44ZAyvnf2JKZPqODYscOI9SJZ5OQY98xbDcBj154aVfgi0s9FeWZzjrv/Y+KFu9eb2bnA/pLNW8BEMzuMIMlcAnwteQEzqwC2unsc+D7wYNojHyDW1u1ifnUt86trePX9Ohp2tQFw1KhSrjztEKZPHMGJh5ZRnN8fT3BFpD+J8lcmZmYF7t4CYGZFQMH+VnD3djO7HpgLxIAH3X2Zmd0BLHD3OcDngDvNzAmq0a7rTVCJQb5uO++YXu5O9mvY1cpr79fxcnVw9rJ26y4ARg0t5KyjR4bXXSoYUbrfr0FEJO2iTDaPAPPM7Nfh66uAh3payd2fBZ7tUnZr0vQTwBMHGtRAGuirpb2DhR/V80qYXJZs2IY7DCnI5ZTDy/nmtEOZPnEER4wowUzXUUQkcyJLNu5+l5ktAc4Mi37s7nOjer/BwN1ZubmR+atrmV9dy5sfbKWprYNYjvHpccO54YyJnD6xguPGDScvlnX364rIIBZpZb27Pwc8F+V7DHSbtzXz8uoa5lfX8kp1HbU7WgA4YkQJF584jmkTKjjl8IMoLczLcKQiIvsWZWu0RiDRCiwfyAN2uvvQqN5zIGhsbuONNVvDC/u1VH+8A4CKIflMm1DB9AkVTJtQwZjhRRmOdLfJY/SVisj+RVmNVpqYtuCCwfnAKVG9X3+V6ApmfnjdZdG6BtrjTmFeDicdVs7FVcHZy1Gjsrdzy4HY2EJE0qtP2ry6uwN/MLPbgL061xxMEl3BJO7Wf31NLTtaOjCDT40dxsywK5jjx5dRmKduX0RkYIiyGu3LSS9zgCqgOar3y2aJrmDmr65l/uoaNm0PrruMy9/Jefnvcfrh+Zz2le/s0RWMiMhAEuWZzXlJ0+3AhwRVaQNeclcw86trWbEp7Aomt43TCj7g+oqPOL1wDeNL2sHjMHwiKNGIyAAW5TWbq6LadrZJ7grmlepaFny4ldYOJ9/inFC0ie8dtIbphR9wbPE2YoUlkJMLhBf4WxozGruISF+IshqtELgaOAYoTJS7+zejes++tG7rrqCfseoaXq2upaGpHYCjCuu5ckg104o+4qSijRQXFUFu4o79YZkLWEQkg6KsRnsYWAmcDdwBXAasiPD9IrVnVzA1rN0ajOEyKq+JswreZ/rBa5lW+AEjimOQVwRmwPCMxiwiki2iTDYT3P0rZnZ+OKrm74CXI3y/tGpp7+DtjxqYX13D/NW1vLthG3GHklg7pxau5ZsVHzK9YA1HFDdjhUPAcoDSHrcrIjIYRZls2sLnBjM7FtgMHNzTSmY2g2BgtBjwgLv/rMv88QR9rA0Pl7k57E+tR/G4c8OZEynOj1HT2LLH2CuJrmASTZLf/KCOprY4MXOmFm7h28PWcHrRhxxXXEte4RCI5RLcq6oL+yIiPYky2cwKx7D5ATAHGALcur8VzCwG3At8gWCUzrfMbI67Jw8L/QPgcXf/dzObTNBp56E9BROPO6u2NHLTk0s6R5X8j6+fwKaGJv57ySbmV9dSu6MVgCMKtvPVomqml6/llMJ1lBYXQKxAVWMiIgcoytZoD4STLwGHp7jaSUC1u68BMLNHCZpLJycbBxL9owwDNqay4bqdrZ3DFwOsr2/i2ocX8sO/nsz8ZR8wrWAN00Z8xPTCDxhT4pBXEiYXXdQXEfmksm2kzrHAuqTX64GTuyzzI+CPZvZtoAQ4ax/vv8ew0K3tHZ2JpnPj9U1MGtLCm4fOIqdgCOTEwk2KiEg6RdkP/TnJwzW7ez1wbhq2eynwG3evDLf3sJnttR/uPsvdq9y9asSIEeTnxqgs27PzysqyIkri28kpGhYmGhERiUKUySZmZp1DQqYyUifBUNDjkl5XhmXJrgYeB3D31wju4anoKZjyknzuv6KqM+FUlhVx/8VHUr7qsZ5WFRGRTyjbRup8C5hoZocRJJlLgK91WWYtwYBsvzGzowmSTU1PweTkGJNGlnLXhVMojnVQ6VsoX3kfOVtX92KXRETkQEQ9Uudidl9T6XGkTndvN7PrgbkEzZofdPdlZnYHsMDd5wDfBe43s78naCzwjbBX6R7l5Bj3zFsNm9/lsSPmht3GiIhI1KL+tX2HYNA0D6d7FN4z82yXsluTppcD09IYo4iIRCyyazZm9lXgTeAi4KvAG2Z2UVTvJyIi2SvKM5tbgBPd/WMAMxsB/Bl4IsL3FBGRLBRla7ScRKIJ1UX8fiIikqUiObMxMyPoamYu8J9h8cV0uRYjIiKDQyTJxt3dzE4i6Attelg8y92fiuL9REQku0V5zWYhsM7d/yHC9xARkX4gymRzMnCZmX0E7EwUuvuUCN9TRESyUJTJ5uwIty0iIv1IlD0IfBTVtkVEpH9RU2QREYlc1iUbM5thZqvMrNrMbu5m/r+Y2aLw8Z6ZNWQgTBER6YWs6okylWGh3f3vk5b/NvDpPg9URER6JdvObDqHhXb3ViAxLPS+XMrum0ZFRCRLZVuy6W5Y6LHdLWhmhwCHAX/pg7hEROQTyKpqtF66BHjC3Tu6m2lmM4GZAOPHj+8snzxmKNT3ONaaiIikUbad2aQyLHTCJeynCs3dZ7l7lbtXjRgxorP8tvOO4baD56cjVhERSVG2JZvOYaHNLJ8goczpupCZHQWUAa/1cXwiInIAsqoaLcVhoSFIQo+mOhz0XorLYdtawNIR9ifjDkXDMx2FiEik7EB/r/uTqqoqX7Bgwe6CjnaIt2cuoK5ieZATy3QU6RZ1Jh/4B65I5qXt7zirzmz6TCw3eIiISJ8YFGc2ZlYDJPfVVgHUZiic3uovsXaNs9bdZ0T1Zmb2fPiePekvn18qtC/ZayDtT/K+pO3veFAkm67MbIG7V2U6jlT0l1izNc5sjetAaF+y10Dan6j2Jdtao4mIyACkZCMiIpEbrMlmVqYD6IX+Emu2xpmtcR0I7Uv2Gkj7E8m+DMprNiIi0rcG65mNiIj0oQGZbMzsQTP72MyWJpUdZGZ/MrPV4XNZWG5mdk84WNsSMzs+w3H+yMw2JA0Qd27SvO+Hca4ys7P7MM5xZvaCmS03s2Vm9r/D8ox8pmb2oZm9G34+Cw40FjO7Mlx+tZldmc4Y9xN7Wo7NfcVuZieEn011uG6kN9em6xi2fQyaGHZd9UZY/ljYjVVU+5K24zzT389+9iVz3427D7gH8BngeGBpUtnPgZvD6ZuBu8Lpc4HnCO6UPQV4I8Nx/gi4sZtlJwOLgQKCoRXeB2J9FOdo4PhwuhR4L4wnI58p8CFQ0aWsV7EABwFrwueycLqsPxyb+4sdeDNc1sJ1z8n2Yzh8vA8cDuSHy0wO13kcuCScvg/422w/zrPh+9nPvmTsuxmQZzbu/hKwtUvx+cBD4fRDwJeSymd74HVguJmNzmCc+3I+QX9wLe7+AVBNMNhc5Nx9k7u/HU43AisIxhnKps+0t7GcDfzJ3be6ez3wJyCym1AT0nRsdht7OG+ou7/uwS/A7KRt9eX+7Mu+juFuB00M/+s/A3giXD/5s0m7NB7nGf9+9rMv+xL5dzMgk80+jHT3TeH0ZmBkOJ3ygG196PrwtPzBxCk7WRKnmR1KMBT3G2TuM3Xgj2a20IJxiziAWLLi8wylK/ax4XTX8kzozTG8r/JyoMHd27uUR+4THudZ9f102RfI0HczmJJNp/C/imxthvfvwBHAVGAT8H8yGk0SMxsCPAl8x923J8/r4890ursfD5wDXGdmn8lgLGnVn2NPkrXHcCqy6Dj/xLrZl4x9N4Mp2WxJVOWEzx+H5b0ZsC1y7r7F3TvcPQ7cz+6qsozGaWZ5BAftI+7++7A4I5+pu28Inz8GniL4jHobSzZ97+mKfUM43bW8Tx3AMbyv8jqCqqncLuWRSdNxnhXfT3f7ksnvZjAlmzlAolXIlcDTSeVXhC1LTgG2JZ0y97ku1zYuABKtfOYAl5hZgZkdBkwkuNjYFzEZ8Ctghbv/ImlWn3+mZlZiZqWJaeCLBJ9Rb2OZC3zRzMrCqoQvhmWZkJbYw3nbzeyU8Du7ImlbfeYAjuFuB00MzyJeAC4K10/+bKKIO13Heca/n33tS0a/m/21HuivD4LhojcBbQR1iVcT1DHOA1YDfwYOCpc14F6CFhfvAlUZjvPhMI4l4QEwOmn5W8I4VxFxK6MucU4nqDpYAiwKH+dm4jMlaBWzOHwsA24Jy3sdC/BNgguh1cBV/enY3FfsQBXBD8j7wC8Jb9zO9mM4PJ7eC+fd0uX7fjPcz/8CCvrDcZ7p72c/+5Kx70Y9CIiISOQGUzWaiIhkiJKNiIhETslGREQip2QjIiKRU7IREZHIKdmkgZl1hD2oLjWzZ8xseFh+qJk1hfOWm9l9ZpYTli/tYbNRxfqPmXhf6R90LEtUlGzSo8ndp7r7sQSdEl6XNO99d58KTCHoWfVLfR/eHvQHKvujY1kioWSTfq/RTYd0HnRY9yowYX8rm9lNFox3sdjMfhaWTTWz18PO856y3eNpvGhmVeF0hZl9GE5/w8x+b2bPWzCexs/D8p8BReF/p4+kcZ9lYNKxLGmjZJNGZhYDziS4M7frvOJw3rv7Wf8cgq6+T3b34wjG0YCgK/Kb3H1KuP5tKYQzFbgY+BRwsZmNc/eb2f2f62Up75gMOjqWJd2UbNKjyMwWsbv78T8lzTsinPcK8P/c/bn9bOcs4NfuvgvA3bea2TBguLv/T7jMQwQDVvVknrtvc/dmYDlwSG92SAYtHcsSidyeF5EUNLn71PA/vrkE9dz3hPMS9dxRaGf3PwyFXea1JE13oO9aUqNjWSKhM5s0Cv+LuwH4ru3uers3/gRcFf6hY2YHufs2oN7MTg+X+TqQ+M/wQ+CEcPoiUtNmQdfjIvukY1nSTckmzdz9HYIeVS89gHWfJ6gjXxBWV9wYzroS+GczW0JQf31HWH438Ldm9g5QkeLbzAKW6KKq9ETHsqSTen0WEZHI6cxGREQip2QjIiKRU7IREZHIKdmIiEjklGxERCRySjYiIhI5JRsREYmcko2IiETu/wOxSVIMmrEjQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x216 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axes = plt.subplots(2,2, figsize = (6,3), gridspec_kw = {'width_ratios': [0.3,0.7], 'height_ratios': [0.6,0.4], 'wspace' : 0.15, 'hspace' : 0.15})\n",
    "\n",
    "metric = 'rocauc'\n",
    "colors = sbn.color_palette(\"tab10\", 2)\n",
    "\n",
    "df = pd.DataFrame(dict(zip(fold_eval_performance.loc[metric,:].index, fold_eval_performance.loc[metric,:].values)))\n",
    "\n",
    "plot_sample_size_upscale_graph(df, metric, axes, \n",
    "                               lineplot_kwgs = {'marker' : 'o', \"err_style\": \"bars\", 'color': colors[0]}, \n",
    "                               fill_between_kwgs = {'alpha' : 0.7, 'color' : colors[1]})\n",
    "\n",
    "# Add axis break symbols on PP\n",
    "# Combine and add \n",
    "# Make sigfigs consistant in PP\n",
    "# Specify that shading is available HIV-1 RPIS for training in PP\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"SampleCurve.png\", dpi = 300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}