scripts/11_calculate_calibrations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate calibration curve points using SciKit Learn\n",
    "\n",
    "### Calculate calibration curve datapoints for subsequent plotting in R\n",
    "\n",
    "Shubhayu Bhattacharyay\n",
    "<br>\n",
    "Ari Ercole"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## I. Initialization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import necessary packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fundamental methods\n",
    "import os\n",
    "import re\n",
    "import sys\n",
    "import time\n",
    "import glob\n",
    "import random\n",
    "import warnings\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle as cp\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "warnings.filterwarnings(action=\"ignore\")\n",
    "\n",
    "# SciKit-Learn methods\n",
    "from sklearn.calibration import calibration_curve\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "# Load custom functions\n",
    "%run -i 'functions/anti_join.py'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II. Calculate probability calibration curve points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load compiled model results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile lists of model predictions\n",
    "mnlr_results_compiled = pd.read_csv('../repeated_cv/compiled_predictions/mnlr.csv')\n",
    "polr_results_compiled = pd.read_csv('../repeated_cv/compiled_predictions/polr.csv')\n",
    "deepMN_results_compiled = pd.read_csv('../repeated_cv/compiled_predictions/deepMN.csv')\n",
    "deepOR_results_compiled = pd.read_csv('../repeated_cv/compiled_predictions/deepOR.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load tuning grid\n",
    "layer_options = range(2,5)\n",
    "units_options = [128,256,512]\n",
    "smote_options = [0,1]\n",
    "\n",
    "layer_smote_combos = np.array(np.meshgrid(layer_options, smote_options)).reshape(2, len(layer_options)*len(smote_options)).T\n",
    "\n",
    "tuning_grid = pd.DataFrame(np.empty((0,3)))\n",
    "for i in range(np.shape(layer_smote_combos)[0]):\n",
    "    curr_tuning_options = pd.DataFrame({\"layers\":layer_smote_combos[i,0],\\\n",
    "                                        \"smote\": layer_smote_combos[i,1],\\\n",
    "                                        \"units\": list(itertools.product(units_options,repeat = layer_smote_combos[i,0]))})\n",
    "    tuning_grid.columns = curr_tuning_options.columns\n",
    "    tuning_grid = tuning_grid.append(curr_tuning_options,ignore_index=True)\n",
    "tuning_grid[\"tune_idx\"] = np.arange(tuning_grid.shape[0]) + 1\n",
    "tuning_grid.to_csv('../repeated_cv/dl_tuning_grid.csv',index=False)\n",
    "\n",
    "# Establish dataframe of all tuning index-GOSE-class combinations for AUC\n",
    "viable_indices = pd.DataFrame(list(itertools.product(tuning_grid.tune_idx.astype('int').unique(), ['1','3','4','5','6','7','8'])),\\\n",
    "                              columns=['tune_idx','class'])\n",
    "\n",
    "# Removed banned tuning indices from both DeepMN and DeepOR\n",
    "deepMN_banned_tuning_indices = pd.read_csv('../repeated_cv/deepMN_banned_tuning_indices.csv')\n",
    "deepMN_banned_tuning_indices.tune_idx = deepMN_banned_tuning_indices.tune_idx.astype('int')\n",
    "deepMN_banned_tuning_class_combos = deepMN_banned_tuning_indices[['tune_idx','class']]\n",
    "\n",
    "deepOR_banned_tuning_indices = pd.read_csv('../repeated_cv/deepOR_banned_tuning_indices.csv')\n",
    "deepOR_banned_tuning_indices.tune_idx = deepOR_banned_tuning_indices.tune_idx.astype('int')\n",
    "deepOR_banned_tuning_class_combos = deepOR_banned_tuning_indices[['tune_idx','class']]\n",
    "\n",
    "# Identify viable columnns for each model type\n",
    "deepMN_viable_indices = pd.concat([viable_indices, deepMN_banned_tuning_class_combos, deepMN_banned_tuning_class_combos]).drop_duplicates(keep=False)\n",
    "deepOR_viable_indices = pd.concat([viable_indices, deepOR_banned_tuning_class_combos, deepOR_banned_tuning_class_combos]).drop_duplicates(keep=False)\n",
    "\n",
    "deepMN_viable_indices = deepMN_viable_indices.rename(columns={\"class\": \"true_labels\"})\n",
    "deepOR_viable_indices = deepOR_viable_indices.rename(columns={\"class\": \"true_labels\"})\n",
    "\n",
    "# Filter out only viable combinations in the compiled prediction dataframes\n",
    "deepMN_results_compiled = deepMN_results_compiled[deepMN_results_compiled.tune_idx.astype('int').isin(deepMN_viable_indices.tune_idx)]\n",
    "deepOR_results_compiled = deepOR_results_compiled[deepOR_results_compiled.tune_idx.astype('int').isin(deepOR_viable_indices.tune_idx)]\n",
    "\n",
    "# Identify columns that hold probability scores\n",
    "prob_cols = [col for col in deepMN_results_compiled if col.startswith('prob_GOSE_')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load bootstrap id's\n",
    "bootstrap_df = pd.read_csv('../repeated_cv/bootstrap_IDs.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loop through result files and calculate probability calibration points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Bootstrap no. 1000 out of 1000 initiated.'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Loop through bootstrap IDs\n",
    "compiled_cal_curves_df = pd.DataFrame(np.empty((0,5)),columns = ['prob_true','prob_pred','Model','class','bs_idx'])\n",
    "bs_message = display('',display_id=True)\n",
    "\n",
    "for curr_bs_idx in bootstrap_df.bs_idx.unique():\n",
    "    bs_message.update('Bootstrap no. ' + str(curr_bs_idx) + ' out of ' + str(len(bootstrap_df.bs_idx.unique())) + ' initiated.')\n",
    "    # Get current entity_id's for the dataframe\n",
    "    curr_bs_ids = bootstrap_df.entity_id[bootstrap_df.bs_idx == curr_bs_idx]\n",
    "    \n",
    "    # Filter out predictions in all the compiled prediction dataframes for in samples in the current bootstrap index\n",
    "    in_sample_mnlr = mnlr_results_compiled[mnlr_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "    in_sample_polr = polr_results_compiled[polr_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "    in_sample_deepMN = deepMN_results_compiled[deepMN_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "    in_sample_deepOR = deepOR_results_compiled[deepOR_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "\n",
    "    # Filter out predictions in all the compiled prediction dataframes for out samples in the current bootstrap index\n",
    "    out_sample_mnlr = mnlr_results_compiled[~mnlr_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "    out_sample_polr = polr_results_compiled[~polr_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "    out_sample_deepMN = deepMN_results_compiled[~deepMN_results_compiled.entity_id.isin(curr_bs_ids)]\n",
    "    out_sample_deepOR = deepOR_results_compiled[~deepOR_results_compiled.entity_id.isin(curr_bs_ids)]   \n",
    "    \n",
    "    # For each model determine the optimal tuning configuration (in terms of minimized MSE of the calibration curve) for each class in the in sample\n",
    "    # DeepMN \n",
    "    deepMN_opt_tune_indices = pd.DataFrame(np.empty((0,2)),columns=['true_labels','tune_idx'])\n",
    "    for curr_deepMN_class in deepMN_viable_indices.true_labels.unique():\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_deepMN_class))][0]\n",
    "        curr_class_tune_indices = deepMN_viable_indices.tune_idx[deepMN_viable_indices.true_labels == curr_deepMN_class].values\n",
    "        if len(curr_class_tune_indices) == 1:\n",
    "            deepMN_opt_tune_indices = deepMN_opt_tune_indices.append(pd.DataFrame({'true_labels':[curr_deepMN_class],'tune_idx':curr_class_tune_indices}),ignore_index = True)\n",
    "            continue\n",
    "        lowest_mse = sys.float_info.max\n",
    "        opt_tune_idx = 0\n",
    "        for curr_tune_idx in curr_class_tune_indices:\n",
    "            filt_in_sample_deepMN = in_sample_deepMN[in_sample_deepMN.tune_idx == curr_tune_idx]\n",
    "            filt_in_sample_deepMN['temp_label'] = (filt_in_sample_deepMN.true_labels == curr_deepMN_class).astype('int')\n",
    "            curr_prob_true, curr_prob_pred =  calibration_curve(filt_in_sample_deepMN['temp_label'].values, np.clip(filt_in_sample_deepMN[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "            curr_mse = mean_squared_error(curr_prob_true, curr_prob_pred)\n",
    "            if curr_mse < lowest_mse:\n",
    "                lowest_mse = curr_mse\n",
    "                opt_tune_idx = curr_tune_idx\n",
    "        deepMN_opt_tune_indices = deepMN_opt_tune_indices.append(pd.DataFrame({'true_labels':[curr_deepMN_class],'tune_idx':[opt_tune_idx]}),ignore_index = True)\n",
    "\n",
    "    # DeepOR    \n",
    "    deepOR_opt_tune_indices = pd.DataFrame(np.empty((0,2)),columns=['true_labels','tune_idx'])\n",
    "    for curr_deepOR_class in deepOR_viable_indices.true_labels.unique():\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_deepOR_class))][0]\n",
    "        curr_class_tune_indices = deepOR_viable_indices.tune_idx[deepOR_viable_indices.true_labels == curr_deepOR_class].values\n",
    "        if len(curr_class_tune_indices) == 1:\n",
    "            deepOR_opt_tune_indices = deepOR_opt_tune_indices.append(pd.DataFrame({'true_labels':[curr_deepOR_class],'tune_idx':curr_class_tune_indices}),ignore_index = True)\n",
    "            continue\n",
    "        lowest_mse = sys.float_info.max\n",
    "        opt_tune_idx = 0\n",
    "        for curr_tune_idx in curr_class_tune_indices:\n",
    "            filt_in_sample_deepOR = in_sample_deepOR[in_sample_deepOR.tune_idx == curr_tune_idx]\n",
    "            filt_in_sample_deepOR['temp_label'] = (filt_in_sample_deepOR.true_labels == curr_deepOR_class).astype('int')\n",
    "            curr_prob_true, curr_prob_pred =  calibration_curve(filt_in_sample_deepOR['temp_label'].values, np.clip(filt_in_sample_deepOR[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "            curr_mse = mean_squared_error(curr_prob_true, curr_prob_pred)\n",
    "            if curr_mse < lowest_mse:\n",
    "                lowest_mse = curr_mse\n",
    "                opt_tune_idx = curr_tune_idx\n",
    "        deepOR_opt_tune_indices = deepOR_opt_tune_indices.append(pd.DataFrame({'true_labels':[curr_deepOR_class],'tune_idx':[opt_tune_idx]}),ignore_index = True)\n",
    "\n",
    "    # MNLR\n",
    "    mnlr_opt_tune_indices = pd.DataFrame(np.empty((0,2)),columns=['true_labels','SMOTE'])\n",
    "    for curr_mnlr_class in in_sample_mnlr['true.labels'].unique():\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_mnlr_class))][0]\n",
    "        lowest_mse = sys.float_info.max\n",
    "        opt_SMOTE = 100\n",
    "        for curr_SMOTE in [0,1]:\n",
    "            filt_in_sample_mnlr = in_sample_mnlr[in_sample_mnlr.SMOTE == curr_SMOTE]\n",
    "            filt_in_sample_mnlr['temp_label'] = (filt_in_sample_mnlr['true.labels'] == curr_mnlr_class).astype('int')\n",
    "            curr_prob_true, curr_prob_pred =  calibration_curve(filt_in_sample_mnlr['temp_label'].values, np.clip(filt_in_sample_mnlr[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "            curr_mse = mean_squared_error(curr_prob_true, curr_prob_pred)\n",
    "            if curr_mse < lowest_mse:\n",
    "                lowest_mse = curr_mse\n",
    "                opt_SMOTE = curr_SMOTE\n",
    "        mnlr_opt_tune_indices = mnlr_opt_tune_indices.append(pd.DataFrame({'true_labels':[curr_mnlr_class],'SMOTE':[opt_SMOTE]}),ignore_index = True)\n",
    "\n",
    "    # POLR\n",
    "    polr_opt_tune_indices = pd.DataFrame(np.empty((0,2)),columns=['true_labels','SMOTE'])\n",
    "    for curr_polr_class in in_sample_polr['true.labels'].unique():\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_polr_class))][0]\n",
    "        lowest_mse = sys.float_info.max\n",
    "        opt_SMOTE = 100\n",
    "        for curr_SMOTE in [0,1]:\n",
    "            filt_in_sample_polr = in_sample_polr[in_sample_polr.SMOTE == curr_SMOTE]\n",
    "            filt_in_sample_polr['temp_label'] = (filt_in_sample_polr['true.labels'] == curr_polr_class).astype('int')\n",
    "            curr_prob_true, curr_prob_pred =  calibration_curve(filt_in_sample_polr['temp_label'].values, np.clip(filt_in_sample_polr[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "            curr_mse = mean_squared_error(curr_prob_true, curr_prob_pred)\n",
    "            if curr_mse < lowest_mse:\n",
    "                lowest_mse = curr_mse\n",
    "                opt_SMOTE = curr_SMOTE\n",
    "        polr_opt_tune_indices = polr_opt_tune_indices.append(pd.DataFrame({'true_labels':[curr_polr_class],'SMOTE':[opt_SMOTE]}),ignore_index = True)\n",
    "        \n",
    "    # Using optimal model configurations, calculate probability calibration bins on out-sample predictions\n",
    "    # DeepMN\n",
    "    for curr_deepMN_class in deepMN_opt_tune_indices.true_labels.astype('int').values:\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_deepMN_class))][0]\n",
    "        curr_opt_tune_idx = deepMN_opt_tune_indices.tune_idx[deepMN_opt_tune_indices.true_labels.astype('int') == curr_deepMN_class].values[0]\n",
    "        filt_out_sample_deepMN = out_sample_deepMN[out_sample_deepMN.tune_idx == curr_opt_tune_idx]\n",
    "        filt_out_sample_deepMN['temp_label'] = (filt_out_sample_deepMN['true_labels'] == curr_deepMN_class).astype('int')\n",
    "        curr_prob_true, curr_prob_pred =  calibration_curve(filt_out_sample_deepMN['temp_label'].values, np.clip(filt_out_sample_deepMN[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "        temp_df = pd.DataFrame({'prob_true':curr_prob_true, 'prob_pred':curr_prob_pred,'Model':'DeepMN','class':curr_deepMN_class,'bs_idx':curr_bs_idx})\n",
    "        compiled_cal_curves_df = compiled_cal_curves_df.append(temp_df,ignore_index = True)\n",
    "        \n",
    "    # DeepOR\n",
    "    for curr_deepOR_class in deepOR_opt_tune_indices.true_labels.astype('int').values:\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_deepOR_class))][0]\n",
    "        curr_opt_tune_idx = deepOR_opt_tune_indices.tune_idx[deepOR_opt_tune_indices.true_labels.astype('int') == curr_deepOR_class].values[0]\n",
    "        filt_out_sample_deepOR = out_sample_deepOR[out_sample_deepOR.tune_idx == curr_opt_tune_idx]\n",
    "        filt_out_sample_deepOR['temp_label'] = (filt_out_sample_deepOR['true_labels'] == curr_deepOR_class).astype('int')\n",
    "        curr_prob_true, curr_prob_pred =  calibration_curve(filt_out_sample_deepOR['temp_label'].values, np.clip(filt_out_sample_deepOR[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "        temp_df = pd.DataFrame({'prob_true':curr_prob_true, 'prob_pred':curr_prob_pred,'Model':'DeepOR','class':curr_deepOR_class,'bs_idx':curr_bs_idx})\n",
    "        compiled_cal_curves_df = compiled_cal_curves_df.append(temp_df,ignore_index = True)\n",
    "\n",
    "    # MNLR\n",
    "    for curr_mnlr_class in mnlr_opt_tune_indices.true_labels.astype('int').values:\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_mnlr_class))][0]\n",
    "        curr_opt_SMOTE = mnlr_opt_tune_indices.SMOTE[mnlr_opt_tune_indices.true_labels.astype('int') == curr_mnlr_class].values[0]\n",
    "        filt_out_sample_mnlr = out_sample_mnlr[out_sample_mnlr.SMOTE == curr_opt_SMOTE]\n",
    "        filt_out_sample_mnlr['temp_label'] = (filt_out_sample_mnlr['true.labels'] == curr_mnlr_class).astype('int')\n",
    "        curr_prob_true, curr_prob_pred =  calibration_curve(filt_out_sample_mnlr['temp_label'].values, np.clip(filt_out_sample_mnlr[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "        temp_df = pd.DataFrame({'prob_true':curr_prob_true, 'prob_pred':curr_prob_pred,'Model':'MNLR','class':curr_mnlr_class,'bs_idx':curr_bs_idx})\n",
    "        compiled_cal_curves_df = compiled_cal_curves_df.append(temp_df,ignore_index = True)\n",
    "        \n",
    "    # POLR\n",
    "    for curr_polr_class in polr_opt_tune_indices.true_labels.astype('int').values:\n",
    "        curr_label_prob_name = [col for col in prob_cols if col.startswith('prob_GOSE_') & col.endswith(str(curr_polr_class))][0]\n",
    "        curr_opt_SMOTE = polr_opt_tune_indices.SMOTE[polr_opt_tune_indices.true_labels.astype('int') == curr_polr_class].values[0]\n",
    "        filt_out_sample_polr = out_sample_polr[out_sample_polr.SMOTE == curr_opt_SMOTE]\n",
    "        filt_out_sample_polr['temp_label'] = (filt_out_sample_polr['true.labels'] == curr_polr_class).astype('int')\n",
    "        curr_prob_true, curr_prob_pred =  calibration_curve(filt_out_sample_polr['temp_label'].values, np.clip(filt_out_sample_polr[curr_label_prob_name].values,0,1), n_bins=10, strategy = 'quantile')            \n",
    "        temp_df = pd.DataFrame({'prob_true':curr_prob_true, 'prob_pred':curr_prob_pred,'Model':'POLR','class':curr_polr_class,'bs_idx':curr_bs_idx})\n",
    "        compiled_cal_curves_df = compiled_cal_curves_df.append(temp_df,ignore_index = True)\n",
    "        \n",
    "compiled_cal_curves_df.to_csv('../metrics/compiled_calib_curves.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sb_venv",
   "language": "python",
   "name": "sb_venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}