train-models-cv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run run-experiment-cv.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = {'MIMIC' : \"~/cohorts/hs_mimic.csv\", 'SINAI' : \"~/cohorts/hs_sinai_preprocessed.csv\", 'DHZB' : \"~/cohorts/hs_dhzb.csv\"}\n",
    "save_path = './experiments/mimic_model_results_calibration.d'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' define options to run experiments on '''\n",
    "options = {\n",
    "    'target': [\"AKI\"],\n",
    "    'cohort': ['MIMIC'],\n",
    "    'test_size': [0.2],\n",
    "    'imputation_method':[imputers.DEFAULT],    \n",
    "    'algorithm': [\n",
    "#        algorithms.LR,\n",
    "#        algorithms.DT,\n",
    "#        algorithms.ADABOOST,\n",
    "#        algorithms.ENLR,\n",
    "#        algorithms.GNBAYES,\n",
    "        algorithms.GBDT,\n",
    "        algorithms.RF\n",
    "    ],\n",
    "    'sampling_method': [samplers.SMOTE],\n",
    "    'sampling_strategy': [0.25],\n",
    "    'scale_method': [None],\n",
    "    'calibration_method':['isotonic','sigmoid'],\n",
    "    'save_pipeline':[False],\n",
    "    'optimize_mode': [False],\n",
    "    'n_splits':[10]\n",
    "}\n",
    "\n",
    "experiments = unpickle(save_path) or {}\n",
    "n_experiments = 0\n",
    "\n",
    "with Timer() as t:\n",
    "    \n",
    "    ''' iterate over different options '''\n",
    "    for combination in product(*options.values()):\n",
    "\n",
    "        ''' initialize parameters '''\n",
    "        params = dict(zip(options.keys(), combination))\n",
    "        print(f\"Running experiment with following parameters: {params}\")\n",
    "        exp_id = str(uuid.uuid1())\n",
    "        experiment = defaultdict(lambda: {})        \n",
    "\n",
    "        ''' load the data '''\n",
    "        data = Load().execute(filename=filenames[params['cohort']])\n",
    "        \n",
    "        ''' split the data '''\n",
    "        train, test = Split().execute(data,test_size=params['test_size'])        \n",
    "        \n",
    "        ''' train model on subset of data '''\n",
    "        experiment = run_experiment(params, train, test)\n",
    "                \n",
    "        ''' after training, our data for subsequent steps becomes train for computing CV statistics '''\n",
    "        data = train.copy()\n",
    "        \n",
    "        features, labels = data.drop(params['target'], axis=1), data[params['target']]                \n",
    "        \n",
    "        ''' initialize metric storage '''\n",
    "        metrics = ['precision', 'recall', 'f1-score', 'auc', 'dor']\n",
    "        cv_experiments = []\n",
    "        cv_metrics = defaultdict(lambda: {})\n",
    "        cv_performance = defaultdict(lambda: {})\n",
    "        \n",
    "        k_params = params\n",
    "        del k_params['calibration_method'] #no need to calibrate within the k-folds\n",
    "        \n",
    "        ''' for each of the k-folds '''\n",
    "        skf = StratifiedKFold(n_splits=params['n_splits'], random_state=None, shuffle=False)        \n",
    "        for train_index, test_index in skf.split(features, labels):\n",
    "            train_data = data.iloc[train_index]\n",
    "            test_data = data.iloc[test_index]\n",
    "            cv_experiments.append(run_experiment(k_params, train_data, test_data))\n",
    "\n",
    "        ''' summarize results of crossvalidation '''        \n",
    "        for metric in metrics:                       \n",
    "            measurements = [exp['performance']['discrimination'][metric] for exp in cv_experiments]\n",
    "            cv_performance[metric]['mean'] = np.mean(measurements)\n",
    "            cv_performance[metric]['std'] = np.std(measurements)\n",
    "            cv_performance[metric]['ci'] = np.std(measurements) * 2 #95% CI\n",
    "        \n",
    "        ''' save everything '''\n",
    "        experiment['parameters'] = params\n",
    "        experiment['performance']['cv'] = cv_performance\n",
    "        experiment['exp_id'] = exp_id\n",
    "        \n",
    "        experiments[exp_id] = experiment\n",
    "        \n",
    "        n_experiments += 1\n",
    "\n",
    "print(f'Running {n_experiments} experiments took {t.interval:.03f} sec.')\n",
    "\n",
    "''' store everything '''\n",
    "if pickle(experiments, save_path): print('Successfully saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}