run-validation-experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_validation_experiment(params, test, model, imputer=None, scaler=None):        \n",
    "    \n",
    "    ''' if model contains features not available in the new dataset, we include them as zeros '''    \n",
    "    for feature in params.get('model_features'):\n",
    "        if feature not in test.columns:\n",
    "            test[feature] = pd.Series(np.zeros(test.shape[0]))    \n",
    "        \n",
    "    ''' reorder columns in order to match the original order '''\n",
    "    test = test[params['model_features']]\n",
    "    \n",
    "    ''' evaluate models '''\n",
    "    test, _ = Impute().execute(test.copy(), imputer=imputer)    \n",
    "    test,_ = Scale().execute(test.copy(), params['target'], scaler=scaler)    \n",
    "    results = Evaluate().execute(test.copy(), target=params['target'], models={params['algorithm']:model})\n",
    "    \n",
    "    ''' obtain performance metrics '''\n",
    "    performance = {}    \n",
    "    performance['discrimination'] = get_discrimination_metrics(**results[params['algorithm']])\n",
    "    performance['calibration'] = get_calibration_metrics(results[params['algorithm']]['y_true'],results[params['algorithm']]['y_probs'])\n",
    "    performance['clinical_usefulness'] = get_clinical_usefulness_metrics(performance['discrimination'])    \n",
    "    \n",
    "    params['datetime'] = datetime.now()\n",
    "    \n",
    "    ''' summarize results '''\n",
    "    experiment = {'parameters' : params,\n",
    "                  'results': results[params['algorithm']],\n",
    "                  'performance' : performance}\n",
    "    return experiment\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}