train-models-cv-rfe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run run-experiment-cv.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run get-feature-rank.ipynb #dict 'feature_rank' is defined here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames = {'MIMIC' : \"~/cohorts/hs_mimic.csv\", 'SINAI' : \"~/cohorts/hs_sinai_preprocessed.csv\", 'DHZB' : \"~/cohorts/hs_dhzb.csv\"}\n",
    "save_path = './experiments/experiments_rfe_sinai_rank_feat_importance.d'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' define options to run experiments on '''\n",
    "\n",
    "#\n",
    "# First run, RFE mode will run using 'weighted_explanations','feature_importance' for both algorithms\n",
    "#\n",
    "options = {\n",
    "    'target': [\"AKI\"],\n",
    "    'cohort': ['SINAI'],\n",
    "    'test_size': [0.2],\n",
    "    'imputation_method': [imputers.DEFAULT],\n",
    "    'algorithm': [algorithms.RF, algorithms.GBDT],\n",
    "    'sampling_method': [samplers.SMOTE],\n",
    "    'sampling_strategy': [0.25],\n",
    "    'rfe_mode' : ['weighted_explanations', 'feature_importance'],\n",
    "    'scale_method': [None],\n",
    "    'optimize_mode': [False],\n",
    "    'num_exps_desired': [50],\n",
    "    'num_features': [100],\n",
    "    'mimic': [Ridge(solver='sag')], #sag is faster for large datasets\n",
    "    #'explainers' : [[explainers.MIMIC, explainers.SHAP, explainers.FEAT_CONTRIB, explainers.LIME]],    \n",
    "    'n_splits':[10],\n",
    "}\n",
    "\n",
    "#\n",
    "# Second run, we comment out the lines below, to run the RFE using the features obtained via the MIMIC models\n",
    "options['rfe_mode'] = ['prefit']\n",
    "#options['feature_rank'] = [feature_rank[algorithms.RF]]\n",
    "options['feature_rank'] = [feature_rank[algorithms.RF], feature_rank[algorithms.GBDT]]\n",
    "\n",
    "#\n",
    "\n",
    "''' retrieve previous experiments as the case may be '''\n",
    "experiments = unpickle(save_path) or {}\n",
    "n_experiments = 0\n",
    "\n",
    "with Timer() as t:\n",
    "    \n",
    "    ''' iterate over different options '''\n",
    "    for combination in product(*options.values()):\n",
    "\n",
    "        ''' initialize parameters '''\n",
    "        params = dict(zip(options.keys(), combination))\n",
    "        #print(f\"Running experiment with following parameters: {params}\")\n",
    "        exp_id = str(uuid.uuid1())\n",
    "        experiment = defaultdict(lambda: {})\n",
    "\n",
    "        ''' load the data '''\n",
    "        data = Load().execute(filename=filenames[params['cohort']])\n",
    "        \n",
    "        ''' split the data '''\n",
    "        train, test = Split().execute(data,test_size=params['test_size'])\n",
    "        \n",
    "        top_explanations = []\n",
    "        \n",
    "        ''' check if we're providing top_explanations as parameter, previous values will be overriden '''\n",
    "        ''' we do this to compare how the metrics change when using top features from previous models, e.g., mimic '''\n",
    "        if params.get('rfe_mode') == 'prefit':\n",
    "            top_explanations = params.get('feature_rank')\n",
    "        else:        \n",
    "            ''' get feature rank for the complete train dataset '''\n",
    "            experiment = run_experiment(params, train, test)\n",
    "\n",
    "            ''' by default, we assume top explanations come from feature contributions '''\n",
    "            top_explanations = list(sorted([(k,v) for k,v in experiment['explanations'][params['algorithm']]['FeatContribExplainer'].items()], key=lambda x: x[1])) # trying now with feature_importances\n",
    "\n",
    "            ''' instead, if we take weighted explanations, then override top_explanations '''\n",
    "            if params.get('rfe_mode') == 'weighted_explanations': top_explanations = experiment['weighted_explanations']         \n",
    "        \n",
    "        top_explanations = top_explanations[-params['num_features']:] #top num_features\n",
    "        params['top_explanations'] = top_explanations\n",
    "        max_features = 10\n",
    "        \n",
    "        ''' remove explainers from this point on, since we do not want to run them in the k-folds '''         \n",
    "        if 'explainers' in params: del params['explainers']\n",
    "        \n",
    "        ''' now my data for subsequent steps becomes train '''\n",
    "        data = train.copy()\n",
    "        \n",
    "        features, labels = data.drop(params['target'], axis=1), data[params['target']]        \n",
    "\n",
    "        ''' initialize storage of metrics  '''\n",
    "        metrics = ['precision', 'recall', 'f1-score', 'auc', 'dor']\n",
    "                \n",
    "        cv_experiments = []\n",
    "        cv_performance = defaultdict(lambda: {})     \n",
    "        rfe_experiments = defaultdict(lambda: [])\n",
    "        rfe_performance = defaultdict(lambda: defaultdict(lambda: {}))\n",
    "\n",
    "        ''' for each of the k-folds '''\n",
    "        skf = StratifiedKFold(n_splits=params['n_splits'], random_state=None, shuffle=False)\n",
    "        for train_index, test_index in skf.split(features, labels):                     \n",
    "            \n",
    "            ''' obtain data for the next experiments '''\n",
    "            train_data = data.iloc[train_index]\n",
    "            test_data = data.iloc[test_index]\n",
    "            k_params = params\n",
    "            \n",
    "            ''' run one CV experiment that we will use for the cross-validated results '''\n",
    "            print('Training model within the K-fold...')\n",
    "            cv_experiments.append(run_experiment(params, train_data, test_data))\n",
    "            \n",
    "            ''' weighted explanations are a tuple of the form (feature, importance, weight, normalized, weight) '''\n",
    "            features_to_select = [exp[0] for exp in top_explanations]\n",
    "            \n",
    "            ''' now for each of the feature sets, up to max_features '''\n",
    "            while len(features_to_select) >= max_features:\n",
    "                \n",
    "                print(f'Training partial model with selected features #: {len(features_to_select)}')\n",
    "                k_params['features_to_select'] = features_to_select\n",
    "                k_params['num_features'] = len(features_to_select)\n",
    "                rfe_experiments[k_params['num_features']].append(run_experiment(k_params, train_data, test_data))\n",
    "                \n",
    "                ''' remove lowest ranked feature (list is sorted A-Z, least to more important) '''\n",
    "                del features_to_select[0]\n",
    "                      \n",
    "        ''' summarize results of crossvalidation '''        \n",
    "        for metric in metrics:\n",
    "            \n",
    "            ''' cv results '''\n",
    "            measurements = [exp['performance']['discrimination'][metric] for exp in cv_experiments]\n",
    "            cv_performance[metric]['mean'] = np.mean(measurements)\n",
    "            cv_performance[metric]['std'] = np.std(measurements)\n",
    "            cv_performance[metric]['ci'] = np.std(measurements) * 2 #95% CI\n",
    "            \n",
    "            ''' rfe results '''\n",
    "            for num_features in rfe_experiments:\n",
    "                measurements = [exp['performance']['discrimination'][metric] for exp in rfe_experiments[num_features]]\n",
    "                rfe_performance[num_features][metric]['mean'] = float(np.mean(measurements))\n",
    "                rfe_performance[num_features][metric]['std'] = float(np.std(measurements))\n",
    "                rfe_performance[num_features][metric]['ci'] = float(np.std(measurements) * 2) #95% CI\n",
    "                rfe_performance[num_features][metric]['raw'] = measurements\n",
    "        \n",
    "        ''' summarize results of experiment '''\n",
    "        experiment['parameters'] = params\n",
    "        experiment['performance']['cv'] = dict(cv_performance)\n",
    "        experiment['performance']['rfe'] = {k : dict(v) for k,v in rfe_performance.items()} #remove lambdas for storage\n",
    "        \n",
    "        ''' append to previously loaded experiments as the case may be '''\n",
    "        experiments[exp_id] = experiment\n",
    "        \n",
    "        n_experiments += 1\n",
    "\n",
    "print(f'Running {n_experiments} experiments took {t.interval:.03f} sec.')\n",
    "\n",
    "''' store everything '''\n",
    "if pickle(experiments, save_path):\n",
    "    print('Successfully saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}