feature-elimination.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run run-experiment.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get explanation features of best performing model\n",
    "# train the model on SINAI using all of the top features from the model\n",
    "# remove the features one by one (recursive feature elimination) (iRFE)\n",
    "# apply recursive feature elimination to current sinai data ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' define options to run experiments on '''\n",
    "options = {\n",
    "    'target' : [\"AKI\"],\n",
    "    'cohort' : ['MIMIC'],\n",
    "    'test_size' : [0.2],\n",
    "    'imputation_method' : [imputers.DEFAULT],\n",
    "    'algorithm' : [algorithms.RF],\n",
    "    'sampling_method': [None],\n",
    "    'scale_method': [None],\n",
    "    'optimize_mode': [False],\n",
    "    'num_features': [50], \n",
    "    'crossval': [True],\n",
    "    'explainers' : [['MimicExplainer', 'ShapExplainer', 'FeatContribExplainer't, 'LimeExplainer']]\n",
    "    #'explainers' : [['MimicExplainer', 'ShapExplainer', 'FeatContribExplainer']]\n",
    "    #'explainers' : [['ShapExplainer', 'FeatContribExplainer']]\n",
    "    #'explainers' : [['FeatContribExplainer']]\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#experiments = unpickle('experiments.json') or {}\n",
    "#exp_id = list(sorted([(k,v['performance']['discrimination']['auc']) for k,v in experiments.items()], key=lambda x: x[1], reverse=True))[:1][0][0]\n",
    "#weighted_explanations = experiments[exp_id]['weighted_explanations']\n",
    "#%run feature-list.ipynb\n",
    "#top_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfe_experiments = unpickle('rfe_experiments.json') or {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_features = 10\n",
    "number_features = 10000\n",
    "previous_experiment = {}\n",
    "n_experiments = 0\n",
    "\n",
    "while number_features > max_features:\n",
    "    ''' iterate over different options '''\n",
    "    try:\n",
    "        params = dict(zip(options.keys(), list(next(product(*options.values())))))        \n",
    "        if previous_experiment:\n",
    "            features_to_select = [exp[0] for exp in previous_experiment['weighted_explanations']]\n",
    "            del features_to_select[0]\n",
    "            print(f'Selected features: {len(features_to_select)}')\n",
    "            params['features_to_select'] = features_to_select\n",
    "            number_features = len(features_to_select)\n",
    "            params['num_features'] = number_features\n",
    "        experiment = run_experiment(params)\n",
    "        exp_id = str(uuid.uuid1())\n",
    "        rfe_experiments[exp_id] = experiment\n",
    "        previous_experiment = experiment            \n",
    "        n_experiments += 1\n",
    "    except Exception as e:\n",
    "        print(f\"Could not run experiment with params {str(params)}. Error: {str(e)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' store everything '''\n",
    "if pickle(rfe_experiments, 'rfe_experiments.json'):\n",
    "    print('Successfully saved.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}