evaluate-experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run default-imports.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments = unpickle('experiments_cv_sinai_80_compare_scalers_imputers.json') or {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_experiments_sinai = list(sorted([(k,v['performance']['cv']['auc']['mean']) for k,v in experiments.items() if v['parameters']['cohort'] == 'SINAI'], key=lambda x: x[1], reverse=True))[:10]\n",
    "#top_experiments_mimic = list(sorted([(k,v['performance']['cv']['auc']['mean']) for k,v in experiments.items() if v['parameters']['cohort'] == 'MIMIC'], key=lambda x: x[1], reverse=True))[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = ['precision', 'recall', 'f1-score', 'auc', 'dor']\n",
    "\n",
    "for exp_id,_ in top_experiments_sinai:    \n",
    "    print(f\"algorithm: {experiments[exp_id]['parameters']['algorithm']}\")\n",
    "    print(f\"imputation: {experiments[exp_id]['parameters']['imputation_method']}\")\n",
    "    print(f\"sampling: {experiments[exp_id]['parameters']['sampling_method']}\")\n",
    "    print(f\"scaling: {experiments[exp_id]['parameters'].get('scaling_method')}\")\n",
    "    for metric in metrics:\n",
    "        m_value = '{:.4f}'.format(experiments[exp_id]['performance']['cv'][metric]['mean']).strip('0')\n",
    "        ci = '{:.4f}'.format(experiments[exp_id]['performance']['cv'][metric]['ci']).strip('0')\n",
    "        print(f\"{metric}: {m_value} ±{ci} 95% CI\")       \n",
    "\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = plt.axes()\n",
    "\n",
    "X = [exp[0] for exp in top_experiments]\n",
    "y = [exp[1] for exp in top_experiments]\n",
    "\n",
    "ax.plot(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiments['f69fde80-179d-11ea-bf3a-000d3a0e06d9']['performance']['discrimination']['auc']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}