preprocess-cohort.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import morpher\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "from collections import defaultdict\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_mimic = pd.read_csv(\"~/cohorts/hs_mimic.csv\",index_col=0)\n",
    "df_sinai = pd.read_csv(\"~/cohorts/hs_sinai.csv\",index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xls = pd.ExcelFile('mapping.xls')\n",
    "mapping = xls.parse(xls.sheet_names[1])\n",
    "mapping.set_index('Field',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy = df_sinai.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_sinai_copy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai[['hypertension_complicated_precondition_from_inf_days_before_to_0_days_after', 'hypertension_uncomplicated_precondition_from_inf_days_before_to_0_days_after']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge_fields = defaultdict(lambda:[])\n",
    "elixhauser = 0\n",
    "series = defaultdict(lambda:[])\n",
    "\n",
    "for _,row in df_sinai_copy.iterrows():\n",
    "    for field in row.keys():        \n",
    "        if mapping.loc[field,'Action'] == 'merge':\n",
    "            merge_fields[mapping.loc[field,'Name']].append(row[field])\n",
    "        if not pd.isna(mapping.loc[field,'Score']):\n",
    "            elixhauser += int(row[field])*mapping.loc[field,'Score']            \n",
    "    \n",
    "    for mfield in merge_fields:\n",
    "        series[mfield].append(any(merge_fields[mfield]))    \n",
    "    series['ELIXHAUSER_SCORE'].append(elixhauser)\n",
    "    \n",
    "    merge_fields = defaultdict(lambda:[])\n",
    "    elixhauser = 0    \n",
    "\n",
    "for mfield in series:\n",
    "    df_sinai_copy[mfield] = pd.Series(series[mfield])  \n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy[['HYPERTENSION']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, column in mapping.iterrows():\n",
    "    if column['Action'] == 'remove' or column['Action'] == 'merge':\n",
    "        df_sinai_copy.drop([index], axis=1, inplace=True)        \n",
    "    if column['Action'] == 'rename':\n",
    "        df_sinai_copy.rename(columns={ index : column['Name'] }, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy.head(26)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy = df_sinai_copy.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AKI'].dtype\n",
    "#df_sinai_copy_copy['HYPERTENSION'] = df_sinai_copy_copy['HYPERTENSION'].astype(bool)\n",
    "#df_sinai_copy_copy['AKI'] = df_sinai_copy_copy['AKI'].astype(bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for column in df_sinai_copy_copy:\n",
    "    print(f\"{column} is type { df_sinai_copy_copy[column].dtype }\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AKI']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column in df_sinai_copy_copy:\n",
    "    if df_sinai_copy_copy[column].dtype == bool:\n",
    "        df_sinai_copy_copy[column] = df_sinai_copy_copy[column].apply(lambda x: x if x == np.NaN else int(x))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy.head(26)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AGE_AT_ADMISSION']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AGE_AT_ADMISSION'] = df_sinai_copy_copy['AGE_AT_ADMISSION'].apply(lambda x: math.ceil(x/365))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AGE_AT_ADMISSION'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AGE_AT_ADMISSION'] = df_sinai_copy_copy['AGE_AT_ADMISSION'].apply(lambda x: x if x <= 90 else 90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['AGE_AT_ADMISSION']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy['GENDER']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn_pandas import DataFrameMapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = DataFrameMapper([('GENDER', LabelBinarizer())], df_out=True, default=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper.fit_transform(df_sinai_copy_copy.copy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sinai_copy_copy.to_csv('~/cohorts/hs_sinai_preprocessed.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}