tutorials/Tutorial1_Hospital.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/pandas/compat/_optional.py:106: UserWarning: Pandas requires version '1.2.1' or newer of 'bottleneck' (version '1.2.0' currently installed).\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "from profiler.core import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Instantiate Engine\n",
    "* workers : number of processes\n",
    "* tol     : tolerance for differences when creating training data (set to 0 if data is completely clean)\n",
    "* eps     : error bound for inverse covariance estimation (since we use conservative calculation when determining minimum sample size, we recommend to set eps <= 0.01)\n",
    "* embedtxt: if set to true, differentiate b/w textual data and categorical data, and use word embedding for the former"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pf = Profiler(workers=2, tol=1e-6, eps=0.05, embedtxt=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load Data\n",
    "* name: any name you like\n",
    "* src: \\[FILE; DF; DB (not implemented)\\]\n",
    "* fpath: required if src == FILE\n",
    "* df: required if src == DF\n",
    "* check_param: print parameters used for data loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:profiler.utility:[0.009931802749633789] Load Data start\n",
      "\n",
      "INFO:profiler.data.dataset:parameters used for data loading:\n",
      " {'na_values': 'empty', 'sep': ',', 'header': 'infer', 'dropcol': None, 'dropna': False, 'encoding': 'utf-8', 'normalize': True, 'min_categories_for_embedding': 10}\n",
      "WARNING:profiler.data.dataset:Dropping the following null column from the dataset: 'Address2'\n",
      "WARNING:profiler.data.dataset:Dropping the following null column from the dataset: 'Address3'\n",
      "INFO:profiler.data.dataset:inferred types of attributes: {\n",
      "    \"ProviderNumber\": \"text\",\n",
      "    \"HospitalName\": \"text\",\n",
      "    \"Address1\": \"text\",\n",
      "    \"City\": \"text\",\n",
      "    \"State\": \"categorical\",\n",
      "    \"ZipCode\": \"text\",\n",
      "    \"CountyName\": \"text\",\n",
      "    \"PhoneNumber\": \"text\",\n",
      "    \"HospitalType\": \"text\",\n",
      "    \"HospitalOwner\": \"text\",\n",
      "    \"EmergencyService\": \"categorical\",\n",
      "    \"Condition\": \"text\",\n",
      "    \"MeasureCode\": \"text\",\n",
      "    \"MeasureName\": \"text\",\n",
      "    \"Score\": \"text\",\n",
      "    \"Sample\": \"text\",\n",
      "    \"Stateavg\": \"text\"\n",
      "}\n",
      "INFO:profiler.data.dataset:(possible types: numeric, categorical, text, date)\n",
      "INFO:profiler.data.dataset:inferred operators of attributes: {'ProviderNumber': ['equal'], 'HospitalName': ['equal'], 'Address1': ['equal'], 'City': ['equal'], 'State': ['equal'], 'ZipCode': ['equal'], 'CountyName': ['equal'], 'PhoneNumber': ['equal'], 'HospitalType': ['equal'], 'HospitalOwner': ['equal'], 'EmergencyService': ['equal'], 'Condition': ['equal'], 'MeasureCode': ['equal'], 'MeasureName': ['equal'], 'Score': ['equal'], 'Sample': ['equal'], 'Stateavg': ['equal']}\n",
      "INFO:profiler.data.dataset:(possible operators: equal, notequal, greater_than, less_than)\n",
      "INFO:profiler.utility:[0.11717534065246582] Load Data execution time: 0.10724353790283203\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# pf.session.load_data(name='hospital', src=FILE, fpath='data/hospital_clean_unflatten.csv', check_param=True, na_values='empty')\n",
    "pf.session.load_data(name='customer', src=FILE, fpath='data/hospital.csv', check_param=True, na_values='empty')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Change Data Types of Attributes\n",
    "* required input:\n",
    "    * a list of attributes\n",
    "    * a list of data types (must match the order of the attributes; can be CATEGORICAL, NUMERIC, TEXT, DATE)\n",
    "* optional input:\n",
    "    * a list of regular expression extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:profiler.data.dataset:updated types of ProviderNumber to 'categorical'\n",
      "INFO:profiler.data.dataset:updated operators of ProviderNumber to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of ZipCode to 'numeric'\n",
      "INFO:profiler.data.dataset:updated operators of ZipCode to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of PhoneNumber to 'categorical'\n",
      "INFO:profiler.data.dataset:updated operators of PhoneNumber to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of State to 'text'\n",
      "INFO:profiler.data.dataset:updated operators of State to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of EmergencyService to 'text'\n",
      "INFO:profiler.data.dataset:updated operators of EmergencyService to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of Score to 'numeric'\n",
      "INFO:profiler.data.dataset:updated operators of Score to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of Sample to 'numeric'\n",
      "INFO:profiler.data.dataset:updated operators of Sample to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of HospitalType to 'text'\n",
      "INFO:profiler.data.dataset:updated operators of HospitalType to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of HospitalOwner to 'text'\n",
      "INFO:profiler.data.dataset:updated operators of HospitalOwner to ['equal']\n",
      "INFO:profiler.data.dataset:updated types of Condition to 'text'\n",
      "INFO:profiler.data.dataset:updated operators of Condition to ['equal']\n",
      "INFO:profiler.data.dataset:updated inferred operators of attributes: {'ProviderNumber': ['equal'], 'HospitalName': ['equal'], 'Address1': ['equal'], 'City': ['equal'], 'State': ['equal'], 'ZipCode': ['equal'], 'CountyName': ['equal'], 'PhoneNumber': ['equal'], 'HospitalType': ['equal'], 'HospitalOwner': ['equal'], 'EmergencyService': ['equal'], 'Condition': ['equal'], 'MeasureCode': ['equal'], 'MeasureName': ['equal'], 'Score': ['equal'], 'Sample': ['equal'], 'Stateavg': ['equal']}\n"
     ]
    }
   ],
   "source": [
    "pf.session.change_dtypes(['ProviderNumber', 'ZipCode', 'PhoneNumber', 'State', 'EmergencyService','Score', 'Sample','HospitalType','HospitalOwner', 'Condition'], \n",
    "                            [CATEGORICAL, NUMERIC, CATEGORICAL, TEXT, TEXT, NUMERIC, NUMERIC, TEXT,TEXT, TEXT],\n",
    "                            [None, None, None, None, None, r'(\\d+)%', r'(\\d+)\\spatients', None, None,None])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2. Load/Train Embeddings for TEXT\n",
    "* path: path to saved/to-save embedding folder\n",
    "* load: set to true -- load saved vec from 'path'; set to false -- train locally\n",
    "* save: (only for load = False) save trained vectors to 'path'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:profiler.utility:[24.145376205444336] Load Embedding start\n",
      "\n",
      "INFO:profiler.data.embedding:[HospitalName] tokenize cell\n",
      "INFO:profiler.data.embedding:[Address1] tokenize cell\n",
      "INFO:profiler.data.embedding:[HospitalName] train language model\n",
      "INFO:profiler.data.embedding:[Address1] train language model\n",
      "INFO:profiler.data.embedding:[HospitalName] compute weights\n",
      "INFO:profiler.data.embedding:[HospitalName] create vector map\n",
      "INFO:profiler.data.embedding:[HospitalName] save vec and vocab\n",
      "INFO:profiler.data.embedding:[City] tokenize cell\n",
      "INFO:profiler.data.embedding:[City] train language model\n",
      "INFO:profiler.data.embedding:[Address1] compute weights\n",
      "INFO:profiler.data.embedding:[Address1] create vector map\n",
      "INFO:profiler.data.embedding:[Address1] save vec and vocab\n",
      "INFO:profiler.data.embedding:[State] tokenize cell\n",
      "INFO:profiler.data.embedding:[State] train language model\n",
      "INFO:profiler.data.embedding:[City] compute weights\n",
      "INFO:profiler.data.embedding:[City] create vector map\n",
      "INFO:profiler.data.embedding:[City] save vec and vocab\n",
      "INFO:profiler.data.embedding:[CountyName] tokenize cell\n",
      "INFO:profiler.data.embedding:[CountyName] train language model\n",
      "INFO:profiler.data.embedding:[State] compute weights\n",
      "INFO:profiler.data.embedding:[State] create vector map\n",
      "INFO:profiler.data.embedding:[State] save vec and vocab\n",
      "INFO:profiler.data.embedding:[HospitalType] tokenize cell\n",
      "INFO:profiler.data.embedding:[HospitalType] train language model\n",
      "INFO:profiler.data.embedding:[CountyName] compute weights\n",
      "INFO:profiler.data.embedding:[CountyName] create vector map\n",
      "INFO:profiler.data.embedding:[CountyName] save vec and vocab\n",
      "INFO:profiler.data.embedding:[HospitalOwner] tokenize cell\n",
      "INFO:profiler.data.embedding:[HospitalOwner] train language model\n",
      "INFO:profiler.data.embedding:[HospitalType] compute weights\n",
      "INFO:profiler.data.embedding:[HospitalType] create vector map\n",
      "INFO:profiler.data.embedding:[HospitalType] save vec and vocab\n",
      "INFO:profiler.data.embedding:[EmergencyService] tokenize cell\n",
      "INFO:profiler.data.embedding:[EmergencyService] train language model\n",
      "INFO:profiler.data.embedding:[HospitalOwner] compute weights\n",
      "INFO:profiler.data.embedding:[HospitalOwner] create vector map\n",
      "INFO:profiler.data.embedding:[HospitalOwner] save vec and vocab\n",
      "INFO:profiler.data.embedding:[Condition] tokenize cell\n",
      "INFO:profiler.data.embedding:[Condition] train language model\n",
      "INFO:profiler.data.embedding:[EmergencyService] compute weights\n",
      "INFO:profiler.data.embedding:[EmergencyService] create vector map\n",
      "INFO:profiler.data.embedding:[EmergencyService] save vec and vocab\n",
      "INFO:profiler.data.embedding:[MeasureCode] tokenize cell\n",
      "INFO:profiler.data.embedding:[MeasureCode] train language model\n",
      "INFO:profiler.data.embedding:[Condition] compute weights\n",
      "INFO:profiler.data.embedding:[Condition] create vector map\n",
      "INFO:profiler.data.embedding:[Condition] save vec and vocab\n",
      "INFO:profiler.data.embedding:[MeasureName] tokenize cell\n",
      "INFO:profiler.data.embedding:[MeasureName] train language model\n",
      "INFO:profiler.data.embedding:[MeasureCode] compute weights\n",
      "INFO:profiler.data.embedding:[MeasureCode] create vector map\n",
      "INFO:profiler.data.embedding:[MeasureCode] save vec and vocab\n",
      "INFO:profiler.data.embedding:[Stateavg] tokenize cell\n",
      "INFO:profiler.data.embedding:[Stateavg] train language model\n",
      "INFO:profiler.data.embedding:[MeasureName] compute weights\n",
      "INFO:profiler.data.embedding:[MeasureName] create vector map\n",
      "INFO:profiler.data.embedding:[MeasureName] save vec and vocab\n",
      "INFO:profiler.data.embedding:[Stateavg] compute weights\n",
      "INFO:profiler.data.embedding:[Stateavg] create vector map\n",
      "INFO:profiler.data.embedding:[Stateavg] save vec and vocab\n",
      "INFO:profiler.utility:[280.13637924194336] Load Embedding execution time: 255.99100303649902\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pf.session.load_embedding(save=True, path='data/', load=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Load Training Data\n",
    "* multiplier: if set to None, will infer the minimal sample size; otherwise, it will create (# samples) * (# attributes) * (multiplier) training samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:profiler.utility:[20.303852796554565] Create Training Data start\n",
      "\n",
      "INFO:profiler.data.transformer:needs multiplier = 1 to bound the error in inv cov estimation <= 0.05000000\n",
      "INFO:profiler.data.transformer:use multiplier = 1, and the bound is 0.00003333\n",
      "INFO:profiler.data.transformer:Draw Pairs\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.31it/s]\n",
      "INFO:profiler.data.transformer:Number of training samples: 1050000\n",
      "INFO:profiler.data.transformer:Computing Differences\n",
      "/root/anaconda3/lib/python3.6/site-packages/pandas/compat/_optional.py:106: UserWarning: Pandas requires version '2.6.2' or newer of 'numexpr' (version '2.6.1' currently installed).\n",
      "  warnings.warn(msg, UserWarning)\n"
     ]
    }
   ],
   "source": [
    "# use simple empirical cov: difference=False\n",
    "# use difference -> cov : difference=True\n",
    "pf.session.load_training_data(multiplier = None, difference=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Learn Structure\n",
    "* sparsity: intensity of L1-regularizer in inverse covariance estimation (glasso)\n",
    "* take_neg: if set to true, consider equal -> equal only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# set sparsity to 0 for exp_reproduce \n",
    "autoregress_matrix = pf.session.learn_structure(sparsity=0, infer_order=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* score: \n",
    "    * \"fit_error\": mse for fitting y = B'X + c for each atttribute y \n",
    "    * \"training_data_fd_vio_ratio\": the higher the score, the more violations of FDs in the training data. (bounded: \\[0,1\\])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "parent_sets = pf.session.get_dependencies(score=\"fit_error\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os, json\n",
    "def read_fds(path='data/fds', f='TECHospital-hyfd'):\n",
    "    all_fds = {}\n",
    "    for line in open(os.path.join(path,f)):\n",
    "        fd = json.loads(line)\n",
    "        right = fd[u'dependant']['columnIdentifier']\n",
    "        left = [l[u'columnIdentifier'] for l in fd[ u'determinant'][u'columnIdentifiers']]\n",
    "        if right not in all_fds:\n",
    "            all_fds[right] = set()\n",
    "        all_fds[right].add(frozenset(left))\n",
    "    return all_fds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gt = read_fds(f='hospital_clean-fun')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tp = 0\n",
    "count = 0\n",
    "for child in parent_sets:\n",
    "    found = parent_sets[child]\n",
    "    if len(found) == 0:\n",
    "        continue\n",
    "    count += 1\n",
    "    match = False\n",
    "    for parent in gt[child]:\n",
    "        if set(parent).issubset(found):\n",
    "            tp += 1\n",
    "            match = True\n",
    "            break\n",
    "    if not match:\n",
    "        print(\"{} -> {} is not valid\".format(found, child))\n",
    "if count > 0:\n",
    "    print(\"Precision: %.4f\"%(float(tp) / count))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pf.session.visualize_covariance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "pf.session.visualize_inverse_covariance()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pf.session.visualize_autoregression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pf.session.timer.get_stat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pf.session.timer.to_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}