faults_dfs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Notes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "* Problem statment does not suggest device independence: these could be networked or interdependent systems, e.g. ordering.\n",
    "* Failure class is HEAVILY imbalanced; modeling is prone to finding local optima that explain the majority class.\n",
    "* [During failure, attributes 2, 4, 7, and 8 report much higher values than normal. This is not true of 1, 3, 5, 6, and 9](#failhard)\n",
    "* [a1 and a6 are very unique](#uniqueness)\n",
    "* [No duplicate data and no missing values](#missing)\n",
    "* [Events may have natural ordering that needs to be preserved](#ordered)\n",
    "* [Devices appear to come in types/classes](#deviceclasses)\n",
    "* Attributes look to be aggregates except for a1\n",
    "* [a2 & a4 ramp-up before failure, and looks like a7 spikes quickly too](#correlation)\n",
    "* Some devices go through \"servicing\" where aggregate value drops\n",
    "* [a3 & a9 are weakly positively correlated](#correlation)\n",
    "* [a7 & a8 are perfectly positively correlated... because they are identical](#a7a8)\n",
    "* [a7 appears to be critical in predicting failure in a naive model](#naivemodel)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-03T10:30:58.624553Z",
     "start_time": "2018-08-03T10:30:53.838670Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# mute warnings\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# science\n",
    "import scipy as sp\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import featuretools as ft\n",
    "import xgboost as xgb\n",
    "from sklearn import model_selection\n",
    "from sklearn import preprocessing\n",
    "from sklearn import metrics\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import h2o\n",
    "from h2o.automl import H2OAutoML\n",
    "from hyperopt import hp\n",
    "from hyperopt import fmin\n",
    "from hyperopt import tpe\n",
    "from hyperopt import STATUS_OK\n",
    "from hyperopt import Trials\n",
    "\n",
    "# system\n",
    "import random\n",
    "import hashlib\n",
    "import pickle\n",
    "import tempfile\n",
    "import time\n",
    "\n",
    "# visuals\n",
    "import seaborn as sns\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pylab import rcParams\n",
    "from IPython import display\n",
    "\n",
    "# options\n",
    "%matplotlib inline\n",
    "rcParams['figure.figsize'] = 15, 6\n",
    "sns.set_style('whitegrid')\n",
    "sns.set_context('notebook')\n",
    "sns_cmap = sns.diverging_palette(10, 220, sep=80, n=5)\n",
    "pd.set_option('display.max_colwidth', -1)\n",
    "\n",
    "# in case nb needs to be re-run\n",
    "try:\n",
    "    %load_ext autoreload\n",
    "except:\n",
    "    %reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions <a id=\"functions\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:38.440834Z",
     "start_time": "2018-08-01T12:53:38.426995Z"
    }
   },
   "outputs": [],
   "source": [
    "def load_data(filename):\n",
    "    '''\n",
    "    Load a generic CSV file into a pandas dataframe\n",
    "    '''\n",
    "    df = pd.read_csv(filename, escapechar='\\\\', encoding='utf-8')\n",
    "    df['uuid'] = df.apply(\n",
    "        lambda x: hashlib.md5(str(x.values).encode('utf-8')).hexdigest(),\n",
    "        axis=1)\n",
    "    assert sum(df.set_index('uuid').index.duplicated()\n",
    "               ) == 0, 'Error loading data: Hash collision; duplicate data'\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:38.455353Z",
     "start_time": "2018-08-01T12:53:38.442127Z"
    }
   },
   "outputs": [],
   "source": [
    "def calculate_nan_values(orig_df):\n",
    "    '''\n",
    "    Parse a dataframe and report nan summary by column\n",
    "    '''\n",
    "    df = orig_df.copy()\n",
    "    df = df.isna().sum(axis=0).reset_index()\n",
    "    df.columns = ['column_name', 'na_count']\n",
    "    df['na_ratio'] = df['na_count'] / orig_df.shape[0]\n",
    "    df = df.loc[df['na_ratio'] > 0]\n",
    "    df = df.sort_values(by='na_ratio')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:38.470310Z",
     "start_time": "2018-08-01T12:53:38.456666Z"
    }
   },
   "outputs": [],
   "source": [
    "def calculate_null_values(orig_df):\n",
    "    '''\n",
    "    Parse a dataframe and report null summary by column\n",
    "    '''\n",
    "    df = orig_df.copy()\n",
    "    df = df.isnull().sum(axis=0).reset_index()\n",
    "    df.columns = ['column_name', 'null_count']\n",
    "    df['null_ratio'] = df['null_count'] / orig_df.shape[0]\n",
    "    df = df.loc[df['null_ratio'] > 0]\n",
    "    df = df.sort_values(by='null_ratio')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:38.827354Z",
     "start_time": "2018-08-01T12:53:38.809930Z"
    }
   },
   "outputs": [],
   "source": [
    "def value_limits(df, field, upper_bound=99, lower_bound=1):\n",
    "    '''\n",
    "    Find upper and lower percentile limits for a dataframe and field\n",
    "    '''\n",
    "    upper_limit = np.percentile(df[field].values, upper_bound)\n",
    "    lower_limit = np.percentile(df[field].values, lower_bound)\n",
    "    return upper_limit, lower_limit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:39.000747Z",
     "start_time": "2018-08-01T12:53:38.982048Z"
    }
   },
   "outputs": [],
   "source": [
    "def visualize_correlations(df, fields):\n",
    "    '''\n",
    "    Simplify the process of looking at a simple all-v-all correlation\n",
    "    '''\n",
    "\n",
    "    # keep a copy\n",
    "    df = df.copy()\n",
    "\n",
    "    # correlation coefficient of each column\n",
    "    for field in fields:\n",
    "        x_cols = [\n",
    "            col for col in df.columns\n",
    "            if col != field and\n",
    "            (df[col].dtype == 'float64' or df[col].dtype == 'int64')\n",
    "        ]\n",
    "        labels = []\n",
    "        values = []\n",
    "        for col in x_cols:\n",
    "            labels.append(col)\n",
    "            values.append(np.corrcoef(df[col].values, df[field].values)[0, 1])\n",
    "\n",
    "        # create dataframe for corr coeffs\n",
    "        corr_df = pd.DataFrame({'col_labels': labels, 'corr_values': values})\n",
    "        corr_df = corr_df.sort_values(by='corr_values')\n",
    "\n",
    "        ind = np.arange(len(labels))\n",
    "        fig, ax = plt.subplots(figsize=(12, 4))\n",
    "        rects = ax.barh(ind, np.array(corr_df.corr_values.values), color='y')\n",
    "        ax.set_yticks(ind)\n",
    "        ax.set_yticklabels(corr_df.col_labels.values, rotation='horizontal')\n",
    "        ax.set_xlabel('correlation coefficient')\n",
    "        ax.set_title('Correlations with {}'.format(field))\n",
    "        ax.set_xlim(-1, 1)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:39.156967Z",
     "start_time": "2018-08-01T12:53:39.139348Z"
    }
   },
   "outputs": [],
   "source": [
    "def visualize_values(df, fields):\n",
    "    '''\n",
    "    Simplify the process of looking at a fields values\n",
    "    '''\n",
    "\n",
    "    # keep a copy\n",
    "    df = df.copy()\n",
    "\n",
    "    # iterate over fields\n",
    "    for field in fields:\n",
    "\n",
    "        # get limits of data\n",
    "        ulimit, llimit = value_limits(df, field)\n",
    "        print('###################')\n",
    "        print('Working on field {}'.format(field))\n",
    "        print('Upper limit: {:.3f}'.format(ulimit))\n",
    "        print('Lower limit: {:.3f}'.format(llimit))\n",
    "\n",
    "        # plot the ordered values for field\n",
    "        plt.figure(figsize=(12, 8))\n",
    "        plt.scatter(range(df.shape[0]), np.sort(df[field].values))\n",
    "        plt.xlabel('index', fontsize=12)\n",
    "        plt.ylabel(field, fontsize=12)\n",
    "        plt.show()\n",
    "\n",
    "        # plot histo of values\n",
    "        sns.distplot(df[field].values, bins=50, kde=True, norm_hist=True)\n",
    "        plt.xlabel(field, fontsize=12)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:39.305286Z",
     "start_time": "2018-08-01T12:53:39.291837Z"
    }
   },
   "outputs": [],
   "source": [
    "def save_hdf5(df, hdf_file, table_name, format=None):\n",
    "    '''\n",
    "    Save a pandas dataframe as compressed HDF5\n",
    "    '''\n",
    "    if format is not None:\n",
    "        df.to_hdf(\n",
    "            hdf_file,\n",
    "            table_name,\n",
    "            mode='w',\n",
    "            format=format,\n",
    "            data_columns=True,\n",
    "            complevel=9,\n",
    "            complib='blosc:lz4')\n",
    "    else:\n",
    "        df.to_hdf(\n",
    "            hdf_file,\n",
    "            table_name,\n",
    "            mode='w',\n",
    "            data_columns=True,\n",
    "            complevel=9,\n",
    "            complib='blosc:lz4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:53:27.005967Z",
     "start_time": "2018-08-01T13:53:26.990080Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_xgb_model(X_train, y_train, params, kfolds=10, random_seed=7):\n",
    "    '''\n",
    "    Wrapper for fitting an xgb model with k-fold cv\n",
    "    '''\n",
    "\n",
    "    # setup\n",
    "    random.seed(random_seed)\n",
    "    np.random.seed(random_seed)\n",
    "\n",
    "    # isolate train features\n",
    "    X_train = X_train.values\n",
    "    y_train = y_train.astype(int).values\n",
    "\n",
    "    # stratified k-fold\n",
    "    skf = StratifiedKFold(n_splits=kfolds, random_state=42, shuffle=False)\n",
    "    best_score = 0\n",
    "    best_model = None\n",
    "    for i, (train_index, cv_index) in enumerate(skf.split(X_train, y_train)):\n",
    "        print('Working on k-fold {}...'.format(i))\n",
    "\n",
    "        # split data\n",
    "        print('Splitting data')\n",
    "        X_ktrain, X_kcv = X_train[train_index], X_train[cv_index]\n",
    "        y_ktrain, y_kcv = y_train[train_index], y_train[cv_index]\n",
    "\n",
    "        # Convert our data into XGBoost format\n",
    "        print('Converting data to xgb format')\n",
    "        d_train = xgb.DMatrix(\n",
    "            X_ktrain, label=y_ktrain, feature_names=feature_names)\n",
    "        d_cv = xgb.DMatrix(X_kcv, label=y_kcv, feature_names=feature_names)\n",
    "        watchlist = [(d_train, 'train'), (d_cv, 'cv')]\n",
    "\n",
    "        # train the model\n",
    "        print('Modeling')\n",
    "        model = xgb.train(\n",
    "            params,\n",
    "            d_train,\n",
    "            num_boost_round=3000,\n",
    "            evals=watchlist,  #feval=cohen_kappa_scorer,\n",
    "            early_stopping_rounds=300,\n",
    "            verbose_eval=200)\n",
    "        if model.best_score > best_score:\n",
    "            best_score = model.best_score\n",
    "            best_model = model\n",
    "            model.save_model('xgboost_tuned.model')\n",
    "        print('Done.')\n",
    "\n",
    "    # testing\n",
    "    return best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cohen_kappa_scorer(preds, dmatrix):\n",
    "    '''\n",
    "    Stub for Cohen's Kappa as scoring metric for xgb\n",
    "    '''\n",
    "    y = dmatrix.get_label()\n",
    "    cohen_kappa_score(preds, y)\n",
    "    return 'kappa', score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:40.739426Z",
     "start_time": "2018-08-01T12:53:40.722111Z"
    }
   },
   "outputs": [],
   "source": [
    "def sensitivity(y_test, y_pred):\n",
    "    '''\n",
    "    Stub for a recall scorer for Keras\n",
    "    Inspired by: http://www.deepideas.net/unbalanced-classes-machine-learning/ \n",
    "    '''\n",
    "    true_positives = K.sum(K.round(K.clip(y_test * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_test, 0, 1)))\n",
    "    return true_positives / (possible_positives + K.epsilon())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:40.907194Z",
     "start_time": "2018-08-01T12:53:40.893183Z"
    }
   },
   "outputs": [],
   "source": [
    "def specificity(y_test, y_pred):\n",
    "    '''\n",
    "    Stub for a precision scorer for Keras\n",
    "    Inspired by: http://www.deepideas.net/unbalanced-classes-machine-learning/\n",
    "    '''\n",
    "    true_negatives = K.sum(K.round(K.clip((1 - y_test) * (1 - y_pred), 0, 1)))\n",
    "    possible_negatives = K.sum(K.round(K.clip(1 - y_test, 0, 1)))\n",
    "    return true_negatives / (possible_negatives + K.epsilon())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-03T10:48:24.371532Z",
     "start_time": "2018-08-03T10:48:24.357254Z"
    }
   },
   "outputs": [],
   "source": [
    "def objective(Xy,\n",
    "              predictors,\n",
    "              predicting,\n",
    "              space=None,\n",
    "              test_size=0.33,\n",
    "              eval_metric=\"auc\",\n",
    "              early_stopping_rounds=300,\n",
    "              random_state=42):\n",
    "    '''\n",
    "    This function trains a model, evaluates it, and returns the error on the cv data.\n",
    "    A param space is fed in for exploration: here, we want to try values from 5 to 30 for\n",
    "    max_depth, from 1 to 10 for min_child_weight and from 0.8 to 1 for subsample.\n",
    "    Inspired by: https://www.dataiku.com/learn/guide/code/python/advanced-xgboost-tuning.html\n",
    "    '''\n",
    "\n",
    "    if space is None:\n",
    "        space = {\n",
    "            'max_depth': hp.quniform(\"x_max_depth\", 5, 30, 1),\n",
    "            'min_child_weight': hp.quniform('x_min_child', 1, 10, 1),\n",
    "            'subsample': hp.uniform('x_subsample', 0.8, 1)\n",
    "        }\n",
    "\n",
    "    # isolate y from X\n",
    "    y = Xy[predicting]\n",
    "    X = Xy[predictors]\n",
    "\n",
    "    # split data into train and test sets\n",
    "    X_train, X_cv, y_train, y_cv = model_selection.train_test_split(\n",
    "        X, y, test_size=test_size, random_state=random_state)\n",
    "    feature_names = X_train.columns.values\n",
    "\n",
    "    # build classifier\n",
    "    clf = xgb.XGBClassifier(\n",
    "        n_estimators=10000,\n",
    "        max_depth=space['max_depth'],\n",
    "        min_child_weight=space['min_child_weight'],\n",
    "        subsample=space['subsample'])\n",
    "\n",
    "    # eval metrics\n",
    "    eval_set = [(X_train, y_train), (X_cv, y_cv)]\n",
    "\n",
    "    # fit the data\n",
    "    clf.fit(\n",
    "        X_train,\n",
    "        y_train,\n",
    "        eval_set=eval_set,\n",
    "        eval_metric=eval_metric,\n",
    "        early_stopping_rounds=early_stopping_rounds)\n",
    "\n",
    "    # make predictions\n",
    "    y_pred = clf.predict_proba(X_cv)[:, 1]\n",
    "    auc = metrics.roc_auc_score(y_cv, y_pred)\n",
    "    print(\"SCORE:\", auc)\n",
    "\n",
    "    return {'loss': 1 - auc, 'status': STATUS_OK}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Data Ingest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:50.545295Z",
     "start_time": "2018-08-01T11:27:45.836775Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# load data\n",
    "faults_df = load_data(\"device_failure.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:50.571847Z",
     "start_time": "2018-08-01T11:27:50.546862Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# see a sample\n",
    "faults_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:50.586059Z",
     "start_time": "2018-08-01T11:27:50.573185Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# rename columns for less typing\n",
    "faults_df.columns = faults_df.columns.str.replace(\"attribute\", \"a\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:50.711940Z",
     "start_time": "2018-08-01T11:27:50.587375Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# odd date format, did load reformat?\n",
    "!head device_failure.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:50.833451Z",
     "start_time": "2018-08-01T11:27:50.713517Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# basic summary for any obvious weirdness\n",
    "faults_df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "This data set appears to have attributes that are either \"codes\" or aggregates. The min/max and stddev for each is large, so there's a spread. If these are codes, one-hot encoding will be... tough."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# EDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## QA/QC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:53.478725Z",
     "start_time": "2018-08-01T11:27:53.363224Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# double-check problem statement, one device per day?\n",
    "check = faults_df.date.astype(str) + faults_df.device.astype(str)\n",
    "assert len(check) == len(check.drop_duplicates()), 'More than one device per day'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:53.597371Z",
     "start_time": "2018-08-01T11:27:53.574542Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# inspect data types\n",
    "dtype_df = faults_df.dtypes.reset_index()\n",
    "dtype_df.columns = [\"Count\", \"Column Type\"]\n",
    "dtype_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:55.220643Z",
     "start_time": "2018-08-01T11:27:55.012770Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# inspect uniqueness\n",
    "uniqueness_df = pd.DataFrame(faults_df.nunique()).reset_index()\n",
    "uniqueness_df.columns = [\"column\", \"num_unique\"]\n",
    "uniqueness_df['perc_unique'] = uniqueness_df.num_unique.apply(lambda x: x/len(faults_df))\n",
    "uniqueness_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "A1 is very unique: device identifier? Can't be. Must be a daily value. <a id=\"uniqueness\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:57.210065Z",
     "start_time": "2018-08-01T11:27:57.150793Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# nulls\n",
    "calculate_null_values(faults_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:27:57.410792Z",
     "start_time": "2018-08-01T11:27:57.358558Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# nan's (just to be sure)\n",
    "calculate_nan_values(faults_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:28:12.181446Z",
     "start_time": "2018-08-01T11:27:57.526823Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# visualize column values\n",
    "fields = [c for c in faults_df.columns if faults_df[c].dtype == 'int64']\n",
    "visualize_values(faults_df, fields)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Some attributes have orders of magnitude higher values than others within same category. Must have units of measure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-31T11:23:03.774106Z",
     "start_time": "2018-07-31T11:23:03.751407Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "cols = faults_df.filter(regex='^a').columns.tolist()\n",
    "temp = faults_df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-31T11:10:06.052630Z",
     "start_time": "2018-07-31T11:08:56.299516Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sns.pairplot(temp);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-31T11:10:06.151005Z",
     "start_time": "2018-07-31T11:10:06.054084Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sns.countplot(x=\"failure\", data=faults_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Yikes, this is an extreme example of imbalanced classes."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:28:51.423562Z",
     "start_time": "2018-08-01T11:28:51.401473Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# summary count of classes\n",
    "100.*faults_df.groupby('failure').failure.count()/len(faults_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "\"Extreme\" is putting it lightly: 99.91% of the time, the data indicates non-failure. **Model #1**: Always predict 'non-failure', you'll be right most of the time :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T10:35:01.332016Z",
     "start_time": "2018-08-01T10:35:01.144795Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# summarize by class\n",
    "temp = faults_df.filter(regex='^a|failure')\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "scaled_values = scaler.fit_transform(temp) \n",
    "temp.loc[:,:] = scaled_values\n",
    "temp_melt = pd.melt(temp, \"failure\", var_name=\"measurement\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T10:35:11.506280Z",
     "start_time": "2018-08-01T10:35:01.894561Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sns.catplot(x=\"measurement\", y=\"value\", hue=\"failure\", kind=\"bar\", data=temp_melt);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Do some attributes have values ONLY when a failure occurs: a2, a4, a7, a8?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T10:35:11.600091Z",
     "start_time": "2018-08-01T10:35:11.507727Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# check this^^^ question\n",
    "subtemp = temp.filter(regex='a2|a3|a9|a4|a7|a8|failure')\n",
    "for c in subtemp.columns:\n",
    "    subtemp.loc[subtemp[c] > 0, c] = 1\n",
    "subtemp_melt = pd.melt(subtemp, \"failure\", var_name=\"measurement\")\n",
    "grouped = subtemp_melt.groupby(['measurement', 'failure']).sum()\n",
    "grouped"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Far more values present when not failing, so there must be a slant toward large values when there is a failure?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T10:35:22.974369Z",
     "start_time": "2018-08-01T10:35:16.558993Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sns.catplot(x=\"measurement\", y=\"value\", hue=\"failure\", kind=\"bar\", data=subtemp_melt);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "<a id=\"failhard\"></a>\n",
    "Looks to be true. When there is a failure, attributes 2, 4, 7, and 8 report much higher values than normal. This is not true of 1, 3, 5, 6, and 9. They should be correlated with failure then..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T10:36:02.929141Z",
     "start_time": "2018-08-01T10:36:02.647923Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Draw the heatmap with the mask and correct aspect ratio\n",
    "corr = temp.corr()\n",
    "\n",
    "# Generate a mask for the upper triangle\n",
    "mask = np.zeros_like(corr, dtype=np.bool)\n",
    "mask[np.triu_indices_from(mask)] = True\n",
    "\n",
    "# Generate a custom diverging colormap\n",
    "cmap = sns.diverging_palette(220, 10, as_cmap=True)\n",
    "\n",
    "# heatmap\n",
    "sns.heatmap(corr, mask=mask, cmap=cmap, center=0,\n",
    "            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "That is confidence building: a2, a4, a7, and a8 are weakly correlated with failure. Dig on A7==A8 in EDA."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:07.692701Z",
     "start_time": "2018-08-01T11:29:07.673307Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# what are the dates representing?\n",
    "dates = faults_df.date\n",
    "dates.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Dates look like an arbitrary sequence <a id=\"ordered\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:10.599172Z",
     "start_time": "2018-08-01T11:29:08.891356Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# is it sorted?\n",
    "all(dates[i] <= dates[i+1] for i in range(len(dates)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:10.619328Z",
     "start_time": "2018-08-01T11:29:10.600593Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# so these are ordered, arbitrarily assigned date representations spaced by...\n",
    "pd.Series(dates.unique()).diff().unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-23T21:13:21.452630Z",
     "start_time": "2018-07-23T21:12:10.360788Z"
    },
    "hidden": true
   },
   "source": [
    " Can't create datetime augmentations: the day of week, time of month, etc. may be important, but I don't know the baseline date, so cannot hardcode unknown information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:13.326960Z",
     "start_time": "2018-08-01T11:29:13.305241Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# add order val in case its useful\n",
    "faults_df.insert(0, 'order', range(0, len(faults_df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "As long as a date transformation is applied uniformly such that relative information is preserved, creating a datetime object with an arbitrary reference is okay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:21.415260Z",
     "start_time": "2018-08-01T11:29:14.097285Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "faults_df['datetime'] = faults_df['date'].apply(lambda x: pd.to_datetime(x, unit='d'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:23.649375Z",
     "start_time": "2018-08-01T11:29:23.627489Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# isolate devices\n",
    "devices = faults_df.device\n",
    "devices.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:24.200968Z",
     "start_time": "2018-08-01T11:29:24.179634Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# num of devices\n",
    "len(devices.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:24.561333Z",
     "start_time": "2018-08-01T11:29:24.365080Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# how many unique chars in each field of device name?\n",
    "maxchars = max(devices.apply(lambda x: len(x)))\n",
    "for nchar in range(0, maxchars):\n",
    "    print('Char position {} :: num unique {}'.format(nchar,\n",
    "                                                     len(devices.apply(lambda x: x[nchar]).unique())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "First four chars don't vary much; are there flavors of naming conventions, e.g. device classes?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:24.925600Z",
     "start_time": "2018-08-01T11:29:24.693165Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# group chars and calc uniqueness\n",
    "maxchars = max(devices.apply(lambda x: len(x)))\n",
    "for nchars in range(1, maxchars):\n",
    "    print('num chars {} :: num unique {}'.format(nchars,\n",
    "                                              len(devices.apply(lambda x: x[:nchars]).unique())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "There appear to be types of devices <a id=\"deviceclasses\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:25.416662Z",
     "start_time": "2018-08-01T11:29:25.377732Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save the class1\n",
    "faults_df['device_class1'] = faults_df.device.apply(lambda x: x[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:26.136353Z",
     "start_time": "2018-08-01T11:29:26.114739Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "faults_df.device_class1.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:26.478495Z",
     "start_time": "2018-08-01T11:29:26.287278Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# remove class1 and look for more\n",
    "sub_devices = devices.apply(lambda x: x[3:])\n",
    "maxchars = max(sub_devices.apply(lambda x: len(x)))\n",
    "for nchars in range(1, maxchars):\n",
    "    print('num chars {} :: num unique {}'.format(nchars,\n",
    "                                              len(sub_devices.apply(lambda x: x[:nchars]).unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:26.521457Z",
     "start_time": "2018-08-01T11:29:26.489161Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# maybe 4th char is a submodel, generation...? small enough to encode\n",
    "faults_df['device_class2'] = faults_df.device.apply(lambda x: x[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:26.894037Z",
     "start_time": "2018-08-01T11:29:26.873556Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "faults_df.device_class2.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:27.591455Z",
     "start_time": "2018-08-01T11:29:27.436026Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# remove class1 and class2 and look for more\n",
    "sub_devices = devices.apply(lambda x: x[4:])\n",
    "maxchars = max(sub_devices.apply(lambda x: len(x)))\n",
    "for nchars in range(1, maxchars):\n",
    "    print('num chars {} :: num unique {}'.format(nchars,\n",
    "                                              len(sub_devices.apply(lambda x: x[:nchars]).unique())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:28.090282Z",
     "start_time": "2018-08-01T11:29:28.049348Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# lump remainder together\n",
    "faults_df['device_class3'] = faults_df.device.apply(lambda x: x[4:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:28.861835Z",
     "start_time": "2018-08-01T11:29:28.839602Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "faults_df.device_class3.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:29.343372Z",
     "start_time": "2018-08-01T11:29:29.297264Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# check that original device == device clases\n",
    "assert sum(faults_df.device != faults_df.device_class1 + faults_df.device_class2 + faults_df.device_class3) == 0, \"Device classification failed\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Trends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:02:29.348096Z",
     "start_time": "2018-08-01T11:00:49.219853Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Rohrshach approach: \"interactively\" pick random device that had a failure and look at attribute trends in time\n",
    "while True:\n",
    "    try:\n",
    "        temp = faults_df[(faults_df.failure == 1)]\n",
    "        rand_device = temp.sample(1).device.values[0]\n",
    "        temp = faults_df[(faults_df.device == rand_device)]\n",
    "        temp = temp.filter(regex='^a|date$')\n",
    "        temp['a1cs'] = temp.a1.cumsum()\n",
    "        scaler = preprocessing.MinMaxScaler()\n",
    "        scaled_values = scaler.fit_transform(temp)\n",
    "        temp.loc[:,:] = scaled_values\n",
    "        temp.plot(x='date', style='.-')\n",
    "        display.display(plt.gcf())\n",
    "        display.clear_output(wait=True)\n",
    "        time.sleep(3)\n",
    "    except KeyboardInterrupt:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "+ a1 looks unaggregated, a daily value possibly -- need to add a1 cumulative sum feature\n",
    "+ All other attributes **_appear_** to be aggregates -- verify below\n",
    "+ a2 and a4 stand-out as ramping up before failure -- need to add linear trend feature at each time step\n",
    "+ A few attributes appear to be correlated -- verify below <a id=\"correlation\"></a>\n",
    "\n",
    "Problem statement doesn't clearly specify if these are:\n",
    "+ daily aggregates\n",
    "+ aggregates of daily"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:40.797308Z",
     "start_time": "2018-08-01T11:29:40.687436Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# add cumulative sum feature just for a1\n",
    "#faults_df['a1_cumsum'] = faults_df.sort_values(by=['device','order']).groupby('device').a1.cumsum()\n",
    "\n",
    "# add for all\n",
    "cols = faults_df.filter(regex='^a.$')\n",
    "for col in cols:\n",
    "    newcol = col+'_cumsum'\n",
    "    faults_df[newcol] = faults_df.sort_values(by=['device','order']).groupby('device')[col].cumsum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:25:54.008439Z",
     "start_time": "2018-08-01T11:25:53.990440Z"
    },
    "hidden": true
   },
   "source": [
    "Do any values besides a1 go down?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:29:42.463537Z",
     "start_time": "2018-08-01T11:29:42.015891Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# sort, diff, and count\n",
    "cols = ['a2', 'a3', 'a4', 'a5', 'a6', 'a7', 'a8', 'a9', 'failure']\n",
    "temp = faults_df.sort_values(by=['device','order']).groupby('device')[cols].diff()\n",
    "temp.lt(0).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "There are a choice few: a2, a5, a7, a8, failure. The failures in particular mean they've been cleared/reset. If these are aggregate attributes as they appear, then values that drop, even if rarely, are experiencing some kind of remediation/service."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T11:31:00.054271Z",
     "start_time": "2018-08-01T11:30:59.997954Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# look at random example\n",
    "a = random.choice(temp[(temp < 0).any(1)].index.tolist())\n",
    "print(temp.loc[a])\n",
    "print('\\n*********************\\n')\n",
    "print('Look at index {}'.format(a))\n",
    "b = faults_df[(faults_df.order == a)].device.values[0]\n",
    "faults_df[(faults_df.device == b)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Need to manually add feature to capture service history:\n",
    "+ Cumulative sum of a* service count\n",
    "+ Cumulative sum of a* service magnitude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:09:51.841356Z",
     "start_time": "2018-08-01T12:09:50.362300Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# service features\n",
    "cols = ['a2', 'a5', 'a7', 'a8']\n",
    "for col in cols:\n",
    "    \n",
    "    # create columns and init\n",
    "    service_mag = col+'_service_mag'\n",
    "    service_count = col+'_service_count'\n",
    "    faults_df[service_mag] = 0\n",
    "    faults_df[service_count] = 0\n",
    "    \n",
    "    # fill with diff and remove natural agg\n",
    "    faults_df[service_mag] = faults_df.sort_values(by=['device', 'order']).groupby('device')[col].diff().fillna(0)\n",
    "    faults_df.loc[faults_df[service_mag] > 0, service_mag] = 0\n",
    "    \n",
    "    # denote service count and cumsum\n",
    "    faults_df.loc[faults_df[service_mag] < 0, service_count] = 1\n",
    "    faults_df[service_count] = faults_df.sort_values(by=['device','order']).groupby('device')[service_count].cumsum()\n",
    "    faults_df[service_mag] = faults_df.sort_values(by=['device','order']).groupby('device')[service_mag].cumsum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:13:42.384880Z",
     "start_time": "2018-08-01T12:13:41.994748Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# handle failure manually to deal with naming convention change   \n",
    "col = 'failure'\n",
    "service_mag = 'f_clear_mag'\n",
    "service_count = 'f_clear_count'\n",
    "faults_df[service_mag] = 0\n",
    "faults_df[service_count] = 0\n",
    "\n",
    "# fill with diff and remove natural agg\n",
    "faults_df[service_mag] = faults_df.sort_values(by=['device', 'order']).groupby('device')[col].diff().fillna(0)\n",
    "faults_df.loc[faults_df[service_mag] > 0, service_mag] = 0\n",
    "\n",
    "# denote service count and cumsum\n",
    "faults_df.loc[faults_df[service_mag] < 0, service_count] = 1\n",
    "faults_df[service_count] = faults_df.sort_values(by=['device','order']).groupby('device')[service_count].cumsum()\n",
    "faults_df[service_mag] = faults_df.sort_values(by=['device','order']).groupby('device')[service_mag].cumsum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:17:52.243531Z",
     "start_time": "2018-08-01T12:17:48.709503Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# look for correlations in numerical values\n",
    "fields = [c for c in faults_df.columns if faults_df[c].dtype == 'int64']\n",
    "visualize_correlations(faults_df, fields)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Indeed, some attributes are correlated, but A7 and A8 identical? <a id=\"a7a8\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:18:47.153990Z",
     "start_time": "2018-08-01T12:18:47.020218Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# are attribute 7 and 8 identical?\n",
    "a7_str = ''.join(faults_df.a7.astype(str).values)\n",
    "a7_hash = hashlib.md5(a7_str.encode('utf-8')).hexdigest()\n",
    "a8_str = ''.join(faults_df.a8.astype(str).values)\n",
    "a8_hash = hashlib.md5(a8_str.encode('utf-8')).hexdigest()\n",
    "a7_hash == a8_hash"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "This is telemetry data, so maybe A7 and A8 are the same measurement with different units, e.g. imperial and metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:19:44.445802Z",
     "start_time": "2018-08-01T12:19:44.416950Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# 7 was my fav num as a kid (Boomer Esiason wore it for the Bengals)\n",
    "# drop attr8\n",
    "drop_cols = faults_df.filter(regex='^a8').columns.tolist()\n",
    "faults_df.drop(drop_cols, inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:20:55.524346Z",
     "start_time": "2018-08-01T12:20:51.280545Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# correlation matrix\n",
    "corrmat = faults_df.corr(method='spearman')\n",
    "\n",
    "# Draw the heatmap using seaborn\n",
    "sns.clustermap(corrmat, vmax=0.6, vmin=-0.6, square=True, cmap=sns_cmap);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Weak similarities, but nothing so glaring we should break attributes apart."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:21:39.460407Z",
     "start_time": "2018-08-01T12:21:39.269936Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "faults_df.to_pickle('faults_df.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Normally, we would put in a great deal of effort and time with SME's to build even more features. For example, cumulative sums of aggregated values over time should be important, and we can engineer such a set of features with:\n",
    "```python\n",
    "for col in ['a'+str(i) for i in range(1,10)]:\n",
    "    newcol = col+'_cumsum'\n",
    "    try:\n",
    "        faults_df.drop([newcol], axis=1, inplace=True)\n",
    "    except:\n",
    "        pass\n",
    "    try:\n",
    "        faults_df[newcol] = faults_df.groupby(['device'])[col].cumsum()\n",
    "    except:\n",
    "        continue\n",
    "```\n",
    "\n",
    "However, what about all the other stats terms that may have embeded info (count, max, min, mean, stddev, etc.). This is laborious to construct all those features. Alternatively, we could use a carefully tuned NN architecture to find those features via the model. This is also laborious. Another option: create an ensemble of models that extract predictive information from features it creates.\n",
    "\n",
    "As a start, let's instead try deep feature synthesis and drive the problem to the left (data) rather than to the right (model). This can be accomplished with time series using [tensor deep feature synthesis][tdfs].\n",
    "\n",
    "[tdfs]: https://docs.featuretools.com/automated_feature_engineering/handling_time.html#creating-a-3-dimensional-feature-tensor-using-multiple-cutoff-times-from-make-temporal-cutoffs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Cut-off Times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:23:32.252141Z",
     "start_time": "2018-08-01T12:23:32.003413Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# list of unique devices\n",
    "unique_devices = faults_df.filter(regex='device').drop_duplicates()\n",
    "instance_ids = sorted(unique_devices.device.tolist())\n",
    "\n",
    "# capture start dates per device -- ORDER MATTERS\n",
    "device_start_dates = faults_df.sort_values(by=['device']).groupby('device')['datetime'].min().tolist()\n",
    "\n",
    "# capture end dates per device -- ORDER MATTERS\n",
    "device_end_dates = faults_df.sort_values(by=['device']).groupby('device')['datetime'].max().tolist()\n",
    "\n",
    "# create reference data frame\n",
    "cutoffs = pd.DataFrame(\n",
    "    {\n",
    "        'device': instance_ids,\n",
    "        'start': device_start_dates,\n",
    "        'end': device_end_dates,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:23:36.395874Z",
     "start_time": "2018-08-01T12:23:34.577087Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# gut check on 100 random entries that reassembly of dates and devices match\n",
    "for n in range(0, 100):\n",
    "    \n",
    "    # pick random device\n",
    "    rand_device = faults_df.sample(1).device.values[0]\n",
    "    \n",
    "    # get original min/max dates in np datetime\n",
    "    min_test = faults_df[(faults_df.device == rand_device)]['datetime'].min().to_datetime64()\n",
    "    max_test = faults_df[(faults_df.device == rand_device)]['datetime'].max().to_datetime64()\n",
    "    \n",
    "    # get cutoff values\n",
    "    rand_min = cutoffs[(cutoffs.device == rand_device)].start.values[0]\n",
    "    rand_max = cutoffs[(cutoffs.device == rand_device)].end.values[0]\n",
    "    \n",
    "    # compare\n",
    "    assert rand_min == min_test, 'Date mismatch'\n",
    "    assert rand_max == max_test, 'Date mismatch'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:23:38.604935Z",
     "start_time": "2018-08-01T12:23:36.968531Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# for each device, begin at start and increment by 1-day until the end\n",
    "temporal_cutoffs = ft.make_temporal_cutoffs(\n",
    "    instance_ids=cutoffs['device'],\n",
    "    start=cutoffs['start'],\n",
    "    cutoffs=cutoffs['end'],\n",
    "    window_size='1d')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:23:39.524144Z",
     "start_time": "2018-08-01T12:23:39.500884Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save\n",
    "temporal_cutoffs.to_pickle('temporal_cutoffs.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Entity Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:33:20.561155Z",
     "start_time": "2018-08-01T12:33:20.548153Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Create new entityset\n",
    "es = ft.EntitySet(id='timeseries')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:33:21.141240Z",
     "start_time": "2018-08-01T12:33:20.747775Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# create entity for faults\n",
    "es = es.entity_from_dataframe(\n",
    "    entity_id='faults',\n",
    "    dataframe=faults_df,\n",
    "    index='uuid',\n",
    "    time_index='datetime')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:33:28.069555Z",
     "start_time": "2018-08-01T12:33:27.778424Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "es.normalize_entity(base_entity_id='faults',\n",
    "                    new_entity_id='unique_devices',\n",
    "                    index='device',\n",
    "                    make_time_index=True,\n",
    "                    additional_variables=[\"device_class1\", \"device_class2\", \"device_class3\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:33:42.229636Z",
     "start_time": "2018-08-01T12:33:42.214459Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "es.entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:33:46.316289Z",
     "start_time": "2018-08-01T12:33:43.082807Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save results\n",
    "es.to_parquet('device_faults_entity_set')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Deep Feature Synthesis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:51.982351Z",
     "start_time": "2018-08-01T12:53:51.399990Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# reload data\n",
    "es = ft.read_parquet('device_faults_entity_set')\n",
    "temporal_cutoffs = pd.read_pickle('temporal_cutoffs.pkl')\n",
    "faults_df = pd.read_pickle('faults_df.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "The below step can be parallelized using Dask (`n_jobs=-1`) since it's an embarassingly parallel problem due to independence:\n",
    "\n",
    "$device_i \\perp cutoff_j \\perp feature matrix_j$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:53:54.184967Z",
     "start_time": "2018-08-01T12:53:54.151396Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# reminder of the dfs primitives\n",
    "ft.primitives.list_primitives()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:54:19.839228Z",
     "start_time": "2018-08-01T12:54:19.824302Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# primitive selections\n",
    "agg_primitives=[\"mean\", \"median\", \"mode\",\n",
    "                \"max\", \"min\", \"std\", \"skew\",\n",
    "                \"count\", \"trend\",\n",
    "                \"last\", \"time_since_last\", \"avg_time_between\"]\n",
    "\n",
    "# not obvious that transforms needed\n",
    "# options, if so: \"percentile\", \"time_since_previous\", \"multiply\", \"mod\", \"add\", \"subtract\", \"diff\", \"divide\"\n",
    "trans_primitives=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T12:54:33.966668Z",
     "start_time": "2018-08-01T12:54:33.949988Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# don't waste time on data that isn't relevant\n",
    "ignore_variables = {'faults': ['uuid',\n",
    "                               'order',\n",
    "                               'date',\n",
    "                               'device_class1',\n",
    "                               'device_class2',\n",
    "                               'device_class3',\n",
    "                               'failure']\n",
    "                   }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "I'm not going \"deep\" with stacking of feature primitives because tests suggest there is no additional information gained at the expense of time-consuming computation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:04:27.351889Z",
     "start_time": "2018-08-01T12:55:18.622434Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# create the feature tensor\n",
    "feature_tensor, feature_defs = ft.dfs(\n",
    "    entityset=es,\n",
    "    target_entity='unique_devices',\n",
    "    cutoff_time=temporal_cutoffs,\n",
    "    cutoff_time_in_index=True,\n",
    "    agg_primitives=agg_primitives,\n",
    "    trans_primitives=trans_primitives,\n",
    "    ignore_variables=ignore_variables,\n",
    "    max_depth=1,\n",
    "    max_features=-1,\n",
    "    chunk_size=\"cutoff time\",\n",
    "    n_jobs=-1,\n",
    "    verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:13:17.801055Z",
     "start_time": "2018-08-01T13:13:17.159390Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save intermediate output\n",
    "hdf_file = 'raw_dfs_feature_tensor.h5'\n",
    "table_name = 'feature_tensor'\n",
    "save_hdf5(feature_tensor, hdf_file, table_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## QA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:13:23.286008Z",
     "start_time": "2018-08-01T13:13:23.023537Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# clean-up data and don't disturb original\n",
    "dfs_qa = feature_tensor.copy().reset_index()\n",
    "dfs_qa.rename(columns={'time':'datetime'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:13:24.147061Z",
     "start_time": "2018-08-01T13:13:24.129031Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# data in == data out\n",
    "orig_len = len(dfs_qa)\n",
    "new_len = len(feature_tensor)\n",
    "assert orig_len == new_len, 'There is new or missing data'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "The '1d' time windows in DFS automatically create steps in time where there may not be data, e.g.:\n",
    "* original: [1 2 3 4 8 9]\n",
    "* dfs: [1 2 3 4 **_5 6 7_** 8 9]\n",
    "\n",
    "There are added values. Simply joing back against original dataframe (need to do anyways) fixes this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:13:28.666766Z",
     "start_time": "2018-08-01T13:13:28.229096Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# per problem statement, one device per day?\n",
    "check = dfs_qa.datetime.astype(str) + dfs_qa.device.astype(str)\n",
    "assert len(check) == len(check.drop_duplicates()), 'More than one device per day'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:13:32.053171Z",
     "start_time": "2018-08-01T13:13:29.737484Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# rid ourselves of the junk datetime features due to arbitrary date assignment\n",
    "dropcols = dfs_qa.filter(regex='DAY|MONTH|WEEKDAY|YEAR').columns.tolist()\n",
    "nunique = dfs_qa.apply(pd.Series.nunique)\n",
    "dropcols.extend(nunique[nunique == 1].index.tolist())\n",
    "list(set(dropcols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:13:51.283047Z",
     "start_time": "2018-08-01T13:13:51.215422Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# drop them\n",
    "dfs_qa.drop(dropcols, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:14:00.301799Z",
     "start_time": "2018-08-01T13:13:59.592896Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# merge dfs with original faults\n",
    "faults_dfs_df = pd.merge(faults_df, dfs_qa, on=['device','datetime'], suffixes=('', '_y'))\n",
    "assert len(faults_dfs_df) == len(faults_df), 'Something went wrong in merge'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:14:01.805749Z",
     "start_time": "2018-08-01T13:14:01.583863Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# some duplicate columns in join\n",
    "dropcols = faults_dfs_df.filter(regex='_y')\n",
    "faults_dfs_df.drop(dropcols, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:14:22.359482Z",
     "start_time": "2018-08-01T13:14:03.294914Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# 10 min runtime for above on laptop, save output just in case\n",
    "hdf_file = 'qa_dfs_feature_tensor.h5'\n",
    "table_name = 'feature_tensor'\n",
    "save_hdf5(faults_dfs_df, hdf_file, table_name, format='table')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:14:59.648572Z",
     "start_time": "2018-08-01T13:14:58.618964Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# reload data\n",
    "hdf_file = 'qa_dfs_feature_tensor.h5'\n",
    "table_name = 'feature_tensor'\n",
    "Xy = pd.read_hdf(hdf_file, table_name)\n",
    "feature_names = Xy.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:03.532326Z",
     "start_time": "2018-08-01T13:15:00.850664Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# some dfs are uniform\n",
    "nunique = Xy.apply(pd.Series.nunique)\n",
    "dropcols = nunique[nunique == 1].index.tolist()\n",
    "sorted(dropcols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:04.267059Z",
     "start_time": "2018-08-01T13:15:04.251497Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# remove uniform cols\n",
    "if len(dropcols) > 0:\n",
    "    Xy.drop(dropcols, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:05.659364Z",
     "start_time": "2018-08-01T13:15:05.626792Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "Xy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:13.336614Z",
     "start_time": "2018-08-01T13:15:13.316852Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:26.351144Z",
     "start_time": "2018-08-01T13:15:26.332906Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# columns to encode\n",
    "encode_cols = [\n",
    "    'device_class1',\n",
    "    'device_class2',\n",
    "    'device_class3'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:28.769830Z",
     "start_time": "2018-08-01T13:15:27.605154Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# one-hot encode the categoricals\n",
    "Xy = pd.concat([Xy, pd.get_dummies(Xy[encode_cols])], axis=1)\n",
    "Xy.drop(encode_cols, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:29.664466Z",
     "start_time": "2018-08-01T13:15:29.492894Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# junk features\n",
    "dropcols = Xy.filter(regex='MODE\\(faults.device_class')\n",
    "Xy.drop(dropcols, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:33.340007Z",
     "start_time": "2018-08-01T13:15:32.444776Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save intermediate output\n",
    "hdf_file = 'model_dfs_feature_tensor.h5'\n",
    "table_name = 'feature_tensor'\n",
    "save_hdf5(Xy, hdf_file, table_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "This is a gnarly imbalanced class problem. Traditional options:\n",
    "* Under-sample majority class: viable option since time-dependent values are encoded at every time step.\n",
    "* Stratified resampling: viable option via xgboost.\n",
    "* Over-sample minority class: this is time series data, so arbitrarily throwing out samples potentially removes valuable information about aggregations or leading indicators to failure.\n",
    "* Synthesis: No. I intensely dislike manufacturing data unless there are strong underlying laws/principles, e.g. in physics, which constrain the fake data.\n",
    "\n",
    "[imblearn][imb] is a good library to automate many of the methods needed and simplify the modeling exercise. Rather than go the route of \"black box,\" I prefer to focus on the data at-hand and drive the problem toward features and not toward models.\n",
    "\n",
    "[imb]:http://contrib.scikit-learn.org/imbalanced-learn/stable/index.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:42.007661Z",
     "start_time": "2018-08-01T13:15:41.653536Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# reload data from prior run as a quick start\n",
    "faults_df = pd.read_pickle('faults_df.pkl')\n",
    "hdf_file = 'model_dfs_feature_tensor.h5'\n",
    "table_name = 'feature_tensor'\n",
    "Xy = pd.read_hdf(hdf_file, table_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:44.820587Z",
     "start_time": "2018-08-01T13:15:43.792013Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# ID cols to use in modeling\n",
    "ignore = ['date', 'device', 'datetime']\n",
    "ignore.extend(Xy.filter(regex='failure').columns)  # anything with failure info\n",
    "predicting = 'failure'\n",
    "predictors = [c for c in Xy.columns if (c != predicting and c not in ignore)]\n",
    "\n",
    "# isolate y from X\n",
    "y = Xy[predicting]\n",
    "X = Xy[predictors]\n",
    "\n",
    "# split data into train and test sets\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size=0.33, random_state=7)\n",
    "\n",
    "# remove uuid and track as index\n",
    "idx_train = X_train.uuid.values\n",
    "idx_test = X_test.uuid.values\n",
    "X_train.drop('uuid', axis=1, inplace=True)\n",
    "X_test.drop('uuid', axis=1, inplace=True)\n",
    "feature_names = X_train.columns.values\n",
    "assert set(feature_names) == set(X_test.columns.values), 'Train/Test feature name mismatch'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:15:51.571420Z",
     "start_time": "2018-08-01T13:15:48.627667Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# make the train test vers for xgb\n",
    "dtrain = xgb.DMatrix(X_train, y_train, feature_names=feature_names, nthread=-1)\n",
    "dtest = xgb.DMatrix(X_test, y_test, feature_names=feature_names, nthread=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Naive Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Before tuning, I want to be informed on what a simple model learns, and what an overfit model learns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:16:18.165774Z",
     "start_time": "2018-08-01T13:16:14.140053Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define a naive model\n",
    "xgb_params = {\n",
    "    'objective': 'binary:logistic',\n",
    "    'seed': 0\n",
    "}\n",
    "model = xgb.train(xgb_params, dtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:16:25.486719Z",
     "start_time": "2018-08-01T13:16:24.897942Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# plot the important features #\n",
    "fig, ax = plt.subplots(figsize=(10, 18))\n",
    "xgb.plot_importance(model, height=0.6, ax=ax, max_num_features=10)\n",
    "fig.savefig('feature_importance.png', bbox_inches='tight', pad_inches=1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:16:30.961174Z",
     "start_time": "2018-08-01T13:16:30.943498Z"
    },
    "hidden": true
   },
   "source": [
    "<a id=\"#naivemodel\"></a>\n",
    "Is a7 really that important?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:17:01.758428Z",
     "start_time": "2018-08-01T13:17:01.729946Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# prediction on test set\n",
    "y_pred = model.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T13:20:01.797362Z",
     "start_time": "2018-08-01T13:20:01.576916Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# let's see if this makes sense\n",
    "temp = faults_df.filter(regex='^a.$|failure')\n",
    "scaler = preprocessing.MinMaxScaler()\n",
    "scaled_values = scaler.fit_transform(temp)\n",
    "temp.loc[:,:] = scaled_values\n",
    "temp_melt = pd.melt(temp, \"failure\", var_name=\"measurement\")\n",
    "temp_melt.groupby(['measurement', 'failure']).std()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Standard deviation for a7 is unusually larger when there's a failure."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Tuned Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:15:33.801117Z",
     "start_time": "2018-08-01T13:53:44.752019Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# control for class imbalance\n",
    "ratio = float(np.sum(y == 0)) / np.sum(y == 1)\n",
    "\n",
    "# model params\n",
    "params = {\n",
    "    'max_depth': 9,\n",
    "    'min_child_weight': 100,\n",
    "    'subsample': 0.9,  # random sample %\n",
    "    'colsample_bytree': 0.4,  # random col %\n",
    "    'eta': 0.01,  # learning rate\n",
    "    'reg_alpha': 0.5,  # L1 reg\n",
    "    'lambda': 0.95,  # L2 reg\n",
    "    'gamma': 0.1,  # loss split\n",
    "    'seed': 6,\n",
    "    'n_estimators': 1000,\n",
    "    'scale_pos_weight': ratio,  # VERY IMPORTANT FOR THIS PARTICULAR PROBLEM\n",
    "    'objective': 'binary:logistic',\n",
    "    'eval_metric': 'auc',\n",
    "    'grow_policy': 'depthwise'\n",
    "}\n",
    "\n",
    "# train the model\n",
    "model = train_xgb_model(X_train, y_train, params, kfolds=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:16:06.201964Z",
     "start_time": "2018-08-01T14:16:05.546140Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# feature importance plot\n",
    "fig, ax = plt.subplots(figsize=(10, 18))\n",
    "xgb.plot_importance(model, height=0.3, ax=ax, max_num_features=50)\n",
    "fig.savefig('feature_importance.png', bbox_inches='tight', pad_inches=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:16:32.485161Z",
     "start_time": "2018-08-01T14:16:32.453216Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# predict\n",
    "y_pred = model.predict(dtest, ntree_limit=model.best_ntree_limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:18:26.075055Z",
     "start_time": "2018-08-01T14:18:10.267872Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# binary predictions\n",
    "target_names = ['no fail', 'fail']\n",
    "for prob_thresh in np.arange(0.48, 0.52, 0.01):\n",
    "    y_pred_binary = [1 if x > prob_thresh else 0 for x in y_pred ]\n",
    "    print(prob_thresh)\n",
    "    print(metrics.classification_report(y_test, y_pred_binary, target_names=target_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "prob_thresh, tn, fn, fp, tp = list(), list(), list(), list(), list()\n",
    "for p in np.arange(0.484, 0.488, 0.0001):\n",
    "    y_pred_binary = [1 if x > p else 0 for x in y_pred ]\n",
    "    prob_thresh.append(p)\n",
    "    cm = metrics.confusion_matrix(y_test, y_pred_binary)\n",
    "    tn.append(cm[0][0])\n",
    "    fp.append(cm[0][1])\n",
    "    fn.append(cm[1][0])\n",
    "    tp.append(cm[1][1])\n",
    "tn = [float(i)/max(tn) for i in tn]\n",
    "fp = [float(i)/max(fp) for i in fp]\n",
    "fn = [float(i)/max(fn) for i in fn]\n",
    "tp = [float(i)/max(tp) for i in tp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "plt.plot(prob_thresh, tn, 'r', label='TN')\n",
    "plt.plot(prob_thresh, tp, 'g', label='TP')\n",
    "plt.plot(prob_thresh, fp, 'b', label='FP')\n",
    "plt.plot(prob_thresh, fn, 'y', label='FN')\n",
    "plt.legend(loc='upper left')\n",
    "plt.xlabel('Probability Threshold')\n",
    "plt.ylabel('%Total')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:23:05.823970Z",
     "start_time": "2018-08-01T14:23:05.757406Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# arbitrarily select threshold\n",
    "y_pred_binary = [1 if x >= 0.485372 else 0 for x in y_pred ]\n",
    "\n",
    "# cm\n",
    "ax = sns.heatmap(metrics.confusion_matrix(y_test, y_pred_binary), annot=True, annot_kws={\"size\": 16});\n",
    "ax.set(title='Confusion Matrix', xlabel='Predicted', ylabel='Actual');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:23:06.381679Z",
     "start_time": "2018-08-01T14:23:06.257171Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# visualize prec-recall\n",
    "average_precision = metrics.average_precision_score(y_test, y_pred)\n",
    "precision, recall, _ = metrics.precision_recall_curve(y_test, y_pred)\n",
    "plt.step(recall, precision, color='b', alpha=0.2, where='post')\n",
    "plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.title('Binary Precision-Recall: Avg Precision={0:0.2f}'.format(average_precision))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Presumably, we'd have acceptance criteria for the quality of the model and be able to discern if a 27% FN rate is \"good\" or \"bad.\" Regardless, that's not a good rate and this model needs more tuning and we need more data/better features. However, if, for example, these are hyper-expensive machines, then we might lean toward a higher FP rate at the cost of unneeded service."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T14:23:08.589051Z",
     "start_time": "2018-08-01T14:23:08.473662Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# roc\n",
    "fpr, tpr, threshold = metrics.roc_curve(y_test, y_pred)\n",
    "roc_auc = metrics.auc(fpr, tpr)\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "plt.legend(loc = 'lower right')\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Hyperopt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Objective function defined in [Functions section][#functions]. Hyperopt will minimize the error in a maximum of 100 experiments. See the Hyperopt documentation for more details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "space = {\n",
    "    'max_depth': hp.quniform(\"x_max_depth\", 5, 30, 1),\n",
    "    'min_child_weight': hp.quniform('x_min_child', 1, 10, 1),\n",
    "    'subsample': hp.uniform('x_subsample', 0.8, 1)\n",
    "}\n",
    "\n",
    "trials = Trials()\n",
    "best = fmin(\n",
    "    fn=objective, space=space, algo=tpe.suggest, max_evals=100, trials=trials)\n",
    "\n",
    "print best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## AutoML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "I'm curious what features AutoML's stacked ensemble models suggest are important versus the tuned xgboost."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:21:16.501581Z",
     "start_time": "2018-08-01T22:21:13.759046Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# init the machine\n",
    "h2o.init(min_mem_size_GB=16)\n",
    "h2o.cluster().show_status()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:25:35.558554Z",
     "start_time": "2018-08-01T22:24:43.023497Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# create H2O's special df format\n",
    "train_df_h2o = h2o.H2OFrame(Xy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:25:47.174627Z",
     "start_time": "2018-08-01T22:25:47.038149Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# split data\n",
    "train, test, valid = train_df_h2o.split_frame(ratios=[0.7, 0.15])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:25:53.914044Z",
     "start_time": "2018-08-01T22:25:51.684005Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# isolate predictors and predicting\n",
    "x = train[predictors].columns\n",
    "y = predicting\n",
    "\n",
    "# make sure we have factors\n",
    "train[y] = train[y].asfactor()\n",
    "valid[y] = valid[y].asfactor()\n",
    "test[y] = test[y].asfactor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T20:25:39.368179Z",
     "start_time": "2018-08-01T15:39:50.260366Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# build a model (4h run time on p2.xlarge; use saved version)\n",
    "#aml = H2OAutoML(max_runtime_secs=3600, balance_classes=True)\n",
    "#aml.train(x=x,\n",
    "#          y=y,\n",
    "#          training_frame=train,\n",
    "#          validation_frame=valid,\n",
    "#          leaderboard_frame=test,\n",
    "#          nfolds=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# check leaderboard\n",
    "aml.leaderboard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "AutoML \"likes\" SKEW(a4) and STD(a7) like an earlier version of the tuned xgboost model harness did."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:04:51.444847Z",
     "start_time": "2018-08-01T22:04:45.120379Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# save the model\n",
    "h2o.save_model(aml.leader, path=\"automl_leader_untuned.model\", force=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:21:29.560415Z",
     "start_time": "2018-08-01T22:21:25.417947Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# reload the model\n",
    "h2o_model = h2o.load_model('automl_leader_untuned_model/StackedEnsemble_AllModels_0_AutoML_20180801_153954')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:23:33.017556Z",
     "start_time": "2018-08-01T22:23:32.934962Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# isolate the metalearner model\n",
    "metalearner = h2o.get_model(h2o_model.metalearner()['name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:23:37.382168Z",
     "start_time": "2018-08-01T22:23:37.275004Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# model details\n",
    "metalearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:23:43.589592Z",
     "start_time": "2018-08-01T22:23:43.567761Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# ensemble contribution\n",
    "metalearner.coef_norm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:23:48.499845Z",
     "start_time": "2018-08-01T22:23:48.207265Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# ensemble \"weights\"\n",
    "metalearner.std_coef_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:36:20.658608Z",
     "start_time": "2018-08-01T22:36:20.322412Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# predictions\n",
    "y_pred = metalearner.predict(test)\n",
    "y_pred = y_pred.as_data_frame().predict.tolist()\n",
    "y_test = test[y].as_data_frame().failure.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:36:21.685078Z",
     "start_time": "2018-08-01T22:36:21.546944Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# visualize prec-recall\n",
    "average_precision = metrics.average_precision_score(y_test, y_pred)\n",
    "precision, recall, _ = metrics.precision_recall_curve(y_test, y_pred)\n",
    "plt.step(recall, precision, color='b', alpha=0.2, where='post')\n",
    "plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.title('Binary Precision-Recall: Avg Precision={0:0.2f}'.format(average_precision))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Interesting exercise, but this model requires tuning as it has no predictive power."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:36:40.705258Z",
     "start_time": "2018-08-01T22:36:40.571205Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# cm\n",
    "ax = sns.heatmap(metrics.confusion_matrix(y_test, y_pred), annot=True, annot_kws={\"size\": 16});\n",
    "ax.set(title='Confusion Matrix', xlabel='Predicted', ylabel='Actual');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:37:05.677713Z",
     "start_time": "2018-08-01T22:37:05.558645Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# roc\n",
    "fpr, tpr, threshold = metrics.roc_curve(y_test, y_pred)\n",
    "roc_auc = metrics.auc(fpr, tpr)\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "plt.legend(loc = 'lower right')\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-01T22:37:09.259189Z",
     "start_time": "2018-08-01T22:37:08.864475Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get sys resources back\n",
    "h2o.cluster().shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Vanilla FFNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Why not? Effort, arch design, tuning, not enough data. Lots of reasons. If I were to go this route..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# define I/O\n",
    "input_size = len(featurenames)\n",
    "output_size = 2\n",
    "\n",
    "# a K.I.S.S. model\n",
    "x = Input(shape=(input_size,))\n",
    "layer1 = Dense(25, activation='relu')(x)\n",
    "layer2 = Dense(10)(layer1)\n",
    "layer3 = Dense(35)(layer2)\n",
    "layer4 = Dense(10)(layer3)\n",
    "layer5 = Dense(25, activation=None)(layer4)\n",
    "layer6 = Dense(output_size)(layer5)\n",
    "model = Model(x, layer6)\n",
    "model.compile(\n",
    "    loss='binary_crossentropy',\n",
    "    optimizer=RMSprop(0.001),\n",
    "    metrics=[sensitivity, specificity]\n",
    ")\n",
    "\n",
    "# weight the class imbalance\n",
    "trade_off = 0.5  # for f-score\n",
    "class_weight={\n",
    "    0: 1,\n",
    "    1: ratio * t\n",
    "}\n",
    "\n",
    "# run in 14 \"day\" batches\n",
    "batch_size = len(X_train) / 14\n",
    "\n",
    "# fit a model\n",
    "model.fit(X_train,\n",
    "          y_train,\n",
    "          nb_epoch=500,\n",
    "          batch_size=batch_size,\n",
    "          class_weight=class_weight)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-07-27T21:23:55.742383Z",
     "start_time": "2018-07-27T21:23:55.703553Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "gist": {
   "data": {
    "description": "faults_dfs.ipynb",
    "public": true
   },
   "id": ""
  },
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  },
  "toc": {
   "base_numbering": 1,
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {},
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "threshold": 4,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "257px"
   },
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "686px",
    "left": "638px",
    "right": "20px",
    "top": "110px",
    "width": "731px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}