H2O_AutoML_IPYNB/Deep Water Custom Network Keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import h2o\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "project_path = \"/gtc-2017\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Connect or Start H2O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking whether there is an H2O instance running at http://localhost:54321. connected.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td>H2O cluster uptime:</td>\n",
       "<td>1 hour 56 mins</td></tr>\n",
       "<tr><td>H2O cluster version:</td>\n",
       "<td>3.11.0.226</td></tr>\n",
       "<tr><td>H2O cluster version age:</td>\n",
       "<td>2 days </td></tr>\n",
       "<tr><td>H2O cluster name:</td>\n",
       "<td>root</td></tr>\n",
       "<tr><td>H2O cluster total nodes:</td>\n",
       "<td>1</td></tr>\n",
       "<tr><td>H2O cluster free memory:</td>\n",
       "<td>26.48 Gb</td></tr>\n",
       "<tr><td>H2O cluster total cores:</td>\n",
       "<td>20</td></tr>\n",
       "<tr><td>H2O cluster allowed cores:</td>\n",
       "<td>20</td></tr>\n",
       "<tr><td>H2O cluster status:</td>\n",
       "<td>locked, healthy</td></tr>\n",
       "<tr><td>H2O connection url:</td>\n",
       "<td>http://localhost:54321</td></tr>\n",
       "<tr><td>H2O connection proxy:</td>\n",
       "<td>None</td></tr>\n",
       "<tr><td>H2O internal security:</td>\n",
       "<td>False</td></tr>\n",
       "<tr><td>Python version:</td>\n",
       "<td>2.7.12 final</td></tr></table></div>"
      ],
      "text/plain": [
       "--------------------------  ----------------------\n",
       "H2O cluster uptime:         1 hour 56 mins\n",
       "H2O cluster version:        3.11.0.226\n",
       "H2O cluster version age:    2 days\n",
       "H2O cluster name:           root\n",
       "H2O cluster total nodes:    1\n",
       "H2O cluster free memory:    26.48 Gb\n",
       "H2O cluster total cores:    20\n",
       "H2O cluster allowed cores:  20\n",
       "H2O cluster status:         locked, healthy\n",
       "H2O connection url:         http://localhost:54321\n",
       "H2O connection proxy:\n",
       "H2O internal security:      False\n",
       "Python version:             2.7.12 final\n",
       "--------------------------  ----------------------"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "h2o.init()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import data into H2O."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n",
      "Parse progress: |█████████████████████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "mnist_training = h2o.import_file(project_path+\"/data/mnist-training.csv\")\n",
    "mnist_testing = h2o.import_file(project_path+\"/data/mnist-testing.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Change the \"label\" column to a factor (i.e. categorical feature)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mnist_training[\"label\"] = mnist_training[\"label\"].asfactor()\n",
    "mnist_testing[\"label\"] = mnist_testing[\"label\"].asfactor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Explore Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>uri                                          </th><th style=\"text-align: right;\">  label</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/6453.png </td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/13970.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/21967.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/11277.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/27350.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/32119.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/44201.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/58770.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/39440.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "<tr><td>/gtc-2017/data/mnist_png/training/6/41988.png</td><td style=\"text-align: right;\">      6</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist_training.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get the number of output classes from the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "num_classes = mnist_training[\"label\"].nlevels()[0]\n",
    "print(num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test GPU Driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon May  8 20:52:54 2017       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 378.13                 Driver Version: 378.13                    |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  GeForce GTX 1080    Off  | 0000:02:00.0      On |                  N/A |\r\n",
      "| 28%   38C    P8    11W / 180W |    796MiB /  8112MiB |     28%      Default |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                       GPU Memory |\r\n",
      "|  GPU       PID  Type  Process name                               Usage      |\r\n",
      "|=============================================================================|\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Custom Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import json\n",
    "from keras.layers.core import Dense, Flatten, Reshape\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.pooling import MaxPooling2D\n",
    "from keras import backend as K\n",
    "from keras.objectives import categorical_crossentropy\n",
    "from tensorflow.python.framework import ops\n",
    "\n",
    "def keras_model(w, h, channels, num_classes):\n",
    "    graph = tf.Graph()\n",
    "    with graph.as_default():\n",
    "        size = w * h * channels\n",
    "        \n",
    "        inp = tf.placeholder(tf.float32, [None, size])\n",
    "        \n",
    "        labels = tf.placeholder(tf.float32, [None, num_classes])\n",
    "        \n",
    "        x = Reshape((w, h, channels))(inp)\n",
    "        x = Conv2D(20, (5,5), padding=\"same\", activation=\"relu\")(x)\n",
    "        x = MaxPooling2D((2,2))(x)\n",
    "        \n",
    "        x = Conv2D(50, (5,5), padding=\"same\", activation=\"relu\")(x)\n",
    "        x = MaxPooling2D((2,2))(x)\n",
    "        \n",
    "        x = Flatten()(x)\n",
    "        \n",
    "        x = Dense(num_classes)(x)\n",
    "        \n",
    "        out = Dense(num_classes)(x)\n",
    "        \n",
    "        predictions = tf.nn.softmax(out)\n",
    "        \n",
    "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=out))\n",
    "        train_step = tf.train.AdamOptimizer(1e-3).minimize(loss)\n",
    "        \n",
    "        init_op = tf.global_variables_initializer()\n",
    "        \n",
    "        tf.add_to_collection(ops.GraphKeys.INIT_OP, init_op.name)\n",
    "        tf.add_to_collection(ops.GraphKeys.TRAIN_OP, train_step)\n",
    "        tf.add_to_collection(\"logits\", out)\n",
    "        tf.add_to_collection(\"predictions\", predictions)\n",
    "        \n",
    "        meta = json.dumps({\n",
    "            \"inputs\": {\"batch_image_input\": inp.name, \"categorical_labels\": labels.name},\n",
    "            \"outputs\": {\"categorical_logits\": out.name, \"layers\": \",\".join([m.name for m in tf.get_default_graph().get_operations()])},\n",
    "            \"parameters\": {}\n",
    "        })\n",
    "        \n",
    "        tf.add_to_collection(\"meta\", meta)\n",
    "        \n",
    "        saver = tf.train.Saver()\n",
    "        filename= project_path+\"/models/keras_tensorflow_lenet.meta\"\n",
    "        tf.train.export_meta_graph(filename, saver_def=saver.as_saver_def())\n",
    "        return(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "network_def_path = keras_model(28, 28, 1, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Deep Water Keras (TensorFlow) Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deepwater Model Build progress: |█████████████████████████████████████████| 100%\n"
     ]
    }
   ],
   "source": [
    "from h2o.estimators.deepwater import H2ODeepWaterEstimator\n",
    "model_mnist_mylenet_keras = H2ODeepWaterEstimator(epochs=80,\n",
    "                                            network_definition_file=network_def_path,\n",
    "                                            backend=\"tensorflow\",\n",
    "                                            image_shape=[28,28],\n",
    "                                            channels=1, model_id=\"model_mnist_mylenet_keras\")\n",
    "model_mnist_mylenet_keras.train(x=[\"uri\"], y=\"label\", training_frame=mnist_training, validation_frame=mnist_testing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Details\n",
      "=============\n",
      "H2ODeepWaterEstimator :  Deep Water\n",
      "Model Key:  model_mnist_mylenet_keras\n",
      "\n",
      "\n",
      "ModelMetricsMultinomial: deepwater\n",
      "** Reported on train data. **\n",
      "\n",
      "MSE: 0.00607328251334\n",
      "RMSE: 0.0779312678797\n",
      "LogLoss: 0.0258161830826\n",
      "Mean Per-Class Error: 0.00759141551164\n",
      "Confusion Matrix: vertical: actual; across: predicted\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>2</b></td>\n",
       "<td><b>3</b></td>\n",
       "<td><b>4</b></td>\n",
       "<td><b>5</b></td>\n",
       "<td><b>6</b></td>\n",
       "<td><b>7</b></td>\n",
       "<td><b>8</b></td>\n",
       "<td><b>9</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>972.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0030769</td>\n",
       "<td>3 / 975</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>1145.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>2.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0052129</td>\n",
       "<td>6 / 1,151</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>1006.0</td>\n",
       "<td>4.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>8.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0156556</td>\n",
       "<td>16 / 1,022</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>3.0</td>\n",
       "<td>1025.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0077444</td>\n",
       "<td>8 / 1,033</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>958.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0</td>\n",
       "<td>6.0</td>\n",
       "<td>0.0113519</td>\n",
       "<td>11 / 969</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>5.0</td>\n",
       "<td>0.0</td>\n",
       "<td>839.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0129412</td>\n",
       "<td>11 / 850</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>992.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0010070</td>\n",
       "<td>1 / 993</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1055.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0037771</td>\n",
       "<td>4 / 1,059</td></tr>\n",
       "<tr><td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>936.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0031949</td>\n",
       "<td>3 / 939</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>7.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>4.0</td>\n",
       "<td>0.0</td>\n",
       "<td>992.0</td>\n",
       "<td>0.0119522</td>\n",
       "<td>12 / 1,004</td></tr>\n",
       "<tr><td>973.0</td>\n",
       "<td>1150.0</td>\n",
       "<td>1011.0</td>\n",
       "<td>1035.0</td>\n",
       "<td>965.0</td>\n",
       "<td>840.0</td>\n",
       "<td>1003.0</td>\n",
       "<td>1065.0</td>\n",
       "<td>951.0</td>\n",
       "<td>1002.0</td>\n",
       "<td>0.0075038</td>\n",
       "<td>75 / 9,995</td></tr></table></div>"
      ],
      "text/plain": [
       "0    1     2     3     4    5    6     7     8    9     Error       Rate\n",
       "---  ----  ----  ----  ---  ---  ----  ----  ---  ----  ----------  ----------\n",
       "972  0     0     0     0    0    2     0     1    0     0.00307692  3 / 975\n",
       "0    1145  1     0     0    0    2     2     1    0     0.00521286  6 / 1,151\n",
       "0    2     1006  4     0    0    1     1     8    0     0.0156556   16 / 1,022\n",
       "0    0     3     1025  0    1    0     1     3    0     0.00774443  8 / 1,033\n",
       "0    1     0     0     958  0    2     2     0    6     0.0113519   11 / 969\n",
       "0    0     0     5     0    839  3     0     1    2     0.0129412   11 / 850\n",
       "0    0     0     0     0    0    992   0     1    0     0.00100705  1 / 993\n",
       "0    1     1     1     0    0    0     1055  0    1     0.00377715  4 / 1,059\n",
       "1    0     0     0     0    0    1     0     936  1     0.00319489  3 / 939\n",
       "0    1     0     0     7    0    0     4     0    992   0.0119522   12 / 1,004\n",
       "973  1150  1011  1035  965  840  1003  1065  951  1002  0.00750375  75 / 9,995"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top-10 Hit Ratios: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>k</b></td>\n",
       "<td><b>hit_ratio</b></td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>0.9924962</td></tr>\n",
       "<tr><td>2</td>\n",
       "<td>0.9989995</td></tr>\n",
       "<tr><td>3</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>4</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>5</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>6</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>7</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>8</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>9</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>10</td>\n",
       "<td>1.0</td></tr></table></div>"
      ],
      "text/plain": [
       "k    hit_ratio\n",
       "---  -----------\n",
       "1    0.992496\n",
       "2    0.999\n",
       "3    1\n",
       "4    1\n",
       "5    1\n",
       "6    1\n",
       "7    1\n",
       "8    1\n",
       "9    1\n",
       "10   1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ModelMetricsMultinomial: deepwater\n",
      "** Reported on validation data. **\n",
      "\n",
      "MSE: 0.0117393816579\n",
      "RMSE: 0.108348427113\n",
      "LogLoss: 0.0580633801598\n",
      "Mean Per-Class Error: 0.0136599369188\n",
      "Confusion Matrix: vertical: actual; across: predicted\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>0</b></td>\n",
       "<td><b>1</b></td>\n",
       "<td><b>2</b></td>\n",
       "<td><b>3</b></td>\n",
       "<td><b>4</b></td>\n",
       "<td><b>5</b></td>\n",
       "<td><b>6</b></td>\n",
       "<td><b>7</b></td>\n",
       "<td><b>8</b></td>\n",
       "<td><b>9</b></td>\n",
       "<td><b>Error</b></td>\n",
       "<td><b>Rate</b></td></tr>\n",
       "<tr><td>968.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>6.0</td>\n",
       "<td>1.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0122449</td>\n",
       "<td>12 / 980</td></tr>\n",
       "<tr><td>1.0</td>\n",
       "<td>1123.0</td>\n",
       "<td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>5.0</td>\n",
       "<td>1.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0105727</td>\n",
       "<td>12 / 1,135</td></tr>\n",
       "<tr><td>6.0</td>\n",
       "<td>2.0</td>\n",
       "<td>1012.0</td>\n",
       "<td>5.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>4.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0193798</td>\n",
       "<td>20 / 1,032</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1002.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>4.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0079208</td>\n",
       "<td>8 / 1,010</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>964.0</td>\n",
       "<td>0.0</td>\n",
       "<td>4.0</td>\n",
       "<td>3.0</td>\n",
       "<td>2.0</td>\n",
       "<td>8.0</td>\n",
       "<td>0.0183299</td>\n",
       "<td>18 / 982</td></tr>\n",
       "<tr><td>1.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>6.0</td>\n",
       "<td>0.0</td>\n",
       "<td>874.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>5.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0201794</td>\n",
       "<td>18 / 892</td></tr>\n",
       "<tr><td>2.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>952.0</td>\n",
       "<td>0.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0062630</td>\n",
       "<td>6 / 958</td></tr>\n",
       "<tr><td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>7.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1017.0</td>\n",
       "<td>1.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0107004</td>\n",
       "<td>11 / 1,028</td></tr>\n",
       "<tr><td>2.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0</td>\n",
       "<td>1.0</td>\n",
       "<td>0.0</td>\n",
       "<td>0.0</td>\n",
       "<td>966.0</td>\n",
       "<td>2.0</td>\n",
       "<td>0.0082136</td>\n",
       "<td>8 / 974</td></tr>\n",
       "<tr><td>1.0</td>\n",
       "<td>4.0</td>\n",
       "<td>0.0</td>\n",
       "<td>5.0</td>\n",
       "<td>7.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>3.0</td>\n",
       "<td>0.0</td>\n",
       "<td>986.0</td>\n",
       "<td>0.0227948</td>\n",
       "<td>23 / 1,009</td></tr>\n",
       "<tr><td>981.0</td>\n",
       "<td>1133.0</td>\n",
       "<td>1021.0</td>\n",
       "<td>1021.0</td>\n",
       "<td>971.0</td>\n",
       "<td>882.0</td>\n",
       "<td>972.0</td>\n",
       "<td>1031.0</td>\n",
       "<td>987.0</td>\n",
       "<td>1001.0</td>\n",
       "<td>0.0136</td>\n",
       "<td>136 / 10,000</td></tr></table></div>"
      ],
      "text/plain": [
       "0    1     2     3     4    5    6    7     8    9     Error       Rate\n",
       "---  ----  ----  ----  ---  ---  ---  ----  ---  ----  ----------  ------------\n",
       "968  0     1     0     0    1    6    1     3    0     0.0122449   12 / 980\n",
       "1    1123  1     1     0    1    5    1     2    0     0.0105727   12 / 1,135\n",
       "6    2     1012  5     0    0    2    4     1    0     0.0193798   20 / 1,032\n",
       "0    0     0     1002  0    2    0    2     4    0     0.00792079  8 / 1,010\n",
       "0    1     0     0     964  0    4    3     2    8     0.0183299   18 / 982\n",
       "1    1     0     6     0    874  3    0     5    2     0.0201794   18 / 892\n",
       "2    1     0     0     0    0    952  0     3    0     0.00626305  6 / 958\n",
       "0    0     7     0     0    0    0    1017  1    3     0.0107004   11 / 1,028\n",
       "2    1     0     2     0    1    0    0     966  2     0.00821355  8 / 974\n",
       "1    4     0     5     7    3    0    3     0    986   0.0227948   23 / 1,009\n",
       "981  1133  1021  1021  971  882  972  1031  987  1001  0.0136      136 / 10,000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top-10 Hit Ratios: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b>k</b></td>\n",
       "<td><b>hit_ratio</b></td></tr>\n",
       "<tr><td>1</td>\n",
       "<td>0.9864</td></tr>\n",
       "<tr><td>2</td>\n",
       "<td>0.9965</td></tr>\n",
       "<tr><td>3</td>\n",
       "<td>0.9994</td></tr>\n",
       "<tr><td>4</td>\n",
       "<td>0.9998</td></tr>\n",
       "<tr><td>5</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>6</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>7</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>8</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>9</td>\n",
       "<td>1.0</td></tr>\n",
       "<tr><td>10</td>\n",
       "<td>1.0</td></tr></table></div>"
      ],
      "text/plain": [
       "k    hit_ratio\n",
       "---  -----------\n",
       "1    0.9864\n",
       "2    0.9965\n",
       "3    0.9994\n",
       "4    0.9998\n",
       "5    1\n",
       "6    1\n",
       "7    1\n",
       "8    1\n",
       "9    1\n",
       "10   1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scoring History: \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"overflow:auto\"><table style=\"width:50%\"><tr><td><b></b></td>\n",
       "<td><b>timestamp</b></td>\n",
       "<td><b>duration</b></td>\n",
       "<td><b>training_speed</b></td>\n",
       "<td><b>epochs</b></td>\n",
       "<td><b>iterations</b></td>\n",
       "<td><b>samples</b></td>\n",
       "<td><b>training_rmse</b></td>\n",
       "<td><b>training_logloss</b></td>\n",
       "<td><b>training_classification_error</b></td>\n",
       "<td><b>validation_rmse</b></td>\n",
       "<td><b>validation_logloss</b></td>\n",
       "<td><b>validation_classification_error</b></td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:52:55</td>\n",
       "<td> 0.000 sec</td>\n",
       "<td>None</td>\n",
       "<td>0.0</td>\n",
       "<td>0</td>\n",
       "<td>0.0</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td>\n",
       "<td>nan</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:52:56</td>\n",
       "<td> 1.256 sec</td>\n",
       "<td>8533 obs/sec</td>\n",
       "<td>0.0170667</td>\n",
       "<td>1</td>\n",
       "<td>1024.0</td>\n",
       "<td>0.5208416</td>\n",
       "<td>1.5063509</td>\n",
       "<td>0.3028514</td>\n",
       "<td>0.5236829</td>\n",
       "<td>1.5059840</td>\n",
       "<td>0.3088</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:06</td>\n",
       "<td>11.631 sec</td>\n",
       "<td>16298 obs/sec</td>\n",
       "<td>2.6624</td>\n",
       "<td>156</td>\n",
       "<td>159744.0</td>\n",
       "<td>0.1070740</td>\n",
       "<td>0.0497958</td>\n",
       "<td>0.0143072</td>\n",
       "<td>0.1310304</td>\n",
       "<td>0.0761871</td>\n",
       "<td>0.0211</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:13</td>\n",
       "<td>18.619 sec</td>\n",
       "<td>15969 obs/sec</td>\n",
       "<td>4.2837333</td>\n",
       "<td>251</td>\n",
       "<td>257024.0</td>\n",
       "<td>0.1051433</td>\n",
       "<td>0.0452353</td>\n",
       "<td>0.0139070</td>\n",
       "<td>0.1229485</td>\n",
       "<td>0.0701109</td>\n",
       "<td>0.0186</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:20</td>\n",
       "<td>25.554 sec</td>\n",
       "<td>15192 obs/sec</td>\n",
       "<td>5.6661333</td>\n",
       "<td>332</td>\n",
       "<td>339968.0</td>\n",
       "<td>0.0915219</td>\n",
       "<td>0.0348545</td>\n",
       "<td>0.0104052</td>\n",
       "<td>0.1186294</td>\n",
       "<td>0.0660772</td>\n",
       "<td>0.0171</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:27</td>\n",
       "<td>32.173 sec</td>\n",
       "<td>14795 obs/sec</td>\n",
       "<td>6.9802667</td>\n",
       "<td>409</td>\n",
       "<td>418816.0</td>\n",
       "<td>0.0779313</td>\n",
       "<td>0.0258162</td>\n",
       "<td>0.0075038</td>\n",
       "<td>0.1083484</td>\n",
       "<td>0.0580634</td>\n",
       "<td>0.0136</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:34</td>\n",
       "<td>39.234 sec</td>\n",
       "<td>14380 obs/sec</td>\n",
       "<td>8.2944</td>\n",
       "<td>486</td>\n",
       "<td>497664.0</td>\n",
       "<td>0.0849103</td>\n",
       "<td>0.0328230</td>\n",
       "<td>0.0088044</td>\n",
       "<td>0.1120859</td>\n",
       "<td>0.0701819</td>\n",
       "<td>0.0147</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:42</td>\n",
       "<td>46.910 sec</td>\n",
       "<td>13986 obs/sec</td>\n",
       "<td>9.6768</td>\n",
       "<td>567</td>\n",
       "<td>580608.0</td>\n",
       "<td>0.0845837</td>\n",
       "<td>0.0349662</td>\n",
       "<td>0.0087044</td>\n",
       "<td>0.1156877</td>\n",
       "<td>0.0840219</td>\n",
       "<td>0.0151</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:49</td>\n",
       "<td>54.660 sec</td>\n",
       "<td>13740 obs/sec</td>\n",
       "<td>11.1104</td>\n",
       "<td>651</td>\n",
       "<td>666624.0</td>\n",
       "<td>0.0649014</td>\n",
       "<td>0.0193741</td>\n",
       "<td>0.0053027</td>\n",
       "<td>0.1103369</td>\n",
       "<td>0.0712966</td>\n",
       "<td>0.0147</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:53:57</td>\n",
       "<td> 1 min  2.148 sec</td>\n",
       "<td>13558 obs/sec</td>\n",
       "<td>12.4928</td>\n",
       "<td>732</td>\n",
       "<td>749568.0</td>\n",
       "<td>0.0635988</td>\n",
       "<td>0.0189405</td>\n",
       "<td>0.0047024</td>\n",
       "<td>0.1131730</td>\n",
       "<td>0.0922341</td>\n",
       "<td>0.014</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:54:04</td>\n",
       "<td> 1 min  9.326 sec</td>\n",
       "<td>13454 obs/sec</td>\n",
       "<td>13.8581333</td>\n",
       "<td>812</td>\n",
       "<td>831488.0</td>\n",
       "<td>0.0766852</td>\n",
       "<td>0.0300768</td>\n",
       "<td>0.0072036</td>\n",
       "<td>0.1144445</td>\n",
       "<td>0.0870827</td>\n",
       "<td>0.0148</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:54:11</td>\n",
       "<td> 1 min 15.991 sec</td>\n",
       "<td>13404 obs/sec</td>\n",
       "<td>15.1381333</td>\n",
       "<td>887</td>\n",
       "<td>908288.0</td>\n",
       "<td>0.0601795</td>\n",
       "<td>0.0161270</td>\n",
       "<td>0.0044022</td>\n",
       "<td>0.1058977</td>\n",
       "<td>0.0685525</td>\n",
       "<td>0.0127</td></tr>\n",
       "<tr><td></td>\n",
       "<td>2017-05-08 20:54:11</td>\n",
       "<td> 1 min 16.883 sec</td>\n",
       "<td>13399 obs/sec</td>\n",
       "<td>15.1381333</td>\n",
       "<td>887</td>\n",
       "<td>908288.0</td>\n",
       "<td>0.0779313</td>\n",
       "<td>0.0258162</td>\n",
       "<td>0.0075038</td>\n",
       "<td>0.1083484</td>\n",
       "<td>0.0580634</td>\n",
       "<td>0.0136</td></tr></table></div>"
      ],
      "text/plain": [
       "    timestamp            duration          training_speed    epochs     iterations    samples    training_rmse    training_logloss    training_classification_error    validation_rmse    validation_logloss    validation_classification_error\n",
       "--  -------------------  ----------------  ----------------  ---------  ------------  ---------  ---------------  ------------------  -------------------------------  -----------------  --------------------  ---------------------------------\n",
       "    2017-05-08 20:52:55  0.000 sec                           0          0             0          nan              nan                 nan                              nan                nan                   nan\n",
       "    2017-05-08 20:52:56  1.256 sec         8533 obs/sec      0.0170667  1             1024       0.520842         1.50635             0.302851                         0.523683           1.50598               0.3088\n",
       "    2017-05-08 20:53:06  11.631 sec        16298 obs/sec     2.6624     156           159744     0.107074         0.0497958           0.0143072                        0.13103            0.0761871             0.0211\n",
       "    2017-05-08 20:53:13  18.619 sec        15969 obs/sec     4.28373    251           257024     0.105143         0.0452353           0.013907                         0.122948           0.0701109             0.0186\n",
       "    2017-05-08 20:53:20  25.554 sec        15192 obs/sec     5.66613    332           339968     0.0915219        0.0348545           0.0104052                        0.118629           0.0660772             0.0171\n",
       "    2017-05-08 20:53:27  32.173 sec        14795 obs/sec     6.98027    409           418816     0.0779313        0.0258162           0.00750375                       0.108348           0.0580634             0.0136\n",
       "    2017-05-08 20:53:34  39.234 sec        14380 obs/sec     8.2944     486           497664     0.0849103        0.032823            0.0088044                        0.112086           0.0701819             0.0147\n",
       "    2017-05-08 20:53:42  46.910 sec        13986 obs/sec     9.6768     567           580608     0.0845837        0.0349662           0.00870435                       0.115688           0.0840219             0.0151\n",
       "    2017-05-08 20:53:49  54.660 sec        13740 obs/sec     11.1104    651           666624     0.0649014        0.0193741           0.00530265                       0.110337           0.0712966             0.0147\n",
       "    2017-05-08 20:53:57  1 min  2.148 sec  13558 obs/sec     12.4928    732           749568     0.0635988        0.0189405           0.00470235                       0.113173           0.0922341             0.014\n",
       "    2017-05-08 20:54:04  1 min  9.326 sec  13454 obs/sec     13.8581    812           831488     0.0766852        0.0300768           0.0072036                        0.114445           0.0870827             0.0148\n",
       "    2017-05-08 20:54:11  1 min 15.991 sec  13404 obs/sec     15.1381    887           908288     0.0601795        0.016127            0.0044022                        0.105898           0.0685525             0.0127\n",
       "    2017-05-08 20:54:11  1 min 16.883 sec  13399 obs/sec     15.1381    887           908288     0.0779313        0.0258162           0.00750375                       0.108348           0.0580634             0.0136"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model_mnist_mylenet_keras.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}