Identifying Hand-written Digits.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 0s 0us/step\n",
      "60000 train samples\n",
      "10000 test samples\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 118,282\n",
      "Trainable params: 118,282\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/20\n",
      "48000/48000 [==============================] - 8s 162us/step - loss: 1.4829 - accuracy: 0.6231 - val_loss: 0.7584 - val_accuracy: 0.8286\n",
      "Epoch 2/20\n",
      "48000/48000 [==============================] - 5s 103us/step - loss: 0.6049 - accuracy: 0.8464 - val_loss: 0.4550 - val_accuracy: 0.8852\n",
      "Epoch 3/20\n",
      "48000/48000 [==============================] - 7s 151us/step - loss: 0.4398 - accuracy: 0.8801 - val_loss: 0.3710 - val_accuracy: 0.9019\n",
      "Epoch 4/20\n",
      "48000/48000 [==============================] - 6s 128us/step - loss: 0.3767 - accuracy: 0.8952 - val_loss: 0.3322 - val_accuracy: 0.9082\n",
      "Epoch 5/20\n",
      "48000/48000 [==============================] - 5s 101us/step - loss: 0.3415 - accuracy: 0.9025 - val_loss: 0.3055 - val_accuracy: 0.9147\n",
      "Epoch 6/20\n",
      "48000/48000 [==============================] - 5s 103us/step - loss: 0.3175 - accuracy: 0.9086 - val_loss: 0.2880 - val_accuracy: 0.9182\n",
      "Epoch 7/20\n",
      "48000/48000 [==============================] - 5s 107us/step - loss: 0.2989 - accuracy: 0.9137 - val_loss: 0.2727 - val_accuracy: 0.9224\n",
      "Epoch 8/20\n",
      "48000/48000 [==============================] - 6s 122us/step - loss: 0.2839 - accuracy: 0.9180 - val_loss: 0.2608 - val_accuracy: 0.9266\n",
      "Epoch 9/20\n",
      "48000/48000 [==============================] - 4s 88us/step - loss: 0.2714 - accuracy: 0.9217 - val_loss: 0.2505 - val_accuracy: 0.9298\n",
      "Epoch 10/20\n",
      "48000/48000 [==============================] - 5s 113us/step - loss: 0.2602 - accuracy: 0.9252 - val_loss: 0.2430 - val_accuracy: 0.9308\n",
      "Epoch 11/20\n",
      "48000/48000 [==============================] - 5s 108us/step - loss: 0.2501 - accuracy: 0.9285 - val_loss: 0.2341 - val_accuracy: 0.9335\n",
      "Epoch 12/20\n",
      "48000/48000 [==============================] - 5s 98us/step - loss: 0.2409 - accuracy: 0.9301 - val_loss: 0.2271 - val_accuracy: 0.9352\n",
      "Epoch 13/20\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 0.2325 - accuracy: 0.9334 - val_loss: 0.2227 - val_accuracy: 0.9367\n",
      "Epoch 14/20\n",
      "48000/48000 [==============================] - 6s 127us/step - loss: 0.2253 - accuracy: 0.9353 - val_loss: 0.2147 - val_accuracy: 0.9396\n",
      "Epoch 15/20\n",
      "48000/48000 [==============================] - 4s 91us/step - loss: 0.2181 - accuracy: 0.9375 - val_loss: 0.2082 - val_accuracy: 0.9411\n",
      "Epoch 16/20\n",
      "48000/48000 [==============================] - 5s 104us/step - loss: 0.2116 - accuracy: 0.9394 - val_loss: 0.2030 - val_accuracy: 0.9431\n",
      "Epoch 17/20\n",
      "48000/48000 [==============================] - 5s 101us/step - loss: 0.2055 - accuracy: 0.9414 - val_loss: 0.1981 - val_accuracy: 0.9445\n",
      "Epoch 18/20\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.1996 - accuracy: 0.9430 - val_loss: 0.1932 - val_accuracy: 0.9458\n",
      "Epoch 19/20\n",
      "48000/48000 [==============================] - 4s 88us/step - loss: 0.1941 - accuracy: 0.9432 - val_loss: 0.1894 - val_accuracy: 0.9467\n",
      "Epoch 20/20\n",
      "48000/48000 [==============================] - 4s 77us/step - loss: 0.1890 - accuracy: 0.9456 - val_loss: 0.1849 - val_accuracy: 0.9498\n",
      "10000/10000 [==============================] - 1s 106us/step\n",
      "Test score: 0.18599770209044217\n",
      "Test accuracy: 0.9463000297546387\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Activation\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils\n",
    "np.random.seed(1671) # for reproducibility\n",
    "# network and training\n",
    "NB_EPOCH = 20\n",
    "BATCH_SIZE = 128\n",
    "VERBOSE = 1\n",
    "NB_CLASSES = 10 # number of outputs = number digits\n",
    "OPTIMIZER = SGD() # optimizer, explained later in this chapter\n",
    "N_HIDDEN = 128\n",
    "VALIDATION_SPLIT=0.2 # how much TRAIN is reserved for VALIDATION\n",
    "# data: shuffled and split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "# X_train is 60000 rows of 28x28 values --> reshaped in 6000 x 784\n",
    "RESHAPED = 784 \n",
    "#\n",
    "X_train = X_train.reshape(60000, RESHAPED)\n",
    "X_test = X_test.reshape(10000, RESHAPED)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "# normalize\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_test.shape[0], 'test samples')\n",
    "# convert class vectors to binary class matrices\n",
    "Y_train = np_utils.to_categorical(y_train, NB_CLASSES)\n",
    "Y_test = np_utils.to_categorical(y_test, NB_CLASSES)\n",
    "# M_HIDDEN hidden layers\n",
    "# 10 outputs\n",
    "# final stage is softmax\n",
    "model = Sequential()\n",
    "model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(N_HIDDEN))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(NB_CLASSES))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])\n",
    "history = model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NB_EPOCH, verbose=VERBOSE, validation_split=VALIDATION_SPLIT) \n",
    "score = model.evaluate(X_test, Y_test, verbose=VERBOSE)\n",
    "print(\"Test score:\", score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000 train samples\n",
      "10000 test samples\n",
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_4 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 118,282\n",
      "Trainable params: 118,282\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/5\n",
      "48000/48000 [==============================] - 4s 83us/step - loss: 1.5102 - accuracy: 0.6051 - val_loss: 0.7603 - val_accuracy: 0.8296\n",
      "Epoch 2/5\n",
      "48000/48000 [==============================] - 3s 72us/step - loss: 0.5955 - accuracy: 0.8511 - val_loss: 0.4469 - val_accuracy: 0.8851\n",
      "Epoch 3/5\n",
      "48000/48000 [==============================] - 3s 73us/step - loss: 0.4300 - accuracy: 0.8843 - val_loss: 0.3666 - val_accuracy: 0.9012\n",
      "Epoch 4/5\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.3702 - accuracy: 0.8971 - val_loss: 0.3301 - val_accuracy: 0.9092\n",
      "Epoch 5/5\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.3371 - accuracy: 0.9059 - val_loss: 0.3053 - val_accuracy: 0.9156\n",
      "10000/10000 [==============================] - 1s 57us/step\n",
      "Test score: 0.30791652681231496\n",
      "Test accuracy: 0.9136999845504761\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1671) # for reproducibility\n",
    "# network and training\n",
    "NB_EPOCH = 5\n",
    "# data: shuffled and split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = X_train.reshape(60000, RESHAPED)\n",
    "X_test = X_test.reshape(10000, RESHAPED)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "# normalize\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_test.shape[0], 'test samples')\n",
    "# convert class vectors to binary class matrices\n",
    "Y_train = np_utils.to_categorical(y_train, NB_CLASSES)\n",
    "Y_test = np_utils.to_categorical(y_test, NB_CLASSES)\n",
    "# M_HIDDEN hidden layers\n",
    "# 10 outputs\n",
    "# final stage is softmax\n",
    "model = Sequential()\n",
    "model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(N_HIDDEN))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(NB_CLASSES))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])\n",
    "history = model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NB_EPOCH, verbose=VERBOSE, validation_split=VALIDATION_SPLIT) \n",
    "score = model.evaluate(X_test, Y_test, verbose=VERBOSE)\n",
    "print(\"Test score:\", score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60000 train samples\n",
      "10000 test samples\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_7 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 118,282\n",
      "Trainable params: 118,282\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/50\n",
      "48000/48000 [==============================] - 4s 87us/step - loss: 1.5284 - accuracy: 0.6085 - val_loss: 0.7963 - val_accuracy: 0.8273\n",
      "Epoch 2/50\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.6185 - accuracy: 0.8482 - val_loss: 0.4710 - val_accuracy: 0.8793\n",
      "Epoch 3/50\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.4466 - accuracy: 0.8826 - val_loss: 0.3841 - val_accuracy: 0.8980\n",
      "Epoch 4/50\n",
      "48000/48000 [==============================] - 4s 77us/step - loss: 0.3823 - accuracy: 0.8958 - val_loss: 0.3427 - val_accuracy: 0.9046\n",
      "Epoch 5/50\n",
      "48000/48000 [==============================] - 3s 68us/step - loss: 0.3465 - accuracy: 0.9034 - val_loss: 0.3163 - val_accuracy: 0.9124\n",
      "Epoch 6/50\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.3222 - accuracy: 0.9090 - val_loss: 0.2980 - val_accuracy: 0.9168\n",
      "Epoch 7/50\n",
      "48000/48000 [==============================] - 3s 62us/step - loss: 0.3035 - accuracy: 0.9145 - val_loss: 0.2823 - val_accuracy: 0.9208\n",
      "Epoch 8/50\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.2885 - accuracy: 0.9181 - val_loss: 0.2699 - val_accuracy: 0.9246\n",
      "Epoch 9/50\n",
      "48000/48000 [==============================] - 5s 101us/step - loss: 0.2761 - accuracy: 0.9208 - val_loss: 0.2596 - val_accuracy: 0.9277\n",
      "Epoch 10/50\n",
      "48000/48000 [==============================] - 5s 95us/step - loss: 0.2650 - accuracy: 0.9244 - val_loss: 0.2515 - val_accuracy: 0.9281\n",
      "Epoch 11/50\n",
      "48000/48000 [==============================] - 4s 87us/step - loss: 0.2550 - accuracy: 0.9276 - val_loss: 0.2428 - val_accuracy: 0.9310\n",
      "Epoch 12/50\n",
      "48000/48000 [==============================] - 4s 78us/step - loss: 0.2459 - accuracy: 0.9299 - val_loss: 0.2354 - val_accuracy: 0.9334\n",
      "Epoch 13/50\n",
      "48000/48000 [==============================] - 4s 86us/step - loss: 0.2377 - accuracy: 0.9325 - val_loss: 0.2315 - val_accuracy: 0.9343\n",
      "Epoch 14/50\n",
      "48000/48000 [==============================] - 4s 84us/step - loss: 0.2305 - accuracy: 0.9346 - val_loss: 0.2236 - val_accuracy: 0.9370\n",
      "Epoch 15/50\n",
      "48000/48000 [==============================] - 4s 77us/step - loss: 0.2233 - accuracy: 0.9366 - val_loss: 0.2170 - val_accuracy: 0.9383\n",
      "Epoch 16/50\n",
      "48000/48000 [==============================] - 4s 83us/step - loss: 0.2168 - accuracy: 0.9381 - val_loss: 0.2115 - val_accuracy: 0.9415\n",
      "Epoch 17/50\n",
      "48000/48000 [==============================] - 4s 84us/step - loss: 0.2107 - accuracy: 0.9400 - val_loss: 0.2064 - val_accuracy: 0.9427\n",
      "Epoch 18/50\n",
      "48000/48000 [==============================] - 4s 76us/step - loss: 0.2046 - accuracy: 0.9413 - val_loss: 0.2018 - val_accuracy: 0.9439\n",
      "Epoch 19/50\n",
      "48000/48000 [==============================] - 4s 83us/step - loss: 0.1990 - accuracy: 0.9434 - val_loss: 0.1978 - val_accuracy: 0.9448\n",
      "Epoch 20/50\n",
      "48000/48000 [==============================] - 4s 79us/step - loss: 0.1939 - accuracy: 0.9449 - val_loss: 0.1931 - val_accuracy: 0.9469\n",
      "Epoch 21/50\n",
      "48000/48000 [==============================] - 4s 82us/step - loss: 0.1888 - accuracy: 0.9460 - val_loss: 0.1894 - val_accuracy: 0.9474\n",
      "Epoch 22/50\n",
      "48000/48000 [==============================] - 3s 66us/step - loss: 0.1838 - accuracy: 0.9474 - val_loss: 0.1869 - val_accuracy: 0.9485\n",
      "Epoch 23/50\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.1795 - accuracy: 0.9489 - val_loss: 0.1828 - val_accuracy: 0.9492\n",
      "Epoch 24/50\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.1753 - accuracy: 0.9498 - val_loss: 0.1792 - val_accuracy: 0.9507\n",
      "Epoch 25/50\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.1709 - accuracy: 0.9513 - val_loss: 0.1759 - val_accuracy: 0.9514\n",
      "Epoch 26/50\n",
      "48000/48000 [==============================] - 3s 66us/step - loss: 0.1669 - accuracy: 0.9525 - val_loss: 0.1723 - val_accuracy: 0.9518\n",
      "Epoch 27/50\n",
      "48000/48000 [==============================] - 3s 68us/step - loss: 0.1630 - accuracy: 0.9532 - val_loss: 0.1695 - val_accuracy: 0.9528\n",
      "Epoch 28/50\n",
      "48000/48000 [==============================] - 3s 66us/step - loss: 0.1592 - accuracy: 0.9546 - val_loss: 0.1674 - val_accuracy: 0.9539\n",
      "Epoch 29/50\n",
      "48000/48000 [==============================] - 3s 63us/step - loss: 0.1558 - accuracy: 0.9553 - val_loss: 0.1636 - val_accuracy: 0.9553\n",
      "Epoch 30/50\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.1524 - accuracy: 0.9569 - val_loss: 0.1613 - val_accuracy: 0.9557\n",
      "Epoch 31/50\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.1490 - accuracy: 0.9576 - val_loss: 0.1584 - val_accuracy: 0.9574\n",
      "Epoch 32/50\n",
      "48000/48000 [==============================] - 3s 72us/step - loss: 0.1458 - accuracy: 0.9585 - val_loss: 0.1574 - val_accuracy: 0.9568\n",
      "Epoch 33/50\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.1429 - accuracy: 0.9595 - val_loss: 0.1546 - val_accuracy: 0.9581\n",
      "Epoch 34/50\n",
      "48000/48000 [==============================] - 3s 69us/step - loss: 0.1399 - accuracy: 0.9602 - val_loss: 0.1525 - val_accuracy: 0.9582\n",
      "Epoch 35/50\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.1371 - accuracy: 0.9610 - val_loss: 0.1492 - val_accuracy: 0.9588\n",
      "Epoch 36/50\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.1342 - accuracy: 0.9618 - val_loss: 0.1483 - val_accuracy: 0.9593\n",
      "Epoch 37/50\n",
      "48000/48000 [==============================] - 3s 64us/step - loss: 0.1317 - accuracy: 0.9622 - val_loss: 0.1459 - val_accuracy: 0.9588\n",
      "Epoch 38/50\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.1291 - accuracy: 0.9632 - val_loss: 0.1446 - val_accuracy: 0.9591\n",
      "Epoch 39/50\n",
      "48000/48000 [==============================] - 3s 65us/step - loss: 0.1266 - accuracy: 0.9641 - val_loss: 0.1414 - val_accuracy: 0.9607\n",
      "Epoch 40/50\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.1241 - accuracy: 0.9646 - val_loss: 0.1396 - val_accuracy: 0.9612\n",
      "Epoch 41/50\n",
      "48000/48000 [==============================] - 4s 76us/step - loss: 0.1218 - accuracy: 0.9654 - val_loss: 0.1392 - val_accuracy: 0.9608\n",
      "Epoch 42/50\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.1196 - accuracy: 0.9659 - val_loss: 0.1366 - val_accuracy: 0.9617\n",
      "Epoch 43/50\n",
      "48000/48000 [==============================] - 3s 71us/step - loss: 0.1174 - accuracy: 0.9667 - val_loss: 0.1359 - val_accuracy: 0.9626\n",
      "Epoch 44/50\n",
      "48000/48000 [==============================] - 3s 68us/step - loss: 0.1150 - accuracy: 0.9677 - val_loss: 0.1341 - val_accuracy: 0.9626\n",
      "Epoch 45/50\n",
      "48000/48000 [==============================] - 3s 70us/step - loss: 0.1130 - accuracy: 0.9681 - val_loss: 0.1319 - val_accuracy: 0.9637\n",
      "Epoch 46/50\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.1109 - accuracy: 0.9683 - val_loss: 0.1305 - val_accuracy: 0.9642\n",
      "Epoch 47/50\n",
      "48000/48000 [==============================] - 3s 67us/step - loss: 0.1089 - accuracy: 0.9694 - val_loss: 0.1298 - val_accuracy: 0.9642\n",
      "Epoch 48/50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.1071 - accuracy: 0.9697 - val_loss: 0.1287 - val_accuracy: 0.9643\n",
      "Epoch 49/50\n",
      "48000/48000 [==============================] - 3s 58us/step - loss: 0.1052 - accuracy: 0.9702 - val_loss: 0.1260 - val_accuracy: 0.9653\n",
      "Epoch 50/50\n",
      "48000/48000 [==============================] - 3s 53us/step - loss: 0.1034 - accuracy: 0.9709 - val_loss: 0.1258 - val_accuracy: 0.9656\n",
      "10000/10000 [==============================] - 0s 43us/step\n",
      "Test score: 0.12274125662110746\n",
      "Test accuracy: 0.9624999761581421\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(1671) # for reproducibility\n",
    "# network and training\n",
    "NB_EPOCH = 50\n",
    "# data: shuffled and split between train and test sets\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = X_train.reshape(60000, RESHAPED)\n",
    "X_test = X_test.reshape(10000, RESHAPED)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "# normalize\n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "print(X_train.shape[0], 'train samples')\n",
    "print(X_test.shape[0], 'test samples')\n",
    "# convert class vectors to binary class matrices\n",
    "Y_train = np_utils.to_categorical(y_train, NB_CLASSES)\n",
    "Y_test = np_utils.to_categorical(y_test, NB_CLASSES)\n",
    "# M_HIDDEN hidden layers\n",
    "# 10 outputs\n",
    "# final stage is softmax\n",
    "model = Sequential()\n",
    "model.add(Dense(N_HIDDEN, input_shape=(RESHAPED,)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(N_HIDDEN))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(NB_CLASSES))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy'])\n",
    "history = model.fit(X_train, Y_train, batch_size=BATCH_SIZE, epochs=NB_EPOCH, verbose=VERBOSE, validation_split=VALIDATION_SPLIT) \n",
    "score = model.evaluate(X_test, Y_test, verbose=VERBOSE)\n",
    "print(\"Test score:\", score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Epoch Modification Test Explanation:\n",
    "\n",
    "The initial code execution highlights the control test for this assignment. It utilizes 20 epochs and results in a training accuracy of 94.56%, a validation accuracy of 94.98%, and a test accuracy of 96.25%. \n",
    "\n",
    "When considering the initial test, the epochs were the paramater to be modified for subsequent testing to identify changes in the training, validation, and test accuracies. As the epoch is the number of iterations training data is passed through the machine learning model, it is a good parameter to understand the relationship between further accuracy gains versus computational time required. This is because the neurons and their associated internal model parameters in the artifical neural network are updated based on the epoch definition. Subsequently, we can determine whether a machine learning model is over-fitting, under-fitting, or precise for the data set.\n",
    "\n",
    "The first epoch modification test saw the number of epochs limited to five. This resulted in a training accuracy of 90.59%, a validation accuracy of 91.56%, and a test accuracy of 91.37%. The second epoch modification test saw the number of epochs expanded to 50. This test resulted in a training accuracy of 97.09%, a validation accuracy of 96.56%, and a test accuracy of 96.25%. With this data in mind, the second test achieved roughly 6.5% higher training accuracy, 5% higher validation accuracy, and roughly 5% higher test accuracy at the cost of 10 times the computational requirement.\n",
    "\n",
    "This increase is solely contributable to the amount of epochs utilized in the machine learning model. This is because as the volume of epochs increases, the model will have more data sets to learn from and modify neuron weights accordingly. Subsequently, the accuracy of all results increased across the board. However, it is important to note that the accuracy increases substantially slowed down as the weights were being decreasingly adjusted for each subsequent epoch. This means that further accuracy gains are achievable but will cost an increasing amount of computational power. During the first epoch modification test, the training accuracy did not surpass the test accuracy indicating the model did not train for long enough and was an under-fit. Alterantively, during the second epoch modification test, the training accuracy surpassed the test accuracy at the 38th epoch highlighting the model has trained for long enough to be accurate and was either precise or becoming over-fitting."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}