computer-vision/CV - Scaling Pixel Methods Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate pixel scaling methods\n",
    "\n",
    "This tutorial evaluates the effectiveness following three pixel scaling methods across the entire training dataset:\n",
    "\n",
    "* Normalization\n",
    "\n",
    "* Centering means\n",
    "\n",
    "* Standardization\n",
    "\n",
    "A simple __CNN__ is built to train a model to recognise images from the __MNIST__ dataset and its accuracy evaluated against each of the data scaling methods above."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load required libs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import MaxPooling2D\n",
    "from keras.layers import Conv2D\n",
    "from keras.layers import Flatten\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib import image\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset():\n",
    "    (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "    # reshape to have single channel\n",
    "    X_train = X_train.reshape((X_train.shape[0], X_train.shape[1], X_train.shape[2], 1))\n",
    "    X_test = X_test.reshape((X_test.shape[0], X_train.shape[1], X_train.shape[2], 1))\n",
    "    # one-hot encode labels\n",
    "    y_train = to_categorical(y_train)\n",
    "    y_test = to_categorical(y_test)\n",
    "    return X_train, y_train, X_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1)\n",
      "(60000, 10)\n"
     ]
    }
   ],
   "source": [
    "trainx, trainy, testx, testy = load_dataset()\n",
    "print(trainx.shape)\n",
    "print(trainy.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, (3,3), activation=\"relu\", input_shape=(28, 28, 1)))\n",
    "    model.add(MaxPooling2D((2,2)))\n",
    "    model.add(Conv2D(64, (3,3), activation=\"relu\"))\n",
    "    model.add(MaxPooling2D((2,2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(64, activation=\"relu\"))\n",
    "    model.add(Dense(10, activation=\"softmax\"))\n",
    "    model.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"acc\"])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# normalize images\n",
    "def normalize_images(train, test):\n",
    "    train_norm = train.astype(\"float32\")\n",
    "    test_norm = test.astype(\"float32\")\n",
    "    train_norm = train_norm / 255.0\n",
    "    test_norm = test_norm / 255.0\n",
    "    return train_norm, test_norm\n",
    "\n",
    "# center mean of images\n",
    "def center_mean(train, test):\n",
    "    train_cent = train.astype(\"float32\")\n",
    "    test_cent = test.astype(\"float32\")\n",
    "    mean = train_cent.mean()\n",
    "    train_cent = train_cent - mean\n",
    "    test_cent = test_cent - mean\n",
    "    return train_cent, test_cent\n",
    "\n",
    "# standardize images\n",
    "def standardize_images(train, test):\n",
    "    # convert from integers to floats\n",
    "    train_stan = train.astype('float32')\n",
    "    test_stan = test.astype('float32')\n",
    "    # calculate statistics\n",
    "    m = train_stan.mean()\n",
    "    s = train_stan.std()\n",
    "    # center datasets\n",
    "    train_stan = (train_stan - m) / s\n",
    "    test_stan = (test_stan - m) / s\n",
    "    \n",
    "    # return normalized images\n",
    "    return train_stan, test_stan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ORIG TRAIN MEAN: 33.318421449829934\n",
      "ORIG TRAIN MAX: 255 TRAIN MIN: 0\n",
      "NORMALIZATION\n",
      "TRAIN MEAN: 0.13066062331199646\n",
      "TRAIN MAX: 1.0 TRAIN MIN: 0.0\n",
      "CENTERING\n",
      "TRAIN MEAN: -1.9512917788233608e-05\n",
      "TRAIN MAX: 221.68154907226562 TRAIN MIN: -33.31844711303711\n",
      "STANDARDIZATION\n",
      "TRAIN MEAN: -0.000\n",
      "TRAIN STD: 1.000\n"
     ]
    }
   ],
   "source": [
    "print(\"ORIG TRAIN MEAN: {}\".format(trainx.mean()))\n",
    "print(\"ORIG TRAIN MAX: {} TRAIN MIN: {}\".format(trainx.max(), trainx.min()))\n",
    "\n",
    "# test normalization\n",
    "x1, _ = normalize_images(trainx, testx)\n",
    "print(\"NORMALIZATION\")\n",
    "print(\"TRAIN MEAN: {}\".format(x1.mean()))\n",
    "print(\"TRAIN MAX: {} TRAIN MIN: {}\".format(x1.max(), x1.min()))\n",
    "\n",
    "# test centering\n",
    "x1, _ = center_mean(trainx, testx)\n",
    "print(\"CENTERING\")\n",
    "print(\"TRAIN MEAN: {}\".format(x1.mean()))\n",
    "print(\"TRAIN MAX: {} TRAIN MIN: {}\".format(x1.max(), x1.min()))\n",
    "\n",
    "# test standardization\n",
    "z, _ = standardize_images(trainx, testx)\n",
    "print(\"STANDARDIZATION\")\n",
    "print(\"TRAIN MEAN: {:.3f}\".format(z.mean()))\n",
    "print(\"TRAIN STD: {:.3f}\".format(z.std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Evaluation loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluation_loop(data_prep_func, n_repeats=10):\n",
    "    X_train, y_train, X_test, y_test = load_dataset()\n",
    "    scores = list()\n",
    "    \n",
    "    model = create_model()\n",
    "    for i in range(n_repeats):\n",
    "        X_train_prep, X_test_prep = data_prep_func(X_train, X_test)\n",
    "        model.fit(X_train_prep, y_train, epochs=5, batch_size=64, verbose=0)\n",
    "        _, acc = model.evaluate(X_test_prep, y_test, verbose=0)\n",
    "        scores.append(acc)\n",
    "        print(\"> {:d} {:.3f}\".format(i, acc))\n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 0 0.990\n"
     ]
    }
   ],
   "source": [
    "all_scores = list()\n",
    "\n",
    "scores = evaluation_loop(normalize_images)\n",
    "print(\"Normalization: {:.3f} ({:.3f})\".format(np.mean(scores), np.std(scores)))\n",
    "all_scores.append(scores)\n",
    "\n",
    "scores = evaluation_loop(center_mean)\n",
    "print(\"Center: {:.3f} ({:.3f})\".format(np.mean(scores), np.std(scores)))\n",
    "all_scores.append(scores)\n",
    "\n",
    "scores = evaluation_loop(standardize_images)\n",
    "print(\"Standardize: {:.3f} ({:.3f})\".format(np.mean(scores), np.std(scores)))\n",
    "all_scores.append(scores)\n",
    "\n",
    "plt.boxplot(allscores, labels=[\"norm\", \"center\", \"std\"])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}