optimize_cifar100.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Maximising classification accuracy via Ensemble Weight optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add the imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from models import wide_residual_net as WRN\n",
    "\n",
    "import numpy as np\n",
    "import sklearn.metrics as metrics\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from keras.datasets import cifar100\n",
    "from keras import backend as K\n",
    "import keras.utils.np_utils as kutils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some variables which we will use in some time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_TESTS = 25\n",
    "n = 2 * 6 + 4\n",
    "k = 4\n",
    "\n",
    "models_filenames = [r\"weights/WRN-CIFAR100-%d-%d-Best.h5\" % (n, k),\n",
    "                    r\"weights/WRN-CIFAR100-%d-%d-1.h5\" % (n, k),\n",
    "                    r\"weights/WRN-CIFAR100-%d-%d-2.h5\" % (n, k),\n",
    "                    r\"weights/WRN-CIFAR100-%d-%d-3.h5\" % (n, k),\n",
    "                    r\"weights/WRN-CIFAR100-%d-%d-4.h5\" % (n, k),\n",
    "                    r\"weights/WRN-CIFAR100-%d-%d-5.h5\" % (n, k)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load up the CIFAR 100 dataset and prepare for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(trainX, trainY), (testX, testY) = cifar100.load_data()\n",
    "nb_classes = len(np.unique(testY))\n",
    "\n",
    "trainX = trainX.astype('float32')\n",
    "trainX /= 255.0\n",
    "testX = testX.astype('float32')\n",
    "testX /= 255.0\n",
    "\n",
    "trainY = kutils.to_categorical(trainY)\n",
    "testY_cat = kutils.to_categorical(testY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the Wide Residual Network (16-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wide Residual Network-16-4 created.\n"
     ]
    }
   ],
   "source": [
    "# Decide dim ordering for Theano and Tensorflow backends\n",
    "if K.image_dim_ordering() == \"th\":\n",
    "    init = (3, 32, 32)\n",
    "else:\n",
    "    init = (32, 32, 3)\n",
    "    \n",
    "model = WRN.create_wide_residual_network(init, nb_classes=100, N=2, k=4, dropout=0.00)\n",
    "model_prefix = 'WRN-CIFAR100-%d-%d' % (n, k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain predictions from each of the Ensemble models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Obtained predictions from model with weights = weights/WRN-CIFAR100-16-4-Best.h5\n",
      "Obtained predictions from model with weights = weights/WRN-CIFAR100-16-4-1.h5\n",
      "Obtained predictions from model with weights = weights/WRN-CIFAR100-16-4-2.h5\n",
      "Obtained predictions from model with weights = weights/WRN-CIFAR100-16-4-3.h5\n",
      "Obtained predictions from model with weights = weights/WRN-CIFAR100-16-4-4.h5\n",
      "Obtained predictions from model with weights = weights/WRN-CIFAR100-16-4-5.h5\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "for fn in models_filenames:\n",
    "    model.load_weights(fn)\n",
    "    yPreds = model.predict(testX, batch_size=128)\n",
    "    preds.append(yPreds)\n",
    "\n",
    "    print(\"Obtained predictions from model with weights = %s\" % (fn))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Define helper function to calculate accuracy and error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calculate_weighted_accuracy(prediction_weights):\n",
    "    weighted_predictions = np.zeros((testX.shape[0], nb_classes), dtype='float32')\n",
    "    for weight, prediction in zip(prediction_weights, preds):\n",
    "        weighted_predictions += weight * prediction\n",
    "    yPred = np.argmax(weighted_predictions, axis=1)\n",
    "    yTrue = testY\n",
    "    accuracy = metrics.accuracy_score(yTrue, yPred) * 100\n",
    "    error = 100 - accuracy\n",
    "    print(\"Accuracy : \", accuracy)\n",
    "    print(\"Error : \", error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Consider a Single Best Model prediction. \n",
    "\n",
    "We can load the weights of the single best model and make predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy :  71.07\n",
      "Error :  28.93\n"
     ]
    }
   ],
   "source": [
    "# Load the weights of the best single model\n",
    "model.load_weights(models_filenames[0])\n",
    "\n",
    "# Get its predictions\n",
    "yPreds = model.predict(testX, batch_size=128)\n",
    "yPred = np.argmax(yPreds, axis=1)\n",
    "yTrue = testY\n",
    "\n",
    "# Calculate accuracy metric\n",
    "accuracy = metrics.accuracy_score(yTrue, yPred) * 100\n",
    "error = 100 - accuracy\n",
    "print(\"Accuracy : \", accuracy)\n",
    "print(\"Error : \", error)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Consider a non weighted ensemble prediction\n",
    "\n",
    "Here, each model has the same weight for predictions. However, this may not lead to optimal results.\n",
    "\n",
    "Notice that ensemble weighting is an improvement over the single best model, by a large margin for CIFAR100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy :  71.78\n",
      "Error :  28.22\n"
     ]
    }
   ],
   "source": [
    "prediction_weights = [1. / len(models_filenames)] * len(models_filenames)\n",
    "\n",
    "calculate_weighted_accuracy(prediction_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now we consider a weighted ensemble\n",
    "\n",
    "In a weighted ensemble, we try to optimize the weights of predictions of each model, so as to minimize the total log loss. This in turn improves the overall accuracy of the predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create the loss metric \n",
    "def log_loss_func(weights):\n",
    "    ''' scipy minimize will pass the weights as a numpy array '''\n",
    "    final_prediction = np.zeros((testX.shape[0], nb_classes), dtype='float32')\n",
    "\n",
    "    for weight, prediction in zip(weights, preds):\n",
    "        final_prediction += weight * prediction\n",
    "\n",
    "    return log_loss(testY_cat, final_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Ensemble Weights: [ 0.          0.26561464  0.04944201  0.05112553  0.59963485  0.03418297]\n",
      "Iteration 1: Accuracy :  71.39\n",
      "Iteration 1: Error :  28.61\n",
      "\n",
      "Best Ensemble Weights: [  4.34944411e-08   3.71529158e-01   5.97871080e-01   0.00000000e+00\n",
      "   9.03772028e-12   3.05997187e-02]\n",
      "Iteration 2: Accuracy :  71.42\n",
      "Iteration 2: Error :  28.58\n",
      "\n",
      "Best Ensemble Weights: [ 0.04306361  0.59534088  0.08286299  0.02587616  0.12175668  0.13109969]\n",
      "Iteration 3: Accuracy :  72.05\n",
      "Iteration 3: Error :  27.95\n",
      "\n",
      "Best Ensemble Weights: [  1.18818912e-02   5.16909311e-01   1.16893620e-01   4.26161061e-06\n",
      "   4.35623601e-02   3.10748556e-01]\n",
      "Iteration 4: Accuracy :  72.22\n",
      "Iteration 4: Error :  27.78\n",
      "\n",
      "Best Ensemble Weights: [ 0.03822819  0.53179849  0.19198791  0.08540319  0.01221369  0.14036853]\n",
      "Iteration 5: Accuracy :  72.1\n",
      "Iteration 5: Error :  27.9\n",
      "\n",
      "Best Ensemble Weights: [  9.90197453e-07   4.81835256e-01   1.47842985e-01   2.06942531e-01\n",
      "   5.64521086e-05   1.63321785e-01]\n",
      "Iteration 6: Accuracy :  72.12\n",
      "Iteration 6: Error :  27.88\n",
      "\n",
      "Best Ensemble Weights: [ 0.15951673  0.56363289  0.09517809  0.01425351  0.01838015  0.14903864]\n",
      "Iteration 7: Accuracy :  72.19\n",
      "Iteration 7: Error :  27.81\n",
      "\n",
      "Best Ensemble Weights: [ 0.02891164  0.57241935  0.10462919  0.04444481  0.05024784  0.19934717]\n",
      "Iteration 8: Accuracy :  72.17\n",
      "Iteration 8: Error :  27.83\n",
      "\n",
      "Best Ensemble Weights: [  0.00000000e+00   2.88115477e-01   1.43776342e-01   1.08759019e-18\n",
      "   1.42627082e-17   5.68108181e-01]\n",
      "Iteration 9: Accuracy :  71.83\n",
      "Iteration 9: Error :  28.17\n",
      "\n",
      "Best Ensemble Weights: [  1.10773384e-18   4.18095712e-01   4.11406337e-01   7.35056301e-03\n",
      "   6.04913870e-02   1.02656002e-01]\n",
      "Iteration 10: Accuracy :  71.96\n",
      "Iteration 10: Error :  28.04\n",
      "\n",
      "Best Ensemble Weights: [  3.04798226e-02   5.06008922e-01   7.27267746e-02   2.59850227e-02\n",
      "   5.66292438e-08   3.64799402e-01]\n",
      "Iteration 11: Accuracy :  72.28\n",
      "Iteration 11: Error :  27.72\n",
      "\n",
      "Best Ensemble Weights: [ 0.05394255  0.59110448  0.07854849  0.03568608  0.0950145   0.1457039 ]\n",
      "Iteration 12: Accuracy :  72.15\n",
      "Iteration 12: Error :  27.85\n",
      "\n",
      "Best Ensemble Weights: [ 0.030531    0.65065528  0.04535307  0.02284668  0.16046094  0.09015303]\n",
      "Iteration 13: Accuracy :  71.8\n",
      "Iteration 13: Error :  28.2\n",
      "\n",
      "Best Ensemble Weights: [  1.41632197e-18   3.72785505e-01   5.46089340e-01   9.63917635e-07\n",
      "   2.50909165e-02   5.60332743e-02]\n",
      "Iteration 14: Accuracy :  71.71\n",
      "Iteration 14: Error :  28.29\n",
      "\n",
      "Best Ensemble Weights: [ 0.06084393  0.55524177  0.12406771  0.0296639   0.09047669  0.139706  ]\n",
      "Iteration 15: Accuracy :  72.09\n",
      "Iteration 15: Error :  27.91\n",
      "\n",
      "Best Ensemble Weights: [  1.77165414e-01   3.47131321e-01   3.55008275e-01   5.39180125e-05\n",
      "   3.32982746e-02   8.73427977e-02]\n",
      "Iteration 16: Accuracy :  71.92\n",
      "Iteration 16: Error :  28.08\n",
      "\n",
      "Best Ensemble Weights: [  9.48731527e-02   5.39117652e-01   1.95170136e-01   8.49831537e-07\n",
      "   1.23173201e-02   1.58520889e-01]\n",
      "Iteration 17: Accuracy :  72.14\n",
      "Iteration 17: Error :  27.86\n",
      "\n",
      "Best Ensemble Weights: [  8.35546250e-18   2.71432493e-01   4.62486367e-01   3.94415085e-02\n",
      "   2.04838066e-01   2.18015660e-02]\n",
      "Iteration 18: Accuracy :  71.86\n",
      "Iteration 18: Error :  28.14\n",
      "\n",
      "Best Ensemble Weights: [ 0.03606518  0.57291738  0.14618661  0.06116694  0.          0.1836639 ]\n",
      "Iteration 19: Accuracy :  72.1\n",
      "Iteration 19: Error :  27.9\n",
      "\n",
      "Best Ensemble Weights: [  2.51167607e-02   4.12042033e-01   3.37370289e-01   1.61105536e-05\n",
      "   1.22579443e-01   1.02875363e-01]\n",
      "Iteration 20: Accuracy :  72.12\n",
      "Iteration 20: Error :  27.88\n",
      "\n",
      "Best Ensemble Weights: [ 0.10496974  0.56644139  0.10074321  0.02321079  0.04623023  0.15840463]\n",
      "Iteration 21: Accuracy :  72.13\n",
      "Iteration 21: Error :  27.87\n",
      "\n",
      "Best Ensemble Weights: [ 0.01168852  0.67141575  0.00406621  0.07710593  0.0752333   0.1604903 ]\n",
      "Iteration 22: Accuracy :  71.71\n",
      "Iteration 22: Error :  28.29\n",
      "\n",
      "Best Ensemble Weights: [ 0.10008157  0.55655323  0.16580167  0.00201891  0.02974157  0.14580305]\n",
      "Iteration 23: Accuracy :  72.08\n",
      "Iteration 23: Error :  27.92\n",
      "\n",
      "Best Ensemble Weights: [  4.57809002e-07   4.99273018e-01   8.00065485e-02   3.06317539e-01\n",
      "   5.42603524e-02   6.01420848e-02]\n",
      "Iteration 24: Accuracy :  72.14\n",
      "Iteration 24: Error :  27.86\n",
      "\n",
      "Best Ensemble Weights: [ 0.05274016  0.64920302  0.0475806   0.0363227   0.02687529  0.18727823]\n",
      "Iteration 25: Accuracy :  71.99\n",
      "Iteration 25: Error :  28.01\n",
      "\n"
     ]
    }
   ],
   "source": [
    "best_acc = 0.0\n",
    "best_weights = None\n",
    "\n",
    "# Parameters for optimization\n",
    "constraints = ({'type': 'eq', 'fun':lambda w: 1 - sum(w)})\n",
    "bounds = [(0, 1)] * len(preds)\n",
    "\n",
    "# Check for NUM_TESTS times\n",
    "for iteration in range(NUM_TESTS):\n",
    "    # Random initialization of weights\n",
    "    prediction_weights = np.random.random(len(models_filenames))\n",
    "    \n",
    "    # Minimise the loss \n",
    "    result = minimize(log_loss_func, prediction_weights, method='SLSQP', bounds=bounds, constraints=constraints)\n",
    "    print('Best Ensemble Weights: {weights}'.format(weights=result['x']))\n",
    "    \n",
    "    weights = result['x']\n",
    "    weighted_predictions = np.zeros((testX.shape[0], nb_classes), dtype='float32')\n",
    "    \n",
    "    # Calculate weighted predictions\n",
    "    for weight, prediction in zip(weights, preds):\n",
    "        weighted_predictions += weight * prediction\n",
    "\n",
    "    yPred = np.argmax(weighted_predictions, axis=1)\n",
    "    yTrue = testY\n",
    "\n",
    "    # Calculate weight prediction accuracy\n",
    "    accuracy = metrics.accuracy_score(yTrue, yPred) * 100\n",
    "    error = 100 - accuracy\n",
    "    print(\"Iteration %d: Accuracy : \" % (iteration + 1), accuracy)\n",
    "    print(\"Iteration %d: Error : \" % (iteration + 1), error)\n",
    "    \n",
    "    # Save current best weights \n",
    "    if accuracy > best_acc:\n",
    "        best_acc = accuracy\n",
    "        best_weights = weights\n",
    "        \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We can now compute the best accuracy ensemble model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Accuracy :  72.25\nBest Weights :  [  3.82715088e-06   5.09113442e-01   8.92782124e-02   2.22301797e-01\n   4.25546797e-02   1.36748041e-01]\n"
     ]
    }
   ],
   "source": [
    "print(\"Best Accuracy : \", best_acc)\n",
    "print(\"Best Weights : \", best_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "calculate_weighted_accuracy(best_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}