tests/onnx_keras_weights.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a25ef000-acc3-4bba-966e-938ce589ae24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num GPUs Available:  0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import onnxruntime as rt\n",
    "import tf2onnx\n",
    "import os\n",
    "import pandas as pd\n",
    "from time import time\n",
    "import numpy as np\n",
    "os.chdir('..')\n",
    "from MakeModelClassifier import MakeModelClassifier as mm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64d16089-0eae-40d0-8ca8-418807ea0cd4",
   "metadata": {},
   "source": [
    "## Instantiate MakeModelClassifier with 50% sample of Stanford car data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4280672c-0ec0-4bfd-ba58-2d627807f11e",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {'img_df': '/Users/josephking/Documents/sponsored_projects/MERGEN/data/vehicle_classifier/stanford_car_data/Bboxes.csv',\n",
    "          'data': '/Users/josephking/Documents/sponsored_projects/MERGEN/data/vehicle_classifier/stanford_car_data',\n",
    "          'output': '/Users/josephking/Documents/sponsored_projects/MERGEN/scripts/output',\n",
    "          'logging': False,\n",
    "          'train': False,\n",
    "          'predict': True,\n",
    "          'weights': '/Users/josephking/Documents/sponsored_projects/MERGEN/output/MakeModelClassifier/2021-11-17-00h53/training_checkpoints',\n",
    "          'min_class_img_count': 0,\n",
    "          'pixel_dilation': 0,\n",
    "          'sample' : 0.5,\n",
    "          'seed': 123,\n",
    "          'img_size': (224, 224),\n",
    "          'batch_size': 32,\n",
    "          'model': 'resnet',\n",
    "          'resnet_size': '50',\n",
    "          'dropout': 0.2,\n",
    "          'units2': 4096,\n",
    "          'units1': 2048,\n",
    "          'optimizer': 'adam',\n",
    "          'learning_rate': 0.0001\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "868f2a05-e95e-4cff-999b-77f83fc2b539",
   "metadata": {},
   "outputs": [],
   "source": [
    "mmc = mm(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c29cbdc4-fa41-4cde-92f3-31c9bb857d23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Reading in and processing images.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "_, _, test = mmc.image_pipeline(predict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ecbc764c-6172-43d3-9202-db05f0ac7165",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<PrefetchDataset shapes: ((None, 224, 224, 3), (None, 574)), types: (tf.float32, tf.int64)>\n"
     ]
    }
   ],
   "source": [
    "print(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3e180ab2-2a77-424d-be61-daba8bff14f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of test images: 2600\n"
     ]
    }
   ],
   "source": [
    "print(f'\\nNumber of test images: {len(mmc.df)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6f5806c-2ebf-406b-a378-75b1c545e104",
   "metadata": {},
   "source": [
    "## Predictions with Tensorflow Keras model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "71fa1312-0c35-4b09-933b-94d91f7aa3b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f9c20004bb0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "latest = tf.train.latest_checkpoint(config['weights'])\n",
    "keras_model = mmc.build_compile_model()\n",
    "keras_model.load_weights(latest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3e16cdff-708b-423e-a105-07786e8b3351",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Total prediction time in seconds: 95.21\n",
      "\n"
     ]
    }
   ],
   "source": [
    "start = time()\n",
    "keras_predictions = keras_model.predict(test)\n",
    "print(\"\\nTotal prediction time in seconds: {:.2f}\\n\".format((time()-start)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eea94ebe-8899-44b4-8e8f-81b23f141d0f",
   "metadata": {},
   "source": [
    "## Convert Keras weights to Onnx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "73205a0e-d24e-493f-b42c-6c4382b5dea9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/josephking/Documents/PythonVenv/MERGEN/lib/python3.9/site-packages/tf2onnx/tf_loader.py:706: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "\n",
      "Total time spent converting weights in seconds: 31.12\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spec = (tf.TensorSpec(((None,) + mmc.config['img_size'] + (3,)), tf.float32, name=\"input\"),)\n",
    "onnx_path = os.path.join(mmc.config['weights'], 'model.onnx')\n",
    "\n",
    "start = time()\n",
    "model_proto, _ = tf2onnx.convert.from_keras(keras_model, input_signature=spec, opset=13, output_path=onnx_path)\n",
    "output_names = [n.name for n in model_proto.graph.output]\n",
    "print(\"\\nTotal time spent converting weights in seconds: {:.2f}\\n\".format((time()-start)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d7630ad6-ae00-41e7-9729-4225a8779c7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['dense_2']\n"
     ]
    }
   ],
   "source": [
    "print(output_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45769678-e19d-448e-a60c-be3e74f59812",
   "metadata": {},
   "source": [
    "## Predictions with Onnx weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a4c5af8-e1b9-4405-9fd9-98886d40e005",
   "metadata": {},
   "source": [
    "#### Convert to Numpy array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "47c48a14-7b47-40f9-ab4f-c0a03bce9074",
   "metadata": {},
   "outputs": [],
   "source": [
    "converted_test = list(test.as_numpy_iterator())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57ab73da-397a-4497-8526-1b0fb014ab5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "array = np.array(converted_test[0][0])\n",
    "for i in range(1, len(converted_test)):\n",
    "    array = np.concatenate((array, converted_test[i][0]), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "acc67cf1-5f08-474c-8c1c-ca31a35284a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Total prediction time in seconds: 745.53\n",
      "\n"
     ]
    }
   ],
   "source": [
    "onnx_model = rt.InferenceSession(onnx_path, providers=['CPUExecutionProvider'])  # run on a MacBook with no GPU\n",
    "\n",
    "start = time()\n",
    "onnx_pred = onnx_model.run(output_names, {\"input\": array})\n",
    "print(\"\\nTotal prediction time in seconds: {:.2f}\\n\".format((time()-start)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "383aa99a-04c4-4e11-a1eb-e6db2ae7030f",
   "metadata": {},
   "source": [
    "##  Compare softmax probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8005da6f-a627-433e-b4b5-52500fd951b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert(keras_predictions.shape == onnx_pred[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7596356f-868a-4011-b0e8-beebe50596d2",
   "metadata": {},
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "\nNot equal to tolerance rtol=1e-05, atol=0\n\nMismatched elements: 514292 / 1492400 (34.5%)\nMax absolute difference: 6.6310167e-06\nMax relative difference: 1.\n x: array([[1.440194e-13, 9.826511e-11, 6.791989e-13, ..., 4.365169e-09,\n        2.501970e-07, 1.148975e-11],\n       [1.994335e-10, 1.379016e-10, 2.057311e-12, ..., 3.600836e-09,...\n y: array([[1.440177e-13, 9.826344e-11, 6.791848e-13, ..., 4.365178e-09,\n        2.501956e-07, 1.148958e-11],\n       [1.994339e-10, 1.379008e-10, 2.057353e-12, ..., 3.600842e-09,...",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-15-7b7e9174474f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtesting\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0massert_allclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkeras_predictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0monnx_pred\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrtol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "    \u001b[0;31m[... skipping hidden 2 frame]\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: \nNot equal to tolerance rtol=1e-05, atol=0\n\nMismatched elements: 514292 / 1492400 (34.5%)\nMax absolute difference: 6.6310167e-06\nMax relative difference: 1.\n x: array([[1.440194e-13, 9.826511e-11, 6.791989e-13, ..., 4.365169e-09,\n        2.501970e-07, 1.148975e-11],\n       [1.994335e-10, 1.379016e-10, 2.057311e-12, ..., 3.600836e-09,...\n y: array([[1.440177e-13, 9.826344e-11, 6.791848e-13, ..., 4.365178e-09,\n        2.501956e-07, 1.148958e-11],\n       [1.994339e-10, 1.379008e-10, 2.057353e-12, ..., 3.600842e-09,..."
     ]
    }
   ],
   "source": [
    "np.testing.assert_allclose(keras_predictions, onnx_pred[0], rtol=1e-5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04311bb2-ac58-49f3-ab99-feba14c08f6e",
   "metadata": {},
   "source": [
    "##  Compare argmax(0) between models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0caa9ed9-f515-410d-923b-81d07baf68ff",
   "metadata": {},
   "source": [
    "#### True labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5a54d2a7-d987-448a-8d16-510ce190fa11",
   "metadata": {},
   "outputs": [],
   "source": [
    "images, labels = tuple(zip(*test))  # Recover labels\n",
    "\n",
    "label_df = pd.DataFrame()\n",
    "for x in range(len(labels)):\n",
    "    label_df = pd.concat([label_df, pd.DataFrame(labels[x].numpy())], axis=0)\n",
    "label_df = label_df.reset_index(drop=True)\n",
    "label_series = label_df.idxmax(axis=1).astype(str)\n",
    "label_series.replace(to_replace=mmc.label_mapping, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "b12b7c99-8b34-4705-8611-5ec95fe03dc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       Hyundai Veracruz\n",
       "1                 BMW X6\n",
       "2          Dodge Charger\n",
       "3           BMW 3 Series\n",
       "4          Dodge Caliber\n",
       "              ...       \n",
       "2595      Hyundai Sonata\n",
       "2596       Buick Enclave\n",
       "2597      Suzuki Kizashi\n",
       "2598       Buick Enclave\n",
       "2599              BMW M3\n",
       "Length: 2600, dtype: object"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_series"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11afee8e-abf9-449d-b8a7-8ac5534adaff",
   "metadata": {},
   "source": [
    "#### Predicted class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "50c20579-705c-4d85-a5fb-3b7f4951d9e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames = []\n",
    "for key in mmc.label_mapping.keys():\n",
    "    colnames.append(mmc.label_mapping[key])\n",
    "keras_pred_df = pd.DataFrame(keras_predictions, columns=colnames)\n",
    "onnx_pred_df = pd.DataFrame(onnx_pred[0], columns=colnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5846ea20-1e9b-4e36-a7b8-d7bce0f75faa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6088461538461538"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(keras_pred_df.idxmax(axis=1).astype(str) == label_series).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7b8e0020-589b-42d6-9d0d-f5db879bfeba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6088461538461538"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(onnx_pred_df.idxmax(axis=1).astype(str) == label_series).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6fd72a49-2a18-4695-8cf6-2e3311735923",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ((keras_pred_df.idxmax(axis=1).astype(str) == onnx_pred_df.idxmax(axis=1).astype(str)).all())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}