notebooks/accuracy_benchmark/benchmark_custom_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "benchmark_custom_model.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMgsUHeKGl67Qn2wLbec3f2",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "aaf52e66bc10465eac1b5066437ca5a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_52300388afc14756896f6498a517ce66",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_143810cc64b649f6a45db658931ccfb0",
              "IPY_MODEL_d93e8c40830d4b10a9de1f638453866a",
              "IPY_MODEL_a377e3c10b4340f8a85a350c8653a099"
            ]
          }
        },
        "52300388afc14756896f6498a517ce66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "143810cc64b649f6a45db658931ccfb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b91d7c3bb31c42868f1218dbb89f59b5",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_03bb36d039414bb08a0dc83f70268505"
          }
        },
        "d93e8c40830d4b10a9de1f638453866a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_76430234ec5c4ccdadf2019df3bd1407",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 170498071,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 170498071,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1572a259c1984ed2be61265cbde63815"
          }
        },
        "a377e3c10b4340f8a85a350c8653a099": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_81503f567e544909863d5d869eb7f9ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170499072/? [00:03&lt;00:00, 58020780.74it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_781466ffbd26492ca6852f20c5dd9c71"
          }
        },
        "b91d7c3bb31c42868f1218dbb89f59b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "03bb36d039414bb08a0dc83f70268505": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "76430234ec5c4ccdadf2019df3bd1407": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1572a259c1984ed2be61265cbde63815": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "81503f567e544909863d5d869eb7f9ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "781466ffbd26492ca6852f20c5dd9c71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jsalbert/biotorch/blob/release/benchmark_custom_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sWpSGLWob-uS"
      },
      "source": [
        "# Install Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nkeKBKXOVGry",
        "outputId": "931570c8-f454-4eee-e2a5-ca45be2bce07"
      },
      "source": [
        "!pip install biotorch --upgrade\n",
        "!git clone https://github.com/jsalbert/biotorch.git"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting biotorch\n",
            "  Downloading biotorch-0.0.6.tar.gz (35.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 35.3 MB 59 kB/s \n",
            "\u001b[?25hCollecting jsonschema>=3.0\n",
            "  Downloading jsonschema-3.2.0-py2.py3-none-any.whl (56 kB)\n",
            "\u001b[K     |████████████████████████████████| 56 kB 3.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.0 in /usr/local/lib/python3.7/dist-packages (from biotorch) (1.9.0+cu102)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from biotorch) (0.10.0+cu102)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from biotorch) (1.19.5)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from biotorch) (3.2.2)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from biotorch) (1.1.5)\n",
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.7/dist-packages (from biotorch) (4.10.1)\n",
            "Requirement already satisfied: jupyter in /usr/local/lib/python3.7/dist-packages (from biotorch) (1.0.0)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from biotorch) (7.1.2)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from biotorch) (3.13)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from biotorch) (1.4.1)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.7/dist-packages (from biotorch) (2.6.0)\n",
            "Collecting torchattacks\n",
            "  Downloading torchattacks-3.1.0-py3-none-any.whl (100 kB)\n",
            "\u001b[K     |████████████████████████████████| 100 kB 9.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from biotorch) (4.62.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from jsonschema>=3.0->biotorch) (57.4.0)\n",
            "Requirement already satisfied: pyrsistent>=0.14.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=3.0->biotorch) (0.18.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from jsonschema>=3.0->biotorch) (4.6.4)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=3.0->biotorch) (21.2.0)\n",
            "Requirement already satisfied: six>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from jsonschema>=3.0->biotorch) (1.15.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.0->biotorch) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->jsonschema>=3.0->biotorch) (3.5.0)\n",
            "Requirement already satisfied: traitlets>=4.1.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->biotorch) (5.0.5)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->biotorch) (5.5.0)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel->biotorch) (5.1.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel->biotorch) (5.3.5)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->biotorch) (1.0.18)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->biotorch) (2.6.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->biotorch) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->biotorch) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->biotorch) (0.8.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=4.0.0->ipykernel->biotorch) (4.4.2)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipykernel->biotorch) (0.2.5)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.1.0->ipykernel->biotorch) (0.2.0)\n",
            "Requirement already satisfied: qtconsole in /usr/local/lib/python3.7/dist-packages (from jupyter->biotorch) (5.1.1)\n",
            "Requirement already satisfied: jupyter-console in /usr/local/lib/python3.7/dist-packages (from jupyter->biotorch) (5.2.0)\n",
            "Requirement already satisfied: notebook in /usr/local/lib/python3.7/dist-packages (from jupyter->biotorch) (5.3.1)\n",
            "Requirement already satisfied: ipywidgets in /usr/local/lib/python3.7/dist-packages (from jupyter->biotorch) (7.6.3)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from jupyter->biotorch) (5.6.1)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter->biotorch) (3.5.1)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter->biotorch) (1.0.0)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets->jupyter->biotorch) (5.1.3)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.7/dist-packages (from nbformat>=4.2.0->ipywidgets->jupyter->biotorch) (4.7.1)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter->biotorch) (0.11.0)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter->biotorch) (1.8.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from notebook->jupyter->biotorch) (2.11.3)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->biotorch) (22.2.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel->biotorch) (2.8.2)\n",
            "Requirement already satisfied: ptyprocess in /usr/local/lib/python3.7/dist-packages (from terminado>=0.8.1->notebook->jupyter->biotorch) (0.7.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->notebook->jupyter->biotorch) (2.0.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->biotorch) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->biotorch) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->biotorch) (1.3.1)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter->biotorch) (0.3)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter->biotorch) (0.8.4)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter->biotorch) (0.7.1)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter->biotorch) (0.5.0)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter->biotorch) (1.4.3)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->jupyter->biotorch) (4.0.0)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter->biotorch) (0.5.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->jupyter->biotorch) (21.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->biotorch) (2018.9)\n",
            "Requirement already satisfied: qtpy in /usr/local/lib/python3.7/dist-packages (from qtconsole->jupyter->biotorch) (1.10.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (0.37.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (2.23.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (1.34.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (3.3.4)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (0.12.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (1.39.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (1.8.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (1.0.1)\n",
            "Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (3.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard->biotorch) (0.4.5)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard->biotorch) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard->biotorch) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard->biotorch) (4.2.2)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->biotorch) (1.3.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard->biotorch) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard->biotorch) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard->biotorch) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard->biotorch) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard->biotorch) (3.0.4)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->biotorch) (3.1.1)\n",
            "Building wheels for collected packages: biotorch\n",
            "  Building wheel for biotorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for biotorch: filename=biotorch-0.0.6-py3-none-any.whl size=78972 sha256=1ccd57e3e70493287133974eebc74b89098b82514fb581566f4dd9d659b1fd0e\n",
            "  Stored in directory: /root/.cache/pip/wheels/bd/03/48/3f2559d4ef6135245075196550fe977448c0a7308cf561f764\n",
            "Successfully built biotorch\n",
            "Installing collected packages: jsonschema, torchattacks, biotorch\n",
            "  Attempting uninstall: jsonschema\n",
            "    Found existing installation: jsonschema 2.6.0\n",
            "    Uninstalling jsonschema-2.6.0:\n",
            "      Successfully uninstalled jsonschema-2.6.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "nbclient 0.5.4 requires jupyter-client>=6.1.5, but you have jupyter-client 5.3.5 which is incompatible.\u001b[0m\n",
            "Successfully installed biotorch-0.0.6 jsonschema-3.2.0 torchattacks-3.1.0\n",
            "Cloning into 'biotorch'...\n",
            "remote: Enumerating objects: 235, done.\u001b[K\n",
            "remote: Counting objects: 100% (235/235), done.\u001b[K\n",
            "remote: Compressing objects: 100% (149/149), done.\u001b[K\n",
            "remote: Total 235 (delta 98), reused 207 (delta 80), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (235/235), 33.72 MiB | 27.32 MiB/s, done.\n",
            "Resolving deltas: 100% (98/98), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lyVwN1jndgmj"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v4HNxJs5VXc7"
      },
      "source": [
        "import yaml\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from biotorch.benchmark.run import Benchmark\n",
        "from biotorch.module.biomodule import BioModule\n"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bU6dwaasdqtE"
      },
      "source": [
        "# Create a PyTorch model and convert it to use Feedback Alignment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z2rE7tq6VXlr"
      },
      "source": [
        "class Model(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Model, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3)\n",
        "        self.relu1 = nn.ReLU()\n",
        "        self.pool1 = nn.MaxPool2d(kernel_size=2)\n",
        "        self.conv2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3)\n",
        "        self.relu2 = nn.ReLU()\n",
        "        self.pool2 = nn.MaxPool2d(kernel_size=2)\n",
        "        self.conv3 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3)\n",
        "        self.relu3 = nn.ReLU()\n",
        "        self.fc = nn.Linear(in_features=128, out_features=10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = self.pool1(self.relu1(self.conv1(x)))\n",
        "        out = self.pool2(self.relu2(self.conv2(out)))\n",
        "        out = self.relu3(self.conv3(out))\n",
        "        out = F.avg_pool2d(out, out.size()[3])\n",
        "        out = out.view(out.size(0), -1)\n",
        "        return self.fc(out)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6gWI-flfVXot",
        "outputId": "baef837a-c8c0-4287-89e2-86ea058651e6"
      },
      "source": [
        "# Create pytorch model\n",
        "model = Model()\n",
        "# Convert model to use Feedback Alignment `fa`\n",
        "biomodel = BioModule(model, mode='fa')\n",
        "# Print model\n",
        "print(biomodel)\n",
        "# Save model\n",
        "torch.save(biomodel, 'biotorch/biomodel.pth')"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Module has been converted to fa mode:\n",
            "\n",
            "The layer configuration was:  {'type': 'fa', 'options': {'constrain_weights': False, 'gradient_clip': False, 'init': 'xavier'}}\n",
            "- All the 3 <class 'torch.nn.modules.conv.Conv2d'> layers were converted successfully.\n",
            "- All the 1 <class 'torch.nn.modules.linear.Linear'> layers were converted successfully.\n",
            "BioModule(\n",
            "  (module): Model(\n",
            "    (conv1): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1))\n",
            "    (relu1): ReLU()\n",
            "    (pool1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (conv2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))\n",
            "    (relu2): ReLU()\n",
            "    (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
            "    (conv3): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1))\n",
            "    (relu3): ReLU()\n",
            "    (fc): Linear(in_features=128, out_features=10, bias=True)\n",
            "  )\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_x4iCihEd6_S"
      },
      "source": [
        "# Modify Configuration File to Append the Model Path"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jGgb0VAXVXrP"
      },
      "source": [
        "# Open an example configuration file\n",
        "with open(\"biotorch/benchmark_configs/cifar10/le_net/sgd/fa.yaml\", 'r') as stream:\n",
        "    try:\n",
        "        file = yaml.safe_load(stream)\n",
        "    except yaml.YAMLError as exc:\n",
        "        print(exc)\n",
        "\n",
        "# Re write to put our custom model path\n",
        "file['model'] = {\n",
        "    'architecture': None,\n",
        "    'loss_function': {'name': 'cross_entropy'},\n",
        "    'mode': {'options': {'constrain_weights': False,\n",
        "    'gradient_clip': False,\n",
        "    'init': 'xavier'},\n",
        "    'type': 'fa'},\n",
        "    'pretrained': False,\n",
        "    'checkpoint': 'biomodel.pth'\n",
        "    }\n",
        "\n",
        "file['experiment']['name'] = 'custom_model_fa'\n",
        "# Deterministic won't work from Colab because we have to set CUDA environments\n",
        "file['experiment']['deterministic'] = False\n",
        "\n",
        "with open(\"biotorch/custom_model.yaml\", 'w') as outfile:\n",
        "    yaml.dump(file, outfile)  "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TIc7I4PkeGCd"
      },
      "source": [
        "# Run Benchmark"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aZnb9uoCHBHB",
        "outputId": "8b358dda-3fc2-4102-8907-01d3801bd470"
      },
      "source": [
        "!ls"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "biotorch  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w_9RHYclHFY0",
        "outputId": "c6413b7d-63d9-4a79-ac50-aca0242dba8f"
      },
      "source": [
        "%cd biotorch"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/biotorch\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "aaf52e66bc10465eac1b5066437ca5a8",
            "52300388afc14756896f6498a517ce66",
            "143810cc64b649f6a45db658931ccfb0",
            "d93e8c40830d4b10a9de1f638453866a",
            "a377e3c10b4340f8a85a350c8653a099",
            "b91d7c3bb31c42868f1218dbb89f59b5",
            "03bb36d039414bb08a0dc83f70268505",
            "76430234ec5c4ccdadf2019df3bd1407",
            "1572a259c1984ed2be61265cbde63815",
            "81503f567e544909863d5d869eb7f9ea",
            "781466ffbd26492ca6852f20c5dd9c71"
          ]
        },
        "id": "HlLhBgjMgqPK",
        "outputId": "731206bb-e4a4-4b9e-d664-e7261b0f87f5"
      },
      "source": [
        "benchmark = Benchmark('custom_model.yaml')\n",
        "benchmark.run()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Preparing CIFAR-10 Benchmark Dataset and storing data in ./datasets/cifar10\n",
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./datasets/cifar10/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "aaf52e66bc10465eac1b5066437ca5a8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/170498071 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./datasets/cifar10/cifar-10-python.tar.gz to ./datasets/cifar10\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Loading model checkpoint from  biomodel.pth\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "\n",
            "Benchmarking model on CIFAR-10 Benchmark Dataset\n",
            "{'display_iterations': 500, 'layer_alignment': True, 'top_k': 5, 'weight_ratio': True}\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/functional.py:718: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  /pytorch/c10/core/TensorImpl.h:1156.)\n",
            "  return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch: [0][  0/351]\tTime  0.224 ( 0.224)\tData  0.107 ( 0.107)\tLoss 2.2935e+00 (2.2935e+00)\tAcc@1  13.28 ( 13.28)\tAcc@5  56.25 ( 56.25)\n",
            " * Acc@1 19.680 Acc@5 63.540\n",
            "New best accuracy reached: 19.68000030517578 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [1][  0/351]\tTime  0.065 ( 0.065)\tData  0.058 ( 0.058)\tLoss 2.2218e+00 (2.2218e+00)\tAcc@1  21.88 ( 21.88)\tAcc@5  59.38 ( 59.38)\n",
            " * Acc@1 21.880 Acc@5 73.900\n",
            "New best accuracy reached: 21.8799991607666 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [2][  0/351]\tTime  0.071 ( 0.071)\tData  0.065 ( 0.065)\tLoss 2.0950e+00 (2.0950e+00)\tAcc@1  24.22 ( 24.22)\tAcc@5  74.22 ( 74.22)\n",
            " * Acc@1 24.460 Acc@5 76.780\n",
            "New best accuracy reached: 24.459999084472656 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [3][  0/351]\tTime  0.061 ( 0.061)\tData  0.055 ( 0.055)\tLoss 1.9832e+00 (1.9832e+00)\tAcc@1  21.88 ( 21.88)\tAcc@5  76.56 ( 76.56)\n",
            " * Acc@1 26.460 Acc@5 80.300\n",
            "New best accuracy reached: 26.459999084472656 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [4][  0/351]\tTime  0.063 ( 0.063)\tData  0.057 ( 0.057)\tLoss 1.9580e+00 (1.9580e+00)\tAcc@1  33.59 ( 33.59)\tAcc@5  86.72 ( 86.72)\n",
            " * Acc@1 28.940 Acc@5 82.340\n",
            "New best accuracy reached: 28.939998626708984 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [5][  0/351]\tTime  0.062 ( 0.062)\tData  0.055 ( 0.055)\tLoss 2.0700e+00 (2.0700e+00)\tAcc@1  22.66 ( 22.66)\tAcc@5  78.91 ( 78.91)\n",
            " * Acc@1 29.600 Acc@5 84.180\n",
            "New best accuracy reached: 29.599998474121094 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [6][  0/351]\tTime  0.061 ( 0.061)\tData  0.055 ( 0.055)\tLoss 1.8953e+00 (1.8953e+00)\tAcc@1  31.25 ( 31.25)\tAcc@5  86.72 ( 86.72)\n",
            " * Acc@1 31.940 Acc@5 85.360\n",
            "New best accuracy reached: 31.939998626708984 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [7][  0/351]\tTime  0.094 ( 0.094)\tData  0.082 ( 0.082)\tLoss 1.9065e+00 (1.9065e+00)\tAcc@1  35.16 ( 35.16)\tAcc@5  83.59 ( 83.59)\n",
            " * Acc@1 33.280 Acc@5 86.060\n",
            "New best accuracy reached: 33.279998779296875 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [8][  0/351]\tTime  0.062 ( 0.062)\tData  0.056 ( 0.056)\tLoss 1.8009e+00 (1.8009e+00)\tAcc@1  31.25 ( 31.25)\tAcc@5  88.28 ( 88.28)\n",
            " * Acc@1 36.540 Acc@5 86.960\n",
            "New best accuracy reached: 36.540000915527344 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [10][  0/351]\tTime  0.063 ( 0.063)\tData  0.057 ( 0.057)\tLoss 2.0088e+00 (2.0088e+00)\tAcc@1  22.66 ( 22.66)\tAcc@5  80.47 ( 80.47)\n",
            " * Acc@1 37.180 Acc@5 87.580\n",
            "New best accuracy reached: 37.18000030517578 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [11][  0/351]\tTime  0.069 ( 0.069)\tData  0.063 ( 0.063)\tLoss 1.7617e+00 (1.7617e+00)\tAcc@1  34.38 ( 34.38)\tAcc@5  85.16 ( 85.16)\n",
            " * Acc@1 38.420 Acc@5 87.720\n",
            "New best accuracy reached: 38.41999816894531 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [12][  0/351]\tTime  0.061 ( 0.061)\tData  0.055 ( 0.055)\tLoss 1.8383e+00 (1.8383e+00)\tAcc@1  36.72 ( 36.72)\tAcc@5  85.16 ( 85.16)\n",
            " * Acc@1 38.860 Acc@5 88.480\n",
            "New best accuracy reached: 38.86000061035156 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [13][  0/351]\tTime  0.060 ( 0.060)\tData  0.054 ( 0.054)\tLoss 1.8721e+00 (1.8721e+00)\tAcc@1  27.34 ( 27.34)\tAcc@5  85.16 ( 85.16)\n",
            " * Acc@1 39.180 Acc@5 88.700\n",
            "New best accuracy reached: 39.18000030517578 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [14][  0/351]\tTime  0.064 ( 0.064)\tData  0.058 ( 0.058)\tLoss 1.7902e+00 (1.7902e+00)\tAcc@1  34.38 ( 34.38)\tAcc@5  85.94 ( 85.94)\n",
            " * Acc@1 40.760 Acc@5 89.140\n",
            "New best accuracy reached: 40.7599983215332 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [15][  0/351]\tTime  0.061 ( 0.061)\tData  0.054 ( 0.054)\tLoss 1.7250e+00 (1.7250e+00)\tAcc@1  31.25 ( 31.25)\tAcc@5  92.19 ( 92.19)\n",
            " * Acc@1 40.040 Acc@5 89.540\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [16][  0/351]\tTime  0.071 ( 0.071)\tData  0.065 ( 0.065)\tLoss 1.6802e+00 (1.6802e+00)\tAcc@1  35.16 ( 35.16)\tAcc@5  89.84 ( 89.84)\n",
            " * Acc@1 40.640 Acc@5 89.800\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [17][  0/351]\tTime  0.061 ( 0.061)\tData  0.055 ( 0.055)\tLoss 1.7489e+00 (1.7489e+00)\tAcc@1  39.06 ( 39.06)\tAcc@5  84.38 ( 84.38)\n",
            " * Acc@1 40.580 Acc@5 89.980\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [18][  0/351]\tTime  0.062 ( 0.062)\tData  0.055 ( 0.055)\tLoss 1.6702e+00 (1.6702e+00)\tAcc@1  42.19 ( 42.19)\tAcc@5  91.41 ( 91.41)\n",
            " * Acc@1 41.780 Acc@5 89.920\n",
            "New best accuracy reached: 41.779998779296875 \n",
            "Saving best accuracy model...\n",
            "Adjusting learning rate of group 0 to 1.0000e-03.\n",
            "Epoch: [19][  0/351]\tTime  0.061 ( 0.061)\tData  0.054 ( 0.054)\tLoss 1.4892e+00 (1.4892e+00)\tAcc@1  45.31 ( 45.31)\tAcc@5  93.75 ( 93.75)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}