BERT_Fine_Tuning_Sentence_Classification_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT Fine-Tuning Sentence Classification v2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c245b5ee1b564034a6c80ee2269daa82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_28a83bb4a458450fbbe5f2ad9600e8b3",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_474afad658004872a0c4e0d94f3d8f21",
              "IPY_MODEL_6bef06bb83984db7a793dfcb930901d3"
            ]
          }
        },
        "28a83bb4a458450fbbe5f2ad9600e8b3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "474afad658004872a0c4e0d94f3d8f21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_9f3f0984b6af48729bacba04f3f7ea11",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b6b6c6c411a44ab88a2b1c260be8e8d2"
          }
        },
        "6bef06bb83984db7a793dfcb930901d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8ff4ca57b55a47fbadde1d61868ac65b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 232k/232k [00:00&lt;00:00, 2.57MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ee2215ae7734487825c7e554dda5646"
          }
        },
        "9f3f0984b6af48729bacba04f3f7ea11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b6b6c6c411a44ab88a2b1c260be8e8d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8ff4ca57b55a47fbadde1d61868ac65b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ee2215ae7734487825c7e554dda5646": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cc5c9f94b6594259863e8e9773a38709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d19a7e6b4db040f798732ad0eaccd0fd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a330c44811e140fa9f66f81c64a94b80",
              "IPY_MODEL_aa0ce42ca15040b38a378d269d5e70f3"
            ]
          }
        },
        "d19a7e6b4db040f798732ad0eaccd0fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a330c44811e140fa9f66f81c64a94b80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_dc8780c86a6e436f940ca9001ff7bf73",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 313,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 313,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_89199719c2524f3f8f2800efae10cd3c"
          }
        },
        "aa0ce42ca15040b38a378d269d5e70f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ae4285ddf3204f9f914d4736187797fc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 313/313 [00:00&lt;00:00, 6.68kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac3f4b71582743e99f06187d077a482d"
          }
        },
        "dc8780c86a6e436f940ca9001ff7bf73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "89199719c2524f3f8f2800efae10cd3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ae4285ddf3204f9f914d4736187797fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ac3f4b71582743e99f06187d077a482d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2a518fe0c37e47eaaeb5aff73c1d0d2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f89dd52490f44cf6bff7084d33781b31",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4112ddcdcd6a4a8ea262b9e6df20d559",
              "IPY_MODEL_4289e54205c5401bb529bc1b3d8bc300"
            ]
          }
        },
        "f89dd52490f44cf6bff7084d33781b31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4112ddcdcd6a4a8ea262b9e6df20d559": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_08231e0a2cad41d3a1905a26b785babe",
            "_dom_classes": [],
            "description": "Downloading",
            "_model_name": "IntProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_be5c4ad6e9be4b2da85695844c7b3ac7"
          }
        },
        "4289e54205c5401bb529bc1b3d8bc300": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f371bb1b8c3f45b186fa49bb7c79a429",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100% 440M/440M [00:10&lt;00:00, 41.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_100c65bc29864e00923a7c348f6aaaa2"
          }
        },
        "08231e0a2cad41d3a1905a26b785babe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "be5c4ad6e9be4b2da85695844c7b3ac7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f371bb1b8c3f45b186fa49bb7c79a429": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "100c65bc29864e00923a7c348f6aaaa2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kevinpulido89/NLP/blob/master/BERT_Fine_Tuning_Sentence_Classification_v2\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKOTlwcmxmej",
        "colab_type": "text"
      },
      "source": [
        "# BERT Fine-Tuning Tutorial with PyTorch\n",
        "\n",
        "By Chris McCormick and Nick Ryan"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MPgpITmdwvX0",
        "colab_type": "text"
      },
      "source": [
        "*Revised on 12/13/19 to use the new [transformers](https://github.com/huggingface/transformers) interface.*\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJR6t_gCQe_x",
        "colab_type": "text"
      },
      "source": [
        "In this tutorial I'll show you how to use BERT with the huggingface PyTorch library to quickly and efficiently fine-tune a model to get near state of the art performance in sentence classification. More broadly, I describe the practical application of transfer learning in NLP to create high performance models with minimal effort on a range of NLP tasks.\n",
        "\n",
        "This post is presented in two forms--as a blog post [here](http://mccormickml.com/2019/07/22/BERT-fine-tuning/) and as a Colab Notebook [here](https://colab.research.google.com/drive/1Y4o3jh3ZH70tl6mCd76vz_IxX23biCPP). \n",
        "\n",
        "The content is identical in both, but: \n",
        "* The blog post includes a comments section for discussion. \n",
        "* The Colab Notebook will allow you to run the code and inspect it as you read through.\n",
        "\n",
        "I've also published a video walkthrough of this post on my YouTube channel! [Part 1](https://youtu.be/x66kkDnbzi4) and [Part 2](https://youtu.be/Hnvb9b7a_Ps).\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jrC9__lXxTJz",
        "colab_type": "text"
      },
      "source": [
        "# Contents"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9MCBOq4xUpr",
        "colab_type": "text"
      },
      "source": [
        "See \"Table of contents\" in the sidebar to the left."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ADkUGTqixRWo",
        "colab_type": "text"
      },
      "source": [
        "# Introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9vxxTBsuL24",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## History\n",
        "\n",
        "2018 was a breakthrough year in NLP. Transfer learning, particularly models like Allen AI's ELMO, OpenAI's Open-GPT, and Google's BERT allowed researchers to smash multiple benchmarks with minimal task-specific fine-tuning and provided the rest of the NLP community with pretrained models that could easily (with less data and less compute time) be fine-tuned and implemented to produce state of the art results. Unfortunately, for many starting out in NLP and even for some experienced practicioners, the theory and practical application of these powerful models is still not well understood.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qCgvR9INuP5q",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## What is BERT?\n",
        "\n",
        "BERT (Bidirectional Encoder Representations from Transformers), released in late 2018, is the model we will use in this tutorial to provide readers with a better understanding of and practical guidance for using transfer learning models in NLP. BERT is a method of pretraining language representations that was used to create models that NLP practicioners can then download and use for free. You can either use these models to extract high quality language features from your text data, or you can fine-tune these models on a specific task (classification, entity recognition, question answering, etc.) with your own data to produce state of the art predictions.\n",
        "\n",
        "This post will explain how you can modify and fine-tune BERT to create a powerful NLP model that quickly gives you state of the art results. \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DaVGdtOkuXUZ",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "## Advantages of Fine-Tuning\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5llwu8GBuqMb",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "In this tutorial, we will use BERT to train a text classifier. Specifically, we will take the pre-trained BERT model, add an untrained layer of neurons on the end, and train the new model for our classification task. Why do this rather than train a train a specific deep learning model (a CNN, BiLSTM, etc.) that is well suited for the specific NLP task you need? \n",
        "\n",
        "1. **Quicker Development**\n",
        "\n",
        "    * First, the pre-trained BERT model weights already encode a lot of information about our language. As a result, it takes much less time to train our fine-tuned model - it is as if we have already trained the bottom layers of our network extensively and only need to gently tune them while using their output as features for our classification task. In fact, the authors recommend only 2-4 epochs of training for fine-tuning BERT on a specific NLP task (compared to the hundreds of GPU hours needed to train the original BERT model or a LSTM from scratch!). \n",
        "\n",
        "2. **Less Data**\n",
        "\n",
        "    * In addition and perhaps just as important, because of the pre-trained weights this method allows us to fine-tune our task on a much smaller dataset than would be required in a model that is built from scratch. A major drawback of NLP models built from scratch is that we often need a prohibitively large dataset in order to train our network to reasonable accuracy, meaning a lot of time and energy had to be put into dataset creation. By fine-tuning BERT, we are now able to get away with training a model to good performance on a much smaller amount of training data.\n",
        "\n",
        "3. **Better Results**\n",
        "\n",
        "    * Finally, this simple fine-tuning procedure (typically adding one fully-connected layer on top of BERT and training for a few epochs) was shown to achieve state of the art results with minimal task-specific adjustments for a wide variety of tasks: classification, language inference, semantic similarity, question answering, etc. Rather than implementing custom and sometimes-obscure architetures shown to work well on a specific task, simply fine-tuning BERT is shown to be a better (or at least equal) alternative.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZEynC5F4u7Nb",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "### A Shift in NLP\n",
        "\n",
        "This shift to transfer learning parallels the same shift that took place in computer vision a few years ago. Creating a good deep learning network for computer vision tasks can take millions of parameters and be very expensive to train. Researchers discovered that deep networks learn hierarchical feature representations (simple features like edges at the lowest layers with gradually more complex features at higher layers). Rather than training a new network from scratch each time, the lower layers of a trained network with generalized image features could be copied and transfered for use in another network with a different task. It soon became common practice to download a pre-trained deep network and quickly retrain it for the new task or add additional layers on top - vastly preferable to the expensive process of training a network from scratch. For many, the introduction of deep pre-trained language models in 2018 (ELMO, BERT, ULMFIT, Open-GPT, etc.) signals the same shift to transfer learning in NLP that computer vision saw.\n",
        "\n",
        "Let's get started!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RX_ZDhicpHkV",
        "colab_type": "text"
      },
      "source": [
        "# 1. Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nSU7yERLP_66",
        "colab_type": "text"
      },
      "source": [
        "## 1.1. Using Colab GPU for Training\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GI0iOY8zvZzL",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "Google Colab offers free GPUs and TPUs! Since we'll be training a large neural network it's best to take advantage of this (in this case we'll attach a GPU), otherwise training will take a very long time.\n",
        "\n",
        "A GPU can be added by going to the menu and selecting:\n",
        "\n",
        "`Edit 🡒 Notebook Settings 🡒 Hardware accelerator 🡒 (GPU)`\n",
        "\n",
        "Then run the following cell to confirm that the GPU is detected."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEfSbAA4QHas",
        "colab_type": "code",
        "outputId": "c27153c6-a1b4-4648-db1a-51a7f7f64d20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 99
        }
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Get the GPU device name.\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# The device name should look like the following:\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqG7FzRVFEIv",
        "colab_type": "text"
      },
      "source": [
        "In order for torch to use the GPU, we need to identify and specify the GPU as the device. Later, in our training loop, we will load data onto the device. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oYsV4H8fCpZ-",
        "colab_type": "code",
        "outputId": "cbe3f5ca-9b88-438d-e519-7283fc2f2be6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "import torch\n",
        "\n",
        "# If there's a GPU available...\n",
        "if torch.cuda.is_available():    \n",
        "\n",
        "    # Tell PyTorch to use the GPU.    \n",
        "    device = torch.device(\"cuda\")\n",
        "\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "\n",
        "# If not...\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ElsnSNUridI",
        "colab_type": "text"
      },
      "source": [
        "## 1.2. Installing the Hugging Face Library\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_N2UDLevYWn",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "Next, let's install the [transformers](https://github.com/huggingface/transformers) package from Hugging Face which will give us a pytorch interface for working with BERT. (This library contains interfaces for other pretrained language models like OpenAI's GPT and GPT-2.) We've selected the pytorch interface because it strikes a nice balance between the high-level APIs (which are easy to use but don't provide insight into how things work) and tensorflow code (which contains lots of details but often sidetracks us into lessons about tensorflow, when the purpose here is BERT!).\n",
        "\n",
        "At the moment, the Hugging Face library seems to be the most widely accepted and powerful pytorch interface for working with BERT. In addition to supporting a variety of different pre-trained transformer models, the library also includes pre-built modifications of these models suited to your specific task. For example, in this tutorial we will use `BertForSequenceClassification`.\n",
        "\n",
        "The library also includes task-specific classes for token classification, question answering, next sentence prediciton, etc. Using these pre-built classes simplifies the process of modifying BERT for your purposes.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0NmMdkZO8R6q",
        "colab_type": "code",
        "outputId": "4ab2a349-9f01-4c47-ebdc-5da2ef2e8c84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 644
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/08/4a6768ca1a7a4fa37e5ee08077c5d02b8d83876bd36caa5fc24d98992ac2/transformers-2.2.2-py3-none-any.whl (387kB)\n",
            "\u001b[K     |████████████████████████████████| 389kB 3.4MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1f/8e/ed5364a06a9ba720fddd9820155cc57300d28f5f43a6fd7b7e817177e642/sacremoses-0.0.35.tar.gz (859kB)\n",
            "\u001b[K     |████████████████████████████████| 860kB 44.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.17.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from transformers) (4.28.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.21.0)\n",
            "Collecting regex\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/0e/84041e5245d0bfc0f6c6431b36f8d7e1668ee0ce5f8184e0e1f9ee1082b6/regex-2019.12.19-cp36-cp36m-manylinux2010_x86_64.whl (689kB)\n",
            "\u001b[K     |████████████████████████████████| 696kB 43.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.10.36)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 34.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.14.1)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2019.11.28)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.8)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.2.1)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (0.9.4)\n",
            "Requirement already satisfied: botocore<1.14.0,>=1.13.36 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers) (1.13.36)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.36->boto3->transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<2.8.1,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.36->boto3->transformers) (2.6.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.35-cp36-none-any.whl size=883999 sha256=224727a36ca806911eba978b9150331daecc6deae8bddf8f171029d137b31c2c\n",
            "  Stored in directory: /root/.cache/pip/wheels/63/2a/db/63e2909042c634ef551d0d9ac825b2b0b32dede4a6d87ddc94\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, regex, sentencepiece, transformers\n",
            "Successfully installed regex-2019.12.19 sacremoses-0.0.35 sentencepiece-0.1.85 transformers-2.2.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lxddqmruamSj",
        "colab_type": "text"
      },
      "source": [
        "The code in this notebook is actually a simplified version of the [run_glue.py](https://github.com/huggingface/transformers/blob/master/examples/run_glue.py) example script from huggingface.\n",
        "\n",
        "`run_glue.py` is a helpful utility which allows you to pick which GLUE benchmark task you want to run on, and which pre-trained model you want to use (you can see the list of possible models [here](https://github.com/huggingface/transformers/blob/e6cff60b4cbc1158fbd6e4a1c3afda8dc224f566/examples/run_glue.py#L69)). It also supports using either the CPU, a single GPU, or multiple GPUs. It even supports using 16-bit precision if you want further speed up.\n",
        "\n",
        "Unfortunately, all of this configurability comes at the cost of *readability*. In this Notebook, we've simplified the code greatly and added plenty of comments to make it clear what's going on. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "guw6ZNtaswKc",
        "colab_type": "text"
      },
      "source": [
        "# 2. Loading CoLA Dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_9ZKxKc04Btk",
        "colab_type": "text"
      },
      "source": [
        "We'll use [The Corpus of Linguistic Acceptability (CoLA)](https://nyu-mll.github.io/CoLA/) dataset for single sentence classification. It's a set of sentences labeled as grammatically correct or incorrect. It was first published in May of 2018, and is one of the tests included in the \"GLUE Benchmark\" on which models like BERT are competing.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4JrUHXms16cn",
        "colab_type": "text"
      },
      "source": [
        "## 2.1. Download & Extract"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ZNVW6xd0T0X",
        "colab_type": "text"
      },
      "source": [
        "We'll use the `wget` package to download the dataset to the Colab instance's file system. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5m6AnuFv0QXQ",
        "colab_type": "code",
        "outputId": "57341e7e-91d9-49fa-cc7d-92475ba50ff4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        }
      },
      "source": [
        "!pip install wget"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting wget\n",
            "  Downloading https://files.pythonhosted.org/packages/47/6a/62e288da7bcda82b935ff0c6cfe542970f04e29c756b0e147251b2fb251f/wget-3.2.zip\n",
            "Building wheels for collected packages: wget\n",
            "  Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for wget: filename=wget-3.2-cp36-none-any.whl size=9681 sha256=a6fdb3b6384387a047c5a76d75d01ecde89a1735ca0a70202174f2064688424c\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/15/30/7d8f7cea2902b4db79e3fea550d7d7b85ecb27ef992b618f3f\n",
            "Successfully built wget\n",
            "Installing collected packages: wget\n",
            "Successfully installed wget-3.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "08pO03Ff1BjI",
        "colab_type": "text"
      },
      "source": [
        "The dataset is hosted on GitHub in this repo: https://nyu-mll.github.io/CoLA/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMtmPMkBzrvs",
        "colab_type": "code",
        "outputId": "d479eb83-76e7-436a-b63b-dc786054e40c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import wget\n",
        "import os\n",
        "\n",
        "print('Downloading dataset...')\n",
        "\n",
        "# The URL for the dataset zip file.\n",
        "url = 'https://nyu-mll.github.io/CoLA/cola_public_1.1.zip'\n",
        "\n",
        "# Download the file (if we haven't already)\n",
        "if not os.path.exists('./cola_public_1.1.zip'):\n",
        "    wget.download(url, './cola_public_1.1.zip')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading dataset...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_mKctx-ll2FB",
        "colab_type": "text"
      },
      "source": [
        "Unzip the dataset to the file system. You can browse the file system of the Colab instance in the sidebar on the left."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Yv-tNv20dnH",
        "colab_type": "code",
        "outputId": "0473ad37-2041-4cf8-f77a-649596d0eb68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 208
        }
      },
      "source": [
        "# Unzip the dataset (if we haven't already)\n",
        "if not os.path.exists('./cola_public/'):\n",
        "    !unzip cola_public_1.1.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  cola_public_1.1.zip\n",
            "   creating: cola_public/\n",
            "  inflating: cola_public/README      \n",
            "   creating: cola_public/tokenized/\n",
            "  inflating: cola_public/tokenized/in_domain_dev.tsv  \n",
            "  inflating: cola_public/tokenized/in_domain_train.tsv  \n",
            "  inflating: cola_public/tokenized/out_of_domain_dev.tsv  \n",
            "   creating: cola_public/raw/\n",
            "  inflating: cola_public/raw/in_domain_dev.tsv  \n",
            "  inflating: cola_public/raw/in_domain_train.tsv  \n",
            "  inflating: cola_public/raw/out_of_domain_dev.tsv  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oQUy9Tat2EF_",
        "colab_type": "text"
      },
      "source": [
        "## 2.2. Parse"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xeyVCXT31EZQ",
        "colab_type": "text"
      },
      "source": [
        "We can see from the file names that both `tokenized` and `raw` versions of the data are available. \n",
        "\n",
        "We can't use the pre-tokenized version because, in order to apply the pre-trained BERT, we *must* use the tokenizer provided by the model. This is because (1) the model has a specific, fixed vocabulary and (2) the BERT tokenizer has a particular way of handling out-of-vocabulary words."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MYWzeGSY2xh3",
        "colab_type": "text"
      },
      "source": [
        "We'll use pandas to parse the \"in-domain\" training set and look at a few of its properties and data points."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_UkeC7SG2krJ",
        "colab_type": "code",
        "outputId": "540f7e5f-1484-4fd2-c1b3-ee880d16068b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 390
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"./cola_public/raw/in_domain_train.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of training sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Display 10 random rows from the data.\n",
        "df.sample(10)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of training sentences: 8,551\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence_source</th>\n",
              "      <th>label</th>\n",
              "      <th>label_notes</th>\n",
              "      <th>sentence</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1942</th>\n",
              "      <td>r-67</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>Every student who ever goes to Europe ever has...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7611</th>\n",
              "      <td>sks13</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Is there anything to do today?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7685</th>\n",
              "      <td>sks13</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>John convinced the rice to be cooked by Bill.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3923</th>\n",
              "      <td>ks08</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>The birds devour the worm.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>722</th>\n",
              "      <td>bc01</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>Home was gone by John.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130</th>\n",
              "      <td>cj99</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>The more John eats, the tighter keep your mout...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5387</th>\n",
              "      <td>b_73</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>She has problem enough as it is.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4566</th>\n",
              "      <td>ks08</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>The roof is leaked.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4480</th>\n",
              "      <td>ks08</td>\n",
              "      <td>0</td>\n",
              "      <td>*</td>\n",
              "      <td>John did not leaving here.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5548</th>\n",
              "      <td>b_73</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Mary has more than two friends.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     sentence_source  ...                                           sentence\n",
              "1942            r-67  ...  Every student who ever goes to Europe ever has...\n",
              "7611           sks13  ...                     Is there anything to do today?\n",
              "7685           sks13  ...      John convinced the rice to be cooked by Bill.\n",
              "3923            ks08  ...                         The birds devour the worm.\n",
              "722             bc01  ...                             Home was gone by John.\n",
              "130             cj99  ...  The more John eats, the tighter keep your mout...\n",
              "5387            b_73  ...                   She has problem enough as it is.\n",
              "4566            ks08  ...                                The roof is leaked.\n",
              "4480            ks08  ...                         John did not leaving here.\n",
              "5548            b_73  ...                    Mary has more than two friends.\n",
              "\n",
              "[10 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfWzpPi92UAH",
        "colab_type": "text"
      },
      "source": [
        "The two properties we actually care about are the the `sentence` and its `label`, which is referred to as the \"acceptibility judgment\" (0=unacceptable, 1=acceptable)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H_LpQfzCn9_o",
        "colab_type": "text"
      },
      "source": [
        "Here are five sentences which are labeled as not grammatically acceptible. Note how much more difficult this task is than something like sentiment analysis!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blqIvQaQncdJ",
        "colab_type": "code",
        "outputId": "61786411-d069-4943-d021-57abf1ec422c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        }
      },
      "source": [
        "df.loc[df.label == 0].sample(5)[['sentence', 'label']]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1651</th>\n",
              "      <td>Here's a knife for you to say was on the table.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2904</th>\n",
              "      <td>I detached the handle and the box.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7409</th>\n",
              "      <td>They are readable of the paper.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3116</th>\n",
              "      <td>Cynthia lunched peaches.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1155</th>\n",
              "      <td>I insist on seeing through all the students wh...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               sentence  label\n",
              "1651    Here's a knife for you to say was on the table.      0\n",
              "2904                 I detached the handle and the box.      0\n",
              "7409                    They are readable of the paper.      0\n",
              "3116                           Cynthia lunched peaches.      0\n",
              "1155  I insist on seeing through all the students wh...      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4SMZ5T5Imhlx",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "Let's extract the sentences and labels of our training set as numpy ndarrays."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GuE5BqICAne2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get the lists of sentences and their labels.\n",
        "sentences = df.sentence.values\n",
        "labels = df.label.values"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ex5O1eV-Pfct",
        "colab_type": "text"
      },
      "source": [
        "# 3. Tokenization & Input Formatting\n",
        "\n",
        "In this section, we'll transform our dataset into the format that BERT can be trained on."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-8kEDRvShcU5",
        "colab_type": "text"
      },
      "source": [
        "## 3.1. BERT Tokenizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bWOPOyWghJp2",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "To feed our text to BERT, it must be split into tokens, and then these tokens must be mapped to their index in the tokenizer vocabulary.\n",
        "\n",
        "The tokenization must be performed by the tokenizer included with BERT--the below cell will download this for us. We'll be using the \"uncased\" version here.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z474sSC6oe7A",
        "colab_type": "code",
        "outputId": "67a34ab4-a607-4c1b-f2af-f8afd4590bb6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "c245b5ee1b564034a6c80ee2269daa82",
            "28a83bb4a458450fbbe5f2ad9600e8b3",
            "474afad658004872a0c4e0d94f3d8f21",
            "6bef06bb83984db7a793dfcb930901d3",
            "9f3f0984b6af48729bacba04f3f7ea11",
            "b6b6c6c411a44ab88a2b1c260be8e8d2",
            "8ff4ca57b55a47fbadde1d61868ac65b",
            "3ee2215ae7734487825c7e554dda5646"
          ]
        }
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c245b5ee1b564034a6c80ee2269daa82",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=231508, style=ProgressStyle(description_wid…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dFzmtleW6KmJ",
        "colab_type": "text"
      },
      "source": [
        "Let's apply the tokenizer to one sentence just to see the output.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dLIbudgfh6F0",
        "colab_type": "code",
        "outputId": "6089066f-dea9-48fe-fa6d-c78d86b336c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "# Print the original sentence.\n",
        "print(' Original: ', sentences[0])\n",
        "\n",
        "# Print the sentence split into tokens.\n",
        "print('Tokenized: ', tokenizer.tokenize(sentences[0]))\n",
        "\n",
        "# Print the sentence mapped to token ids.\n",
        "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(sentences[0])))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Original:  Our friends won't buy this analysis, let alone the next one we propose.\n",
            "Tokenized:  ['our', 'friends', 'won', \"'\", 't', 'buy', 'this', 'analysis', ',', 'let', 'alone', 'the', 'next', 'one', 'we', 'propose', '.']\n",
            "Token IDs:  [2256, 2814, 2180, 1005, 1056, 4965, 2023, 4106, 1010, 2292, 2894, 1996, 2279, 2028, 2057, 16599, 1012]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WeNIc4auFUdF",
        "colab_type": "text"
      },
      "source": [
        "When we actually convert all of our sentences, we'll use the `tokenize.encode` function to handle both steps, rather than calling `tokenize` and `convert_tokens_to_ids` separately. \n",
        "\n",
        "Before we can do that, though, we need to talk about some of BERT's formatting requirements."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "viKGCCh8izww",
        "colab_type": "text"
      },
      "source": [
        "## 3.2. Required Formatting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yDcqNlvVhL5W",
        "colab_type": "text"
      },
      "source": [
        "The above code left out a few required formatting steps that we'll look at here.\n",
        "\n",
        "*Side Note: The input format to BERT seems \"over-specified\" to me... We are required to give it a number of pieces of information which seem redundant, or like they could easily be inferred from the data without us explicity providing it. But it is what it is, and I suspect it will make more sense once I have a deeper understanding of the BERT internals.*\n",
        "\n",
        "We are required to:\n",
        "1. Add special tokens to the start and end of each sentence.\n",
        "2. Pad & truncate all sentences to a single constant length.\n",
        "3. Explicitly differentiate real tokens from padding tokens with the \"attention mask\".\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V6mceWWOjZnw",
        "colab_type": "text"
      },
      "source": [
        "### Special Tokens\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ykk0P9JiKtVe",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "**`[SEP]`**\n",
        "\n",
        "At the end of every sentence, we need to append the special `[SEP]` token. \n",
        "\n",
        "This token is an artifact of two-sentence tasks, where BERT is given two separate sentences and asked to determine something (e.g., can the answer to the question in sentence A be found in sentence B?). \n",
        "\n",
        "I am not certain yet why the token is still required when we have only single-sentence input, but it is!\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "86C9objaKu8f",
        "colab_type": "text"
      },
      "source": [
        "**`[CLS]`**\n",
        "\n",
        "For classification tasks, we must prepend the special `[CLS]` token to the beginning of every sentence.\n",
        "\n",
        "This token has special significance. BERT consists of 12 Transformer layers. Each transformer takes in a list of token embeddings, and produces the same number of embeddings on the output (but with the feature values changed, of course!).\n",
        "\n",
        "![Illustration of CLS token purpose](http://www.mccormickml.com/assets/BERT/CLS_token_500x606.png)\n",
        "\n",
        "On the output of the final (12th) transformer, *only the first embedding (corresponding to the [CLS] token) is used by the classifier*.\n",
        "\n",
        ">  \"The first token of every sequence is always a special classification token (`[CLS]`). The final hidden state\n",
        "corresponding to this token is used as the aggregate sequence representation for classification\n",
        "tasks.\" (from the [BERT paper](https://arxiv.org/pdf/1810.04805.pdf))\n",
        "\n",
        "I'm not sure why the authors took this strategy instead of some kind of pooling of all the final vectors, but I'm sure that if pooling were better they would have gone that route. \n",
        "\n",
        "Also, because BERT is trained to only use this [CLS] token for classification, we know that the model has been motivated to encode everything it needs for the classification step into that single 768-value embedding vector.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u51v0kFxeteu",
        "colab_type": "text"
      },
      "source": [
        "### Sentence Length & Attention Mask\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qPNuwqZVK3T6",
        "colab_type": "text"
      },
      "source": [
        "The sentences in our dataset obviously have varying lengths, so how does BERT handle this?\n",
        "\n",
        "BERT has two constraints:\n",
        "1. All sentences must be padded or truncated to a single, fixed length.\n",
        "2. The maximum sentence length is 512 tokens.\n",
        "\n",
        "Padding is done with a special `[PAD]` token, which is at index 0 in the BERT vocabulary. The below illustration demonstrates padding out to a \"MAX_LEN\" of 8 tokens.\n",
        "\n",
        "<img src=\"http://www.mccormickml.com/assets/BERT/padding_and_mask.png\" width=\"600\">\n",
        "\n",
        "The \"Attention Mask\" is simply an array of 1s and 0s indicating which tokens are padding and which aren't (seems kind of redundant, doesn't it?! Again, I don't currently know why).\n",
        "\n",
        "I've experimented with running this notebook with two different values of MAX_LEN, and it impacted both the training speed and the  test set accuracy.\n",
        "\n",
        "With a Tesla K80 and:\n",
        "\n",
        "```\n",
        "MAX_LEN = 128  -->  Training epochs take ~5:28 each, score is 0.535\n",
        "MAX_LEN = 64   -->  Training epochs take ~2:57 each, score is 0.566\n",
        "```\n",
        "These results suggest to me that the padding tokens aren't simply skipped over--that they are in fact fed through the model and incorporated in the results (thereby impacting both model speed and accuracy). I'll have to dig into the architecture more to understand this.\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l6w8elb-58GJ",
        "colab_type": "text"
      },
      "source": [
        "## 3.2. Sentences to IDs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1M296yz577fV",
        "colab_type": "text"
      },
      "source": [
        "The `tokenizer.encode` function combines multiple steps for us:\n",
        "1. Split the sentence into tokens.\n",
        "2. Add the special `[CLS]` and `[SEP]` tokens.\n",
        "3. Map the tokens to their IDs.\n",
        "\n",
        "Oddly, this function can perform truncating for us, but doesn't handle padding. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bBdb3pt8LuQ",
        "colab_type": "code",
        "outputId": "3e801684-e4f8-4e26-e0cd-a9024dabd969",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "\n",
        "                        # This function also supports truncation and conversion\n",
        "                        # to pytorch tensors, but we need to do padding, so we\n",
        "                        # can't use these features :( .\n",
        "                        #max_length = 128,          # Truncate all sentences.\n",
        "                        #return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "    \n",
        "    # Add the encoded sentence to the list.\n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "# Print sentence 0, now as a list of IDs.\n",
        "print('Original: ', sentences[0])\n",
        "print('Token IDs:', input_ids[0])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Original:  Our friends won't buy this analysis, let alone the next one we propose.\n",
            "Token IDs: [101, 2256, 2814, 2180, 1005, 1056, 4965, 2023, 4106, 1010, 2292, 2894, 1996, 2279, 2028, 2057, 16599, 1012, 102]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WhwCKszh6ych",
        "colab_type": "text"
      },
      "source": [
        "## 3.3. Padding & Truncating"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xytsw1oIfnX0",
        "colab_type": "text"
      },
      "source": [
        "Pad and truncate our sequences so that they all have the same length, `MAX_LEN`."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zqiWTDrn_nGB",
        "colab_type": "text"
      },
      "source": [
        "First, what's the maximum sentence length in our dataset?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JhUZO9vc_l6T",
        "colab_type": "code",
        "outputId": "5cf10568-1550-4d27-a4db-abe59cca1992",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print('Max sentence length: ', max([len(sen) for sen in input_ids]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Max sentence length:  47\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hp-54FcQ_p3h",
        "colab_type": "text"
      },
      "source": [
        "Given that, let's choose MAX_LEN = 64 and apply the padding."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cp9BPRd1tMIo",
        "colab_type": "code",
        "outputId": "81e338e9-167a-46f5-9b93-c61dddcde17b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "# We'll borrow the `pad_sequences` utility function to do this.\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "# Set the maximum sequence length.\n",
        "# I've chosen 64 somewhat arbitrarily. It's slightly larger than the\n",
        "# maximum training sentence length of 47...\n",
        "MAX_LEN = 64\n",
        "\n",
        "print('\\nPadding/truncating all sentences to %d values...' % MAX_LEN)\n",
        "\n",
        "print('\\nPadding token: \"{:}\", ID: {:}'.format(tokenizer.pad_token, tokenizer.pad_token_id))\n",
        "\n",
        "# Pad our input tokens with value 0.\n",
        "# \"post\" indicates that we want to pad and truncate at the end of the sequence,\n",
        "# as opposed to the beginning.\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", \n",
        "                          value=0, truncating=\"post\", padding=\"post\")\n",
        "\n",
        "print('\\nDone.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Padding/truncating all sentences to 64 values...\n",
            "\n",
            "Padding token: \"[PAD]\", ID: 0\n",
            "\n",
            "Done.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kDs-MYtYH8sL",
        "colab_type": "text"
      },
      "source": [
        "## 3.4. Attention Masks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KhGulL1pExCT",
        "colab_type": "text"
      },
      "source": [
        "The attention mask simply makes it explicit which tokens are actual words versus which are padding. \n",
        "\n",
        "The BERT vocabulary does not use the ID 0, so if a token ID is 0, then it's padding, and otherwise it's a real token."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cDoC24LeEv3N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create attention masks\n",
        "attention_masks = []\n",
        "\n",
        "# For each sentence...\n",
        "for sent in input_ids:\n",
        "    \n",
        "    # Create the attention mask.\n",
        "    #   - If a token ID is 0, then it's padding, set the mask to 0.\n",
        "    #   - If a token ID is > 0, then it's a real token, set the mask to 1.\n",
        "    att_mask = [int(token_id > 0) for token_id in sent]\n",
        "    \n",
        "    # Store the attention mask for this sentence.\n",
        "    attention_masks.append(att_mask)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aRp4O7D295d_",
        "colab_type": "text"
      },
      "source": [
        "## 3.5. Training & Validation Split\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qu0ao7p8rb06",
        "colab_type": "text"
      },
      "source": [
        "Divide up our training set to use 90% for training and 10% for validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFbE-UHvsb7-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Use train_test_split to split our data into train and validation sets for\n",
        "# training\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Use 90% for training and 10% for validation.\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n",
        "                                                            random_state=2018, test_size=0.1)\n",
        "# Do the same for the masks.\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, labels,\n",
        "                                             random_state=2018, test_size=0.1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7LzSbTqW9_BR",
        "colab_type": "text"
      },
      "source": [
        "## 3.6. Converting to PyTorch Data Types"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6p1uXczp-Je4",
        "colab_type": "text"
      },
      "source": [
        "Our model expects PyTorch tensors rather than numpy.ndarrays, so convert all of our dataset variables."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jw5K2A5Ko1RF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Convert all inputs and labels into torch tensors, the required datatype \n",
        "# for our model.\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "\n",
        "train_labels = torch.tensor(train_labels)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "\n",
        "train_masks = torch.tensor(train_masks)\n",
        "validation_masks = torch.tensor(validation_masks)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dD9i6Z2pG-sN",
        "colab_type": "text"
      },
      "source": [
        "We'll also create an iterator for our dataset using the torch DataLoader class. This helps save on memory during training because, unlike a for loop, with an iterator the entire dataset does not need to be loaded into memory."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GEgLpFVlo1Z-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "# The DataLoader needs to know our batch size for training, so we specify it \n",
        "# here.\n",
        "# For fine-tuning BERT on a specific task, the authors recommend a batch size of\n",
        "# 16 or 32.\n",
        "\n",
        "batch_size = 32\n",
        "\n",
        "# Create the DataLoader for our training set.\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# Create the DataLoader for our validation set.\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8bwa6Rts-02-",
        "colab_type": "text"
      },
      "source": [
        "# 4. Train Our Classification Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3xYQ3iLO08SX",
        "colab_type": "text"
      },
      "source": [
        "Now that our input data is properly formatted, it's time to fine tune the BERT model. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6TKgyUzPIQc",
        "colab_type": "text"
      },
      "source": [
        "## 4.1. BertForSequenceClassification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1sjzRT1V0zwm",
        "colab_type": "text"
      },
      "source": [
        "For this task, we first want to modify the pre-trained BERT model to give outputs for classification, and then we want to continue training the model on our dataset until that the entire model, end-to-end, is well-suited for our task. \n",
        "\n",
        "Thankfully, the huggingface pytorch implementation includes a set of interfaces designed for a variety of NLP tasks. Though these interfaces are all built on top of a trained BERT model, each has different top layers and output types designed to accomodate their specific NLP task.  \n",
        "\n",
        "Here is the current list of classes provided for fine-tuning:\n",
        "* BertModel\n",
        "* BertForPreTraining\n",
        "* BertForMaskedLM\n",
        "* BertForNextSentencePrediction\n",
        "* **BertForSequenceClassification** - The one we'll use.\n",
        "* BertForTokenClassification\n",
        "* BertForQuestionAnswering\n",
        "\n",
        "The documentation for these can be found under [here](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BXYitPoE-cjH",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "\n",
        "We'll be using [BertForSequenceClassification](https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#bertforsequenceclassification). This is the normal BERT model with an added single linear layer on top for classification that we will use as a sentence classifier. As we feed input data, the entire pre-trained BERT model and the additional untrained classification layer is trained on our specific task. \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WnQW9E-bBCRt",
        "colab_type": "text"
      },
      "source": [
        "OK, let's load BERT! There are a few different pre-trained BERT models available. \"bert-base-uncased\" means the version that has only lowercase letters (\"uncased\") and is the smaller version of the two (\"base\" vs \"large\").\n",
        "\n",
        "The documentation for `from_pretrained` can be found [here](https://huggingface.co/transformers/v2.2.0/main_classes/model.html#transformers.PreTrainedModel.from_pretrained), with the additional parameters defined [here](https://huggingface.co/transformers/v2.2.0/main_classes/configuration.html#transformers.PretrainedConfig)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFsCTp_mporB",
        "colab_type": "code",
        "outputId": "02fb6128-30ac-47e9-90d3-cb15b95c3f31",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "cc5c9f94b6594259863e8e9773a38709",
            "d19a7e6b4db040f798732ad0eaccd0fd",
            "a330c44811e140fa9f66f81c64a94b80",
            "aa0ce42ca15040b38a378d269d5e70f3",
            "dc8780c86a6e436f940ca9001ff7bf73",
            "89199719c2524f3f8f2800efae10cd3c",
            "ae4285ddf3204f9f914d4736187797fc",
            "ac3f4b71582743e99f06187d077a482d",
            "2a518fe0c37e47eaaeb5aff73c1d0d2c",
            "f89dd52490f44cf6bff7084d33781b31",
            "4112ddcdcd6a4a8ea262b9e6df20d559",
            "4289e54205c5401bb529bc1b3d8bc300",
            "08231e0a2cad41d3a1905a26b785babe",
            "be5c4ad6e9be4b2da85695844c7b3ac7",
            "f371bb1b8c3f45b186fa49bb7c79a429",
            "100c65bc29864e00923a7c348f6aaaa2"
          ]
        }
      },
      "source": [
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "\n",
        "# Load BertForSequenceClassification, the pretrained BERT model with a single \n",
        "# linear classification layer on top. \n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "    num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "                    # You can increase this for multi-class tasks.   \n",
        "    output_attentions = False, # Whether the model returns attentions weights.\n",
        "    output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        ")\n",
        "\n",
        "# Tell pytorch to run this model on the GPU.\n",
        "model.cuda()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cc5c9f94b6594259863e8e9773a38709",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=313, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2a518fe0c37e47eaaeb5aff73c1d0d2c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=0, description='Downloading', max=440473133, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e0Jv6c7-HHDW",
        "colab_type": "text"
      },
      "source": [
        "Just for curiosity's sake, we can browse all of the model's parameters by name here.\n",
        "\n",
        "In the below cell, I've printed out the names and dimensions of the weights for:\n",
        "\n",
        "1. The embedding layer.\n",
        "2. The first of the twelve transformers.\n",
        "3. The output layer.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PIiVlDYCtSq",
        "colab_type": "code",
        "outputId": "142adf6b-cf57-4059-8b41-4a2de17aa844",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 624
        }
      },
      "source": [
        "# Get all of the model's parameters as a list of tuples.\n",
        "params = list(model.named_parameters())\n",
        "\n",
        "print('The BERT model has {:} different named parameters.\\n'.format(len(params)))\n",
        "\n",
        "print('==== Embedding Layer ====\\n')\n",
        "\n",
        "for p in params[0:5]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== First Transformer ====\\n')\n",
        "\n",
        "for p in params[5:21]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))\n",
        "\n",
        "print('\\n==== Output Layer ====\\n')\n",
        "\n",
        "for p in params[-4:]:\n",
        "    print(\"{:<55} {:>12}\".format(p[0], str(tuple(p[1].size()))))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The BERT model has 201 different named parameters.\n",
            "\n",
            "==== Embedding Layer ====\n",
            "\n",
            "bert.embeddings.word_embeddings.weight                  (30522, 768)\n",
            "bert.embeddings.position_embeddings.weight                (512, 768)\n",
            "bert.embeddings.token_type_embeddings.weight                (2, 768)\n",
            "bert.embeddings.LayerNorm.weight                              (768,)\n",
            "bert.embeddings.LayerNorm.bias                                (768,)\n",
            "\n",
            "==== First Transformer ====\n",
            "\n",
            "bert.encoder.layer.0.attention.self.query.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.query.bias                (768,)\n",
            "bert.encoder.layer.0.attention.self.key.weight            (768, 768)\n",
            "bert.encoder.layer.0.attention.self.key.bias                  (768,)\n",
            "bert.encoder.layer.0.attention.self.value.weight          (768, 768)\n",
            "bert.encoder.layer.0.attention.self.value.bias                (768,)\n",
            "bert.encoder.layer.0.attention.output.dense.weight        (768, 768)\n",
            "bert.encoder.layer.0.attention.output.dense.bias              (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.weight        (768,)\n",
            "bert.encoder.layer.0.attention.output.LayerNorm.bias          (768,)\n",
            "bert.encoder.layer.0.intermediate.dense.weight           (3072, 768)\n",
            "bert.encoder.layer.0.intermediate.dense.bias                 (3072,)\n",
            "bert.encoder.layer.0.output.dense.weight                 (768, 3072)\n",
            "bert.encoder.layer.0.output.dense.bias                        (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.weight                  (768,)\n",
            "bert.encoder.layer.0.output.LayerNorm.bias                    (768,)\n",
            "\n",
            "==== Output Layer ====\n",
            "\n",
            "bert.pooler.dense.weight                                  (768, 768)\n",
            "bert.pooler.dense.bias                                        (768,)\n",
            "classifier.weight                                           (2, 768)\n",
            "classifier.bias                                                 (2,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qRWT-D4U_Pvx",
        "colab_type": "text"
      },
      "source": [
        "## 4.2. Optimizer & Learning Rate Scheduler"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8o-VEBobKwHk",
        "colab_type": "text"
      },
      "source": [
        "Now that we have our model loaded we need to grab the training hyperparameters from within the stored model.\n",
        "\n",
        "For the purposes of fine-tuning, the authors recommend choosing from the following values:\n",
        "- Batch size: 16, 32  (We chose 32 when creating our DataLoaders).\n",
        "- Learning rate (Adam): 5e-5, 3e-5, 2e-5  (We'll use 2e-5).\n",
        "- Number of epochs: 2, 3, 4  (We'll use 4).\n",
        "\n",
        "The epsilon parameter `eps = 1e-8` is \"a very small number to prevent any division by zero in the implementation\" (from [here](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n",
        "\n",
        "You can find the creation of the AdamW optimizer in `run_glue.py` [here](https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L109)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLs72DuMODJO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
        "# I believe the 'W' stands for 'Weight Decay fix\"\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
        "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
        "                )\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-p0upAhhRiIx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 4\n",
        "\n",
        "# Total number of training steps is number of batches * number of epochs.\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# Create the learning rate scheduler.\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RqfmWwUR_Sox",
        "colab_type": "text"
      },
      "source": [
        "## 4.3. Training Loop"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_QXZhFb4LnV5",
        "colab_type": "text"
      },
      "source": [
        "Below is our training loop. There's a lot going on, but fundamentally for each pass in our loop we have a trianing phase and a validation phase. At each pass we need to:\n",
        "\n",
        "Training loop:\n",
        "- Unpack our data inputs and labels\n",
        "- Load data onto the GPU for acceleration\n",
        "- Clear out the gradients calculated in the previous pass. \n",
        "    - In pytorch the gradients accumulate by default (useful for things like RNNs) unless you explicitly clear them out.\n",
        "- Forward pass (feed input data through the network)\n",
        "- Backward pass (backpropagation)\n",
        "- Tell the network to update parameters with optimizer.step()\n",
        "- Track variables for monitoring progress\n",
        "\n",
        "Evalution loop:\n",
        "- Unpack our data inputs and labels\n",
        "- Load data onto the GPU for acceleration\n",
        "- Forward pass (feed input data through the network)\n",
        "- Compute loss on our validation data and track variables for monitoring progress\n",
        "\n",
        "So please read carefully through the comments to get an understanding of what's happening. If you're unfamiliar with pytorch a quick look at some of their [beginner tutorials](https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py) will help show you that training loops really involve only a few simple steps; the rest is usually just decoration and logging.  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pE5B99H5H2-W",
        "colab_type": "text"
      },
      "source": [
        "Define a helper function for calculating accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9cQNvaZ9bnyy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Function to calculate the accuracy of our predictions vs labels\n",
        "def flat_accuracy(preds, labels):\n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KNhRtWPXH9C3",
        "colab_type": "text"
      },
      "source": [
        "Helper function for formatting elapsed times.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpt6tR83keZD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "\n",
        "def format_time(elapsed):\n",
        "    '''\n",
        "    Takes a time in seconds and returns a string hh:mm:ss\n",
        "    '''\n",
        "    # Round to the nearest second.\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # Format as hh:mm:ss\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cfNIhN19te3N",
        "colab_type": "text"
      },
      "source": [
        "We're ready to kick off the training!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6J-FYdx6nFE_",
        "colab_type": "code",
        "outputId": "71d66aec-105d-4cd2-defc-a8836947c64e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "import random\n",
        "\n",
        "# This training code is based on the `run_glue.py` script here:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L128\n",
        "\n",
        "\n",
        "# Set the seed value all over the place to make this reproducible.\n",
        "seed_val = 42\n",
        "\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# Store the average loss after each epoch so we can plot them.\n",
        "loss_values = []\n",
        "\n",
        "# For each epoch...\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    # Perform one full pass over the training set.\n",
        "\n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # Measure how long the training epoch takes.\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Reset the total loss for this epoch.\n",
        "    total_loss = 0\n",
        "\n",
        "    # Put the model into training mode. Don't be mislead--the call to \n",
        "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
        "    # `dropout` and `batchnorm` layers behave differently during training\n",
        "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
        "    model.train()\n",
        "\n",
        "    # For each batch of training data...\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "\n",
        "        # Progress update every 40 batches.\n",
        "        if step % 40 == 0 and not step == 0:\n",
        "            # Calculate elapsed time in minutes.\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "            # Report progress.\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # Unpack this training batch from our dataloader. \n",
        "        #\n",
        "        # As we unpack the batch, we'll also copy each tensor to the GPU using the \n",
        "        # `to` method.\n",
        "        #\n",
        "        # `batch` contains three pytorch tensors:\n",
        "        #   [0]: input ids \n",
        "        #   [1]: attention masks\n",
        "        #   [2]: labels \n",
        "        b_input_ids = batch[0].to(device)\n",
        "        b_input_mask = batch[1].to(device)\n",
        "        b_labels = batch[2].to(device)\n",
        "\n",
        "        # Always clear any previously calculated gradients before performing a\n",
        "        # backward pass. PyTorch doesn't do this automatically because \n",
        "        # accumulating the gradients is \"convenient while training RNNs\". \n",
        "        # (source: https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch)\n",
        "        model.zero_grad()        \n",
        "\n",
        "        # Perform a forward pass (evaluate the model on this training batch).\n",
        "        # This will return the loss (rather than the model output) because we\n",
        "        # have provided the `labels`.\n",
        "        # The documentation for this `model` function is here: \n",
        "        # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "        outputs = model(b_input_ids, \n",
        "                    token_type_ids=None, \n",
        "                    attention_mask=b_input_mask, \n",
        "                    labels=b_labels)\n",
        "        \n",
        "        # The call to `model` always returns a tuple, so we need to pull the \n",
        "        # loss value out of the tuple.\n",
        "        loss = outputs[0]\n",
        "\n",
        "        # Accumulate the training loss over all of the batches so that we can\n",
        "        # calculate the average loss at the end. `loss` is a Tensor containing a\n",
        "        # single value; the `.item()` function just returns the Python value \n",
        "        # from the tensor.\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        # Perform a backward pass to calculate the gradients.\n",
        "        loss.backward()\n",
        "\n",
        "        # Clip the norm of the gradients to 1.0.\n",
        "        # This is to help prevent the \"exploding gradients\" problem.\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # Update parameters and take a step using the computed gradient.\n",
        "        # The optimizer dictates the \"update rule\"--how the parameters are\n",
        "        # modified based on their gradients, the learning rate, etc.\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the learning rate.\n",
        "        scheduler.step()\n",
        "\n",
        "    # Calculate the average loss over the training data.\n",
        "    avg_train_loss = total_loss / len(train_dataloader)            \n",
        "    \n",
        "    # Store the loss value for plotting the learning curve.\n",
        "    loss_values.append(avg_train_loss)\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "    # After the completion of each training epoch, measure our performance on\n",
        "    # our validation set.\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    t0 = time.time()\n",
        "\n",
        "    # Put the model in evaluation mode--the dropout layers behave differently\n",
        "    # during evaluation.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables \n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "    # Evaluate data for one epoch\n",
        "    for batch in validation_dataloader:\n",
        "        \n",
        "        # Add batch to GPU\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # Unpack the inputs from our dataloader\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        # Telling the model not to compute or store gradients, saving memory and\n",
        "        # speeding up validation\n",
        "        with torch.no_grad():        \n",
        "\n",
        "            # Forward pass, calculate logit predictions.\n",
        "            # This will return the logits rather than the loss because we have\n",
        "            # not provided labels.\n",
        "            # token_type_ids is the same as the \"segment ids\", which \n",
        "            # differentiates sentence 1 and 2 in 2-sentence tasks.\n",
        "            # The documentation for this `model` function is here: \n",
        "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
        "            outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "        \n",
        "        # Get the \"logits\" output by the model. The \"logits\" are the output\n",
        "        # values prior to applying an activation function like the softmax.\n",
        "        logits = outputs[0]\n",
        "\n",
        "        # Move logits and labels to CPU\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "        # Calculate the accuracy for this batch of test sentences.\n",
        "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        \n",
        "        # Accumulate the total accuracy.\n",
        "        eval_accuracy += tmp_eval_accuracy\n",
        "\n",
        "        # Track the number of batches\n",
        "        nb_eval_steps += 1\n",
        "\n",
        "    # Report the final accuracy for this validation run.\n",
        "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    241.    Elapsed: 0:00:11.\n",
            "  Batch    80  of    241.    Elapsed: 0:00:21.\n",
            "  Batch   120  of    241.    Elapsed: 0:00:31.\n",
            "  Batch   160  of    241.    Elapsed: 0:00:42.\n",
            "  Batch   200  of    241.    Elapsed: 0:00:52.\n",
            "  Batch   240  of    241.    Elapsed: 0:01:03.\n",
            "\n",
            "  Average training loss: 0.50\n",
            "  Training epcoh took: 0:01:03\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.79\n",
            "  Validation took: 0:00:02\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    241.    Elapsed: 0:00:11.\n",
            "  Batch    80  of    241.    Elapsed: 0:00:21.\n",
            "  Batch   120  of    241.    Elapsed: 0:00:32.\n",
            "  Batch   160  of    241.    Elapsed: 0:00:42.\n",
            "  Batch   200  of    241.    Elapsed: 0:00:52.\n",
            "  Batch   240  of    241.    Elapsed: 0:01:03.\n",
            "\n",
            "  Average training loss: 0.32\n",
            "  Training epcoh took: 0:01:03\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.82\n",
            "  Validation took: 0:00:02\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    241.    Elapsed: 0:00:11.\n",
            "  Batch    80  of    241.    Elapsed: 0:00:21.\n",
            "  Batch   120  of    241.    Elapsed: 0:00:32.\n",
            "  Batch   160  of    241.    Elapsed: 0:00:42.\n",
            "  Batch   200  of    241.    Elapsed: 0:00:52.\n",
            "  Batch   240  of    241.    Elapsed: 0:01:03.\n",
            "\n",
            "  Average training loss: 0.20\n",
            "  Training epcoh took: 0:01:03\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.82\n",
            "  Validation took: 0:00:02\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch    40  of    241.    Elapsed: 0:00:10.\n",
            "  Batch    80  of    241.    Elapsed: 0:00:21.\n",
            "  Batch   120  of    241.    Elapsed: 0:00:31.\n",
            "  Batch   160  of    241.    Elapsed: 0:00:42.\n",
            "  Batch   200  of    241.    Elapsed: 0:00:52.\n",
            "  Batch   240  of    241.    Elapsed: 0:01:03.\n",
            "\n",
            "  Average training loss: 0.14\n",
            "  Training epcoh took: 0:01:03\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.82\n",
            "  Validation took: 0:00:02\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1-G03mmwH3aI",
        "colab_type": "text"
      },
      "source": [
        "Let's take a look at our training loss over all batches:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68xreA9JAmG5",
        "colab_type": "code",
        "outputId": "7e4cc7cc-1327-47da-dae3-6095fce65551",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 427
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "# Use plot styling from seaborn.\n",
        "sns.set(style='darkgrid')\n",
        "\n",
        "# Increase the plot size and font size.\n",
        "sns.set(font_scale=1.5)\n",
        "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
        "\n",
        "# Plot the learning curve.\n",
        "plt.plot(loss_values, 'b-o')\n",
        "\n",
        "# Label the plot.\n",
        "plt.title(\"Training loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.ylabel(\"Loss\")\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvAAAAGaCAYAAABpIXfbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeViWZd7/8ffNLpsIAqKsbqAoqLhr\nrqSouGRqbrnk+Ng0TU09TubYYlZjqe3r2IylprnkLuVapiWJoIEomrsgmgiCorIo/P7oJ88QqKDo\ndQOf13F4HN3ntZzfi2/oh4vzvm5TYWFhISIiIiIiUilYGF2AiIiIiIiUnQK8iIiIiEglogAvIiIi\nIlKJKMCLiIiIiFQiCvAiIiIiIpWIAryIiIiISCWiAC8iUk3NmTOHwMBA0tLS7uj43NxcAgMDeeml\nlyq4svL56quvCAwM5JdffjG0DhGR+8XK6AJERKqzwMDAMu+7detWvL2972E1IiJSGSjAi4gYaNas\nWcVex8XFsXTpUh555BHCwsKKbXN1da3Quf/2t7/x17/+FVtb2zs63tbWloSEBCwtLSu0LhERuTUF\neBERAw0cOLDY6+vXr7N06VJatGhRYtvNFBYWcvXqVezt7cs1t5WVFVZWd/fPwJ2GfxERuXNaAy8i\nUols376dwMBA1q9fz/z584mIiKB58+Z8+eWXAOzZs4fnnnuOXr16ERoaSqtWrRg1ahTff/99iXOV\ntgb+xlhycjJvvvkmDzzwAM2bN+ehhx7ip59+KnZ8aWvg/3ts9+7djBgxgtDQUNq3b89LL73E1atX\nS9Sxc+dOhg4dSvPmzencuTNvvPEGBw4cIDAwkLlz597x1+r8+fO89NJLdOnShWbNmtG9e3dee+01\nsrKyiu135coV3nnnHXr37k1ISAht2rShf//+vPPOO8X227JlCyNGjKBdu3aEhITQvXt3nnrqKZKT\nk++4RhGRO6E78CIildBnn33GpUuXePjhh3Fzc8PHxweADRs2kJycTN++falbty4ZGRmsWrWKxx9/\nnA8++IBevXqV6fz/+7//i62tLX/605/Izc3liy++4M9//jObN2/G09Pztsfv27ePjRs3MmTIEAYM\nGEB0dDRLly7FxsaGF154oWi/6OhoJk6ciKurK5MmTcLR0ZGoqChiYmLu7Avz/2VmZvLII4+QmprK\n0KFDCQoKYt++fXz55Zfs2rWLZcuWUaNGDQBefPFFoqKieOihh2jRogX5+fmcOHGCn3/+ueh8P/74\nI08++SRNmzbl8ccfx9HRkd9++42ffvqJlJSUoq+/iMj9oAAvIlIJnTt3jm+//RYXF5di43/7299K\nLKV59NFHGTBgAJ988kmZA7ynpyfvv/8+JpMJoOhO/vLly3nyySdve/yhQ4f4+uuvadq0KQAjRoxg\n7NixLF26lOeeew4bGxsAZs6cibW1NcuWLcPLywuAkSNHMnz48DLVeTOffvopKSkpvP766wwZMqRo\nvFGjRrz55ptFP5AUFhby3XffER4ezsyZM296vi1btgAwf/58nJycisbL8rUQEaloWkIjIlIJPfzw\nwyXCO1AsvF+9epULFy6Qm5tL27ZtSUpKIi8vr0znHzt2bFF4BwgLC8Pa2poTJ06U6fg2bdoUhfcb\n2rdvT15eHmfOnAHg9OnTHDp0iN69exeFdwAbGxvGjBlTpnlu5sZvCgYPHlxsfPTo0Tg5ObF582YA\nTCYTDg4OHDp0iKNHj970fE5OThQWFrJx40auX79+V7WJiNwt3YEXEamE/P39Sx0/d+4c77zzDt9/\n/z0XLlwosf3SpUu4ubnd9vx/XBJiMpmoWbMmmZmZZaqvtCUlN37gyMzMxM/Pj5SUFAACAgJK7Fva\nWFkVFhaSmppK+/btsbAofp/KxsYGX1/forkBpk2bxj/+8Q/69u2Ln58f7dq1o0ePHnTr1q3oh5ix\nY8eybds2pk2bxhtvvEHr1q154IEH6Nu3L7Vq1brjWkVE7oQCvIhIJXRj/fZ/u379OuPGjSMlJYUx\nY8YQHByMk5MTFhYWLFmyhI0bN1JQUFCm8/8x+N5QWFh4V8eX5xz3S58+fWjXrh3bt28nJiaGH3/8\nkWXLltGhQwf+/e9/Y2VlRe3atVm1ahW7d+9m586d7N69m9dee43333+f//znPzRr1szoyxCRakQB\nXkSkikhMTOTo0aM8++yzTJo0qdi2G0+pMSf16tUD4Pjx4yW2lTZWViaTiXr16nHs2DEKCgqK/TCR\nl5fHqVOn8PX1LXaMq6srgwYNYtCgQRQWFvLPf/6TBQsWsH37dnr06AH8/tjNDh060KFDB+D3r/eQ\nIUP417/+xQcffHDH9YqIlJfWwIuIVBE3guof73Dv37+fH374wYiSbsnb25vGjRuzcePGonXx8HvI\nXrBgwV2dOzw8nLNnz7J69epi44sXL+bSpUs8+OCDAOTn55OdnV1sH5PJRJMmTQCKHjmZkZFRYo6G\nDRtiY2NT5mVFIiIVRXfgRUSqiMDAQPz9/fnkk0+4ePEi/v7+HD16lGXLlhEYGMj+/fuNLrGE559/\nnokTJzJs2DCGDx+Og4MDUVFRxd5Aeycef/xxNm3axAsvvEB8fDyBgYEkJiaycuVKGjduzLhx44Df\n1+OHh4cTHh5OYGAgrq6uJCcn89VXX1GrVi26du0KwHPPPcfFixfp0KED9erV48qVK6xfv57c3FwG\nDRp0t18GEZFyUYAXEakibGxs+Oyzz5g1axYrVqwgNzeXxo0b8/bbbxMXF2eWAb5Tp07MnTuXd955\nh08//ZSaNWsSGRlJeHg4o0aNws7O7o7O6+LiwtKlS/nggw/YunUrK1aswM3NjdGjR/PXv/616D0E\nTk5OjB49mujoaHbs2MHVq1dxd3enV69eTJo0CVdXVwAGDx7MmjVrWLlyJRcuXMDJyYlGjRrx8ccf\n07Nnzwr7eoiIlIWp0NzeTSQiItXe2rVr+fvf/85HH31EeHi40eWIiJgVrYEXERHDFBQUlHg2fV5e\nHvPnz8fGxobWrVsbVJmIiPnSEhoRETFMdnY2ffv2pX///vj7+5ORkUFUVBSHDx/mySefLPXDqkRE\nqjsFeBERMYydnR2dOnVi06ZNnD9/HoD69evz6quvMmzYMIOrExExT1oDLyIiIiJSiWgNvIiIiIhI\nJaIALyIiIiJSiWgNfDlduHCZgoL7v+rIzc2R9PTs2+8o9416Yp7UF/Ojnpgn9cX8qCfmyYi+WFiY\nqFXL4abbFeDLqaCg0JAAf2NuMS/qiXlSX8yPemKe1Bfzo56YJ3Pri5bQiIiIiIhUIgrwIiIiIiKV\niAK8iIiIiEglogAvIiIiIlKJGBrg8/LymD17Np07dyYkJIRhw4YRHR192+M++OADAgMDS/zp1KlT\nqfsvX76cPn360Lx5c3r37s2iRYsq+lJERERERO4LQ59C8/zzz7Np0ybGjBmDn58fq1atYuLEiSxc\nuJCWLVve9vgZM2ZgZ2dX9Pq///uGJUuW8PLLLxMREcH48eOJjY1lxowZ5Obm8thjj1Xo9YiIiIiI\n3GuGBfiEhASioqKYOnUq48aNA2DQoEFERkYyZ86cMt0l79OnD87OzjfdnpOTwzvvvEPPnj157733\nABg2bBgFBQV8+OGHDB06FCcnpwq5HhERERGR+8GwJTQbNmzA2tqaoUOHFo3Z2toyZMgQ4uLiOHfu\n3G3PUVhYSHZ2NoWFpT+bc9euXWRmZjJy5Mhi46NGjeLy5cts37797i5CREREROQ+MyzAJyUlERAQ\ngIND8U+ZCgkJobCwkKSkpNueo1u3boSFhREWFsbUqVPJzMwstv3AgQMANGvWrNh4cHAwFhYWRdtF\nRERERCoLw5bQpKWl4enpWWLc3d0d4JZ34J2dnXn00UcJDQ3F2tqan3/+maVLl3LgwAGWL1+OjY1N\n0Rw2Nja4uLgUO/7GWFnu8hstev9ZVv5wlIyLubg62zK4awM6BNcxuiwRERERMYhhAT4nJwdra+sS\n47a2tgDk5ube9NixY8cWex0REUGjRo2YMWMGq1evZtiwYbec48Y8t5rjZtzcHMt9zJ3aFpfMgg2H\nyM2/DkD6xVwWbDiEs5Md3cJ87lsdcnPu7noPhTlSX8yPemKe1Bfzo56YJ3Pri2EB3s7Ojvz8/BLj\nN0L1jSBfViNGjGD27NlER0cXBXg7Ozvy8vJK3T83N7fccwCkp2dTUFD6mvuK9sX6/UXh/Ybc/Ot8\nsX4/wb4uNzlK7hd3dyfS0i4ZXYb8gfpiftQT86S+mB/1xDwZ0RcLC9Mtbxobtgbe3d291CUsaWlp\nAHh4eJTrfBYWFnh6epKVlVVsjvz8/BJr4/Py8sjMzCz3HPdb+sXSf0Nws3ERERERqfoMC/BBQUEc\nP36cy5cvFxuPj48v2l4e+fn5nDlzhlq1ahWNNWnSBIDExMRi+yYmJlJQUFC03Vy5OZf+GwKnGqUv\nCxIRERGRqs+wAB8REUF+fj7Lly8vGsvLy2PlypW0atWq6A2uqampHD16tNixGRkZJc73n//8h9zc\nXB544IGisfbt2+Pi4sLixYuL7fvVV19hb29Ply5dKvKSKtzgrg2wsSreIhNw6Wo+S7Ye5tr1AmMK\nExERERHDGLYGPjQ0lIiICObMmUNaWhq+vr6sWrWK1NRUZs6cWbTflClTiImJ4dChQ0Vj3bt3p2/f\nvjRu3BgbGxt27drFxo0bCQsLIzIysmg/Ozs7nnrqKWbMmMHTTz9N586diY2NZe3atUyePPmWHwJl\nDm48bea/n0IzsHMAJ89ms2l3Mr8mZ/L4wGA8atkbXKmIiIiI3C+GBXiAWbNm8e6777JmzRqysrII\nDAxk7ty5hIWF3fK4/v37s2fPHjZs2EB+fj716tXjiSeeYNKkSVhZFb+kUaNGYW1tzbx589i6dSte\nXl5MmzaNMWPG3MtLqzAdguvQIbhOsTdQdA6BIL9afP5NEq98sZuxEUG0bVLykZwiIiIiUvWYCm/2\nMaZSqvv5FJr/Vto7oM9nXeVfa/ZzNPUiXVvUZUTPRthYW9732qorPS3APKkv5kc9MU/qi/lRT8yT\nnkIjFap2zRpMGdWKPu19+eGXVF5dEMvp85dvf6CIiIiIVFoK8JWclaUFQ7s15JlhoVy8nMer83ez\nIyEV/WJFREREpGpSgK8imtd3Y/r4ttT3cubzbw7y2foDXM29ZnRZIiIiIlLBFOCrkFpOtkwe3pJB\nnQPYdeA3Znyxm1O/aS2diIiISFWiAF/FWFiYGNA5gOdGtCQ3/zqvLYhla1yKltSIiIiIVBEK8FVU\noG8tpj/Wlqb+riza/CsfrUrkck6+0WWJiIiIyF1SgK/CnO1teGpICI/0aEj8kfNMn7ebo6ezjC5L\nRERERO6CAnwVZ2Ey0butL1NHh2EywRuL9vDtzycp0JIaERERkUpJAb6aqF/Xmenj29CyUW2WbzvK\nu8viuXg5z+iyRERERKScFOCrEXs7a/48qBmP9g7k4KlMXv48hqSTF4wuS0RERETKQQG+mjGZTHRv\nWY8XxoRRw8aKOV/tZfWOYxQUaEmNiIiISGWgAF9N+Xo68dK41nRsVoe1P51g9ld7uXAp1+iyRERE\nROQ2FOCrMTsbKyZENmVCvyacOHuJl+fFkHD0vNFliYiIiMgtKMALnZp78dK41rg42vLu8gSWfXeE\na9cLjC5LREREREqhAC8AeLk58OLYMLq3qseGmFPM/HIPaZlXjS5LRERERP5AAV6KWFtZ8mivQJ4Y\n1IyzGVeY/vluYg+eM7osEREREfkvCvBSQusgD6aPb0MdV3s+Xp3Iwo2HyL923eiyRERERAQFeLkJ\nd5caTB3dioi2vny/9zSvzo/jTPplo8sSERERqfYU4OWmrCwtGNajIX8bGkJmdi4zvojlp31njC5L\nREREpFpTgJfbCmlQm1cea4tfHSf+E5XEv9cfICfvmtFliYiIiFRLCvBSJrWcbPn7iBYM6ORPdOJZ\nZnwRS/K5bKPLEhEREal2FOClzCwtLBj0QH0mj2jJ1bxrvDo/lu/3nqawsNDo0kRERESqDQV4Kbcm\nfrV4ZXxbgvxcWLjxEJ+sTuRKTr7RZYmIiIhUCwrwckecHWz429BQhnZvwN7D55n++W6OpV40uiwR\nERGRKk8BXu6YhclEn3Z+PD+qFYWFMPPLODbsOkWBltSIiIiI3DMK8HLXGtSryfTH2hDasDbLvj/C\n+18ncOlKntFliYiIiFRJCvBSIRzsrPnLQ80Y9WBjDpzI4OV5MRw6dcHoskRERESqHAV4qTAmk4me\nYd5Me7Q1ttaWzPpqL2t/PE5BgZbUiIiIiFQUQwN8Xl4es2fPpnPnzoSEhDBs2DCio6PLfZ6JEycS\nGBjI66+/XmJbYGBgqX+++uqrirgEKYVfHSdeGteGdk09Wf3jceYs2Utmdq7RZYmIiIhUCVZGTv78\n88+zadMmxowZg5+fH6tWrWLixIksXLiQli1blukc27ZtIzY29pb7dO7cmQEDBhQbCw0NveO65fZq\n2FoxMbIpTfxqsWjzr7w8L4aJkU1pVt/N6NJEREREKjXDAnxCQgJRUVFMnTqVcePGATBo0CAiIyOZ\nM2cOixYtuu058vLymDlzJhMmTOCDDz646X7169dn4MCBFVW6lJHJZOKBkLo0qFuTT9Yk8vayePq0\n9+WhB+pjZanVWyIiIiJ3wrAUtWHDBqytrRk6dGjRmK2tLUOGDCEuLo5z587d9hwLFiwgJyeHCRMm\n3HbfnJwccnO1jMMIdWs78OKY1nRrUZdvfz7Fm4v3cD7rqtFliYiIiFRKhgX4pKQkAgICcHBwKDYe\nEhJCYWEhSUlJtzw+LS2Njz/+mGeeeYYaNWrcct+vv/6aFi1aEBISQv/+/dm8efNd1y/lY2NtyZiI\nIB4fGEzq+ctMn7ebuENpRpclIiIiUukYtoQmLS0NT0/PEuPu7u4At70D//bbbxMQEHDbpTEtW7ak\nb9++eHt7c+bMGRYsWMCTTz7JW2+9RWRkZLnrdnNzLPcxFcXd3cmwuStKP3cnwoK9eHNhLB+t2kdk\npwDG9w/GxtrS6NLuSFXoSVWkvpgf9cQ8qS/mRz0xT+bWF8MCfE5ODtbW1iXGbW1tAW653CUhIYHV\nq1ezcOFCTCbTLedZsmRJsdcPPfQQkZGRzJ49m379+t32+D9KT8825LGI7u5OpKVduu/z3guWwHPD\nW/D1tqOs/+k4CYfTeHxQM+q42htdWrlUpZ5UJeqL+VFPzJP6Yn7UE/NkRF8sLEy3vGls2BIaOzs7\n8vPzS4zfCO43gvwfFRYW8vrrr9OrVy9at25d7nnt7e0ZPnw4Z8+e5dixY+U+XiqGlaUFw3s24qmH\nQ0i/mMMrX+wmev9Zo8sSERERMXuGBXh3d/dSl8mkpf2+LtrDw6PU4zZv3kxCQgIjRowgJSWl6A9A\ndnY2KSkp5OTk3HJuLy8vALKysu7mEqQCtGhUm1cea4uvhyOfrTvAvKgkcvOuG12WiIiIiNkyLMAH\nBQVx/PhxLl++XGw8Pj6+aHtpUlNTKSgoYOzYsfTs2bPoD8DKlSvp2bMnMTExt5w7OTkZAFdX17u9\nDKkArs52PDeyJZEd/flp3xlmzN9NSlq20WWJiIiImCXD1sBHREQwb948li9fXvQc+Ly8PFauXEmr\nVq2K3uCamprK1atXadCgAQA9evTA29u7xPn+8pe/0L17d4YMGUJwcDAAGRkZJUL6hQsXWLx4Md7e\n3vj7+9+7C5RysbSwYHCX+gT6uvDZugO8Oj+WkeGN6BJat9zvUxARERGpygwL8KGhoURERDBnzhzS\n0tLw9fVl1apVpKamMnPmzKL9pkyZQkxMDIcOHQLA19cXX1/fUs/p4+NDeHh40etFixaxdetWunXr\nRt26dfntt99YunQpGRkZfPTRR/f2AuWOBPu78spjbfn3uv3M33CIpJMXGBsRRA1bQz80WERERMRs\nGJqKZs2axbvvvsuaNWvIysoiMDCQuXPnEhYWViHnb9myJXv27GH58uVkZWVhb29PixYtmDRpUoXN\nIRWvpoMNzzzSgm9/Psmq7cc5fuYijw9sRoCXs9GliYiIiBjOVFhYeP+fiViJ6TGS99fhlEz+tXY/\nWdl5DO3ekAdbe5vNkprq2hNzp76YH/XEPKkv5kc9MU96jKRIOTXydmH6+LY0r+/Gkq2H+WDFPrKv\nlnz8qIiIiEh1oQAvZs+xhjV/fbg5I8IbkXg8nZfnxfBrcqbRZYmIiIgYQgFeKgWTycSDrX34x6Nh\nWFtaMGvxXtbtPGHIciYRERERIynAS6XiX8eZl8e3oXWQO6u2H+PtZb+QlZ1rdFkiIiIi940CvFQ6\nNWytmDQgmHF9gjiSksXL82LYfzzD6LJERERE7gsFeKmUTCYTXULr8uLY1jja2/D20l9Y8cNRrhcU\nGF2aiIiIyD2lAC+VWj13R14c25oHQr2Iij7Jm4v3knExx+iyRERERO4ZBXip9GytLRnXpwn/M6Ap\nyeeyeXleDHsPpxldloiIiMg9oQAvVUb7pnWYPr4NtWvW4IMV+1i85Vfyr2lJjYiIiFQtCvBSpXjW\nsucfj4YRHubNltgU/rkwjt8uXDG6LBEREZEKowAvVY61lQUjH2zMk4Obcz7rKq98vptdB34zuiwR\nERGRCqEAL1VWq8buTB/fFm93R/61dj9ffJtEbv51o8sSERERuSsK8FKludW047mRLenXwY8d8Wd4\nbX4sp89fNrosERERkTumAC9VnpWlBQ93bcAzj4Ry6Uoer36xmx3xqRQWFhpdmoiIiEi5KcBLtdEs\nwI1XHmtLg3o1+fzbg3y27gBXc68ZXZaIiIhIuSjAS7VS09GW/32kBQ91qc+upN945YvdnDx7yeiy\nRERERMpMAV6qHQsLE/07+jNlZCvyrxXw+sJYtsQma0mNiIiIVAoK8FJtNfZxYfr4NgT7u7J4y2E+\nXLmP7Kv5RpclIiIicksK8FKtOdnb8NSQEIb3aEjC0XRe+TyGIylZRpclIiIiclMK8FLtmUwmerX1\n5R+PhmEymXhj0R6iok9QoCU1IiIiYoYU4EX+vwAvZ6aPb0tYoDsrfjjGO8viybqcZ3RZIiIiIsUo\nwIv8F3s7Kx4fGMyYiEB+Tc5k+rwYkk5kGF2WiIiISBEFeJE/MJlMdGtRjxfHtMbezoo5S35h1fZj\nXC8oMLo0EREREQV4kZvx9nDkpbFt6NTci3U7TzB78V4yLuYYXZaIiIhUcwrwIrdga2PJY/2aMDGy\nKSd/y2b657v55ch5o8sSERGRakwBXqQMOjSrw8vj2+DqZMv7XyewZOth8q9pSY2IiIjcfwrwImVU\nx9WeaWPC6NGqHpt2JzPlwx2cy7xqdFkiIiJSzRga4PPy8pg9ezadO3cmJCSEYcOGER0dXe7zTJw4\nkcDAQF5//fVSty9fvpw+ffrQvHlzevfuzaJFi+62dKmmrK0sGd0rkL881IzUtGxe+TyGmKTfjC5L\nREREqhFDA/zzzz/P/PnzGTBgANOmTcPCwoKJEyeyd+/eMp9j27ZtxMbG3nT7kiVLeOGFF2jcuDEv\nvvgioaGhzJgxg3nz5lXEJUg1FRbowXv/2526bg58umY/CzYcJC//utFliYiISDVgWIBPSEggKiqK\nyZMn89xzz/HII48wf/58vLy8mDNnTpnOkZeXx8yZM5kwYUKp23NycnjnnXfo2bMn7733HsOGDWPW\nrFn079+fDz/8kEuXLlXkJUk14+lqz5RRrejTzpdtv6Ty2oJYzqRfNrosERERqeIMC/AbNmzA2tqa\noUOHFo3Z2toyZMgQ4uLiOHfu3G3PsWDBAnJycm4a4Hft2kVmZiYjR44sNj5q1CguX77M9u3b7+4i\npNqzsrRgaPeGPDMslKzLebzyxW5+2nfG6LJERESkCjMswCclJREQEICDg0Ox8ZCQEAoLC0lKSrrl\n8WlpaXz88cc888wz1KhRo9R9Dhw4AECzZs2KjQcHB2NhYVG0XeRuNa/vxvTxbanv5cx/opL4bN0B\ncvKuGV2WiIiIVEGGBfi0tDQ8PDxKjLu7uwPc9g7822+/TUBAAAMHDrzlHDY2Nri4uBQbvzFWlrv8\nImVVy8mWycNbMqhzAD8fOMsrX8Ry6jct0xIREZGKZWXUxDk5OVhbW5cYt7W1BSA3N/emxyYkJLB6\n9WoWLlyIyWQq9xw35rnVHDfj5uZY7mMqiru7k2FzS+lK68mEh0JoG1KXOV/G8frCOCb0D6Zvp4Bb\n/r8qFUvfK+ZHPTFP6ov5UU/Mk7n1xbAAb2dnR35+fonxG6H6RpD/o8LCQl5//XV69epF69atbztH\nXl5eqdtyc3NvOsetpKdnU1BQWO7j7pa7uxNpabqba05u1ZM6zra8NK41/1mfxKer9rF7/1nG9w3C\n3q70Hyil4uh7xfyoJ+ZJfTE/6ol5MqIvFhamW940NmwJjbu7e6lLWNLS0gBKXV4DsHnzZhISEhgx\nYgQpKSlFfwCys7NJSUkhJyenaI78/HwyMzOLnSMvL4/MzMybziFSEZztbXh6aAjDujfklyPneXne\nbo6ezjK6LBEREankDAvwQUFBHD9+nMuXiz92Lz4+vmh7aVJTUykoKGDs2LH07Nmz6A/AypUr6dmz\nJzExMQA0adIEgMTExGLnSExMpKCgoGi7yL1iYTIR0c6X50e3wmSCNxbt4dtdJykovP+/xREREZGq\nwbAlNBEREcybN4/ly5czbtw44Pc74ytXrqRVq1Z4enoCvwf2q1ev0qBBAwB69OiBt7d3ifP95S9/\noXv37gwZMoTg4GAA2rdvj4uLC4sXL6Zz585F+3711VfY29vTpUuXe3yVIr9rULcm08e34fNvD7L8\n+6McPJnJhMgmONvbGF2aiIiIVDKGBfjQ0FAiIiKYM2cOaWlp+Pr6smrVKlJTU5k5c2bRflOmTCEm\nJoZDhw4B4Ovri6+vb6nn9PHxITw8vOi1nZ0dTz31FDNmzODpp5+mc+fOxMbGsnbtWiZPnoyzs/O9\nvUiR/2JvZ80Tg5qxbe9pvtp6hOnzYvif/sEE+dUyujQRERGpRAwL8ACzZs3i3XffZc2aNWRlZREY\nGMjcuXMJCwursDlGjRqFtZMS20cAACAASURBVLU18+bNY+vWrXh5eTFt2jTGjBlTYXOIlJXJZKJ7\nK28a1KvJJ2v2M3vJXgZ0CqB/R38sLPSUGhEREbk9U2GhFuOWh55CIzfcbU9y8q7x5aZf2Zl4liBf\nFyb2D6aWU/mfjCTF6XvF/Kgn5kl9MT/qiXnSU2hEpIidjRV/imzKhH5NOHbmIi/PiyHhaLrRZYmI\niIiZU4AXMVin5l68PK4NLo62vLs8nmXfH+Ha9QKjyxIREREzpQAvYga83Bx4YUwY3VrWY8OuU7yx\naA/nM68aXZaIiIiYIQV4ETNhY23JmN6B/HlQM86kX+blz3cTe7Dkh52JiIhI9aYAL2Jm2gR58PL4\nttRxrcHHqxNZuOkQ+deuG12WiIiImAkFeBEz5OFSg6mjw+jd1ofv95zmtQVxnM24YnRZIiIiYgYU\n4EXMlJWlBY/0aMTTQ0K4cCmXVz7fzc7EM0aXJSIiIgZTgBcxc6ENazN9fBv86jjx7/VJ/CfqALl5\nWlIjIiJSXSnAi1QCrs52/H1ECwZ08mfnvrPMmL+b5HPZRpclIiIiBlCAF6kkLC0sGPRAfSYPb8GV\nnGu8Oj+W7/eeRh+mLCIiUr0owItUMk38XXnlsbYE+rqwcOMhPlmznys514wuS0RERO4TBXiRSsjZ\nwYZnhoUypFsD9hxKY/rnMRw/c9HoskREROQ+UIAXqaQsTCb6tvfj+dGtKCws5J8L49gYc0pLakRE\nRKo4BXiRSq5hvZpMf6wtoQ1rs/S7I7z3dQKXruQZXZaIiIjcIwrwIlWAg501f3moGaMebMyBExlM\n/3w3vyZnGl2WiIiI3AMK8CJVhMlkomeYN9MebY21lQVvLt7Dup+OU1CgJTUiIiJViQK8SBXjV8eJ\nl8e1oV1TT1btOM5bS38hMzvX6LJERESkgijAi1RBNWytmBjZlPF9gzh6Oovp82JIPJZudFkiIiJS\nARTgRaook8nEAyF1eXFcG5wcbHh7WTxfbzvKtesFRpcmIiIid0EBXqSKq1fbgRfGtKZLaF2++fkk\nby7ew/msq0aXJSIiIndIAV6kGrC1tmRcnyAeHxjM6bTLTJ+3mz2/phldloiIiNwBBXiRaqRtE0+m\nj2+De60afLhyH4s2/0r+NS2pERERqUwU4EWqGY9a9vxjdBgPtvZha1wK/1wYx28ZV4wuS0RERMpI\nAV6kGrK2smBEeCOeejiE81lXmf7Fbn7ef9boskRERKQMFOBFqrEWjWrzymNt8fFwZO66A3z+TRK5\n+deNLktERERuQQFepJpzdbZjysiWRHb048eEM7w6P5aUtGyjyxIREZGbUIAXESwtLBjcpQHPDm9B\n9tV8Xp0fyw+/nKawsNDo0kREROQPrIycPC8vj/fee481a9Zw8eJFgoKCeOaZZ+jQocMtj1u7di1f\nf/01R48eJSsrCw8PD9q1a8eTTz5JvXr1iu0bGBhY6jmmT5/OiBEjKuxaRKqCYH9XXhnfhs/WH2D+\nhkMknbzA2Iggatga+leFiIiI/BdD/1V+/vnn2bRpE2PGjMHPz49Vq1YxceJEFi5cSMuWLW963MGD\nB/H09KRr167UrFmT1NRUli1bxrZt21i7di3u7u7F9u/cuTMDBgwoNhYaGnpPrkmksqvpaMuzj7Tg\nm+iTrN5xnBNnLvH4oGD86zgbXZqIiIhgYIBPSEggKiqKqVOnMm7cOAAGDRpEZGQkc+bMYdGiRTc9\n9rnnnisx1rNnTwYPHszatWuZMGFCsW3169dn4MCBFVq/SFVmYTIR2dGfQF8X/rV2P68viGNY94aE\nt/bGZDIZXZ6IiEi1Ztga+A0bNmBtbc3QoUOLxmxtbRkyZAhxcXGcO3euXOerW7cuABcvXix1e05O\nDrm5uXdesEg11Mjbhenj29K8vhtfbT3MByv2kX013+iyREREqjXDAnxSUhIBAQE4ODgUGw8JCaGw\nsJCkpKTbniMzM5P09HT27dvH1KlTAUpdP//111/TokULQkJC6N+/P5s3b66YixCpBhxrWPPXh5sz\nomcj9h1LZ/rnMRxOyTS6LBERkWrLsCU0aWlpeHp6lhi/sX69LHfge/fuTWbm70HCxcWFl156ifbt\n2xfbp2XLlvTt2xdvb2/OnDnDggULePLJJ3nrrbeIjIysgCsRqfpMJhMPtvGhoXdN/rVmP28u2stD\nXQLo094PCy2pERERua8MC/A5OTlYW1uXGLe1tQUo03KXDz/8kCtXrnD8+HHWrl3L5cuXS+yzZMmS\nYq8feughIiMjmT17Nv369Sv3el43N8dy7V+R3N2dDJtbSlfdeuLu7kSzxh58tDyeFT8c4+iZSzw7\nshW1nOyMLq2Y6taXykA9MU/qi/lRT8yTufXFsABvZ2dHfn7JtbQ3gvuNIH8rbdq0AaBr16707NmT\n/v37Y29vz+jRo296jL29PcOHD+ett97i2LFjNGjQoFx1p6dnU1Bw/5+N7e7uRFrapfs+r9xcde7J\n2N6Nqe/lxKLNv/Lk7O+Z2L8pwf6uRpcFVO++mCv1xDypL+ZHPTFPRvTFwsJ0y5vGhq2Bd3d3L3WZ\nTFpaGgAeHh7lOp+Pjw/BwcGsW7futvt6eXkBkJWVVa45ROR3JpOJLqF1eXFsaxxrWPP2kl9Yuf0o\n1wsKjC5NRESkyjMswAcFBXH8+PESy17i4+OLtpdXTk4Oly7d/iek5ORkAFxdzeOOoUhl5e3uyItj\nWtMpxIv1O08ya/FeMi7mGF2WiIhIlWZYgI+IiCA/P5/ly5cXjeXl5bFy5UpatWpV9AbX1NRUjh49\nWuzYjIyMEudLTEzk4MGDBAcH33K/CxcusHjxYry9vfH396+gqxGpvmxtLHmsbxP+p39TTp3L5uV5\nMfxy+LzRZYmIiFRZhq2BDw0NJSIigjlz5pCWloavry+rVq0iNTWVmTNnFu03ZcoUYmJiOHToUNFY\n9+7d6dOnD40bN8be3p4jR46wYsUKHBwceOKJJ4r2W7RoEVu3bqVbt27UrVuX3377jaVLl5KRkcFH\nH310X69XpKprH1yHAC9nPlmTyPsrEniwtQ9DuzfAytKw+wQiIiJVkmEBHmDWrFm8++67rFmzhqys\nLAIDA5k7dy5hYWG3PG7kyJFER0ezZcsWcnJycHd3JyIigieeeAIfH5+i/Vq2bMmePXtYvnw5WVlZ\n2Nvb06JFCyZNmnTbOUSk/Dxd7Zn2aGuWfX+EzbHJHE7J5PGBwXjUsje6NBERkSrDVFhYeP8fqVKJ\n6Sk0coN6cmt7fk1jXlQSBYWFjOsTRNsmJT/34V5QX8yPemKe1Bfzo56YJz2FRkSqjVaN3Zn+WBvq\nuTvw6Zr9zN9wkLz860aXJSIiUukpwIvIPVO7Zg2mjGxF3/Z+/PBLKq8uiOX0+ZIfuCYiIiJlV+4A\nf/LkSbZv315sLD4+nscff5zhw4ezdOnSCitORCo/K0sLhnRrwLPDQrl4OY9Xv9jNjvhUtHpPRETk\nzpT7Taxz5swhMzOTLl26AL8/qnHixIlcuXIFW1tbpk+fjpubG+Hh4RVerIhUXs3qu/HKY235bN0B\nPv/2IEmnLvBor0Bq2Br6XnoREZFKp9x34BMTE+nYsWPR66ioKLKzs1m5ciXR0dGEhoYyf/78Ci1S\nRKoGF0db/veRFgx6IIBdB35jxhe7OXlWb9gSEREpj3IH+IyMDDw8PIpe79ixg1atWtG4cWNsbGzo\n27dviQ9eEhG5wcLCxIBOATw3oiV51wp4fWEsW+NStKRGRESkjMod4GvUqMGlS7/fMbt+/TpxcXG0\nbt26aLudnR3Z2dkVV6GIVEmBvrWYPr4NTf1dWbT5Vz5alcjlnHyjyxIRETF75Q7wjRo1YvXq1Vy4\ncIFly5Zx5coVOnXqVLT99OnTuLq6VmiRIlI1Odnb8PSQEB7p0ZD4I+eZPm83R05nGV2WiIiIWSt3\ngJ8wYQK//vorHTt2ZMaMGTRp0qTYHfiffvqJpk2bVmiRIlJ1mUwmerf1ZeroMEwmeOPLPXz780kK\ntKRGRESkVOV+/EO3bt2YP38+W7duxdHRkdGjR2MymQC4cOECderUYdCgQRVeqIhUbfXrOjN9fFu+\n2HCQ5duOknTyAn+KbIqzg43RpYmIiJgVU6HeOVYu6enZFBTc/y+ZPl7Z/Kgn90ZhYSE//JLK4i2H\ncbCz4n/6N6WJf9mX5akv5kc9MU/qi/lRT8yTEX2xsDDh5uZ48+0VMcm1a9fYuHEjy5YtIy0trSJO\nKSLVlMlkolvLerw4tjX2dlbMWfILq7Yf43pBgdGliYiImIVyL6GZNWsWu3btYsWKFcDvd8vGjx9P\nbGwshYWFuLi4sGzZMnx9fSu8WBGpPnw8HHlpbBu+3HyIdTtPcCg5k0kDgqnlZGt0aSIiIoYq9x34\nHTt2FHvT6nfffcfu3buZMGECb731FgBz586tuApFpNqytbFkQr+m/CmyCSfPXuLleTHEHzlvdFki\nIiKGKvcd+LNnz+Ln51f0+vvvv8fb25vJkycDcPjwYdatW1dxFYpItdexmRcBXs58umY/732dQO+2\nPjzctQFWlhWyClBERKRSKfe/fvn5+VhZ/V/u37VrFx07dix67ePjo3XwIlLhvNwceGFMGN1b1WNj\nTDIzv9xDWuZVo8sSERG578od4OvUqcPevXuB3++2Jycn06ZNm6Lt6enp2NvbV1yFIiL/n7WVJY/2\nCuSJQc04m3GF6Z/vJvbgOaPLEhERua/KvYSmX79+fPzxx2RkZHD48GEcHR3p2rVr0fakpCS9gVVE\n7qnWQR741XHiX2v38/HqRLq3rEeAlxNrfjxOxsVcXJ1tGdy1AR2C6xhdqoiISIUrd4CfNGkSZ86c\nKfogpzfffBNnZ2cALl26xHfffce4ceMquk4RkWLcXWrw/KhWrNx+jA27TrFtL9z4hIb0i7nM//Yg\ngEK8iIhUOeUO8DY2Nvzzn/8sdZuDgwM//vgjdnZ2d12YiMjtWFlaMKx7Q37ad4ZLV/KLbcu7VsDK\nH44qwIuISJVT7gB/KxYWFjg5OVXkKUVEbuuP4f2G9Iu597kSERGRe++OAvyVK1f497//zebNm0lJ\nSQHA29ubXr16MWHCBL2JVUTuKzdn21LDuglYuf0YD7b2xsne5v4XJiIicg+U+yk0mZmZDB06lI8/\n/pj09HSaNGlCkyZNSE9P56OPPmLo0KFkZmbei1pFREo1uGsDbKyK/3VmbWmBfx0nonae4LlPoln2\n3RGysnVHXkREKr9y34F///33OXbsGC+++CLDhw/H0tISgOvXr7N06VJee+01PvzwQ1544YUKL1ZE\npDQ31rmv/OFoiafQnD5/mW+iT7Bx9ym2xKXQJdSLPu38cKup9+qIiEjlZCosLCy8/W7/p1u3bnTp\n0oUZM2aUuv3FF19kx44dbNu2rSLqMzvp6dkUFJTrS1Yh3N2dSEu7dN/nlZtTT8zTzfry24UrfPvz\nSX7adxaAjs3q0LeDH561tOTvXtP3inlSX8yPemKejOiLhYUJNzfHm28v7wnPnz9PkyZNbrq9adOm\nnD9/vrynFRG5pzxr2TOuTxPemNSBbi3qEb3/N/4x92fmrtvP6fOXjS5PRESkzMq9hKZ27dokJSXd\ndHtSUhK1a9e+q6JERO4Vt5p2jOrVmH4d/dgUk8z3e0/z8/7fCAt0J7KDP3519CQtERExb+W+A9+9\ne3e+/vprlixZQkFBQdF4QUEBS5cuZcWKFfTo0aNM58rLy2P27Nl07tyZkJAQhg0bRnR09G2PW7t2\nLWPGjKFTp040a9aMHj16MHXqVE6fPl3q/suXL6dPnz40b96c3r17s2jRorJdrIhUWS6Otgzr0ZBZ\nf+5AZEd/DpzI4JUvdvPu8niOnM4yujwREZGbKvca+AsXLjB8+HBOnTqFq6srAQEBABw/fpyMjAx8\nfX1ZsmQJtWrVuu25nn32WTZt2sSYMWPw8/Nj1apVJCYmsnDhQlq2bHnT42bNmkVaWhpBQUHUrFmT\n1NRUli1bxvXr11m7di3u7u5F+y5ZsoSXX36ZiIgIOnXqRGxsLGvWrGHKlCk89thj5bl0QGvg5f+o\nJ+bpTvtyJeca3+1JYdPuZLKv5tPErxb9O/oT6OuCyWS6B5VWH/peMU/qi/lRT8yTOa6BL3eAB8jO\nzuazzz5jy5YtRc+B9/HxoWfPnkycOBFHx5tPeENCQgJDhw5l6tSpjBs3DoDc3FwiIyPx8PAo913y\n/fv3M3jwYJ577jkmTJgAQE5ODl27diUsLIyPP/64aN/Jkyfz3Xff8cMPP5T7g6cU4OUG9cQ83W1f\ncvKu8cMvqWzYdYqsy3k0rFeTyI7+NK/vqiB/h/S9Yp7UF/Ojnpgncwzw5V5CA+Do6MgzzzxDVFQU\n8fHxxMfHs379ep555hnWr19P3759b3uODRs2YG1tzdChQ4vGbG1tGTJkCHFxcZw7d65cNdWtWxeA\nixcvFo3t2rWLzMxMRo4cWWzfUaNGcfnyZbZv316uOUSk6rOzsaJ3W19m/bkDo3s15sKlHN5dHs+M\nL2KJO5RGQfnveYiIiFSoO/ok1lu5cOECx48fv+1+SUlJBAQE4ODgUGw8JCSEwsJCkpKS8PDwuOU5\nMjMzuX79OqmpqXz00UcAdOjQoWj7gQMHAGjWrFmx44KDg7GwsODAgQP069evTNclItWLtZUlPVp5\n0yW0LtGJZ4mKPslHq/ZRr7YD/Tr60TbIEwsL3ZEXEZH7r8IDfFmlpaXh6elZYvzG+vWy3IHv3bt3\n0ae+uri48NJLL9G+fftic9jY2ODi4lLsuBtj5b3LLyLVj5WlBQ+E1qVj8zrsTjrH+uiTzF17gDU7\njtO3gx8dgutgZXlHv8wUERG5I4YF+JycHKytrUuM29raAr+vh7+dDz/8kCtXrnD8+HHWrl3L5cvF\nn+V8szluzFOWOf7oVuuR7jV3dz3eztyoJ+bpXvWlv2dN+nVpyM+JZ1i65Vc+/+YgUdEnebhHI8Lb\n+GJjbXlP5q0K9L1intQX86OemCdz64thAd7Ozo78/PwS4zdC9Y0gfytt2rQBoGvXrvTs2ZP+/ftj\nb2/P6NGji+bIy8sr9djc3NwyzfFHehOr3KCemKf70ZdGXk5MG92KfcfSWbfzBJ+sSGDxxoP0aetL\n1xb1sLVRkP9v+l4xT+qL+VFPzFOVeRNrRXB3dy91CUtaWhrAbde//5GPjw/BwcGsW7eu2Bz5+flF\ny2xuyMvLIzMzs9xziIjcYDKZCGlQm3+MDuPvI1pS182BJd8d4e+f7GT9zhNcyblmdIkiIlJFlekO\n/Oeff17mE+7Zs6dM+wUFBbFw4UIuX75c7I2s8fHxRdvLKycnh6tXrxa9btKkCQCJiYl07ty5aDwx\nMZGCgoKi7SIid8pkMtHErxZN/GpxJCWL9dEnWLn9GN/uOkV4mDcPtvHBsUbpS/lERETuRJkC/Jtv\nvlmuk5blWckRERHMmzeP5cuXFz0HPi8vj5UrV9KqVauiN7impqZy9epVGjRoUHRsRkYGrq6uxc6X\nmJjIwYMHiz3Csn379ri4uLB48eJiAf6rr77C3t6eLl26lOu6RERupaF3Tf42NJQTZy8StfMk63ae\nYNPuZLq3qkfvNj7UdCz/sj0REZE/KlOAX7BgQYVPHBoaSkREBHPmzCEtLQ1fX19WrVpFamoqM2fO\nLNpvypQpxMTEcOjQoaKx7t2706dPHxo3boy9vT1HjhxhxYoVODg48MQTTxTtZ2dnx1NPPcWMGTN4\n+umn6dy5M7Gxsaxdu5bJkyfj7Oxc4dclIuJfx5m/DG5OSlo230SfZGPMKbbGpdAltC592vni6mxn\ndIkiIlKJlSnAt23b9p5MPmvWLN59913WrFlDVlYWgYGBzJ07l7CwsFseN3LkSKKjo9myZQs5OTm4\nu7sTERHBE088gY+PT7F9R40ahbW1NfPmzWPr1q14eXkxbdo0xowZc0+uSUTkBm93R/5nQDADOwcQ\n9fNJtu09zba9p+nU3Iu+HfzwcKlhdIkiIlIJmQoL9bGC5aGn0MgN6ol5Mue+nM+6yre7TrEj/gwF\nBYW0a+pJvw5+1K3tcPuDKzFz7kl1pr6YH/XEPJnjU2gMe4ykiEh1U7tmDR7tFUj/jv5sjDnF93tP\n8/P+s4QFeRDZwQ9fT/N6zrCIiJgnBXgRkfvMxdGWR3o0om97PzbHJrM1LoXYg+cIbeBGZCd/GtSt\naXSJIiJixhTgRUQM4mRvw+AuDYho68vWuBQ27U7m9QVxNPWvRf+O/jT2cSnTU71ERKR6UYAXETGY\nvZ01/TsF8GAbH7btTWVDzCneXLyXRt416d/Rn+AAVwV5EREpogAvImIm7GysiGjnS49W9diRcIZv\nfj7J28vi8a/jRP+O/oQ2qo2FgryISLWnAC8iYmZsrC3pGeZN1xZ12Zl4lqjoE3ywch/e7g5EdvSn\ndaAHFhYK8iIi1ZUCvIiImbKytKBLaF06Na9DzIFzrI8+wadr9uPpepzIDn60a+qJlaWF0WWKiMh9\npgAvImLmLC0s6NCsDu2CPdlzKI31O0/wn6gk1vx4nD7t/ejc3AtrKwV5EZHqQgFeRKSSsDCZaB3k\nQVigO/FH01m/8wQLNx5i3U/HiWjnR9cWdbG1tjS6TBERuccU4EVEKhmTyUSLhrUJbeBG0skLrN95\ngiVbDxMVfYJebXzo0cqbGrb6611EpKrS3/AiIpWUyWSiqb8rTf1d+TU5k/XRJ1jxwzG+/fkU4a29\nCW/tg2MNa6PLFBGRCqYALyJSBTT2ceFZnxYcP3OR9TtPsPanE2zcnUyPVvXo3cYXZwcbo0sUEZEK\nogAvIlKFBHg589eHQ0g5l8366BNs2HWKrbEpdGlRl4i2vrg62xldooiI3CUFeBGRKsjbw5HHBzZj\n0ANXiIo+wXdxp9m29zSdmnvRp70fHi41jC5RRETukAK8iEgVVsfVngn9mjKwUwDf7DrFjwmp7Ig/\nQ/tgT/p18MPLzcHoEkVEpJwU4EVEqoHaLjUY0zuQ/h392Rhzim17TxOdeJY2TTzo18EfHw9Ho0sU\nEZEyUoAXEalGajnZMrxnI/q292NzbDJb41KISTpHi4a1iezoT/26zkaXKCIit6EALyJSDTk72PBw\n1wZEtPNla2wKm2OTeW1BLMEBrvTv6E9jHxejSxQRkZtQgBcRqcYc7KwZ0DmAB9v4sG3vaTbGnOKN\nRXto7ONC/47+NPWvhclkMrpMERH5LwrwIiJCDVsr+rT3o0eYN9vjU9mw6xRvLf2FAC9nIjv60aJh\nbQV5EREzoQAvIiJFbK0tebC1D91a1OOnxDN8E32SD1bsw9vdkf6d/Alr7I6FhYK8iIiRFOBFRKQE\naysLurWoxwMhXuw68Bvrd57kk9WJeLnZ06+DH+2aemJpYWF0mSIi1ZICvIiI3JSlhQUdm3nRvmkd\n4n5NY91PJ/j3+iRW7zhO3w5+dGrmhbWVgryIyP2kAC8iIrdlYWGiTZAHrQPdiT+SzrqdJ1iw4RDr\nfjpBRDtfuoTWxdba0ugyRUSqBQV4EREpM5PJRItGtQlt6MaBExdYt/MEX205TNTOE/Ru60u3lvWo\nYat/WkRE7iX9LSsiIuVmMpkIDnAlOMCVQ6cusD76JMu3HeWbn0/yYGsferb2xsHO2ugyRUSqJAV4\nERG5K4G+tQj0rcWx1Ius33mC1T8eZ0PMKXqGefNgGx/cjS5QRKSKMTTA5+Xl8d5777FmzRouXrxI\nUFAQzzzzDB06dLjlcZs2beKbb74hISGB9PR0vLy86N69O0888QROTk7F9g0MDCz1HNOnT2fEiBEV\ndi0iItVd/brOPDUkhORz2URFn+Cb6JNs3p1Mn44BdGleh1pOtkaXKCJSJZgKCwsLjZr82WefZdOm\nTYwZMwY/Pz9WrVpFYmIiCxcupGXLljc9rl27dnh4eBAeHk7dunU5dOgQS5Yswd/fnxUrVmBr+3//\nSAQGBtK5c2cGDBhQ7ByhoaH4+/uXu+b09GwKCu7/l8zd3Ym0tEv3fV65OfXEPKkv5uNM+mW+iT5J\n9IHfsDBB55C69G3nS22XGkaXJuh7xRypJ+bJiL5YWJhwc3O86XbD7sAnJCQQFRXF1KlTGTduHACD\nBg0iMjKSOXPmsGjRopse+/7779OuXbtiY82aNWPKlClERUUxePDgYtvq16/PwIEDK/waRETk5rzc\nHJgQ2ZRxA5qx6JsD/JiQyo74VNoHe9Kvgz91XO2NLlFEpFIy7OG9GzZswNramqFDhxaN2draMmTI\nEOLi4jh37txNj/1jeAcIDw8H4OjRo6Uek5OTQ25u7l1WLSIi5VXHzYExEUG8MakD3VvVIybpHNM+\n+5lP1ySSci7b6PJERCodwwJ8UlISAQEBODg4FBsPCQmhsLCQpKSkcp3v/PnzANSqVavEtq+//poW\nLVoQEhJC//792bx5850XLiIid8TV2Y6R4Y2Z9eeORLTzJf5oOi/Ni+GDFQkcP3PR6PJERCoNw5bQ\npKWl4enpWWLc3f335xXc6g58aT777DMsLS3p1atXsfGWLVvSt29fvL29OXPmDAsWLODJJ5/krbfe\nIjIy8s4vQERE7khNBxuGdmtIn3Z+bIlNZktsCnsPx9Ksviv9O/rTyNvF6BJFRMyaYW9iDQ8Pp2HD\nhnz66afFxpOTkwkPD+fFF19k9OjRZTrXunXrmDx5MpMmTeLZZ5+95b5XrlwhMjKS69evs23bNkwm\n0x1fg4iI3L0rOflE/XScNduPkpWdR/MGtXkkvDEhjWrr72gRkVIYdgfezs6O/Pz8EuM31qn/95Nk\nbiU2NpZp06bRrVs3nn766dvub29vz/Dhw3nrrbc4duwYDRo0KFfdegqN3KCemCf1xfyUpSfdQrzo\n0MSD7b+k8u2uk7zw/9q784Coznv/4+8ZGPYdBmRHkcUVERNFozEaU+JSYxKTJi5Zqs16m9ib/IzN\nbW+bexPvL/U2sWnzOVvrIwAAIABJREFUq1uaJk2axlRD1LhFbUzEpWLEFVFEAXFBFFBEQJnfHyNT\nCeDGMgN8Xn+FZ87hPCdfjufD4XmeMy+D2DAfxgyOISk2UEG+FehacTyqiWPSKjRXMZvNjQ6TKS4u\nBiA4OPi63yM7O5tnnnmGhIQE3nrrLZycnG7o2KGhoQCUlZXdRI9FRKQ1uZqcGHVbJMOTw9m0+zhf\nbjnK7z7bRVSwF2MHx9A/wYxRQV5ExH6TWBMTE8nLy6OioqJee1ZWlu3za8nPz2fatGkEBAQwb948\nPDxufDmygoICAAICAm6y1yIi0tpMzkaGJ4fzxk8G8eMxPai6VMu7n+/hFwu3snnPCS7X1tq7iyIi\ndmW3AJ+WlkZNTQ2LFy+2tVVXV7NkyRL69+9vm+BaVFTUYGnI4uJinnzySQwGA4sWLWoyiJ85c6ZB\n29mzZ/n444+JiIi4pRc5iYhI23B2MjKkTyivTxvI0+N7YTQaWLB8Hz+fv4WNWUVcuqwgLyKdk92G\n0CQlJZGWlsacOXMoLi4mKiqKpUuXUlRUxOzZs23bzZw5k23btnHgwAFb27Rp0ygoKGDatGlkZmaS\nmZlp+ywqKsr2FtePPvqIdevWMXz4cMLCwjh58iR/+9vfOHPmDH/4wx/a7mRFROSWGY0Gbu8RwoDE\nYLIOnmZZxhHeX5lN+rd5jB4UzdC+obiYbmwIpYhIR2C3AA/w5ptv8vbbb5Oenk5ZWRkJCQnMnz+f\nlJSUa+6XnZ0NwMKFCxt8NmHCBFuAT05OZseOHSxevJiysjI8PDzo168fTz311HWPISIijsVoMJAc\nb6ZfXBB7886wLOMIH63NYVnGEdJuj2J4chhuLna9rYmItAm7LSPZXmkVGqmjmjgm1cXxtGZNDuSf\nZXnGEfYeOYunmzP33BbJyJQIPNxMrXK8jkTXiuNRTRyTVqERERFpQQlR/iRE+ZNbVMaKjKMs/SaP\nVdvyGdE/glG3ReLj4WLvLoqItDgFeBERafdiw3z56YN9yT95juWbj/Ll5qOs3V7A8H7hpA2Mws/r\nxt4tIiLSHijAi4hIhxEV4s2z9/Wm6HQFKzYf5avthazfcYyhSaHcOzCKIF93e3dRRKTZFOBFRKTD\nCQvyZPq4noy/I4Yvt+SzcWcRG3cWkdq7C2MGRRMScOPvDhERcTQK8CIi0mEF+3vw+L2J/HBIDCu3\n5rMxq4hNu48zsEcIY1KjCTc3PUlMRMRRKcCLiEiHF+DjxqRR8YwdHMOabfms33GMLftOkhJvZuzg\nGKK7eNu7iyIiN0wBXkREOg1fTxcm3tWdewdF89X2AtZuLyQzp5g+3QIZNziG7hG+9u6iiMh1KcCL\niEin4+Vu4r6h3bjntig2fFfI6m0FvPGXTBKj/Bg3OIbEaH8MBoO9uyki0igFeBER6bQ83JwZkxrD\n3SmRfL3zGCu35fObT3YSG+7DuMEx9OkWqCAvIg5HAV5ERDo9Vxcn7rk9irv6h/PtruN8ueUoby/e\nRVSIF+MGx5Acb8aoIC8iDkIBXkRE5AqTsxN39Y9gaFIYm/eeYMXmo/xh6R7CgjwZmxrNbT2CcTIa\n7d1NEenkFOBFRES+x9nJyNC+YQzpHcq27JOsyDjK/GX7+PzbPMYMiia1dxecnRTkRcQ+FOBFRESa\nYDQaGNSzC7f3CGHnwdMsyzjCn1Zm88WmPNIGRjMsKRSTs5O9uykinYwCvIiIyHUYDQb6x5tJjgti\nT94Zlm06wkdrc1iecYQf3B7F8OQw3Fx0SxWRtqF/bURERG6QwWCgT7dAencN4EB+KcsyjvDphkN8\nueUoo26LZGT/CDzcdGsVkdalf2VERERuksFgIDHan8Rofw4dK2N5xhGWbjzMqq35jEyJYNSACLw9\nXOzdTRHpoBTgRUREmqF7uC8vTkzi6IlzLN98hOUZR1j7zwLuSg7nB7dH4uvlau8uikgHowAvIiLS\nAqK7ePPchD4cKz7Pii1HWf3PfL7KLOTOpDDSBkYR6Otm7y6KSAehAC8iItKCws1e/GRcL8bf0ZWV\nW47yj53H+MfOYwzp04XRg6IJ9vewdxdFpJ1TgBcREWkFIf4ePH5vD8YN7sqqrfl8nVXEN7uOM7Bn\nCGNSYwgP8rR3F0WknVKAFxERaUWBvm5MuieeMYOjWbOtgA3fHWPr3pP0TzAzNjWG6C7e9u6iiLQz\nCvAiIiJtwM/LlYdGdOfeQVGs3V7IuswCMg8U0zc2kHGDY4gN97V3F0WknVCAFxERaUPeHi7cP6wb\nabdHsm7HMdb+s4DXP8ykR7Q/4wbHkBDlh8FgsHc3RcSBKcCLiIjYgYebiXGDYxg1IIJ/fFfE6m35\nvPnX7+ge4cu4wTH07hqgIC8ijVKAFxERsSM3F2fSBkYxon843+w6zsqtR3nr0yyiu3gzbnAM/eKC\nMCrIi8hVFOBFREQcgIvJiZEpEdzZL4yMPSf4cvNRfr9kN+FBnowZHM3tiSEYjQryIqIALyIi4lCc\nnYwMSwpjSJ8ubNt/ihWbjzL/i32kf5PH6NRoUnt1wdnJaO9uiogd2TXAV1dXM3fuXNLT0ykvLycx\nMZEZM2aQmpp6zf3WrFnDl19+ya5duygpKSE0NJS77rqLZ599Fm/vhstxLV68mPfee4/CwkLCwsKY\nOnUqkyZNaq3TEhERaTYno5HUXl0Y2DOE73KKWZZxhD99mc0X3x5h9KAo7ugbisnZyd7dFBE7cPrV\nr371K3sd/OWXX2bJkiU89NBDjBs3jgMHDrBo0SJSU1MJDQ1tcr9HH32U6upqRo8ezZgxY/D09OTj\njz9m3bp1PPDAAzg7/+v3kk8++YRf/vKXDBw4kMmTJ1NbW8v8+fPx9PQkOTn5pvtcWVmNxXJLp9ss\nnp6uXLhQ3fYHliapJo5JdXE8qknzGAwGwoI8ubNfGN3CfMg/eY5/7LS+FMoIRJi9bumJvOrieFQT\nx2SPuhgMBjw8XJr+3GKxRxyFXbt2MXHiRGbNmsXjjz8OQFVVFWPHjiU4OJiPPvqoyX23bt3KwIED\n67V9/vnnzJw5k9mzZ3P//fcDcPHiRe68805SUlJ49913bdu+9NJLrF+/nq+//rrRJ/bXUlJyntra\ntv9fZjZ7U1x8rs2PK01TTRyT6uJ4VJOWZbFYyD56lmUZR8jOL8XL3cQPbo9kRP8I3F1v/A/rqovj\nUU0ckz3qYjQaCAz0avrzNuxLPatWrcJkMjFx4kRbm6urKw8++CCZmZmcOnWqyX2/H94B7r77bgBy\nc3NtbVu3bqW0tJRHH3203raTJk2ioqKCjRs3Nvc0RERE2pTBYKBHTAD/59H+zJrcn66hPvz968O8\n/G4Gn39zmPOVNfbuooi0MrsF+P3799O1a1c8PT3rtfft2xeLxcL+/ftv6vudPn0aAH9/f1vbvn37\nAOjdu3e9bXv16oXRaLR9LiIi0h7FRfgx46Ekfvn4ABKj/fli0xFe/n8ZfLrhEGXnq+zdPRFpJXab\nxFpcXExISEiDdrPZDHDNJ/CNWbBgAU5OTtxzzz31juHi4oKfn1+9bevabvYYIiIijiimiw/P39+H\nwuLzrNh8lNXb8lmXWciwpDDuHRhFgI+bvbsoIi3IbgH+4sWLmEymBu2urq6AdTz8jVq2bBmfffYZ\nTz31FFFRUdc9Rt1xbuYYda41Hqm1mc03N15fWp9q4phUF8ejmrQNs9mb5J6hFBWf57P1B1m/vYCv\ndx5j5G1RPDgiji6Bnvwjs4APVu7n9NlKgvzdmXpvD4anRNq763KFrhXH5Gh1sVuAd3Nzo6am4Ti9\nulBdF+SvZ/v27bz66qsMHz6cF154ocExqqsbnzVcVVV1w8e4miaxSh3VxDGpLo5HNWl7JuCREd0Z\nlRLOyq35rPtnAWu35tMtzJujJ85Tc7kWgOKzlbzz6U7Kz10ktVcX+3ZadK04KE1ivYrZbG50CEtx\ncTEAwcHB1/0e2dnZPPPMMyQkJPDWW2/h5FR/PVyz2UxNTQ2lpaX12qurqyktLb2hY4iIiLRXQb7u\nTLkngf/7dCp3D4jg0LFyW3ivU32pliVf5zbxHUTEEdktwCcmJpKXl0dFRUW99qysLNvn15Kfn8+0\nadMICAhg3rx5eHh4NNimR48eAOzZs6de+549e6itrbV9LiIi0pH5e7vyo5FxTX5eUl7Fpt3HKS6t\nxE6rS4vITbBbgE9LS6OmpobFixfb2qqrq1myZAn9+/e3TXAtKiqqtzQkWJ/SP/nkkxgMBhYtWkRA\nQECjxxg0aBB+fn58/PHH9dr/+te/4uHhwbBhw1r4rERERBxXoE/jQ0cNwKIV+5n5x8289G4Gf0zf\nw/odhRSeOk+tAr2Iw7HbGPikpCTS0tKYM2cOxcXFREVFsXTpUoqKipg9e7Ztu5kzZ7Jt2zYOHDhg\na5s2bRoFBQVMmzaNzMxMMjMzbZ9FRUXZ3rDq5ubGT3/6U1577TVeeOEF7rjjDrZv384XX3zBSy+9\nhI+PT9udsIiIiJ3df2csf16ZTfWlfw2jcXE2MjUtgchgb3IKSjlYWEpOQSnb9luHuXq4OtM9wpf4\nSD/iI/yICfW+pTe/ikjLsVuAB3jzzTd5++23SU9Pp6ysjISEBObPn09KSso198vOzgZg4cKFDT6b\nMGGCLcCD9aVNJpOJ9957j3Xr1hEaGsqrr77K1KlTW/ZkREREHFzdRNUlX+dypryKAB9X7r8z1tYe\nGezFyJQILBYLp8suXhXoy9iVWwKAydlI11Af4iN9iY/wIzbc96beACsizWewaLDbTdEqNFJHNXFM\nqovjUU0c083WpbyimoOFpRwsLCOnoJT8k9bhNQYDRAV7E3cl0MdF+uHr6dKKPe+4dK04JkdchUa/\nMouIiMh1+Xi6kJIQTEqCdQW3yqpLHC4qtz2l37iziK+2FwIQ4u9O3JUhN/GRvpj93DEYDPbsvkiH\nogAvIiIiN83d1ZleXQPo1dW6kMSly7UcPXGOnMJSDhaU8V1OMd/uOg6Ar5cLcRF+xF8ZSx9h9sJo\nVKAXuVUK8CIiItJszk5GYsN9iQ335d6BUGuxcPx0BTmFZRwsKCWnsJTt2daJse6uTsSG+155Qu9H\n11BvTM5O1zmCiNRRgBcREZEWZzQYCDd7EW724q7kcABOl1Vy0Bboy1iy8TAAzk6GKxNj/YiL8KV7\nuB8eboooIk3R1SEiIiJtIsjXnSBfd9uqN+cuVHOosMw6MbawlFVb81mx2YIBiAj2ujIp1pe4CD/8\nvRtfw16kM1KAFxEREbvw9nAhOd5McrwZgKrqyxwuKiPnyko33+wuYt0O68RYs5+bbZWb+Eg/Qvw1\nMVY6LwV4ERERcQiuLk70iAmgR8y/JsYWnDpPToH15VJZuSVs2nMCAB8PE3G2QO9LZLAXTka9YEo6\nBwV4ERERcUjOTtaXRnUN9eEHt0dhsVg4XnLB9nKpg4WlZOYUA9bw3z3cl7gI6+TYbmE+uJg0MVY6\nJgV4ERERaRcMBgNhQZ6EBXlyZz/rxNgz5RdtY+gPFpSS/k0eFsDJaCAm1Ns27KZ7uC9e7ib7noBI\nC1GAFxERkXYrwMeNgT3dGNgzBICKizW2lW4OFpax5p8FrNyaD0C42dM2MTY+wo8AHzd7dl3klinA\ni4iISIfh6WaiX/cg+nUPAqC65jJ5x61vjM0pLCNj7wk2fHcMgCBfN+IifG1vjQ0N9NDEWGkXFOBF\nRESkw3IxOZEQ5U9ClD8Al2trKTxVcSXQl7I37wyb954EwMvdZA30V14wFRXihbOTJsaK41GAFxER\nkU7DyWgkuos30V28GXVbJBaLhVNnK22B/mBBGd8dPA2Aq8mJbmHWF0zFR/jSLcwXVxdNjBX7U4AX\nERGRTstgMBAS4EFIgAdDk8IAOHuuioOFpbax9F98+6+JsVEh3sRfGUMfF+mnibFiFwrwIiIiIlfx\n93bl9h4h3N7DOjH2wsVLHDpWdmX5ylLWZRayelsBAGFBnralK+MifQnydbdn16WTUIAXERERuQYP\nN2f6xgbSNzYQgJpLl8k7fs62Hv22/Sf5emcRAAE+rtYwf2VybFiQJ0ZNjJUWpgAvIiIichNMzk7W\ncfGRfoxJhdpaC4XF563r0ReUsj//LFv2WSfGero5X3ljrPUpfXQXb02MlWZTgBcRERFpBuOVsfFR\nId6MTInAYrFQXFppC/Q5hWXsPGSdGOvibKRbmI9tpZvYcB/cXBTH5OboJ0ZERESkBRkMBoL9PQj2\n92BIn1AAyiqqOVi30k1hGcs3H8GSAUaDgcgQL+Ij/BjQqwshPq74eLrY9wTE4SnAi4iIiLQyX08X\nBiQGMyAxGIDKqkvkFpWRU2Bd6eYfO4+xdrt1YmyXAA/iI32vDL3xw+zrphdMST0K8CIiIiJtzN3V\nmd5dA+nd1Tox9tLlWsouXmbb7iJyCkrZnl3MxqzjAPh5uRAf6WcN9BG+RAR7aWJsJ6cALyIiImJn\nzk5GEmN8CfQ0ce+gaGotFoqKK6wr3VwZS79t/ynAGv6tb4z1JT7Sj5guPpicNTG2M1GAFxEREXEw\nRoOBiGAvIoK9uKu/dWJsSdlF2xj6nIJSduWWANbw3y3Um7grK+N0D/fF3VURryNTdUVEREQcnMFg\nIMjPnSA/dwb3tk6MLb9QzaErYf5gYSkrt+SzYvNRDAaIDPayrXQTH+GLr5ernc9AWpICvIiIiEg7\n5OPhQv94M/3jzQBUVV++MjHW+pT+m11FrMssBCDY3932ttj4CD+C/d01MbYdU4AXERER6QBcXZzo\nGRNAz5gAwDoxNv/kedsT+p2HTvPtbuvEWF9PF9vbYuMj/IgM9sJoVKBvLxTgRURERDogZyfrS6O6\nhfmQNjCKWouF4yUXOFhYal2TvqCM7QeKAXBzcaJ7eF2g96VbmA8mZyc7n4E0xa4Bvrq6mrlz55Ke\nnk55eTmJiYnMmDGD1NTUa+63a9culixZwq5du8jJyaGmpoYDBw402K6wsJCRI0c2+j0WLFjAsGHD\nWuQ8RERERByd0WAgPMiT8CBPhvcLB+BM+UXbkJucwlKWbjwMgLOTgZhQH+tKN1eWr/RwM9mz+3IV\nuwb4V155hTVr1jB16lSio6NZunQp06dP58MPPyQ5ObnJ/b7++msWL15MQkICkZGRHD58+JrH+eEP\nf8gdd9xRry0xMbFFzkFERESkvQrwcWNQry4M6tUFgPOVNdaJsVee0q/ZVsDKLfkYgHCzl+0FU/GR\nfvh7a2KsvdgtwO/atYsVK1Ywa9YsHn/8cQDuu+8+xo4dy5w5c/joo4+a3PeRRx5h+vTpuLm58frr\nr183wPfq1Yvx48e3ZPdFREREOhwvdxP94oLoFxcEQFXNZfKKym2BftOeE6zfcQyAIF836yo3kdYn\n9F0CPDQxto3YLcCvWrUKk8nExIkTbW2urq48+OCDvPXWW5w6dYrg4OBG9w0KCrrp4124cAFnZ2dc\nXFxuuc8iIiIinYmryYnEaH8So/0BuFxrnRh7sLCMgwWl7D5cQsaeEwB4e5isT+evTI6NCvHCyagX\nTLUGuwX4/fv307VrVzw9Peu19+3bF4vFwv79+5sM8Ddr7ty5zJ49G4PBQFJSEi+99BK33XZbi3xv\nERERkc7CyWika6gPXUN9uOe2SCwWCyfOXLAF+pzCUnbkWCfGupqciA33ubJ8pR/dwnxwNWlibEuw\nW4AvLi4mJCSkQbvZbF3L9NSpU80+htFo5I477mDUqFEEBwdz9OhRFi1axBNPPMH777/PgAEDmn0M\nERERkc7KYDAQGuhJaKAnw5LCADh7roqDhaW2ybHp3+ZhAZyMBmK6WN8YGxdhHUvv5a6JsbfCbgH+\n4sWLmEwNi+bqap0QUVVV1exjhIWFsWjRonpto0ePZsyYMcyZM4dPPvnkpr9nYKBXs/t1q8xmb7sd\nWxqnmjgm1cXxqCaOSXVxPB2hJmazN/Hdghhz5evzlTVkHznD3sMl7D1cwlfbC1m1NR+AqC7e9Ooa\nSM9ugfTqGojZ391+Hb8GR6uL3QK8m5sbNTU1DdrrgntdkG9pISEhjBkzhk8//ZTKykrc3W/uB6Wk\n5Dy1tZZW6du1mM3eFBefa/PjStNUE8ekujge1cQxqS6OpyPXJDrIg+ggD0bfHknNpcvkHT9HzpUh\nN//YUcDKzUcACPRxtb1cKi7Sj7BA+0+MtUddjEbDNR8a2y3Am83mRofJFBdbx0211Pj3xoSGhlJb\nW0t5eflNB3gRERERuXUmZyfb6jUAtbUWCk6dtw67KSxj/5GzbNl7ErCuitM93Ne60k2kL9Eh3jg7\naWKs3QJ8YmIiH374IRUVFfUmsmZlZdk+by0FBQU4OTnh6+vbascQERERkeszGg1Ed/Emuos3dw+w\nTow9VVppHUNfUMbBwlJ2HjoNgIvJSGyYr3UMfaQfsWE+uLnY9bVGdmG3M05LS+O9995j8eLFtnXg\nq6urWbJkCf3797dNcC0qKqKyspLY2NibPsaZM2cICAio13b06FFWrFjBgAEDcHNza/Z5iIiIiEjL\nMRgMhPh7EOLvwdC+1omxZeerrG+LvTLsZlnGESwW69tlo7t42V4uFRfhi7dHx18y3G4BPikpibS0\nNObMmUNxcTFRUVEsXbqUoqIiZs+ebdtu5syZbNu2jQMHDtjajh07Rnp6OgC7d+8G4N133wWsT+5H\njBgBwG9+8xsKCgoYNGgQwcHB5Ofn2yauzpw5s03OU0RERESax9fLlQGJwQxItA6xrqy6RO4x6xtj\ncwrKWL/jGGv+WQBAaKDHlUDvS3yEH4G+bnYfR9/S7Po3hzfffJO3336b9PR0ysrKSEhIYP78+aSk\npFxzv8LCQubOnVuvre7rCRMm2AL8kCFD+OSTT/jLX/7CuXPn8PHxYciQITz//PPExcW1zkmJiIiI\nSKtyd3Wmd7dAencLBKDmUi1HTpTbntJvzz7FxqwiAPy9XYmLsI6jj4/wI8zsibGdB3qDxWJp+yVV\n2jGtQiN1VBPHpLo4HtXEMakujkc1aTm1FgvHiiuurEVvXY/+7DnrSoeebs7/mhgb4UdMaOMTYzfv\nPcGSr3M5U15FgI8r998ZS2qvLm3Sf4ddhUZEREREpDUYDQYig72IDPZiZEoEFouF02UXbYE+p6CM\nrNwSAEzORrqF+liXr4z0JTbMl52HTvPnldlUX6oFoKS8ij+vzAZosxB/LQrwIiIiItKhGQwGzH7u\nmP3cGdInFIDyC9W2VW5yCkr5cvNRlmdYMBisvwBc/t6Ii+pLtSz5OlcBXkRERETEHnw8XEhJMJOS\nYAbgYvUlcovKOVhQyhebjjS6T0l5VRv2sGlaCV9EREREOj03F2d6xQRw39BuBPq4NrpNU+1tTQFe\nREREROQq998Zi4tz/Zjs4mzk/jtv/r1ErUFDaERERERErlI3zt1eq9BcjwK8iIiIiMj3pPbqQmqv\nLg65vKeG0IiIiIiItCMK8CIiIiIi7YgCvIiIiIhIO6IALyIiIiLSjijAi4iIiIi0IwrwIiIiIiLt\niAK8iIiIiEg7ogAvIiIiItKOKMCLiIiIiLQjehPrTTIaDZ3y2NI41cQxqS6ORzVxTKqL41FNHFNb\n1+V6xzNYLBZLG/VFRERERESaSUNoRERERETaEQV4EREREZF2RAFeRERERKQdUYAXEREREWlHFOBF\nRERERNoRBXgRERERkXZEAV5EREREpB1RgBcRERERaUcU4EVERERE2hEFeBERERGRdsTZ3h3ozKqr\nq5k7dy7p6emUl5eTmJjIjBkzSE1Nve6+J0+e5I033mDTpk3U1tYyaNAgZs2aRWRkZBv0vOO61Zq8\n8847/P73v2/QHhQUxKZNm1qru53CqVOn+OCDD8jKymLPnj1cuHCBDz74gIEDB97Q/rm5ubzxxhvs\n2LEDk8nEXXfdxcyZMwkICGjlnndszanLK6+8wtKlSxu0JyUl8emnn7ZGdzuFXbt2sXTpUrZu3UpR\nURF+fn4kJyfz4osvEh0dfd39dV9pec2pie4rrWf37t388Y9/ZN++fZSUlODt7U1iYiLPPfcc/fv3\nv+7+jnCtKMDb0SuvvMKaNWuYOnUq0dHRLF26lOnTp/Phhx+SnJzc5H4VFRVMnTqViooKnn76aZyd\nnXn//feZOnUqn3/+Ob6+vm14Fh3LrdakzmuvvYabm5vt66v/W25NXl4eCxYsIDo6moSEBL777rsb\n3vfEiRNMmjQJHx8fZsyYwYULF3jvvffIycnh008/xWQytWLPO7bm1AXA3d2dX//61/Xa9EtV8yxc\nuJAdO3aQlpZGQkICxcXFfPTRR9x333189tlnxMbGNrmv7iutozk1qaP7SssrKCjg8uXLTJw4EbPZ\nzLlz51i2bBmTJ09mwYIFDBkypMl9HeZasYhdZGVlWeLj4y1/+tOfbG0XL1603H333ZZHH330mvvO\nnz/fkpCQYNm7d6+t7dChQ5YePXpY3n777dbqcofXnJr87ne/s8THx1vKyspauZedz7lz5yxnzpyx\nWCwWy9q1ay3x8fGWLVu23NC+//mf/2np16+f5cSJE7a2TZs2WeLj4y2LFy9ulf52Fs2py8yZMy0p\nKSmt2b1OKTMz01JVVVWvLS8vz9K7d2/LzJkzr7mv7iutozk10X2lbV24cMEyePBgy09+8pNrbuco\n14rGwNvJqlWrMJlMTJw40dbm6urKgw8+SGZmJqdOnWpy39WrV9OvXz969uxpa4uNjSU1NZWVK1e2\nar87subUpI7FYuH8+fNYLJbW7Gqn4uXlhb+//y3tu2bNGkaMGEFISIitbfDgwcTExOhaaabm1KXO\n5cuXOX/+fAv1SPr374+Li0u9tpiYGOLi4sjNzb3mvrqvtI7m1KSO7ittw93dnYCAAMrLy6+5naNc\nKwrwdrJ//366du2Kp6dnvfa+fftisVjYv39/o/vV1tZy4MABevfu3eCzPn36cOTIESorK1ulzx3d\nrdbkasOHDye+/q22AAALAUlEQVQlJYWUlBRmzZpFaWlpa3VXruPkyZOUlJQ0eq307dv3huopraei\nosJ2rQwcOJDZs2dTVVVl7251OBaLhdOnT1/zly3dV9rWjdTkarqvtJ7z589z5swZDh8+zG9/+1ty\ncnKuOefNka4VjYG3k+Li4npPBeuYzWaAJp/2lpaWUl1dbdvu+/taLBaKi4uJiopq2Q53ArdaEwAf\nHx+mTJlCUlISJpOJLVu28Le//Y19+/axePHiBk9gpPXV1aupa6WkpITLly/j5OTU1l3r9MxmM9Om\nTaNHjx7U1tayYcMG3n//fXJzc1m4cKG9u9ehfPHFF5w8eZIZM2Y0uY3uK23rRmoCuq+0hZ///Oes\nXr0aAJPJxI9+9COefvrpJrd3pGtFAd5OLl682OgEOldXV4Amn0TVtTd24dbte/HixZbqZqdyqzUB\neOyxx+p9nZaWRlxcHK+99hqff/45Dz30UMt2Vq7rRq+V7//FRVrfv//7v9f7euzYsYSEhLBo0SI2\nbdp0zQlkcuNyc3N57bXXSElJYfz48U1up/tK27nRmoDuK23hueee4+GHH+bEiROkp6dTXV1NTU1N\nk78cOdK1oiE0duLm5kZNTU2D9rofjrofhO+ra6+urm5yX81QvzW3WpOmPPLII7i7u7N58+YW6Z/c\nHF0r7cuTTz4JoOulhRQXF/PUU0/h6+vL3LlzMRqbvt3rWmkbN1OTpui+0rISEhIYMmQIDzzwAIsW\nLWLv3r3MmjWrye0d6VpRgLcTs9nc6JCM4uJiAIKDgxvdz8/PDxcXF9t239/XYDA0+qcdub5brUlT\njEYjISEhlJWVtUj/5ObU1aupayUwMFDDZxxIUFAQJpNJ10sLOHfuHNOnT+fcuXMsXLjwuvcE3Vda\n383WpCm6r7Qek8nEyJEjWbNmTZNP0R3pWlGAt5PExETy8vKoqKio156VlWX7vDFGo5H4+Hj27NnT\n4LNdu3YRHR2Nu7t7y3e4E7jVmjSlpqaG48ePN3ulDrk1ISEhBAQENHmt9OjRww69kqacOHGCmpoa\nrQXfTFVVVTz99NMcOXKEefPm0a1bt+vuo/tK67qVmjRF95XWdfHiRSwWS4McUMeRrhUFeDtJS0uj\npqaGxYsX29qqq6tZsmQJ/fv3t02mLCoqarDU1A9+8AN27tzJvn37bG2HDx9my5YtpKWltc0JdEDN\nqcmZM2cafL9FixZRVVXF0KFDW7fjAkB+fj75+fn12u655x7Wr1/PyZMnbW2bN2/myJEjulbayPfr\nUlVV1ejSke+++y4Ad9xxR5v1raO5fPkyL774Ijt37mTu3Ln069ev0e10X2k7zamJ7iutp7H/t+fP\nn2f16tWEhoYSGBgIOPa1YrBoYVG7eeGFF1i3bh2PPfYYUVFRLF26lD179vDnP/+ZlJQUAKZMmcK2\nbds4cOCAbb/z588zYcIEKisreeKJJ3BycuL999/HYrHw+eef6zfzZrjVmiQlJTF69Gji4+NxcXFh\n69atrF69mpSUFD744AOcnTVfvDnqwl1ubi7Lly/ngQceICIiAh8fHyZPngzAiBEjAFi/fr1tv+PH\nj3Pffffh5+fH5MmTuXDhAosWLSI0NFSrOLSAW6lLYWEhEyZMYOzYsXTr1s22Cs3mzZsZPXo0b731\nln1OpgN4/fXX+eCDD7jrrru49957633m6enJ3XffDei+0paaUxPdV1rP1KlTcXV1JTk5GbPZzPHj\nx1myZAknTpzgt7/9LaNHjwYc+1pRgLejqqoq3n77bZYtW0ZZWRkJCQn87Gc/Y/DgwbZtGvvhAeuf\nm9944w02bdpEbW0tAwcO5NVXXyUyMrKtT6NDudWa/Md//Ac7duzg+PHj1NTUEB4ezujRo3nqqac0\n+asFJCQkNNoeHh5uC4aNBXiAgwcP8j//8z9kZmZiMpkYPnw4s2bN0lCNFnArdSkvL+e//uu/yMrK\n4tSpU9TW1hITE8OECROYOnWq5iU0Q92/TY25uia6r7Sd5tRE95XW89lnn5Gens6hQ4coLy/H29ub\nfv368eSTT3L77bfbtnPka0UBXkRERESkHdEYeBERERGRdkQBXkRERESkHVGAFxERERFpRxTgRURE\nRETaEQV4EREREZF2RAFeRERERKQdUYAXEREREWlHFOBFRMThTZkyxfZSKBGRzk7v4RUR6aS2bt3K\n1KlTm/zcycmJffv2tWGPRETkRijAi4h0cmPHjmXYsGEN2o1G/ZFWRMQRKcCLiHRyPXv2ZPz48fbu\nhoiI3CA9XhERkWsqLCwkISGBd955h+XLlzNu3Dj69OnD8OHDeeedd7h06VKDfbKzs3nuuecYOHAg\nffr0YfTo0SxYsIDLly832La4uJj//u//ZuTIkfTu3ZvU1FSeeOIJNm3a1GDbkydP8rOf/YzbbruN\npKQkfvzjH5OXl9cq5y0i4qj0BF5EpJOrrKzkzJkzDdpdXFzw8vKyfb1+/XoKCgqYNGkSQUFBrF+/\nnt///vcUFRUxe/Zs23a7d+9mypQpODs727bdsGEDc+bMITs7m//93/+1bVtYWMgjjzxCSUkJ48eP\np3fv3lRWVpKVlUVGRgZDhgyxbXvhwgUmT55MUlISM2bMoLCwkA8++IBnn32W5cuX4+Tk1Er/h0RE\nHIsCvIhIJ/fOO+/wzjvvNGgfPnw48+bNs32dnZ3NZ599Rq9evQCYPHkyzz//PEuWLOHhhx+mX79+\nALz++utUV1fzySefkJiYaNv2xRdfZPny5Tz44IOkpqYC8Otf/5pTp06xcOFChg4dWu/4tbW19b4+\ne/YsP/7xj5k+fbqtLSAggN/85jdkZGQ02F9EpKNSgBcR6eQefvhh0tLSGrQHBATU+3rw4MG28A5g\nMBiYNm0aX331FWvXrqVfv36UlJTw3XffMWrUKFt4r9v2mWeeYdWqVaxdu5bU1FRKS0v55ptvGDp0\naKPh+/uTaI1GY4NVcwYNGgTA0aNHFeBFpNNQgBcR6eSio6MZPHjwdbeLjY1t0Na9e3cACgoKAOuQ\nmKvbr9atWzeMRqNt2/z8fCwWCz179ryhfgYHB+Pq6lqvzc/PD4DS0tIb+h4iIh2BJrGKiEi7cK0x\n7haLpQ17IiJiXwrwIiJyQ3Jzcxu0HTp0CIDIyEgAIiIi6rVf7fDhw9TW1tq2jYqKwmAwsH///tbq\nsohIh6QALyIiNyQjI4O9e/favrZYLCxcuBCAu+++G4DAwECSk5PZsGEDOTk59badP38+AKNGjQKs\nw1+GDRvGxo0bycjIaHA8PVUXEWmcxsCLiHRy+/btIz09vdHP6oI5QGJiIo899hiTJk3CbDazbt06\nMjIyGD9+PMnJybbtXn31VaZMmcKkSZN49NFHMZvNbNiwgW+//ZaxY8faVqAB+MUvfsG+ffuYPn06\n9913H7169aKqqoqsrCzCw8N5+eWXW+/ERUTaKQV4EZFObvny5SxfvrzRz9asWWMbez5ixAi6du3K\nvHnzyMvLIzAwkGeffZZnn3223j59+vThk08+4Xe/+x1//etfuXDhApGRkbz00ks8+eST9baNjIzk\n73//O3/4wx/YuHEj6enp+Pj4kJiYyMMPP9w6Jywi0s4ZLPobpYiIXENhYSEjR47k+eef59/+7d/s\n3R0RkU5PY+BFRERERNoRBXgRERERkXZEAV5EREREpB3RGHgRERERkXZET+BFRERERNoRBXgRERER\nkXZEAV5EREREpB1RgBcRERERaUcU4EVERERE2hEFeBERERGRduT/A1v6lxggwyeuAAAAAElFTkSu\nQmCC\n",
            "text/plain": [
              "<Figure size 864x432 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mkyubuJSOzg3",
        "colab_type": "text"
      },
      "source": [
        "# 5. Performance On Test Set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DosV94BYIYxg",
        "colab_type": "text"
      },
      "source": [
        "Now we'll load the holdout dataset and prepare inputs just as we did with the training set. Then we'll evaluate predictions using [Matthew's correlation coefficient](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html) because this is the metric used by the wider NLP community to evaluate performance on CoLA. With this metric, +1 is the best score, and -1 is the worst score. This way, we can see how well we perform against the state of the art models for this specific task."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tg42jJqqM68F",
        "colab_type": "text"
      },
      "source": [
        "### 5.1. Data Preparation\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xWe0_JW21MyV",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "We'll need to apply all of the same steps that we did for the training data to prepare our test data set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAN0LZBOOPVh",
        "colab_type": "code",
        "outputId": "975b17e9-c906-4f59-bf93-a9b55374548e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Load the dataset into a pandas dataframe.\n",
        "df = pd.read_csv(\"./cola_public/raw/out_of_domain_dev.tsv\", delimiter='\\t', header=None, names=['sentence_source', 'label', 'label_notes', 'sentence'])\n",
        "\n",
        "# Report the number of sentences.\n",
        "print('Number of test sentences: {:,}\\n'.format(df.shape[0]))\n",
        "\n",
        "# Create sentence and label lists\n",
        "sentences = df.sentence.values\n",
        "labels = df.label.values\n",
        "\n",
        "# Tokenize all of the sentences and map the tokens to thier word IDs.\n",
        "input_ids = []\n",
        "\n",
        "# For every sentence...\n",
        "for sent in sentences:\n",
        "    # `encode` will:\n",
        "    #   (1) Tokenize the sentence.\n",
        "    #   (2) Prepend the `[CLS]` token to the start.\n",
        "    #   (3) Append the `[SEP]` token to the end.\n",
        "    #   (4) Map tokens to their IDs.\n",
        "    encoded_sent = tokenizer.encode(\n",
        "                        sent,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                   )\n",
        "    \n",
        "    input_ids.append(encoded_sent)\n",
        "\n",
        "# Pad our input tokens\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, \n",
        "                          dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "# Create attention masks\n",
        "attention_masks = []\n",
        "\n",
        "# Create a mask of 1s for each token followed by 0s for padding\n",
        "for seq in input_ids:\n",
        "  seq_mask = [float(i>0) for i in seq]\n",
        "  attention_masks.append(seq_mask) \n",
        "\n",
        "# Convert to tensors.\n",
        "prediction_inputs = torch.tensor(input_ids)\n",
        "prediction_masks = torch.tensor(attention_masks)\n",
        "prediction_labels = torch.tensor(labels)\n",
        "\n",
        "# Set the batch size.  \n",
        "batch_size = 32  \n",
        "\n",
        "# Create the DataLoader.\n",
        "prediction_data = TensorDataset(prediction_inputs, prediction_masks, prediction_labels)\n",
        "prediction_sampler = SequentialSampler(prediction_data)\n",
        "prediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of test sentences: 516\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "16lctEOyNFik",
        "colab_type": "text"
      },
      "source": [
        "## 5.2. Evaluate on Test Set\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rhR99IISNMg9",
        "colab_type": "text"
      },
      "source": [
        "\n",
        "With the test set prepared, we can apply our fine-tuned model to generate predictions on the test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hba10sXR7Xi6",
        "colab_type": "code",
        "outputId": "418390e2-b264-402f-9834-87d2e56f3ff7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# Prediction on test set\n",
        "\n",
        "print('Predicting labels for {:,} test sentences...'.format(len(prediction_inputs)))\n",
        "\n",
        "# Put model in evaluation mode\n",
        "model.eval()\n",
        "\n",
        "# Tracking variables \n",
        "predictions , true_labels = [], []\n",
        "\n",
        "# Predict \n",
        "for batch in prediction_dataloader:\n",
        "  # Add batch to GPU\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  \n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  \n",
        "  # Telling the model not to compute or store gradients, saving memory and \n",
        "  # speeding up prediction\n",
        "  with torch.no_grad():\n",
        "      # Forward pass, calculate logit predictions\n",
        "      outputs = model(b_input_ids, token_type_ids=None, \n",
        "                      attention_mask=b_input_mask)\n",
        "\n",
        "  logits = outputs[0]\n",
        "\n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  label_ids = b_labels.to('cpu').numpy()\n",
        "  \n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  true_labels.append(label_ids)\n",
        "\n",
        "print('    DONE.')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicting labels for 516 test sentences...\n",
            "    DONE.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-5jscIM8R4Gv",
        "colab_type": "text"
      },
      "source": [
        "Accuracy on the CoLA benchmark is measured using the \"[Matthews correlation coefficient](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html)\" (MCC).\n",
        "\n",
        "We use MCC here because the classes are imbalanced:\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hWcy0X1hirdx",
        "colab_type": "code",
        "outputId": "3dd0a61a-70cd-4bb7-a0b3-90ee6992ff0f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print('Positive samples: %d of %d (%.2f%%)' % (df.label.sum(), len(df.label), (df.label.sum() / len(df.label) * 100.0)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Positive samples: 354 of 516 (68.60%)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cRaZQ4XC7kLs",
        "colab_type": "code",
        "outputId": "13a30575-38c2-44e4-ec79-cb69a6e8d6a7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "source": [
        "from sklearn.metrics import matthews_corrcoef\n",
        "\n",
        "matthews_set = []\n",
        "\n",
        "# Evaluate each test batch using Matthew's correlation coefficient\n",
        "print('Calculating Matthews Corr. Coef. for each batch...')\n",
        "\n",
        "# For each input batch...\n",
        "for i in range(len(true_labels)):\n",
        "  \n",
        "  # The predictions for this batch are a 2-column ndarray (one column for \"0\" \n",
        "  # and one column for \"1\"). Pick the label with the highest value and turn this\n",
        "  # in to a list of 0s and 1s.\n",
        "  pred_labels_i = np.argmax(predictions[i], axis=1).flatten()\n",
        "  \n",
        "  # Calculate and store the coef for this batch.  \n",
        "  matthews = matthews_corrcoef(true_labels[i], pred_labels_i)                \n",
        "  matthews_set.append(matthews)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Calculating Matthews Corr. Coef. for each batch...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:872: RuntimeWarning: invalid value encountered in double_scalars\n",
            "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IUM0UA1qJaVB",
        "colab_type": "text"
      },
      "source": [
        "The final score will be based on the entire test set, but let's take a look at the scores on the individual batches to get a sense of the variability in the metric between batches. \n",
        "\n",
        "Each batch has 32 sentences in it, except the last batch which has only (516 % 32) = 4 test sentences in it.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xytAr_C48wnu",
        "colab_type": "code",
        "outputId": "7464e6cb-df84-4029-e7dd-8933260a93ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        }
      },
      "source": [
        "matthews_set"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.049286405809014416,\n",
              " -0.21684543705982773,\n",
              " 0.4040950971038548,\n",
              " 0.41179801403140964,\n",
              " 0.25365601296401685,\n",
              " 0.6777932975034471,\n",
              " 0.4879500364742666,\n",
              " 0.0,\n",
              " 0.8320502943378436,\n",
              " 0.8246211251235321,\n",
              " 0.9229582069908973,\n",
              " 0.647150228929434,\n",
              " 0.8150678894028793,\n",
              " 0.7141684885491869,\n",
              " 0.3268228676411533,\n",
              " 0.5844155844155844,\n",
              " 0.0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oCYZa1lQ8Jn8",
        "colab_type": "code",
        "outputId": "0f041f56-ffca-4097-a1c9-4878918e83a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# Combine the predictions for each batch into a single list of 0s and 1s.\n",
        "flat_predictions = [item for sublist in predictions for item in sublist]\n",
        "flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "\n",
        "# Combine the correct labels for each batch into a single list.\n",
        "flat_true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "# Calculate the MCC\n",
        "mcc = matthews_corrcoef(flat_true_labels, flat_predictions)\n",
        "\n",
        "print('MCC: %.3f' % mcc)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MCC: 0.529\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jXx0jPc4HUfZ",
        "colab_type": "text"
      },
      "source": [
        "Cool! In about half an hour and without doing any hyperparameter tuning (adjusting the learning rate, epochs, batch size, ADAM properties, etc.) we are able to get a good score. I should also mention we didn't train on the entire training dataset, but set aside a portion of it as our validation set for legibililty of code.\n",
        "\n",
        "The library documents the expected accuracy for this benchmark [here](https://huggingface.co/transformers/examples.html#glue).\n",
        "\n",
        "You can also look at the official leaderboard [here](https://gluebenchmark.com/leaderboard/submission/zlssuBTm5XRs0aSKbFYGVIVdvbj1/-LhijX9VVmvJcvzKymxy). \n",
        "\n",
        "Note that (due to the small dataset size?) the accuracy can vary significantly with different random seeds.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GfjYoa6WmkN6",
        "colab_type": "text"
      },
      "source": [
        "# Conclusion"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xlQG7qgkmf4n",
        "colab_type": "text"
      },
      "source": [
        "This post demonstrates that with a pre-trained BERT model you can quickly and effectively create a high quality model with minimal effort and training time using the pytorch interface, regardless of the specific NLP task you are interested in."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YUmsUOIv8EUO",
        "colab_type": "text"
      },
      "source": [
        "# Appendix\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q2079Qyn8Mt8",
        "colab_type": "text"
      },
      "source": [
        "## A1. Saving & Loading Fine-Tuned Model\n",
        "\n",
        "This first cell (taken from `run_glue.py` [here](https://github.com/huggingface/transformers/blob/35ff345fc9df9e777b27903f11fa213e4052595b/examples/run_glue.py#L495)) writes the model and tokenizer out to disk."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ulTWaOr8QNY",
        "colab_type": "code",
        "outputId": "005899af-7702-4f8f-f956-b00cd3a5934a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "import os\n",
        "\n",
        "# Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()\n",
        "\n",
        "output_dir = './model_save/'\n",
        "\n",
        "# Create output directory if needed\n",
        "if not os.path.exists(output_dir):\n",
        "    os.makedirs(output_dir)\n",
        "\n",
        "print(\"Saving model to %s\" % output_dir)\n",
        "\n",
        "# Save a trained model, configuration and tokenizer using `save_pretrained()`.\n",
        "# They can then be reloaded using `from_pretrained()`\n",
        "model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training\n",
        "model_to_save.save_pretrained(output_dir)\n",
        "tokenizer.save_pretrained(output_dir)\n",
        "\n",
        "# Good practice: save your training arguments together with the trained model\n",
        "# torch.save(args, os.path.join(output_dir, 'training_args.bin'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving model to ./model_save/\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('./model_save/vocab.txt',\n",
              " './model_save/special_tokens_map.json',\n",
              " './model_save/added_tokens.json')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z-tjHkR7lc1I",
        "colab_type": "text"
      },
      "source": [
        "Let's check out the file sizes, out of curiosity."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqMzI3VTCZo5",
        "colab_type": "code",
        "outputId": "e4e435b9-c22f-4226-8bd1-d01cf090f4fa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "!ls -l --block-size=K ./model_save/"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 427964K\n",
            "-rw-r--r-- 1 root root      1K Dec 19 17:33 added_tokens.json\n",
            "-rw-r--r-- 1 root root      1K Dec 19 17:33 config.json\n",
            "-rw-r--r-- 1 root root 427719K Dec 19 17:33 pytorch_model.bin\n",
            "-rw-r--r-- 1 root root      1K Dec 19 17:33 special_tokens_map.json\n",
            "-rw-r--r-- 1 root root      1K Dec 19 17:33 tokenizer_config.json\n",
            "-rw-r--r-- 1 root root    227K Dec 19 17:33 vocab.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fr_bt2rFlgDn",
        "colab_type": "text"
      },
      "source": [
        "The largest file is the model weights, at around 418 megabytes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-WUFUIQ8Cu8D",
        "colab_type": "code",
        "outputId": "c4490ecf-ffce-49f7-fe2e-fba9d3ebdd9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "!ls -l --block-size=M ./model_save/pytorch_model.bin"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 418M Dec 19 17:33 ./model_save/pytorch_model.bin\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dzGKvOFAll_e",
        "colab_type": "text"
      },
      "source": [
        "To save your model across Colab Notebook sessions, download it to your local machine, or ideally copy it to your Google Drive."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Trr-A-POC18_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Mount Google Drive to this Notebook instance.\n",
        "from google.colab import drive\n",
        "    drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NxlZsafTC-V5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Copy the model files to a directory in your Google Drive.\n",
        "!cp -r ./model_save/ \"./drive/Shared drives/ChrisMcCormick.AI/Blog Posts/BERT Fine-Tuning/\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W0vstijw85SZ",
        "colab_type": "text"
      },
      "source": [
        "The following functions will load the model back from disk."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nskPzUM084zL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Load a trained model and vocabulary that you have fine-tuned\n",
        "model = model_class.from_pretrained(output_dir)\n",
        "tokenizer = tokenizer_class.from_pretrained(output_dir)\n",
        "\n",
        "# Copy the model to the GPU.\n",
        "model.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NIWouvDrGVAi",
        "colab_type": "text"
      },
      "source": [
        "## A.2. Weight Decay\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f123ZAlF1OyW",
        "colab_type": "text"
      },
      "source": [
        "The huggingface example includes the following code block for enabling weight decay, but the default decay rate is \"0.0\", so I moved this to the appendix.\n",
        "\n",
        "This block essentially tells the optimizer to not apply weight decay to the bias terms (e.g., $ b $ in the equation $ y = Wx + b $ ). Weight decay is a form of regularization--after calculating the gradients, we multiply them by, e.g., 0.99."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QxSMw0FrptiL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# This code is taken from:\n",
        "# https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L102\n",
        "\n",
        "# Don't apply weight decay to any parameters whose names include these tokens.\n",
        "# (Here, the BERT doesn't have `gamma` or `beta` parameters, only `bias` terms)\n",
        "no_decay = ['bias', 'LayerNorm.weight']\n",
        "\n",
        "# Separate the `weight` parameters from the `bias` parameters. \n",
        "# - For the `weight` parameters, this specifies a 'weight_decay_rate' of 0.01. \n",
        "# - For the `bias` parameters, the 'weight_decay_rate' is 0.0. \n",
        "optimizer_grouped_parameters = [\n",
        "    # Filter for all parameters which *don't* include 'bias', 'gamma', 'beta'.\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.1},\n",
        "    \n",
        "    # Filter for parameters which *do* include those.\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]\n",
        "\n",
        "# Note - `optimizer_grouped_parameters` only includes the parameter values, not \n",
        "# the names."
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}