bert_classification_bbc_news.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SNLP Classification BBC News Category.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "915d22ce7c4d4c3292b269eec1fe2355": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c263e2e71d1c4a358c592794c00e5500",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e5194f0d4559456faba8580ca56ff165",
              "IPY_MODEL_0f9699c76dbf405bafa7837e57e79a1f"
            ]
          }
        },
        "c263e2e71d1c4a358c592794c00e5500": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5194f0d4559456faba8580ca56ff165": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f5a9010232f048d7a7cff401e0caaef1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_279b29895d814e98a1c9a66d373bd2fa"
          }
        },
        "0f9699c76dbf405bafa7837e57e79a1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_66266322c4094cd89d5f6f8697b0fff3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 627kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2b6af9d289fe4093b4c55e7b79108ae3"
          }
        },
        "f5a9010232f048d7a7cff401e0caaef1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "279b29895d814e98a1c9a66d373bd2fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "66266322c4094cd89d5f6f8697b0fff3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2b6af9d289fe4093b4c55e7b79108ae3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b7f9430615941edb0f9aa49be2aac1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bdaf276727d447bebbb2c2df2bdec6df",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_914d35d3c1c14111af9ce4976cc8c10f",
              "IPY_MODEL_b005b2af56814a8885018f0447dade37"
            ]
          }
        },
        "bdaf276727d447bebbb2c2df2bdec6df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "914d35d3c1c14111af9ce4976cc8c10f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_293c0a3f8b1e4705af733d26ac75a5c5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 28,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 28,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_52cb5c4d99ba49668e9438f7880e79b1"
          }
        },
        "b005b2af56814a8885018f0447dade37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1681527f5c184593ac93b752b247d9a6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 28.0/28.0 [00:00&lt;00:00, 57.7B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c995cfe06214573af1a08173aeac62d"
          }
        },
        "293c0a3f8b1e4705af733d26ac75a5c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "52cb5c4d99ba49668e9438f7880e79b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1681527f5c184593ac93b752b247d9a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c995cfe06214573af1a08173aeac62d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e2e8188aff7d4273bea83793556591d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7feee761460d476497e8ed49357da91a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a2db329e322f446e8282f6af535b781e",
              "IPY_MODEL_cd9d00ca5a814f4eb91279d422ea2658"
            ]
          }
        },
        "7feee761460d476497e8ed49357da91a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a2db329e322f446e8282f6af535b781e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7dadecf1d67e40a681b96082667d3092",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f66d1050ddf494186b0cc04ebcd3465"
          }
        },
        "cd9d00ca5a814f4eb91279d422ea2658": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d79f5856df534d30985ce8004262c313",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 3.18MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_39c06d7612354f389737ee9693ffb25d"
          }
        },
        "7dadecf1d67e40a681b96082667d3092": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f66d1050ddf494186b0cc04ebcd3465": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d79f5856df534d30985ce8004262c313": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "39c06d7612354f389737ee9693ffb25d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "63b05f1d3e6d4ca985ecbba3d01b8785": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1e65ef1565dc409fbae4578dd699198e",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c59f128efe5f44aaaeed777a5bb780a3",
              "IPY_MODEL_1a5be1af8e2745948d9c84960a2b0039"
            ]
          }
        },
        "1e65ef1565dc409fbae4578dd699198e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c59f128efe5f44aaaeed777a5bb780a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_faad4ef4e1b840519f97f92244300caf",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e392a529666a4c52a345062ca6551f0c"
          }
        },
        "1a5be1af8e2745948d9c84960a2b0039": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1db3751d31f9482fbce9913998eb5771",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4/4 [05:15&lt;00:00, 78.81s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_47273906bb844f6e86b8ba8f7d4ca425"
          }
        },
        "faad4ef4e1b840519f97f92244300caf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e392a529666a4c52a345062ca6551f0c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1db3751d31f9482fbce9913998eb5771": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "47273906bb844f6e86b8ba8f7d4ca425": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef58eddd1a3644dbabcd36ca4bf122ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_dd749f8ac4c34539b658ce538ab76df1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_63425a82d4f64dedb825fc28cbec5cb0",
              "IPY_MODEL_7a1a2139365e4018bc5d6a6a6cf0ce8a"
            ]
          }
        },
        "dd749f8ac4c34539b658ce538ab76df1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "63425a82d4f64dedb825fc28cbec5cb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7e3031b48f6e49fd9a3d1db216038595",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e57fb649f9a47c19927a1f34d168b61"
          }
        },
        "7a1a2139365e4018bc5d6a6a6cf0ce8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0c1112023a244b69b5d13737a98e1bf2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [02:39&lt;00:00,  1.42s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0bc50285861a46019b20171d7f040f75"
          }
        },
        "7e3031b48f6e49fd9a3d1db216038595": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4e57fb649f9a47c19927a1f34d168b61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0c1112023a244b69b5d13737a98e1bf2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0bc50285861a46019b20171d7f040f75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "54ce4b435410465892575b56b8aaf361": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5c28e2c2e21b4355813ad67ae070e951",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6d5986f80c9b471eafeb336f7c98247e",
              "IPY_MODEL_19ab9db25661411abb86c19f43000679"
            ]
          }
        },
        "5c28e2c2e21b4355813ad67ae070e951": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6d5986f80c9b471eafeb336f7c98247e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6f83485d6e0441fead9ff3e71619cd1a",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68f0410899dc434ab803827c357e5743"
          }
        },
        "19ab9db25661411abb86c19f43000679": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_04ff7e2e815a4e3fa369d20e5833f29b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [01:16&lt;00:00,  1.46it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_38347f446ccc494d86793c11b172c833"
          }
        },
        "6f83485d6e0441fead9ff3e71619cd1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68f0410899dc434ab803827c357e5743": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "04ff7e2e815a4e3fa369d20e5833f29b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "38347f446ccc494d86793c11b172c833": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "653182eaa70a4879b13a43ddbce42053": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_194b7065cd424be084e55eaa359cbaeb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fb7f1b0d4e4a43b4a11061e42caf1798",
              "IPY_MODEL_928ba74a822b4d279a221f6496fe48e7"
            ]
          }
        },
        "194b7065cd424be084e55eaa359cbaeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb7f1b0d4e4a43b4a11061e42caf1798": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1f15894f2b804501bac9cb2d1b203278",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_668a8b33d7234a25b2964e06410b28e6"
          }
        },
        "928ba74a822b4d279a221f6496fe48e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d33579127be941019f257d711c472029",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [01:17&lt;00:00,  1.45it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d0641135f1c4a1da93bbc2f2e83cb00"
          }
        },
        "1f15894f2b804501bac9cb2d1b203278": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "668a8b33d7234a25b2964e06410b28e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d33579127be941019f257d711c472029": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d0641135f1c4a1da93bbc2f2e83cb00": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eff1969225b744aca48e5cc28ac4f164": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bc2405ba4b344e10abf09aa795268ba0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a4127f7eff2d4af38517ff2e7ae35aae",
              "IPY_MODEL_74ef4eee0d494e20abdbb60e1732570d"
            ]
          }
        },
        "bc2405ba4b344e10abf09aa795268ba0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4127f7eff2d4af38517ff2e7ae35aae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_742b858658314552a3f7b0cd0aaee241",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 112,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 112,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_76786725f3434541bddc7f172c4cad03"
          }
        },
        "74ef4eee0d494e20abdbb60e1732570d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0045957db07c44129321bfbc6ce477ee",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 112/112 [01:17&lt;00:00,  1.45it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_de8bb269ad534ab5947fdf6bf1c1f87b"
          }
        },
        "742b858658314552a3f7b0cd0aaee241": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "76786725f3434541bddc7f172c4cad03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0045957db07c44129321bfbc6ce477ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "de8bb269ad534ab5947fdf6bf1c1f87b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1cecd84f0390422a934468e920adca79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a23e67de2f854867a6db9b43000a58ff",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c13a461ce76e4295b364ceddc5f2e71f",
              "IPY_MODEL_8971a145f7ea43dfa397e8493cb8bd1a"
            ]
          }
        },
        "a23e67de2f854867a6db9b43000a58ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c13a461ce76e4295b364ceddc5f2e71f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d69cdb7d4ec74f4a890e5ab51688a1ce",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 15,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 15,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f601e873e8b40fca4e7c24ffd30f80b"
          }
        },
        "8971a145f7ea43dfa397e8493cb8bd1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bffbbd81e5b7487a8308fb481ff0d828",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 15/15 [00:02&lt;00:00,  5.04it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6e6f6132d6ba4d9f88164bfe7532cf88"
          }
        },
        "d69cdb7d4ec74f4a890e5ab51688a1ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f601e873e8b40fca4e7c24ffd30f80b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bffbbd81e5b7487a8308fb481ff0d828": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6e6f6132d6ba4d9f88164bfe7532cf88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x6GB9DPqy-fx",
        "outputId": "38b0a50c-4785-4801-dd42-e2ec17b38b22"
      },
      "source": [
        "!pip install transformers torch"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d5/43/cfe4ee779bbd6a678ac6a97c5a5cdeb03c35f9eaebbb9720b036680f9a2d/transformers-4.6.1-py3-none-any.whl (2.2MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 4.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.1+cu101)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 24.0MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 39.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting huggingface-hub==0.0.8\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.0.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Installing collected packages: sacremoses, tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.0.8 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.6.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pydajSTmC_TE",
        "outputId": "f1cd500e-4c39-4457-bf5e-5750f9aba53b"
      },
      "source": [
        "from tqdm.notebook import tqdm\n",
        "tqdm.pandas()\n",
        "\n",
        "import pickle\n",
        "import time\n",
        "import string\n",
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import torch\n",
        "from torchtext.legacy import data\n",
        "import torch.nn as nn\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "import torch.optim as optim\n",
        "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n",
        "from sklearn.model_selection import train_test_split\n",
        "import seaborn as sns\n",
        "\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
        "print(device)\n",
        "\n",
        "PATH = 'data'\n",
        "\n",
        "RANDOM_STATE = 1"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tqdm/std.py:658: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
            "  from pandas import Panel\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "cpu\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZA1P_cYIszI3"
      },
      "source": [
        "#### Creating dataset from bbc news"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9hs5ROqGAJD"
      },
      "source": [
        "# data = []\n",
        "# for category in tqdm(categories):\n",
        "#   summary_path = PATH + 'BBC News Summary/Summaries/' + category\n",
        "#   article_path = PATH + 'BBC News Summary/News Articles/' + category\n",
        "#   for filename in tqdm(os.listdir(article_path)):\n",
        "#     article = ''\n",
        "#     summary = ''\n",
        "#     with open(article_path + \"/\" + filename, encoding='latin1') as f:\n",
        "#       article = f.read()\n",
        "#     with open(summary_path + \"/\" + filename, encoding='latin1') as f:\n",
        "#       summary = f.read()\n",
        "#     data.append([article, summary, category])\n",
        "# print(data[:3])\n",
        "\n",
        "# new_data = []\n",
        "# for row in data:\n",
        "#   article = row[0].splitlines()\n",
        "#   body = \" \".join([sent for sent in article[2:] if sent != \"\"])\n",
        "#   print(body)\n",
        "#   break\n",
        "#   new_data.append([article[0], body, row[1], row[2]])\n",
        "\n",
        "# df = pd.DataFrame(new_data, columns = ['Title', 'Body', 'Summary', 'Category'])\n",
        "# df.to_csv(PATH + 'BBC News Summary/bbc_news_dataset.csv', index=False, header=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z3VkUe5Ks7_8"
      },
      "source": [
        "# Loading and processing dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oXROtpdOAktr"
      },
      "source": [
        "### Load raw dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "exfGXLWvFLNl"
      },
      "source": [
        "raw_df = pd.read_csv(PATH + '/bbc_news_dataset.csv')"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2pEjTyQCDAaa"
      },
      "source": [
        "### Pre-process text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-bkX99gDCiv"
      },
      "source": [
        "MAX_SEQ_LEN = 200\n",
        "\n",
        "def preprocess_text(text):\n",
        "  text = text.split()\n",
        "  text = ' '.join(text[:MAX_SEQ_LEN])\n",
        "  text = text.lower()\n",
        "  return text"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e-ubxgz-DFLY"
      },
      "source": [
        "raw_df['Title'] = raw_df['Title'].apply(lambda x: preprocess_text(x))\n",
        "raw_df['Body'] = raw_df['Body'].apply(lambda x: preprocess_text(x))\n",
        "raw_df['Summary'] = raw_df['Summary'].apply(lambda x: preprocess_text(x))"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R9aOka5b7s-z"
      },
      "source": [
        "### Prepare columns for concatenated texts and integer labels"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nBeu5Hig71gd"
      },
      "source": [
        "labels = ['business', 'entertainment', 'politics', 'sport', 'tech']\n",
        "df = pd.DataFrame(columns=['label', 'titlebody'])\n",
        "df['label'] = raw_df['Category'].apply(lambda x: labels.index(x))\n",
        "df['titlebody'] = raw_df['Title'] + \". \" + raw_df['Body']"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "szqVJFmf9ckG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "939d9999-383c-4a8a-86b6-a6f818518d61"
      },
      "source": [
        "df"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>titlebody</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>fed chief warning on us deficit. federal reser...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>saudi ministry to employ women. women will be ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "      <td>deutsche telekom sees mobile gain. german tele...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>steel firm 'to cut' 45,000 jobs. mittal steel,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>eu aiming to fuel development aid. european un...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2220</th>\n",
              "      <td>4</td>\n",
              "      <td>digital uk driven by net and tv. the uk's adop...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2221</th>\n",
              "      <td>4</td>\n",
              "      <td>gizmondo gadget hits the shelves. the gizmondo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2222</th>\n",
              "      <td>4</td>\n",
              "      <td>what price for 'trusted pc security'?. you can...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2223</th>\n",
              "      <td>4</td>\n",
              "      <td>nintendo ds makes its euro debut. nintendo's d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2224</th>\n",
              "      <td>4</td>\n",
              "      <td>chip maker backs net phone calls. rich templet...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2225 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      label                                          titlebody\n",
              "0         0  fed chief warning on us deficit. federal reser...\n",
              "1         0  saudi ministry to employ women. women will be ...\n",
              "2         0  deutsche telekom sees mobile gain. german tele...\n",
              "3         0  steel firm 'to cut' 45,000 jobs. mittal steel,...\n",
              "4         0  eu aiming to fuel development aid. european un...\n",
              "...     ...                                                ...\n",
              "2220      4  digital uk driven by net and tv. the uk's adop...\n",
              "2221      4  gizmondo gadget hits the shelves. the gizmondo...\n",
              "2222      4  what price for 'trusted pc security'?. you can...\n",
              "2223      4  nintendo ds makes its euro debut. nintendo's d...\n",
              "2224      4  chip maker backs net phone calls. rich templet...\n",
              "\n",
              "[2225 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "duMXEpm_hGz_",
        "outputId": "97475ec3-ea76-4357-caf4-845a8207e5f3"
      },
      "source": [
        "category, values = np.unique(df['label'].values, return_counts=True)\n",
        "plt.title(\"Class distribution of BBC News Dataset\")\n",
        "plt.bar(category, values)\n",
        "plt.xticks(np.arange(5), labels)\n",
        "plt.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAcGUlEQVR4nO3debwcVZ338c8XwqIsCZCYgSQSh0UNOiBmEAYdURRZVFABEUYCwhMdQUHEERVHeB50EBUcxjUKL4IgsoxKBFRiMIDsiUDCIhoQDGFJCCRsgiC/54/za6lcb98lt29ucvJ9v179StWp03VOdVV/u+pU344iAjMzq8saQ90BMzPrPIe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO4rkKQTJJ0zhO3PlHR4Th8k6fIOrvt2SbvkdEe3U9JnJX2/U+vrR7vvkTRf0pOSXrei2zcbCId7h0k6UNKsDIQHJf1c0huHul9dRcS5EbFbb/UknSXppD6sb5uImDnQfknaRdL9Xdb9pYg4fKDrXg5fBY6MiPUj4uauCyWFpKdyXz8i6TxJIxrLZ0p6JpcvlXSVpNd2Wcc7svwJSYskXSnp3d11Jj80Q9L+jbJhWTa+Y1vdR5LGZ9tP5uNhSZdIens/1nGIpN8MZj9XZDsrE4d7B0k6Bvg68CVgNPBy4FvA3kPZr8EkadhQ92EQbQ7c3kudbSNifeAfgY2AE7osPzKXbwzMBH7QWiBpX+BC4GxgLOWY+U/gXT209yhwoqQ1+7wVg29EbuO2wHTgJ5IOGdouGRHhRwcewHDgSWC/HuqcAJzTmL8QeAhYClwFbNNYtidwB/AEsAA4NstHApcASyhv9KuBNdq093bgd7n+bwBXAofnskOA3+S0gNOAhcDjwFzgNcBk4DngL7ltP8v69wKfBuYAzwLDsuxtje28CDg/+/9bSgi2+hXAlo35s4CTgPWAPwMvZHtPApt187q9mxK6SyiB+erGsnuBY7NvS7MP67Z5fdYAjgfuy20/O/fjOtl2AE8Bd7d5ftft+ChweWN+Zuv1zvkJwF8ar/mfgE/14xg7ATgXuBWYlGXDsh/jc34dyhXHn4CHge8AL8llVwLvy+md83l75fyuwC05vWXWXQo8Apzfpj/jcx3DupQfm22vkfPHAXfnsXAH8J4sfzXwDPDXfL2XZPlewM2UY3E+cEJj3esC5wCLc//fBIxuvAfPAB6kvGdOAtZs107tD5+5d85OlAPvJ/14zs+BrYCXUQLw3MayM4APR8QGlKC9Iss/CdwPjKKc6X2W8gZbhqSRwI8p4TWS8ubauU0/dgP+Fdia8gbZH1gcEVOyT6dEGZponlF+gPImHBERz3ezzr0pH14bAz8EfipprbavBBARTwF7AA9ke+tHxANdtmtr4Dzg6HwNLgN+JmntRrX9gd2BVwD/RPkg684h+XgL5cx7feAbEfFslDNRKB9KW/TU7+zXRsA+wPVtlq8NHNRY/kpgHOVDsD8C+DzwhTav58mU/bgdJaTHUK4GoAT2Ljn9ZuAeyn5vzV+Z0/8PuJxyJTIW+J9+9vHHlGP6lTl/N/AmyrF1InCOpE0j4k7gI8B1ua9bQ1pPAQcDIyjH2L9L2ieXTcr1jAM2yef/OZedBTyf2/06ynF9eA/tVM3h3jmbAI+0CbpuRcSZEfFERDxLOSvbVtLwXPwcMEHShhHxWET8tlG+KbB5RDwXEVdHnrZ0sSdwe0RcFBHPUYaLHmrTleeADYBXAYqIOyPiwV66f3pEzI+IP7dZPrvR9qmUD74de1lnX7wfuDQipue6vwq8BPiXLn17ICIeBX5GCbruHAScGhH3RMSTwGeAA/o51PRbSUsoZ7gvB77bZfnpufwJ4EhKuEE5XqCcZfZLREwDFgHL3IeQJMrV1ici4tGIeIIyRHhAVrmSEuJQQv2/GvPNcH+OMiS1WUQ8ExH9HatufSBvnP29MPfHCxFxPvAHYIcetm9mRMzN+nMoH+atfj5Hee22jIi/RsTsiHhc0mjKMX90RDwVEQspV6MHdNvIasDh3jmLgZF9DQZJa0o6WdLdkh6nDCdAOcsGeB/lYL0vb7LtlOVfAeYBl0u6R9JxbZrYjHJJC0B+AMzvrmJEXEEZtvkmsFDSFEkb9rIJ3a6ru+UR8QLlamOzXp7TF5tRhlGa655POUNtaX6IPU05I+91XTk9jHJF1Ffb55ngusC3gaslrdtY/vFc/hLgncBFkv6JcrxA+aBeHscDn8t2W0YBLwVmS1qSHyq/yHKA64CtMwi3owxDjcurvB0oQ4MA/0EZNroxvwX1oX72rbUvHgWQdLCkWxp9eg0vHud/R9IbJP06bzAvpZx1t+r/APgl8CNJD0g6Ja9gNgfWAh5stPNdyhXEasnh3jnXUcaf9+mtYjqQMnTxNspl5vgsF0BE3BQRe1MOzp8CF2T5ExHxyYj4R8rY8zGSdu1m/Q9SLl3LSstZ3bhu6pHrPT0iXk8ZF94a+FRrUbun9LJ9zbbXoFzet87onqaEUMs/9GO9D1DeyK11t7ZrQS/P63VdlDPv5ynjxf2SVxHfpwwFvaab5S9ExNWUD+bdgLsoH0rv63+3ISKm57o+2ih+hDJEsU1EjMjH8NYQU0Q8DcwGjgJui4i/ANcCx1DuKzyS9R6KiP8TEZsBHwa+JWnLfnTvPZR7GHdJ2hz4HuWqZZP8oLuNPM7pfn//EJgGjIuI4ZT7Bq33xXMRcWJETKBcrb2TMoQzn/L+G9nY9g0jYpse2qmaw71DImIpZWzzm5L2kfRSSWtJ2kPSKd08ZQPKwbiYEnRfai2QtLbK99CHZ2g8TrnJiKR3StoyQ20p5SbRC92s/1JgG0nvzauJj7NsiP6NpH/Os6W1KOOdzzTW+TBlPLq/Xt9o++jc1tZ48y3AgXn1sjsvXnK32tukMTzV1QXAXpJ2zf5+Mtd97XL08TzgE5JeIWl9yj44vz9Day357ZVDKeF6T5s6O1E+PG/PK6ljgM9LOlTShpLWkPRGSVP62OznKGfZwN+uYr4HnCbpZdnmGEnvaDznSkrQtoZgZnaZR9J+ksbm7GOUYOzuGOu6faMlHQl8AfhM9me9fP6irHMoy374PQyM7XLPZAPg0Yh4RtIOlBOhVhtvkfTafL0fpwzTvJDDiJcDX2u8lltIenMP7VTN4d5BEfE1yhv2eMrBPJ/yxvlpN9XPpgwDLKB8g6DrjbgPAvfmkM1HKOPDUG7A/opy1/864FsR8etu+vIIsB/lBtvifN41bbq+ISUUHss+LaYM/0C5sTshL3W72452LqaMjz+W2/Le/KCCcub4Lsq3HQ6i8fpExO8ooXtPtrnMUE5E3AX8G+Um3yO5nnflWWh/nUm5zL8K+CPlQ+1j/VzHrZKepGznJMo3QR5tLP+G8nvg2dbxEfHz3JaLKK/RhyhXEQ9TvuFxcV8ajohrgBu7FH+ackZ/fR47v+LFG5tQQnwDXhyC6ToP8M/ADdnnacBREdHtB1ZaIukpyres9qR8Y+zM7OMdwNcox+rDwGtZ9ji8gvLNp4ckPZJlHwX+r6QnKCdMFzTq/wPlJvTjwJ3Z/9bXSw8G1qa8nx7Leq1hr+7aqZq6vxdnZmarMp+5m5lVyOFuZlYhh7uZWYUc7mZmFVopfvRp5MiRMX78+KHuhpnZKmX27NmPRMSo7patFOE+fvx4Zs2aNdTdMDNbpUi6r90yD8uYmVXI4W5mViGHu5lZhRzuZmYVcribmVXI4W5mViGHu5lZhRzuZmYVcribmVVopfgL1YEYf9ylQ92Fjrn35L2Gugu2Cqnl2PdxPzj6dOYu6V5Jc/M/uZ2VZRtLmi7pD/nvRlkuSadLmidpjqTtB3MDzMzs7/VnWOYtEbFdREzM+eOAGRGxFTAj5wH2oPyXblsBkyn/I7yZma1AAxlz3xuYmtNTgX0a5WdHcT0wQtKm3a3AzMwGR1/DPYDLJc2WNDnLRuf/OA7wEDA6p8dQ/mPolvuzbBmSJkuaJWnWokWLlqPrZmbWTl9vqL4xIhZIehkwXdLvmgsjIiT163/ajogpwBSAiRMn+n/pNjProD6duUfEgvx3IfATYAfg4dZwS/67MKsvAMY1nj42y8zMbAXpNdwlrSdpg9Y0sBtwGzANmJTVJgEX5/Q04OD81syOwNLG8I2Zma0AfRmWGQ38RFKr/g8j4heSbgIukHQYcB+wf9a/DNgTmAc8DRza8V6bmVmPeg33iLgH2Lab8sXArt2UB3BER3pnZmbLxT8/YGZWIYe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO5mZhVyuJuZVcjhbmZWIYe7mVmFHO5mZhVyuJuZVWjYUHfAbHmNP+7Soe5Cx9x78l5D3QWrjM/czcwq5HA3M6uQw93MrEJ9DndJa0q6WdIlOf8KSTdImifpfElrZ/k6OT8vl48fnK6bmVk7/TlzPwq4szH/ZeC0iNgSeAw4LMsPAx7L8tOynpmZrUB9+raMpLHAXsAXgWMkCXgrcGBWmQqcAHwb2DunAS4CviFJERGd67aZrc78Tane9fXM/evAfwAv5PwmwJKIeD7n7wfG5PQYYD5ALl+a9ZchabKkWZJmLVq0aDm7b2Zm3ek13CW9E1gYEbM72XBETImIiRExcdSoUZ1ctZnZaq8vwzI7A++WtCewLrAh8N/ACEnD8ux8LLAg6y8AxgH3SxoGDAcWd7znZmbWVq9n7hHxmYgYGxHjgQOAKyLiIODXwL5ZbRJwcU5Py3ly+RUebzczW7EG8j33T1Nurs6jjKmfkeVnAJtk+THAcQPropmZ9Ve/flsmImYCM3P6HmCHbuo8A+zXgb5ZH9TyrQH/topZZ/kvVM3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCvYa7pHUl3SjpVkm3Szoxy18h6QZJ8ySdL2ntLF8n5+fl8vGDuwlmZtZVX87cnwXeGhHbAtsBu0vaEfgycFpEbAk8BhyW9Q8DHsvy07KemZmtQL2GexRP5uxa+QjgrcBFWT4V2Cen9855cvmuktSxHpuZWa/6NOYuaU1JtwALgenA3cCSiHg+q9wPjMnpMcB8gFy+FNikm3VOljRL0qxFixYNbCvMzGwZfQr3iPhrRGwHjAV2AF410IYjYkpETIyIiaNGjRro6szMrKFf35aJiCXAr4GdgBGShuWiscCCnF4AjAPI5cOBxR3prZmZ9Ulfvi0zStKInH4J8HbgTkrI75vVJgEX5/S0nCeXXxER0clOm5lZz4b1XoVNgamS1qR8GFwQEZdIugP4kaSTgJuBM7L+GcAPJM0DHgUOGIR+m5lZD3oN94iYA7yum/J7KOPvXcufAfbrSO/MzGy5+C9Uzcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCjnczcwq5HA3M6uQw93MrEIOdzOzCvUa7pLGSfq1pDsk3S7pqCzfWNJ0SX/IfzfKckk6XdI8SXMkbT/YG2FmZsvqy5n788AnI2ICsCNwhKQJwHHAjIjYCpiR8wB7AFvlYzLw7Y732szMetRruEfEgxHx25x+ArgTGAPsDUzNalOBfXJ6b+DsKK4HRkjatOM9NzOztvo15i5pPPA64AZgdEQ8mIseAkbn9BhgfuNp92dZ13VNljRL0qxFixb1s9tmZtaTPoe7pPWB/wWOjojHm8siIoDoT8MRMSUiJkbExFGjRvXnqWZm1os+hbuktSjBfm5E/DiLH24Nt+S/C7N8ATCu8fSxWWZmZitIX74tI+AM4M6IOLWxaBowKacnARc3yg/Ob83sCCxtDN+YmdkKMKwPdXYGPgjMlXRLln0WOBm4QNJhwH3A/rnsMmBPYB7wNHBoR3tsZma96jXcI+I3gNos3rWb+gEcMcB+mZnZAPgvVM3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCDnczswo53M3MKuRwNzOrkMPdzKxCvYa7pDMlLZR0W6NsY0nTJf0h/90oyyXpdEnzJM2RtP1gdt7MzLrXlzP3s4Ddu5QdB8yIiK2AGTkPsAewVT4mA9/uTDfNzKw/eg33iLgKeLRL8d7A1JyeCuzTKD87iuuBEZI27VRnzcysb5Z3zH10RDyY0w8Bo3N6DDC/Ue/+LPs7kiZLmiVp1qJFi5azG2Zm1p0B31CNiABiOZ43JSImRsTEUaNGDbQbZmbWsLzh/nBruCX/XZjlC4BxjXpjs8zMzFag5Q33acCknJ4EXNwoPzi/NbMjsLQxfGNmZivIsN4qSDoP2AUYKel+4AvAycAFkg4D7gP2z+qXAXsC84CngUMHoc9mZtaLXsM9Ij7QZtGu3dQN4IiBdsrMzAbGf6FqZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVyOFuZlYhh7uZWYUc7mZmFXK4m5lVaFDCXdLuku6SNE/ScYPRhpmZtdfxcJe0JvBNYA9gAvABSRM63Y6ZmbU3GGfuOwDzIuKeiPgL8CNg70Fox8zM2lBEdHaF0r7A7hFxeM5/EHhDRBzZpd5kYHLOvhK4q6Md6byRwCND3Ykh4m1ffa3O278qbPvmETGquwXDVnRPWiJiCjBlqNrvL0mzImLiUPdjKHjbV89th9V7+1f1bR+MYZkFwLjG/NgsMzOzFWQwwv0mYCtJr5C0NnAAMG0Q2jEzszY6PiwTEc9LOhL4JbAmcGZE3N7pdobAKjOENAi87auv1Xn7V+lt7/gNVTMzG3r+C1Uzswo53M3MKlRluEsaL+m2Aa5jM0kXdapPg03SPsvzl8CSdpH0L32o9+6h+ikJSSMkfXQFtTVT0sScvizbXqb9Ve3YGGx9PYZWFgM5niSdlX/Ls9KrMtw7ISIeiIhVYiemfSg/99BnkoYBuwC9vjEjYlpEnLx8XRuwEcAKCfemiNgzIpZ0bX8VPDYGTX+OoZXIkBxPK1xEVPcAxgO/A84F7gQuAl4K3AuMzDoTgZk5/WbglnzcDGyQ67gtlx8C/Bj4BfAH4JRGW7sB1wG/BS4E1s/yk4E7gDnAV7NsP+A24Fbgqj5sx78BN2a/vkv59tGTwBdzHdcDoylvrEeBP2bdLfLxC2A2cDXwqlznWcB3gBtymx6i/B3CLcCbgHflspuBXwGjG6/BNxrrOB24FrgH2DfLdwGuBC7O8pOBg3Ib5gJbZL1RwP9SvjZ7E7Bzlp8AnAnMzOd/PMt/BPw5+/iVDh0Lu+Y2zs0218n6M4GJOX0v5a8Ul2m/y7GxJvDV3K9zgI+12/8rwwNYD7g0j5/bgPfndp6Sr8WNwJaN1+6K3IYZwMv7cgwN9Tb24TXouj8/lcfhHODERr2Ds+xW4Ac9Hfsr42PIOzBIO288EI3QOBM4lvbh/rNG3fUpXxFtvoEPyR05HFgXuI/yh1ojgauA9bLep4H/BDah/JxC69tII/LfucCYZlkP2/Dq7NdaOf+tPNgCeFeWnQIc3zjo9m08fwawVU6/AbiiUe8SYM2cPwE4tvG8jRr9Phz4WuM1aIb7hZQrvwmU3xKCEu5LgE2BdfINf2IuOwr4ek7/EHhjTr8cuLPRl2vzuSOBxcBazX3RoWPheGA+sHWWnQ0cndMz+ftwX6b9LsfGv1M+MIbl/Mbt9v/K8ADeB3yvMT88t/NzOX8wcEnjfTEppz8E/LQvx9DK/uiy/3ajfOVReTxfAvwrsA3we17Mi417OvZXxseQ/fzACjA/Iq7J6XOAj/dQ9xrgVEnnAj+OiPslda0zIyKWAki6A9iccnk3Abgm669NOYtfCjwDnCHpEsoB02rnLEkXUM54erIr8Hrgplz3S4CFwF8a65sNvL3rEyWtTzmbv7CxHes0qlwYEX9t0+5Y4HxJm+b2/LFNvZ9GxAvAHZJGN8pviogHsx93A5dn+VzgLTn9NmBCo28bZp8BLo2IZ4FnJS2kXJkMVNdj4fPAHyPi91k2FTgC+PpyrPttwHci4nmAiHg0hyq62/8rg7nA1yR9mRLiV+d+OC+XnwecltM7Ae/N6R9QTiZaejqGViW75ePmnF8f2ArYlrKNj0DZr43ntDv2Vyo1h3vXL/AH8Dwv3mdY928LIk6WdCmwJyWo30F5czY925j+K+W1EzA9Ij7QtXFJO1ACel/gSOCtEfERSW8A9gJmS3p9RCxu038BUyPiM13We2zkKUSjH12tASyJiO3arPupNuUA/wOcGhHTJO1COSvrTvP1UJvyFxrzLzT6ugawY0Qs8xpnyHT3Og9U12NhCeXselBE+UO+v9v/g9Vef0TE7yVtTznWT5I0o7WoWa0Pq+rpGFqVCPiviPjuMoXSx3p4Trtjf6VS8w3Vl0vaKacPBH5Dufx8fZa9r1VR0hYRMTcivkwZe3tVH9u4HthZ0pa5nvUkbZ1nocMj4jLgE5SzgFY7N0TEfwKLWPY3eLqaAewr6WX53I0lbd5D/Sco9wqIiMeBP0raL58rSdv29rw0nBd/C2hSD+0NxOXA3948ktp9CLV07WN/dT0WZgHjW/sN+CDlXsHytD8d+HCerbf2U7f7f2UgaTPg6Yg4hzLevH0uen/j3+ty+lrKz4dAuXdydZvVDnT/rGjN/v4S+FDrylHSmHzPXQHsJ2mTLN94SHo6ADWH+13AEZLupIwjfxs4EfhvSbMoZ4UtR0u6TdIc4Dng531pICIWUcaiz8vnXkf5YNgAuCTLfgMck0/5iqS5+TXNayk3atqt+w7K2PDluZ7plLHsdn4EfErSzZK2oLwZD5N0K3A77X9T/2fAeyTdIulNlDP1CyXNZvB+7vTjwERJc3KI6yM9Vc6rm2tyH31lOdrreiycBhxK2c65lKuK7yxn+98H/gTMydf6QNrv/5XBa4EbJd0CfAE4Kcs3yv4eRflAgvIBfGiWfzCXdafrMbRSa+5PyrDmD4Hr8li4CNggyk+mfBG4MvfrqUPW4eXknx+wqkkaTxlbfs0Qd2WlJeleyk3klf23y60faj5zNzNbbfnM3cysQj5zNzOrkMPdzKxCDnczswo53M3MKuRwNzOr0P8HpLQJb5gNHbAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2HgOEAzd7O7b"
      },
      "source": [
        "### Split dataset into train, valid, test"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2OzX9oBQ7SYp"
      },
      "source": [
        "df_business = df[df['label'] == 0]\n",
        "df_entertainment = df[df['label'] == 1]\n",
        "df_politics = df[df['label'] == 2]\n",
        "df_sport = df[df['label'] == 3]\n",
        "df_tech = df[df['label'] == 4]\n",
        "\n",
        "# Train-valid-test split 8:1:1\n",
        "\n",
        "df_business_train, df_business_rem = train_test_split(df_business, train_size=0.8, random_state=RANDOM_STATE)\n",
        "df_entertainment_train, df_entertainment_rem = train_test_split(df_entertainment, train_size=0.8, random_state=RANDOM_STATE)\n",
        "df_politics_train, df_politics_rem = train_test_split(df_politics, train_size=0.8, random_state=RANDOM_STATE)\n",
        "df_sport_train, df_sport_rem = train_test_split(df_sport, train_size=0.8, random_state=RANDOM_STATE)\n",
        "df_tech_train, df_tech_rem = train_test_split(df_tech, train_size=0.8, random_state=RANDOM_STATE)\n",
        "\n",
        "df_business_valid, df_business_test = train_test_split(df_business_rem, train_size=0.5, random_state=RANDOM_STATE)\n",
        "df_entertainment_valid, df_entertainment_test = train_test_split(df_entertainment_rem, train_size=0.5, random_state=RANDOM_STATE)\n",
        "df_politics_valid, df_politics_test = train_test_split(df_politics_rem, train_size=0.5, random_state=RANDOM_STATE)\n",
        "df_sport_valid, df_sport_test = train_test_split(df_sport_rem, train_size=0.5, random_state=RANDOM_STATE)\n",
        "df_tech_valid, df_tech_test = train_test_split(df_tech_rem, train_size=0.5, random_state=RANDOM_STATE)\n",
        "\n",
        "\n",
        "df_train = pd.concat([df_business_train, df_entertainment_train, df_politics_train, df_sport_train, df_tech_train], ignore_index=True, sort=False)\n",
        "df_valid = pd.concat([df_business_valid, df_entertainment_valid, df_politics_valid, df_sport_valid, df_tech_valid], ignore_index=True, sort=False)\n",
        "df_test = pd.concat([df_business_test, df_entertainment_test, df_politics_test, df_sport_test, df_tech_test], ignore_index=True, sort=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mB5S-UwcjtgV"
      },
      "source": [
        "# Save to file\n",
        "df_train.to_csv(PATH + '/train.csv', index=False)\n",
        "df_valid.to_csv(PATH + '/valid.csv', index=False)\n",
        "df_test.to_csv(PATH + '/test.csv', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rhixHzNczpJZ"
      },
      "source": [
        "# Fine-tuning a pre-trained Bert model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yxmp-RrzEn8K"
      },
      "source": [
        "Uses “bert-base-uncased” version of BERT, which is pre-trained on lower-cased English text\n",
        "\n",
        "(with 12-layer, 768-hidden, 12-heads, 110M parameters)\n",
        "\n",
        "For the purposes of fine-tuning, the authors recommend choosing from the following values (from Appendix A.3 of the BERT paper: https://arxiv.org/pdf/1810.04805.pdf):\n",
        "\n",
        "*   **Batch size:** 16, 32\n",
        "*   **Learning rate (Adam):** 5e-5, 3e-5, 2e-5\n",
        "*   **Number of epochs:** 2, 3, 4"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SgUJE0-DzoJK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "915d22ce7c4d4c3292b269eec1fe2355",
            "c263e2e71d1c4a358c592794c00e5500",
            "e5194f0d4559456faba8580ca56ff165",
            "0f9699c76dbf405bafa7837e57e79a1f",
            "f5a9010232f048d7a7cff401e0caaef1",
            "279b29895d814e98a1c9a66d373bd2fa",
            "66266322c4094cd89d5f6f8697b0fff3",
            "2b6af9d289fe4093b4c55e7b79108ae3",
            "4b7f9430615941edb0f9aa49be2aac1b",
            "bdaf276727d447bebbb2c2df2bdec6df",
            "914d35d3c1c14111af9ce4976cc8c10f",
            "b005b2af56814a8885018f0447dade37",
            "293c0a3f8b1e4705af733d26ac75a5c5",
            "52cb5c4d99ba49668e9438f7880e79b1",
            "1681527f5c184593ac93b752b247d9a6",
            "7c995cfe06214573af1a08173aeac62d",
            "e2e8188aff7d4273bea83793556591d1",
            "7feee761460d476497e8ed49357da91a",
            "a2db329e322f446e8282f6af535b781e",
            "cd9d00ca5a814f4eb91279d422ea2658",
            "7dadecf1d67e40a681b96082667d3092",
            "0f66d1050ddf494186b0cc04ebcd3465",
            "d79f5856df534d30985ce8004262c313",
            "39c06d7612354f389737ee9693ffb25d"
          ]
        },
        "outputId": "2ac49009-a26f-4c1d-b967-b51c75683505"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Model parameters\n",
        "BATCH_SIZE = 16\n",
        "MAX_SEQ_LEN = 200\n",
        "PAD_INDEX = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)\n",
        "UNK_INDEX = tokenizer.convert_tokens_to_ids(tokenizer.unk_token)\n",
        "\n",
        "# Fields\n",
        "label_field = data.Field(sequential=False, use_vocab=False, batch_first=True, dtype=torch.float)\n",
        "text_field = data.Field(use_vocab=False, tokenize=tokenizer.encode, lower=False, include_lengths=False, batch_first=True, fix_length=MAX_SEQ_LEN, pad_token=PAD_INDEX, unk_token=UNK_INDEX)\n",
        "fields = [('label', label_field), ('titlebody', text_field)]\n",
        "\n",
        "# TabularDataset\n",
        "train, valid, test = data.TabularDataset.splits(path=PATH, train='train.csv', validation='valid.csv', test='test.csv', format='CSV', fields=fields, skip_header=True)\n",
        "\n",
        "# Iterators\n",
        "train_iter = data.BucketIterator(train, batch_size=BATCH_SIZE, sort_key=lambda x: len(x.titlebody), device=device, train=True, sort=True, sort_within_batch=True)\n",
        "valid_iter = data.BucketIterator(valid, batch_size=BATCH_SIZE, sort_key=lambda x: len(x.titlebody), device=device, train=True, sort=True, sort_within_batch=True)\n",
        "test_iter = data.Iterator(test, batch_size=BATCH_SIZE, device=device, train=False, shuffle=False, sort=False)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "915d22ce7c4d4c3292b269eec1fe2355",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4b7f9430615941edb0f9aa49be2aac1b",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=28.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e2e8188aff7d4273bea83793556591d1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ixSgLA_6Svw"
      },
      "source": [
        "class BertModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(BertModel, self).__init__()\n",
        "        self.encoder = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=5)\n",
        "\n",
        "    def forward(self, text, label):\n",
        "        loss, text_fea = self.encoder(text, labels=label)[:2]\n",
        "        return loss, text_fea"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a5vg9dj7FEbe"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "loShN7ZrFF--"
      },
      "source": [
        "def save_model(path, model, valid_loss):\n",
        "    state_dict = {'model_state_dict': model.state_dict(),\n",
        "                  'valid_loss': valid_loss}\n",
        "    torch.save(state_dict, path)\n",
        "\n",
        "def load_model(path, model):\n",
        "    state_dict = torch.load(path, map_location=device)\n",
        "    model.load_state_dict(state_dict['model_state_dict'])\n",
        "    return state_dict['valid_loss']\n",
        "\n",
        "def save_metrics(path, train_loss_list, valid_loss_list, global_steps_list):\n",
        "    state_dict = {'train_loss_list': train_loss_list,\n",
        "                  'valid_loss_list': valid_loss_list,\n",
        "                  'global_steps_list': global_steps_list}\n",
        "    torch.save(state_dict, path)\n",
        "\n",
        "def load_metrics(path):\n",
        "    state_dict = torch.load(path, map_location=device)\n",
        "    return state_dict['train_loss_list'], state_dict['valid_loss_list'], state_dict['global_steps_list']"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x5O2SBIvFmLP"
      },
      "source": [
        "def train(model, optimizer, train_set=train_iter, valid_set=valid_iter,\n",
        "          num_epochs=5, eval_every = len(train_iter) // 2,\n",
        "          file_path = PATH, best_valid_loss = float(\"Inf\")):\n",
        "    \n",
        "    running_loss = 0.0\n",
        "    valid_running_loss = 0.0\n",
        "    global_step = 0\n",
        "    train_loss_list = []\n",
        "    valid_loss_list = []\n",
        "    global_steps_list = []\n",
        "\n",
        "    inference_time = 0.0\n",
        "    inference_count = 0\n",
        "\n",
        "    # Training loop\n",
        "    model.train()\n",
        "    for epoch in tqdm(range(num_epochs)):\n",
        "      # Perform one full pass over the train set\n",
        "      print('======= Epoch [{}/{}] ======='.format(epoch+1, num_epochs))\n",
        "\n",
        "      for (labels, titlebody), _ in tqdm(train_set):\n",
        "        labels = labels.type(torch.LongTensor)           \n",
        "        labels = labels.to(device)\n",
        "        titlebody = titlebody.type(torch.LongTensor)  \n",
        "        titlebody = titlebody.to(device)\n",
        "        start_time = time.time()\n",
        "        loss, _ = model(titlebody, labels)\n",
        "        torch.cuda.synchronize()\n",
        "        time_taken = time.time() - start_time\n",
        "        inference_time += time_taken\n",
        "        inference_count += 1\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update running values\n",
        "        running_loss += loss.item()\n",
        "        global_step += 1\n",
        "\n",
        "        # Evaluation on valid set\n",
        "        if global_step % eval_every == 0:\n",
        "          model.eval()\n",
        "          with torch.no_grad():\n",
        "            # Validation loop\n",
        "            for (labels, titlebody), _ in valid_set:\n",
        "                labels = labels.type(torch.LongTensor)\n",
        "                labels = labels.to(device)\n",
        "                titlebody = titlebody.type(torch.LongTensor)  \n",
        "                titlebody = titlebody.to(device)\n",
        "                loss, _ = model(titlebody, labels)\n",
        "                \n",
        "                valid_running_loss += loss.item()\n",
        "\n",
        "          # Compute metrics\n",
        "          average_train_loss = running_loss / eval_every\n",
        "          average_valid_loss = valid_running_loss / len(valid_set)\n",
        "          train_loss_list.append(average_train_loss)\n",
        "          valid_loss_list.append(average_valid_loss)\n",
        "          global_steps_list.append(global_step)\n",
        "\n",
        "          # Reset running values\n",
        "          running_loss = 0.0                \n",
        "          valid_running_loss = 0.0\n",
        "          model.train()\n",
        "\n",
        "          # Log progress\n",
        "          print('Epoch [{}/{}], Step [{}/{}], Train Loss: {:.4f}, Valid Loss: {:.4f}'\n",
        "                .format(epoch+1, num_epochs, global_step, num_epochs*len(train_set),\n",
        "                        average_train_loss, average_valid_loss))\n",
        "          \n",
        "          # Save Model\n",
        "          if best_valid_loss > average_valid_loss:\n",
        "              best_valid_loss = average_valid_loss\n",
        "              save_model(file_path + '/model.pt', model, best_valid_loss)\n",
        "              save_metrics(file_path + '/metrics.pt', train_loss_list, valid_loss_list, global_steps_list)\n",
        "    \n",
        "    save_metrics(file_path + '/metrics.pt', train_loss_list, valid_loss_list, global_steps_list)\n",
        "    inference_time = inference_time / inference_count\n",
        "    print(\"Inference Time: %.4f s\" % inference_time)\n",
        "    print('Training complete!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 605,
          "referenced_widgets": [
            "63b05f1d3e6d4ca985ecbba3d01b8785",
            "1e65ef1565dc409fbae4578dd699198e",
            "c59f128efe5f44aaaeed777a5bb780a3",
            "1a5be1af8e2745948d9c84960a2b0039",
            "faad4ef4e1b840519f97f92244300caf",
            "e392a529666a4c52a345062ca6551f0c",
            "1db3751d31f9482fbce9913998eb5771",
            "47273906bb844f6e86b8ba8f7d4ca425",
            "ef58eddd1a3644dbabcd36ca4bf122ed",
            "dd749f8ac4c34539b658ce538ab76df1",
            "63425a82d4f64dedb825fc28cbec5cb0",
            "7a1a2139365e4018bc5d6a6a6cf0ce8a",
            "7e3031b48f6e49fd9a3d1db216038595",
            "4e57fb649f9a47c19927a1f34d168b61",
            "0c1112023a244b69b5d13737a98e1bf2",
            "0bc50285861a46019b20171d7f040f75",
            "54ce4b435410465892575b56b8aaf361",
            "5c28e2c2e21b4355813ad67ae070e951",
            "6d5986f80c9b471eafeb336f7c98247e",
            "19ab9db25661411abb86c19f43000679",
            "6f83485d6e0441fead9ff3e71619cd1a",
            "68f0410899dc434ab803827c357e5743",
            "04ff7e2e815a4e3fa369d20e5833f29b",
            "38347f446ccc494d86793c11b172c833",
            "653182eaa70a4879b13a43ddbce42053",
            "194b7065cd424be084e55eaa359cbaeb",
            "fb7f1b0d4e4a43b4a11061e42caf1798",
            "928ba74a822b4d279a221f6496fe48e7",
            "1f15894f2b804501bac9cb2d1b203278",
            "668a8b33d7234a25b2964e06410b28e6",
            "d33579127be941019f257d711c472029",
            "1d0641135f1c4a1da93bbc2f2e83cb00",
            "eff1969225b744aca48e5cc28ac4f164",
            "bc2405ba4b344e10abf09aa795268ba0",
            "a4127f7eff2d4af38517ff2e7ae35aae",
            "74ef4eee0d494e20abdbb60e1732570d",
            "742b858658314552a3f7b0cd0aaee241",
            "76786725f3434541bddc7f172c4cad03",
            "0045957db07c44129321bfbc6ce477ee",
            "de8bb269ad534ab5947fdf6bf1c1f87b"
          ]
        },
        "id": "Wxl9-a3zHPGT",
        "outputId": "5f0c2094-0139-4d9e-ce31-02439c5d2ae4"
      },
      "source": [
        "model = BertModel().to(device)\n",
        "optimizer = optim.Adam(model.parameters(), lr=2e-5)\n",
        "train(model=model, optimizer=optimizer, num_epochs=4)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "63b05f1d3e6d4ca985ecbba3d01b8785",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "======= Epoch [1/4] =======\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ef58eddd1a3644dbabcd36ca4bf122ed",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=112.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [1/4], Step [56/448], Train Loss: 1.1124, Valid Loss: 0.5490\n",
            "Epoch [1/4], Step [112/448], Train Loss: 0.2879, Valid Loss: 0.1886\n",
            "\n",
            "======= Epoch [2/4] =======\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "54ce4b435410465892575b56b8aaf361",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=112.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [2/4], Step [168/448], Train Loss: 0.1427, Valid Loss: 0.1668\n",
            "Epoch [2/4], Step [224/448], Train Loss: 0.0859, Valid Loss: 0.1421\n",
            "\n",
            "======= Epoch [3/4] =======\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "653182eaa70a4879b13a43ddbce42053",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=112.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [3/4], Step [280/448], Train Loss: 0.0537, Valid Loss: 0.1546\n",
            "Epoch [3/4], Step [336/448], Train Loss: 0.0338, Valid Loss: 0.1569\n",
            "\n",
            "======= Epoch [4/4] =======\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eff1969225b744aca48e5cc28ac4f164",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=112.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch [4/4], Step [392/448], Train Loss: 0.0374, Valid Loss: 0.1938\n",
            "Epoch [4/4], Step [448/448], Train Loss: 0.0394, Valid Loss: 0.1552\n",
            "\n",
            "\n",
            "Inference Time: 0.1959 s\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c7qa46bAJJ46"
      },
      "source": [
        "### Plot Training and Validation Loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pvhqyZ5JLLU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "outputId": "3e5cbc00-22be-4e98-d1b9-feaa12960927"
      },
      "source": [
        "train_loss_list, valid_loss_list, global_steps_list = load_metrics(PATH + '/metrics.pt')\n",
        "plt.plot(global_steps_list, train_loss_list, label='Train')\n",
        "plt.plot(global_steps_list, valid_loss_list, label='Valid')\n",
        "plt.xlabel('Global Steps')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "plt.show() "
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9b3v/9cnOxMZybDDFCQgEESLDBEHPJLUnuNYqPfYgXt6br3trdXa29rJo57W2vbn7572aOu1tae1tz3e9pxK7Tmts7WtBaWOBEUEZBKCBFAyAAmBzN/7x1qBnZA5e2XvZL+fj8d+7DWvT5ayPvv7Xd/1/ZpzDhERSVxJsQ5ARERiS4lARCTBKRGIiCQ4JQIRkQSnRCAikuCSYx3AUBUWFrqSkpJYhyEiMqZs2LCh1jkX7m3dmEsEJSUlVFZWxjoMEZExxcz29rVOVUMiIglOiUBEJMEpEYiIJLgx94xARGSo2traqK6uprm5OdahBC49PZ3i4mJSUlIGvY8SgYiMe9XV1WRnZ1NSUoKZxTqcwDjnqKuro7q6mpkzZw56P1UNici419zcTEFBwbhOAgBmRkFBwZBLPkoEIpIQxnsS6DKcvzNhEsGGvYf5zu+3xToMEZG4kzCJYMuBo/zL2rfZU9sU61BEJIHU1dWxcOFCFi5cyOTJk5k2bdrJ+dbW1n73rays5POf/3zgMSbMw+LyuUXAFtZsO8TMiwf/EEVEZCQKCgrYuHEjAHfeeSdZWVl85StfObm+vb2d5OTeb8VlZWWUlZUFHmPClAjOKMhgVjiTtTtqYh2KiCS46667jhtuuIHzzz+fW265hVdffZULL7yQRYsWcdFFF7F9+3YA1q5dy9VXXw14SeSTn/wk5eXlzJo1i/vuuy9q8SRMiQCgorSIX768l+Ot7WSkJtSfLiK+bz6+ha0HGqJ6zPlTc/jGB88e0j7V1dW8+OKLhEIhGhoaWLduHcnJyfzpT3/i9ttv5z//8z9P22fbtm2sWbOGxsZGSktLufHGG4f0vkBfEupuWFFaxM/+soeX3q7j0rMmxTocEUlgH/7whwmFQgAcPXqUT3ziE+zcuRMzo62trdd9rrrqKtLS0khLS6OoqIj33nuP4uLiEceSUIngvJl5ZKSGWLu9RolAJEEN9Zd7UDIzM09Of/3rX6eiooLf/e53VFVVUV5e3us+aWlpJ6dDoRDt7e1RiSVhnhEApCWHuOjMQtZsP4RzLtbhiIgAXolg2rRpADz44IOjfv6ESgQAFfPCVB8+wds1x2IdiogIALfccgu33XYbixYtitqv/KGwsfbLuKyszI1kYJr9R06w7J/+zNeuOov/8VezohiZiMSrt956i7POOivWYYya3v5eM9vgnOu1LWrClQimTZzA3ElZrNl+KNahiIjEhYRLBOC1Hnp1Tz3HWka/CCYiEm8SMhEsLw3T1uF4cVdtrEMREYm5hEwEZTPyyUpLZs12vWUsIpKQiSA1OYmLZxeyVs1IRUQSMxEAlJeGOXi0mR3vqRmpiCS2BE4ERQBqPSQigauoqOCZZ57ptuzee+/lxhtv7HX78vJyuprJX3nllRw5cuS0be68807uvvvuqMSXsIlgcm46Z03JYa0SgYgEbNWqVaxevbrbstWrV7Nq1aoB933qqaeYOHFiUKEBCZwIwKseqqw6TENz7x08iYhEw7XXXsuTTz55ciCaqqoqDhw4wEMPPURZWRlnn3023/jGN3rdt6SkhNpar4XjXXfdxdy5c7n44otPdlUdDYF1OmdmPweuBg45587pZb0B/xu4EjgOXOecey2oeHpTUVrEv6x9mxd21nLF+6aM5qlFJFaevhXefTO6x5z8Prjin/pcnZ+fz9KlS3n66adZuXIlq1ev5iMf+Qi33347+fn5dHR0cOmll7Jp0yYWLFjQ6zE2bNjA6tWr2bhxI+3t7SxevJglS5ZEJfwgSwQPApf3s/4KYI7/uR74lwBj6dXiMyaSnZ7MWjUjFZGARVYPdVULPfzwwyxevJhFixaxZcsWtm7d2uf+69at45prriEjI4OcnBxWrFgRtdgCKxE45543s5J+NlkJ/MJ57TdfNrOJZjbFOXcwqJh6Sg4lccmc8MneSL1CioiMa/38cg/SypUr+eIXv8hrr73G8ePHyc/P5+6772b9+vXk5eVx3XXX0dzcHJPYYvmMYBqwL2K+2l92GjO73swqzayypia6v97LS8Mcamxh68HojlgkIhIpKyuLiooKPvnJT7Jq1SoaGhrIzMwkNzeX9957j6effrrf/S+55BIeeeQRTpw4QWNjI48//njUYhsTA9M45x4AHgCv99FoHnt5aRiAtdtrOHtqbjQPLSLSzapVq7jmmmtYvXo18+bNY9GiRcybN4/p06ezbNmyfvddvHgxH/3oRzn33HMpKirivPPOi1pcgXZD7VcNPdHHw+KfAGudcw/589uB8oGqhkbaDXVvrv7BOiakhPjNDRdF9bgiEh/UDXX8dkP9GPDfzHMBcHQ0nw9EqigtYsPewxw9rmakIpJ4AksEZvYQ8BJQambVZvYpM7vBzG7wN3kK2A3sAn4KfDaoWAZSXlpEp4N1u9R6SEQST5Cthvp9Zc5vLXRTUOcfioXTJzIxI4U122q4esHUWIcjIgFIlJaBw6nuT+g3i7uEkoxL5oR5bschOjvVG6nIeJOenk5dXd24723YOUddXR3p6elD2m9MtBoaDRXzwjz2xgG2HGjgfcVqPSQynhQXF1NdXU20m5/Ho/T0dIqLi4e0jxKB75I5Ycy83kiVCETGl5SUFGbOnBnrMOKWqoZ8BVlpLCieqG6pRSThKBFEqCgNs3HfEeqbWmMdiojIqFEiiFBeWoRzsG7n+K9HFBHpokQQYcG0XAoyU1mzTdVDIpI4lAgiJCUZy+eGeX5nLR1qRioiCUKJoIflpWHqm1rZVH36GKEiIuOREkEPl8wJk2SwRoPViEiCUCLoIS8zlUVn5PGcmpGKSIJQIuhF+dwwb1QfpaaxJdahiIgETomgFxXzigB4foeqh0Rk/FMi6MX8KTmEs9NYq0QgIglAiaAXJ5uR7qihvaMz1uGIiARKiaAPFaVFHD3RxsZ9akYqIuObEkEfLp5TSCjJWKtmpCIyzikR9CF3QgpLzshTb6QiMu4pEfSjfF6YLQcaONTQHOtQREQCo0TQj4pSrxmpWg+JyHimRNCPeZOzmZyTzlpVD4nIOKZE0A8zo7w0zLodtbSpGamIjFNKBAMoLy2isaWd1/YejnUoIiKBUCIYwLLZBSQnmXojFZFxS4lgANnpKZxXkq/nBCIybikRDELFvDDb3m3k4NETsQ5FRCTqlAgGobyrGamqh0RkHAo0EZjZ5Wa23cx2mdmtvaw/w8zWmNnrZrbJzK4MMp7hmlOUxbSJEzSovYiMS4ElAjMLAfcDVwDzgVVmNr/HZl8DHnbOLQI+BvwoqHhGoqsZ6Qu7amltVzNSERlfgiwRLAV2Oed2O+dagdXAyh7bOCDHn84FDgQYz4iUlxbR1NpBZVV9rEMREYmqIBPBNGBfxHy1vyzSncDHzawaeAr4n70dyMyuN7NKM6usqYlNPf1FZxaQGkpSJ3QiMu7E+mHxKuBB51wxcCXwSzM7LSbn3APOuTLnXFk4HB71IAEy05I5f1a+HhiLyLgTZCLYD0yPmC/2l0X6FPAwgHPuJSAdKAwwphFZPjfMzkPH2Fd/PNahiIhETZCJYD0wx8xmmlkq3sPgx3ps8w5wKYCZnYWXCOL2J3fXoPbqjVRExpPAEoFzrh34HPAM8BZe66AtZvYtM1vhb/Zl4NNm9gbwEHCdc84FFdNIzSrM5Iz8DJ7TcwIRGUeSgzy4c+4pvIfAkcvuiJjeCiwLMoZo6mpG+pvKaprbOkhPCcU6JBGREYv1w+Ixp6K0iBNtHby6R81IRWR8UCIYogtmFZCWnKTWQyIybigRDNGE1BAXzCpQb6QiMm4oEQxDRWmY3bVN7K1rinUoIiIjpkQwDOqNVETGEyWCYSgpzGRmYaa6mxCRcUGJYJjKS8O89HYdzW0dsQ5FRGRElAiGqaK0iJb2Tl7aXRfrUERERkSJYJiWzsxnQkqItRqsRkTGOCWCYUpPCXHRmQWs2V5DHPeKISIyICWCESifV8Q79cfZU6tmpCIydikRjED5XG9shDVqRioiY5gSwQhMz89gdlGW3jIWkTFNiWCEKkrDvLK7nuOt7bEORURkWJQIRqi8tIjWjk5e3KVmpCIyNikRjFBZSR6ZqSG9ZSwiY5YSwQilJYdYNruQtWpGKiJjlBJBFJSXFrH/yAl2HToW61BERIZMiSAKyku7mpGqekhExh4lgiiYOnEC8yZnq1tqERmTlAiiZHlpmPVV9TQ2t8U6FBGRIVEiiJKK0iLaOhwvqBmpiIwxSgRRsmRGHtlpyTy3Q88JRGRsUSKIkpRQEhfPKWTNNjUjFZGxRYkgiipKi3i3oZlt7zbGOhQRkUFTIoii5X4zUrUeEpGxRIkgiiblpDN/So7eJxCRMUWJIMoq5oXZsPcwR0+oGamIjA2DSgRmlmlmSf70XDNbYWYpg9jvcjPbbma7zOzWPrb5iJltNbMtZvaroYUffypKi+jodLywqzbWoYiIDMpgSwTPA+lmNg34A/D3wIP97WBmIeB+4ApgPrDKzOb32GYOcBuwzDl3NnDzkKKPQwunTyQnPZk1GtReRMaIwSYCc84dB/4L8CPn3IeBswfYZymwyzm32znXCqwGVvbY5tPA/c65wwDOuTF/90wOJXHJ3DBrd9TQ2almpCIS/wadCMzsQuDvgCf9ZaEB9pkG7IuYr/aXRZoLzDWzF8zsZTO7vI+TX29mlWZWWVMT/y1yKkqLqGlsYevBhliHIiIyoMEmgpvxqnB+55zbYmazgDVROH8yMAcoB1YBPzWziT03cs494Jwrc86VhcPhKJw2WJfM7WpGOuYLOCKSAAaVCJxzzznnVjjnvuM/NK51zn1+gN32A9Mj5ov9ZZGqgcecc23OuT3ADrzEMKaFs9NYUJzLGr1PICJjwGBbDf3KzHLMLBPYDGw1s68OsNt6YI6ZzTSzVOBjwGM9tnkErzSAmRXiVRXtHkL8cau8tIjX3znMkeOtsQ5FRKRfg60amu+cawA+BDwNzMRrOdQn51w78DngGeAt4GG/WulbZrbC3+wZoM7MtuJVNX3VOTcuuu8sLw3T6eD5nWpGKiLxLXmQ26X47w18CPihc67NzAZsEuOcewp4qseyOyKmHfAl/zOunFs8kbyMFNZuO8SKc6fGOhwRkT4NtkTwE6AKyASeN7MZgJrE9COUZCyfG+Y5NSMVkTg32IfF9znnpjnnrnSevUBFwLGNeeWlRdQ1tfLm/qOxDkVEpE+DfVica2bf62rLb2b34JUOxo62Ztj5x1E95SVzw5hpUHsRiW+DrRr6OdAIfMT/NAD/GlRQgXj+u/Crj8D+DaN2yvzMVBZOn6huqUUkrg02EZzpnPuG313EbufcN4FZQQYWdRd9HrImwaOfg/aWUTtt+dwi3qg+Qt2x0TuniMhQDDYRnDCzi7tmzGwZcCKYkAIyYSJcfS8c2grr7hm101bMC+McPL9TpQIRiU+DTQQ3APebWZWZVQE/BD4TWFRBKb0cFnzUSwQHN43KKc+ZmkthVqqqh0Qkbg221dAbzrlzgQXAAufcIuD9gUYWlMv/CSbkw6M3QUfwg8ckJRmX+M1IO9SMVETi0JBGKHPONfhvGMNYfQksIx+uugfe3QQv3Dsqp6woLeLI8TY27jsyKucTERmKkQxVaVGLYrTNXwHzPwTPfRcOvRX46S6ZEybJ4Dk1IxWRODSSRDC26zmuvBtSs/wqovZAT5WbkcLiM/LUG6mIxKV+E4GZNZpZQy+fRmBsd6CTFYYr/9l7r+DlHwV+uop5Rby5/yiHGpsDP5eIyFD0mwicc9nOuZxePtnOucF2WBe/zvlbKL0K1twFtbsCPVV5qTdYzfM71BupiMSXkVQNjX1mcPX3IDnNqyLq7AzsVPOn5FCUnabuJkQk7iR2IgDInuw1Kd33Mrz6QGCnMTPKS8Os21FDe0dwCUdEZKiUCADOXQWz/xqe/SbU7wnsNBWlRTQ0t/O6mpGKSBxRIgCviuiD94KF4LH/GVgV0bI5hSQnGWu2qXpIROKHEkGX3GL4m29D1Tp47cFATpGTnsKSGWpGKiLxRYkg0pLrYOZy+MMdcGRfIKeomFfEWwcbePeompGKSHxQIohkBivuA9cJj38BXPTfmetqRvrcDlUPiUh8UCLoKa8EPnAnvP0sbPz3qB++dFI2U3LTWbNN1UMiEh+UCHpz3v+AMy6C398ODQejemivGWkRf9lVS5uakYpIHFAi6E1SEqz8IXS0wBNfjHoVUXlpmGMt7VRWHY7qcUVEhkOJoC8FZ8L7vw47noY3/yOqh142u5CUkLFWbxmLSBxQIujPBTdC8Xnw9FfhWPRu2llpySydma9Ry0QkLigR9CcpBCvvh9YmeOorUT10+dwitr/XyP4jY2voZxEZf5QIBhIuhfJbYeujsOWRqB22Yp7XjFTVQyISa0oEg3HRF2DKQq9U0FQXlUOeGc6iOG+CqodEJOYCTQRmdrmZbTezXWZ2az/b/a2ZOTMrCzKeYQsle1VEJ47A7/8hKofs6o30hV21tLR3ROWYIiLDEVgiMLMQcD9wBTAfWGVm83vZLhv4AvBKULFExeRz4JKvwJu/gW1PReWQFaVFHG/tYP0eNSMVkdgJskSwFNjlnNvtnGsFVgMre9nu28B3gPjvfOfiL8Gkc7x3C06M/OZ94ZkFpCYn6TmBiMRUkIlgGhDZc1u1v+wkM1sMTHfOPdnfgczsejOrNLPKmpoY1qknp3ovmjXVwDNfG/HhMlKTOX9mvkYtE5GYitnDYjNLAr4HfHmgbZ1zDzjnypxzZeFwOPjg+jN1ESz7Amz8N9j5pxEfrqK0iLdrmthXfzwKwYmIDF2QiWA/MD1ivthf1iUbOAdYa2ZVwAXAY3H7wDjS8n+AwlKvh9LmhhEdqmJeEaBmpCISO0EmgvXAHDObaWapwMeAx7pWOueOOucKnXMlzrkS4GVghXOuMsCYoiMl3WtF1HgA/njHiA41szCTGQUZGqxGRGImsETgnGsHPgc8A7wFPOyc22Jm3zKzFUGdd9RMPw8u+Cxs+FfY/dyIDlVRWsSLb9fS3KZmpCIy+gJ9RuCce8o5N9c5d6Zz7i5/2R3Oucd62bZ8TJQGIlX8I+TP8sY5bjk27MOUl4ZpbuvklT31UQxORGRw9GbxSKRmeFVER/bCn7897MNcMKuAtOQkDWovIjGhRDBSMy6CpdfDKz+BvS8N6xDpKSEuOrNAD4xFJCaUCKLh0m/AxOnw6E3QNrzeRCvmFVFVd5w9tU1RDk5EpH9KBNGQlgUrfgD1b8Oau4Z1iPK5akYqIrGhRBAts8phyXXw0v1QPfRn3mcUZDArnKlmpCIy6pQIoumvvwXZU7wqovaWIe9eUVrEy7vrONGqZqQiMnqUCKIpPRc+eB/UbIPnvjvk3ctLw7S2d/LS7toAghMR6Z0SQbTN+QAs/Dv4y/fhwMYh7bp0Zj4TUkKs2abqIREZPUoEQbjsLsgM+1VErYPeLS05xLLZhazZfgjnXIABioicokQQhAl5cPX34b3NXslgCMpLw1QfPsHbNWpGKiKjQ4kgKPOuhHOuhef/Gd7bMujdyks1qL2IjC4lgiBd8V3vAfIjn4WO9kHtUpyXwdxJWRrUXkRGjRJBkDIL4Kq74eBGePG+Qe9WXlrEK3vqaGoZXPIQERkJJYKgnX0NnLUC1v4T1Gwf1C7lpWHaOhzrdqpUICLBUyIYDVfd4/VU+uhN0Dnwy2JlM/IJZ6dx86838qO1u2ht7xyFIEUkUSkRjIasIu95QfV6eOXHA26empzEIzctY/ncMN/9/XauvG8dL71dNwqBikgiUiIYLe/7MMy9HJ79NtS9PeDm0yZO4Cd/X8bPPlFGc1sHq376Ml/69UZqGofedYWISH+UCEaLmfduQSjVG9Gsc3DVPZeeNYk/fnE5n6uYzeObDnDpPWv55ct76ejUC2ciEh1KBKMpZ6r31vHeF6DyZ4PebUJqiK9cVsrTX7iEs6fm8vVHNvNffvQCm/cfDTBYEUkUSgSjbdHH4cz3wx+/AYf3DmnX2UVZ/OrT53PvRxey/0gzK374F77x6GYamtsCClZEEoESwWgz83ooNYPHPw9D7FPIzPjQomk8++XlfPyCGfzi5b1ces9zPLpxv/onEpFhUSKIhYnTvbELdq+F134xrEPkTkjhWyvP4dGbljE5J50vrN7Ix3/2Cm/XHIturCIy7ikRxMqS/w4lfwV/+Boc3T/swywonsgjNy3j2yvPZlP1Ua64dx33/GE7zW0a3EZEBkeJIFaSkmDFfdDZDk/cPOQqokihJOPvLyzh2S8v56oFU/jBn3fx199/jjXb1HGdiAxMiSCW8mfBpXfAzj/AG6tHfLii7HS+/9GF/OrT55MaSuK/P7ieG365gQNHTkQhWBEZr5QIYm3pZ2D6BfD7f4DGd6NyyIvOLOTpL1zCVy8rZc32Q3zge8/x0+d309ahripE5HRKBLGWlAQrf+gNdv/kl0dURRQpNTmJmypm86cvLeeCWQXc9dRbfPAHf6Gyqj4qxxeR8SPQRGBml5vZdjPbZWa39rL+S2a21cw2mdmzZjYjyHjiVuEcqLgdtj0BW34b1UNPz8/gZ58o4yd/v4SGE21c++OXuOU/3qC+afBDaIrI+BZYIjCzEHA/cAUwH1hlZvN7bPY6UOacWwD8B/DdoOKJexfcBFMXw1NfhabaqB7azLjs7Mn86cvL+czyWfz2tf28/561rH71HTrVVYVIwguyRLAU2OWc2+2cawVWAysjN3DOrXHOHfdnXwaKA4wnvoWS4UM/gpZGLxkEICM1mduuOIsnP/9XzC3K5tbfvsm1P36RrQcaAjmfiIwNQSaCacC+iPlqf1lfPgU83dsKM7vezCrNrLKmZhwP1lJ0Fiy/xaseeuvxwE5TOjmbX3/mAu7+8LlU1R3ngz/8C99+YivHNCKaJJLmo14DDb2RT3KsAwAws48DZcDy3tY75x4AHgAoKysb3//Vlt0MWx+DJ74EhaWQW+wNahNlZsa1S4r5wFlFfOf32/nZX/bw5KaD3PHB+VxxzmTMLOrnFBl1nZ1wdB/U7YTanVC7w//eCcf8VnppOVA4F8LzIFx66pN7hteYIwFYUP3TmNmFwJ3Oucv8+dsAnHP/q8d2HwB+ACx3zg34BlRZWZmrrKwMIOI4cnAT/LTCe9kMIDXbG9ym65NZBFmTICvsfWdGrEtOG9YpX3vnMF/73Wa2Hmxg+dww31xxNiWFmVH8o0QC1NoEdbtO3eS7bvh1u6A94j2a9IneTb9wrtdIIzXT27ZmmzeU7LH3Tm2bPMHbpluCmAd5M72q3DHGzDY458p6XRdgIkgGdgCXAvuB9cB/dc5tidhmEd5D4sudczsHc9yESAQA722BAxu9/zGPHYKmQ973sUPesuYjve+Xnnt6oug1eYQhlNJt1/aOTn7x0l6+98cdtHZ0clP5bD6zfBbpKaFR+INFBuCcV5VTuyPiRu/f+I9G1EJbEkyc4d3Eu274XTf/jAKvw8e+nDgMNX5iiEwQkcdPSoGC2acSQ9gvTRTMHvYPsdEQk0Tgn/hK4F4gBPzcOXeXmX0LqHTOPWZmfwLeBxz0d3nHObeiv2MmTCIYSHsLNNX4icL/7pYsIpJHSx8Pgyfk91rKOBrK5982n+DJ3R1MyJvCzSsv5K9Kp4zu3yeJq73FG8WvLrIqZwfU7oLWxlPbpWZ1v9kX+NP5syAlPboxtRzzE8N2qN3ufddsg8NV4PwXNS3JKy30rGIqnOuVPGIsZokgCEoEw9B2okdy6Ct5vAdtx0/bvdMZTcm5pOdNISVn0qkSRVdpI8tPItlTYEJe/7+4RMD7dX+8rvuv+64b/pG9p26uALnTvV/b3X7dz/H+f4v1/2ttzV71U1fJoStJ1O06VbUL3vOGyOQQnuf9HRMmjlqoSgQyeC3HuiWHtqPvsmHrdqr27iFsR3lfbgthO4I11UB78+n7J6dD9mTvH2nXJ6eX6ZQJo/+3yejraPMGYOp2w9/h/do/cfjUdsnp/i/6OT1+5c+Oi1/TQ9bRBvV7eiSIbd7fH/nvJntKxIPqru95kFkY9ZCUCGTEqmqbuOOxLTy/o4azp+bw/608m0WTQqdKFsfe9epvGw9Cw0Hvu2u6vZdO79InekN3Zk+GbP87Z0rE9FSv1JGk5xOn6ez0fjG7Du+7syNi3nWf729dt307Tz9Wf+tOzrvu851tcOSdUzf8+t3dfxlnTer+y77r5p87PTFa6HR2eCWemh2nlyJaI8YSySjwWg32fA4xglKQEoFEhXOOp958l289sYVDjS2sWnoGt1xWysSM1P528ttrRySGxgNe0ohMGMfe614dAGAh78YRWaLoShKRJYy0nNhXEfTknPfLr+WY94ym9Zj3smCL/93aeGq+1d/m5LqIbVsbofV49xtwvEtKgYIzI6pzuj6zvcYMcjrnoGG//+xh+6kkUbOte8OQy/4XXPjZYZ1CiUCi6lhLO9//4w4efLGKiRNSuO3Ks/jbxdNG9u5BZ4dXHXUyYRw4VcKILGX01loqJcNPClO7V0tFljCyJw/couPkzbvx1KfbDbzh9Jt0t5v7se77usEMDmSQlu19UrMgLStiOsebT5ngJcWkkPdA0kLer+euaUvqvs6sx3zk+p7b9ljX53rrMd/PusyiMdm8Mi455zUK6UoMJRd7L54OgxKBBGLrgQa+9sibvPbOERYU51JeWsTSknwWz5hIRmpAN4LW435y6CVJRCaQjpbT980o8BPDJO+XdbcbuP+LfFg37+yIG3h2j/msXraNmE/NjL/SjIxLSgQSmM5Ox8OV+/jly3t562ADnQ6Sk4yzp+WytCSP80ryOa8kn7zMfqqPos0570FkX0ni2LveL9h+b945vc/r5i1jlBKBjIqG5jZe23uYV/fUs76qnjf2HaXVHwxn7qQszivJZ3n7yMUAAA4OSURBVOlM7zMlV62GREaTEoHERHNbB2/sO8L6qnperTrMhqp6mlq9qpfivAks9RPDeTPzmVWYqf6NRALUXyLQEx0JTHpKiPNnFXD+rALA68LirYONvFpVz/o99Ty3o4bfvr4fgMKs1JPVSEtn5nPWlBxCSUoMIqNBJQKJGeccb9c0sd5PDK/sqWf/Ee+dg+y0ZBbPyDtZlbSgOJe0ZL1TIDJcqhqSMePAkROsr/KSwvo99ew85L1kk5qcxMLiiZw3M4+lMwtYMiOPrDQVaEUGS4lAxqz6ptaTJYb1VfVsPtBAR6cjyWD+1ByWlhSwdGYeZSX5FGbFb8+PIrGmRCDjRlNLO6+9c5j1e+p5taqe1985Qku71zLpzHCm9/DZf85QnBf9AX1ExiolAhm3Wto72Lz/KK/uOeyVHKrqaWz2+raZmpvOeX5iOH9mPrOLstQySRKWEoEkjI5Ox/Z3G70mq36poabRe8s4LyOFspJ8lpbkUzo5mxkFGUydOIGUUAJ0diYJT4lAEpZzjr11x08mhfVV9eytOzXmQijJmDoxnRn5mZxRkMEZ+RnMyM/gjIIMZhRk6oG0jBt6j0ASlplRUphJSWEmHzlvOgCHGpvZU9PE3vrjvFN3nHfqj7O3/jhPv3mQw8fbuu2fn5nqJQc/SXjTmcwoyCCclUaS3nWQcUCJQBJOUXY6RdnpJ190i9TQ3HYqOfjf79Q3sWHvYR5/4wCdEQXotOSkiCSRyRn5E5hR4JUsivMm6L0HGTOUCEQi5KSncM60XM6Zdnq/+a3tnRw4csIvSTRFJIrjvLCrjhNtp3ouNYMpOeleFVNktZM/n5uRMpp/lki/lAhEBik1OelkNROEu61zzlF7rJV36r0EsbfuOPv8Kqdntx2i9lj3brFz0pNPlh56PpeYnJOu7jVkVCkRiESBmRHOTiOcncaSGfmnrW9qaWffYb+6qe44e+ubeKf+BFv2H+WZze/SHlHnlBpKojhvwskkUZw3gYLMNPKzUin0vwsyU0lPUdWTRIcSgcgoyExLZt7kHOZNzjltXXtHJwePNp98LrG3vskrTdQdZ0PVYRpb2ns5ImSmhvykkEZBZir5makUZHnTBVn+fGbayWklDumLEoFIjCWHkpien8H0/AyWze6+zjlHY0s79cdaqWtqpe5YC/VNXdOt1De1UNfUysGjzWw+cJT6plbaOnpvEp6Vlky+nzAKs7onjt6SiB52Jw4lApE4ZmbkpKeQk57iP5von3OOhuZ26pu8JFF7rNVLHMe8hFHvJ5D9R5rZVO0ljshqqUhZacndSxaZqSerpQr8kkj+CBKHc46OTkd7Z/fvjpPznd3Xd/jrnKOjs/PkfG/7n7Zvp6Ojo5MOh7dvp6OjwztWp/NG2uv0p507Nd3R6fx5Ti7rc1vnb9tJt/UdPbc9ub77cbvWd3R2P26nczg/li//zVxWLpw2pOs8GEoEIuOImZE7IYXcCSnMHELi6CppdCWObkmkqYXqw8fZVH2k38SRnZZMXmYqoSTr/YbcEXGz9pNAvEgy7+VCMyPJIMmMJDPMX57kL++5Pimpx7YR0z3X9zxuclKSf066rTczQj2O27U+qI4VlQhEElhk4pgVHnh75xwNJ9qpbWo5Wbqoa2o5WXVV39SKA0IGoaQkkpOMUMi876Sub295UlLP5f53yN/P/GWh0/ftmu9+jKRetj/1Obk+cn8zvRSIEoGIDIGZkZuRQm5GCmcOInHI2KDetkREElygicDMLjez7Wa2y8xu7WV9mpn92l//ipmVBBmPiIicLrBEYGYh4H7gCmA+sMrM5vfY7FPAYefcbOD7wHeCikdERHoXZIlgKbDLObfbOdcKrAZW9thmJfB//en/AC41jRwiIjKqgkwE04B9EfPV/rJet3HOtQNHgdO6hDSz682s0swqa2pqAgpXRCQxjYmHxc65B5xzZc65snBYTRVERKIpyESwH5geMV/sL+t1GzNLBnKBugBjEhGRHoJMBOuBOWY208xSgY8Bj/XY5jHgE/70tcCf3VgbO1NEZIwLdMxiM7sSuBcIAT93zt1lZt8CKp1zj5lZOvBLYBFQD3zMObd7gGPWAHsHGUIhUDvsPyA48RoXKLbhUmzDE6+xxWtcMPzYZjjneq1bH3OD1w+FmVX2NVhzLMVrXKDYhkuxDU+8xhavcUEwsY2Jh8UiIhIcJQIRkQQ33hPBA7EOoA/xGhcotuFSbMMTr7HFa1wQQGzj+hmBiIgMbLyXCEREZABKBCIiCW7cJAIzqzKzN81so5lV+svyzeyPZrbT/84bpVh+bmaHzGxzxLJeYzHPfX5X3JvMbHEMYrvTzPb7126j//5H17rb/Ni2m9llAcY13czWmNlWM9tiZl/wl8f8uvUTWzxct3Qze9XM3vBj+6a/fKbftfsuv6v3VH/5qHX93k9sD5rZnojrttBfPtr/FkJm9rqZPeHPx/ya9RNbsNfM+QMuj/UPUAUU9lj2XeBWf/pW4DujFMslwGJg80CxAFcCTwMGXAC8EoPY7gS+0su284E3gDRgJvA2EAoorinAYn86G9jhnz/m162f2OLhuhmQ5U+nAK/41+NhvBc0AX4M3OhPfxb4sT/9MeDXAV63vmJ7ELi2l+1H+9/Cl4BfAU/48zG/Zv3EFug1Gzclgj5EdnP9f4EPjcZJnXPP470pPZhYVgK/cJ6XgYlmNmWUY+vLSmC1c67FObcH2IXXvXgQcR10zr3mTzcCb+H1Thvz69ZPbH0ZzevmnHPH/NkU/+OA9+N17Q6nX7dR6fq9n9j6Mmr/Tc2sGLgK+D/+vBEH16y32AYQlWs2nhKBA/5gZhvM7Hp/2STn3EF/+l1gUmxC6zeWwXTXPRo+5xctf26nqtBiEptf9F6E9wsyrq5bj9ggDq6bX42wETgE/BGvBHLEeV279zz/oLp+Dyo251zXdbvLv27fN7O0nrH1Ene03QvcAnT68wXEyTXrJbYugV2z8ZQILnbOLcYbEe0mM7skcqXzylFx0VY2nmLx/QtwJrAQOAjcE6tAzCwL+E/gZudcQ+S6WF+3XmKLi+vmnOtwzi3E6+F3KTAvFnH0pmdsZnYOcBtejOcB+cA/jGZMZnY1cMg5t2E0zzsY/cQW6DUbN4nAObff/z4E/A7vH8R7XcUk//tQ7CLsM5bBdNcdKOfce/4/2E7gp5yqxhjV2MwsBe9G++/Oud/6i+PiuvUWW7xcty7OuSPAGuBCvCqC5F7OH5Ou3yNiu9yvanPOuRbgXxn967YMWGFmVXgjJ74f+N/ExzU7LTYz+7egr9m4SARmlmlm2V3TwN8Am+nezfUngEdjEyH0E8tjwH/zn/5fAByNqAoZFT3qFK/Bu3ZdsX3MbzUxE5gDvBpQDAb8DHjLOfe9iFUxv259xRYn1y1sZhP96QnAX+M9w1iD17U7nH7dRqXr9z5i2xaR2A2vHj7yugX+39Q5d5tzrtg5V4L38PfPzrm/Iw6uWR+xfTzwazacJ8zx9gFm4bXSeAPYAvyjv7wAeBbYCfwJyB+leB7Cqypow6uz+1RfseA97b8fr173TaAsBrH90j/3Jv9/rCkR2/+jH9t24IoA47oYr9pnE7DR/1wZD9etn9ji4botAF73Y9gM3BHxb+JVvAfVvwHS/OXp/vwuf/2sGMT2Z/+6bQb+jVMti0b134J/znJOtcyJ+TXrJ7ZAr5m6mBARSXDjompIRESGT4lARCTBKRGIiCQ4JQIRkQSnRCAikuCUCGTcMbNJZvYrM9vtdznykpld468r7+rRsZ/97zSzrwzxnMf6WP6P5vW8ucnvNfJ8f/nNZpYxlHOIBEWJQMYV/4WbR4DnnXOznHNL8F7MKY5BLBcCV+P1XLoA+ACn+oW5GVAikLigRCDjzfuBVufcj7sWOOf2Oud+0HND88Y6eMT/tf6ymS2IWH2uX5LYaWaf9rfPMrNnzew188a+WDlALFOAWud1C4BzrtY5d8DMPg9MBdaY2Rr/2H/jn+81M/uN369R1zgb3/XP96qZzfaXf9jMNpvX1//zw79cIkoEMv6cDbw2yG2/Cbzu/1q/HfhFxLoFeEnlQuAOM5sKNAPXOK9zwwrgHr8E0pc/ANPNbIeZ/cjMlgM45+4DDgAVzrkKMysEvgZ8wD92JV5/9F2OOufeB/wQr2dKgDuAy5xz5wIrBvn3ivRKiUDGNTO73//VvL6X1RfjdROBc+7PQIGZ5fjrHnXOnXDO1eL1QbMU73X+/9/MNuF1dzGNfro2d15f/EuA64Ea4Ndmdl0vm16AN5jNC+Z12fwJYEbE+ocivi/0p18AHvRLK6F+LoHIgJIH3kRkTNkC/G3XjHPuJv8Xd+UQj9Oz7xUH/B0QBpY459r8HiLT+z2Icx3AWmCtmb2Jd5N/sMdmhtdX/6pBxOL8497gP3i+CthgZkucc4H3Iirjk0oEMt78GUg3sxsjlvX1UHYd3s0dMyvHq8/vGgNhpXlj7hbgdf61Hq/74UN+Eqig+6/205hZqZnNiVi0ENjrTzfiDXsJ8DKwLKL+P9PM5kbs99GI75f8bc50zr3inLsDr7QR2RWxyJCoRCDjinPOmdmHgO+b2S14N8kmeh/I407g535Vz3FOdTUMXo+Za4BC4Nv+Q95/Bx73f9lXAtsGCCcL+IHfFXM7Xu+VXaPnPQD83swO+M8JrgMeslMjT30Nb2xkgDw/xhagq9Twz36SMbzeWd8YIBaRPqn3UZE45lc/lfnPKkQCoaohEZEEpxKBiEiCU4lARCTBKRGIiCQ4JQIRkQSnRCAikuCUCEREEtz/A5TxH6ERDTLbAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OHDjnTSByjk0"
      },
      "source": [
        "### Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYE-bXduykx9"
      },
      "source": [
        "def evaluate(model, test_set=test_iter):\n",
        "  y_pred = []\n",
        "  y_true = []\n",
        "  model.eval()\n",
        "  with torch.no_grad():\n",
        "    for (labels, titlebody), _ in tqdm(test_set):\n",
        "        labels = labels.type(torch.LongTensor)           \n",
        "        labels = labels.to(device)\n",
        "        titlebody = titlebody.type(torch.LongTensor)  \n",
        "        titlebody = titlebody.to(device)\n",
        "        output = model(titlebody, labels)\n",
        "        _, output = output\n",
        "        y_pred.extend(torch.argmax(output, 1).tolist())\n",
        "        y_true.extend(labels.tolist())\n",
        "\n",
        "  print('Classification Report:')\n",
        "  print(classification_report(y_true, y_pred, labels=[0,1,2,3,4], digits=4))\n",
        "\n",
        "  cm = confusion_matrix(y_true, y_pred, labels=[0,1,2,3,4])\n",
        "  ax = plt.subplot()\n",
        "  sns.heatmap(cm, annot=True, ax=ax, cmap='Blues', fmt=\"d\")\n",
        "\n",
        "  ax.set_title('Confusion Matrix')\n",
        "\n",
        "  ax.set_xlabel('Predicted Labels')\n",
        "  ax.set_ylabel('True Labels')\n",
        "\n",
        "  ax.xaxis.set_ticklabels(['business', 'entertainment', 'politics', 'sport', 'tech'])\n",
        "  ax.yaxis.set_ticklabels(['business', 'entertainment', 'politics', 'sport', 'tech'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 670,
          "referenced_widgets": [
            "1cecd84f0390422a934468e920adca79",
            "a23e67de2f854867a6db9b43000a58ff",
            "c13a461ce76e4295b364ceddc5f2e71f",
            "8971a145f7ea43dfa397e8493cb8bd1a",
            "d69cdb7d4ec74f4a890e5ab51688a1ce",
            "1f601e873e8b40fca4e7c24ffd30f80b",
            "bffbbd81e5b7487a8308fb481ff0d828",
            "6e6f6132d6ba4d9f88164bfe7532cf88"
          ]
        },
        "id": "C5CNjova0VHk",
        "outputId": "3280bbcc-d6dd-4db8-fff7-356c65196d23"
      },
      "source": [
        "bert_model = BertModel().to(device)\n",
        "load_model(PATH + '/model.pt', bert_model)\n",
        "evaluate(bert_model, test_iter)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1cecd84f0390422a934468e920adca79",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=15.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Classification Report:\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0     0.9796    0.9412    0.9600        51\n",
            "           1     0.9750    1.0000    0.9873        39\n",
            "           2     1.0000    1.0000    1.0000        42\n",
            "           3     1.0000    1.0000    1.0000        52\n",
            "           4     0.9524    0.9756    0.9639        41\n",
            "\n",
            "    accuracy                         0.9822       225\n",
            "   macro avg     0.9814    0.9834    0.9822       225\n",
            "weighted avg     0.9824    0.9822    0.9822       225\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5wU9f3H8df77mhKEVQOImfFjiWBqNhQEUWxYE8ssQZ7id1obFFjj4kGuwa7YjQiIOqPiAVRQUVAsSuIwIEooKICd5/fHzML68ltu53d2b3Pk8c+bmd25jvf783dh+995zufkZnhnHMufiqKXQHnnHMr5gHaOediygO0c87FlAdo55yLKQ/QzjkXUx6gnXMupjxAuyaT1EbS05IWSBrahHIOk/RcPutWDJKekXRksevhSp8H6GZE0qGSJkj6TtKsMJBsn4eiDwSqgVXN7KBcCzGzB81stzzU52ck7STJJD3ZYP0W4foxGZZzqaQH0m1nZnuY2ZAcq+vcMh6gmwlJZwI3AVcRBNM1gcHAvnkofi3gQzNbmoeyojIX6C1p1aR1RwIf5usACvjvlMsb/2FqBiR1AC4HTjazJ8zsezNbYmZPm9k54TatJN0kaWb4uklSq/CznSTNkHSWpDlh7/vo8LPLgIuBQ8Ke+bENe5qS1g57qlXh8lGSPpX0raTPJB2WtP6VpP22lTQ+HDoZL2nbpM/GSPqrpLFhOc9JWi3Ft2Ex8F/gd+H+lcAhwIMNvlf/kPSFpIWS3pS0Q7i+P/DnpHa+k1SPKyWNBRYB64brjgs/v1XSf5LKv0bSaEnK+AS6ZssDdPPQG2gNPJlimwuBbYAtgS2ArYCLkj7vAnQA1gCOBf4lqaOZXULQK3/UzNqa2d2pKiJpZeCfwB5m1g7YFpi4gu06ASPCbVcFbgRGNOgBHwocDXQGWgJnpzo2cB/wh/D97sAUYGaDbcYTfA86AQ8BQyW1NrNRDdq5RdI+RwCDgHbAtAblnQVsFv7nswPB9+5I8xwLLgMeoJuHVYGv0gxBHAZcbmZzzGwucBlB4ElYEn6+xMxGAt8BG+ZYn3qgh6Q2ZjbLzN5dwTYDgI/M7H4zW2pmDwPvA3snbXOvmX1oZj8AjxEE1kaZ2atAJ0kbEgTq+1awzQNmNi885g1AK9K3899m9m64z5IG5S0i+D7eCDwAnGpmM9KU5xzgAbq5mAeslhhiaMSv+Hnvb1q4blkZDQL8IqBtthUxs+8JhhZOAGZJGiFpowzqk6jTGknLs3Ooz/3AKcDOrOAvCklnS5oaDqvMJ/irIdXQCcAXqT40s9eBTwER/EfiXEY8QDcP44CfgIEptplJcLEvYU1++ed/pr4HVkpa7pL8oZk9a2b9gK4EveI7M6hPok5f5linhPuBk4CRYe92mXAI4lzgYKCjma0CLCAIrACNDUukHK6QdDJBT3xmWL5zGfEA3QyY2QKCC3n/kjRQ0kqSWkjaQ9K14WYPAxdJWj282HYxwZ/kuZgI7ChpzfAC5QWJDyRVS9o3HIv+iWCopH4FZYwENginBlZJOgTYBBieY50AMLPPgD4EY+4NtQOWEsz4qJJ0MdA+6fNaYO1sZmpI2gC4AjicYKjjXEkph2KcS/AA3UyE46lnElz4m0vwZ/kpBDMbIAgiE4BJwGTgrXBdLsd6Hng0LOtNfh5UK8J6zAS+JgiWJ66gjHnAXgQX2eYR9Dz3MrOvcqlTg7JfMbMV/XXwLDCKYOrdNOBHfj58kbgJZ56kt9IdJxxSegC4xszeMbOPCGaC3J+YIeNcKvKLyc45F0/eg3bOuZjyAO2cczHlAdo552LKA7RzzsVUqhsXiqpNv2vK7urlVyPKcwpsZYWnlSgVPy1Z0YzG0tehTdN/CNv8+pSMY84Pb99SkB9670E751xMxbYH7ZxzBRXDTLEeoJ1zDqCistg1+AUP0M45BxDDFN0eoJ1zDnyIwznnYiuPPWhJnwPfAnXAUjPrFT6E4lFgbeBz4GAz+yZVOfH7L8M554pBFZm/MrOzmW1pZr3C5fOB0Wa2PjA6XE7JA7RzzkHQg870lZt9gcTT3oeQOj874AHaOecCFZUZvyQNkjQh6TWoQWkGPBc+eDjxWbWZzQrfzwaq01XJx6Cdcw6yukhoZncAd6TYZHsz+1JSZ+B5Se832N8kpb1z0XvQzjkHeR3iMLMvw69zCJ59uRVQK6lrcCh1BeakK8cDtHPOQd4uEkpaWVK7xHtgN2AKMAw4MtzsSOCpdFXyIQ7nnIN8zoOuBp5U0NOuAh4ys1GSxgOPSTqW4JFqB6cryAO0c84BVObnVm8z+xTYYgXr5wF9synLA7RzzoHf6u2cc7Hlt3o751xMxbAHHb//MgqgokKMu/Uo/vPXAwDY6ddr8ergI3nttqMY/ffDWPdXqxS5hrm79C9/pm+fbTlov72LXZW8G/vyS+wzYHf26t+Pu+9MNQW1dJRjm2pnz+LE447kkP334pD99+KRB+8rdpUyk/9bvZusWQboU/brxQfT5y1b/udpu3H01cPZ5oR/8+j/3uP8w7YtYu2aZu999+OWW+8sdjXyrq6ujquuvJzBt93Fk8NGMGrkcD75+ONiV6tJyrFNAJWVlZx+1rk8+sRw7rn/UYY++hCfflIC7Yr+Vu+sNbsAvcZq7ei/9brc+8w7y9aZGe1XaglA+5VbMWved8WqXpP17PVbOnToUOxq5N2UyZOoqVmLbjU1tGjZkv57DmDMC6OLXa0mKcc2Aay2emc22nhTAFZeeWXWWXc95s6pLXKtMpDFrd6FUrAxaEkdgRozm1SoY67IdSf25cI7x9C2Tctl6066cRRPXnkQP/60lIWLfqLPafcXsYZuRebU1tKla5dly52rq5k8qag/Sk1Wjm1qaOaXX/LB+1PZdLNfzDqLnxheJIy0RpLGSGof5kF9C7hT0o0ptl+WgGTpjNfzXp89tl6POfO/5+2Pfv6/+akH9GK/C4fS/dDB3P/sZK45YZe8H9u55mbRou85/+zTOPOc82nbtm2xq5NeDIc4ou5BdzCzhZKOA+4zs0skNdpFSE5A0qbfNRk/Aj1TvTddg716r0//rdajVctK2q/UiieuOJANazox/v0gydTjY6by1N/S3uDjCqxzdTWzZ81etjyntpbq6rTJwGKtHNuUsHTJEs4763R233Nvdu67W7Grk5nm1oMGqsKkIAcDwyM+VloX3/MS3Q8dzEZH3MYfrhzGmInTOOji/9B+5VZ0X6MjALv0XOdnFxBdPGzaYzOmT/+cGTO+YMnixYwaOYI+O5f2Xzrl2CYIrun89bKLWGeddTnsiKOKXZ3MxXAWR9Q96MuBZ4FXzGy8pHWBjyI+Zlbq6o2T/z6Khy/Zj/p6Y/53P3L89SOLXa2cXXDumbw5fjzz539D/759OOHkUxm4/4HFrlaTVVVVccGFF3PioOOor69j4H4H0L37+sWuVpOUY5sA3pn4Fs8MH0b39TfgsIP3A+CkU89gux36FLlmacTwqd4yy/tIQl5EMcRRbF+NOLfYVYhEZUX8Jvi7FftpSX2xqxCJDm2a/kPYZuAdGcecH/47qCA/9FFfJLw2vEjYQtJoSXMlHR7lMZ1zLicxHOKI+ki7mdlCYC+Cp9h2B86J+JjOOZe9ZjiLI1H+AGComS1QDO93d865OMamqAP08PBZXD8AJ0paHfgx4mM651zWml2ANrPzJV0LLDCzOkmLCB497pxzsaIYXuyO+iLhSsBJwK3hql8BvaI8pnPO5UJSxq9Cifoi4b3AYiCRHu5L4IqIj+mcc1lrjgF6PTO7FlgCYGaLgPj9HeGca/biGKCjvki4WFIbwAAkrQf8FPExnXMuezHsOkYdoC8BRgE1kh4EtgOOiviYzjmXteY4i+N5SW8B2xD8/3S6mX0V5TGdcy4XFRXxy2ZXiIT9rYFvwmNtIgkze6kAx3XOuYw1ux60pGuAQ4B3gUSWFgM8QDvn4iV+8TnyHvRAYEMz8wuDzrlYa3Y9aOBToAU+c8M5F3PNMUAvAiZKGk1SkDaz0yI+rnPOZSWOt3pHHaCHha+sffPMeXmuSvFtfM6IYlchElOvG1DsKrgMtWoRv5kKcdHsetBmNiTK8p1zLl+aTYCW9JiZHSxpMuFdhImPADOzzaM4rnPO5arZBGjg9PDrXhGV75xzeRXHAB3JgJSZzQrffgV8YWbTgFbAFsDMKI7pnHNNoixemRQnVUp6W9LwcHkdSa9L+ljSo5Japisj6isGLwGtJa0BPAccAfw74mM651zWKioqMn5l6HRgatLyNcDfzaw7wd3Vx6atU9atyI7CFKP7A4PN7CBg04iP6ZxzWctnulFJ3QiexXpXuCxgF+DxcJMhBDfypRR5gJbUGzgMSMwxq4z4mM45l738DnHcBJzL8hQXqwLzzWxpuDwDWCNdIVEH6DOAC4AnzexdSesCL0R8TOecy1o2PWhJgyRNSHoNSipnL2COmb3Z1DpFPQ/6ReDFpOVPAb+L0DkXO9nM4jCzO4A7Gvl4O2AfSXsSZPNsD/wDWEVSVdiL7kbwCMCUos5m9wI/nwcNgJntEuVxnXMuW/maZmdmFxCMHCBpJ+BsMztM0lDgQOAR4EjgqXRlRX2r99lJ71sDBwBLG9nWOeeKpgC5OM4DHpF0BfA2cHe6HaIe4mg4BjNW0htRHtM553IRxY0qZjYGGBO+/xTYKpv9ox7i6JS0WAH0AjpEeUznnMtFHO8kjHqI402Wj0EvBT4ng8nZzjlXaDGMz5EH6E2Ak4DtCQL1y8CEiI/pnHNZa4496CHAQuCf4fKhwP3AQREf1znnslLRDBP29zCzTZKWX5D0XsTHdM65rMWwAx35nYRvSdomsSBpa2I0xDH25ZfYZ8Du7NW/H3ff2dic8/hrWVXBf8/YjpFn78Cz5+3IGf3XB6B391V5+qztGXXujlx/6BZUxrCHkI1yOV/JyrFNUJrtqqhQxq9CiSphfyJRfwvgVUnTw+W1gPejOGa26urquOrKy7n9znuprq7m0EMOZKedd2G97t2LXbWsLV5az6GDX2PR4jqqKsTQ03rz0vtfcf2hW3D4ra/z2dzv+VP/DTjgt9147PUvil3dnJTT+UooxzZB6barOfWg9wL2BvoD6wB9gJ3C93tEdMysTJk8iZqatehWU0OLli3pv+cAxrwwutjVytmixXUAVFWKqsoK6uuNJXX1fDb3ewBe+XAu/TfvUswqNkm5nS8ozzZB6bYrn9ns8iWqhP3TwiT9JyXeJ6+L4pjZmlNbS5euywNW5+pqamtri1ijpqkQjDh7eyb8tR+vfPAVE6fPp6pCbFYTTDvfY4uudF2ldZFrmbtyO19Qnm2C0m2XlPmrUKK+SNiP4PbGZHusYJ1ronqDAde/QrvWVdx+TC826NKWU+97m78M3ISWlRW8/MFc6u0XaVGcc6EsEvEXTCQ1knRiOA69oaRJSa/PgEkp9luWwi/qCwudq6uZPWv2suU5tbVUV1dHesxC+PbHpYz7+Cv6bNSZt6fN5+CbxzHwprG88enXy4Y7SlE5nq9ybBOUbrvi2IOO6r+MhwjGoIeFXxOvnmZ2eGM7mdkdZtbLzHod+8dBjW2WF5v22Izp0z9nxowvWLJ4MaNGjqDPzqWZZK/Tyi1p1zr4Y6hViwp22HB1PpnzHau2DR551rKyguN3WY8Hx04vZjWbpJzOV0I5tglKt11xHIOOZIjDzBYAC4DfS6oEqsNjtZXU1syKHimqqqq44MKLOXHQcdTX1zFwvwPo3n39YlcrJ53bt1o2jU4SIybO5H/vzeGCvTdil02rqRA8MHYa4z6eV+yq5qyczldCObYJSrddcZzFIYtwXFLSKcClQC3LH/1iZrZ5un1/XPrLPNKlbuNzRqTfqARNvW5AsavgmrnWVZk+a7txPf/6QsYx582/7FyQcB71RcIzgA3NrHS7bs65ZiGOPeioA/QXBEMdzjkXa80xF8enwBhJI4CfEivN7MaIj+ucc1lpjtnspoevluHLOediKYbxOfJHXl0GIGklM1sU5bGcc64p4tiDjvTWGUm9w/Si74fLW0gaHOUxnXMuF83pRpWEm4DdgXkAZvYOsGPEx3TOuaw1m3SjyczsiwZ/OtRFfUznnMtWHIc4Ip9mJ2lbwCS1AE4HpkZ8TOecy1ocA3TUQxwnACcDawBfAluGy845FytxHIOOehbHV8BhUR7DOefyIY496EgDtKR1gFOBtZOPZWb7RHlc55zLVgzjc+Rj0P8F7gaeZnmyJOeci52SvNVb0unAvcC3wF3Ar4Hzzey5DMr/0cz+2bQqOudc9Cpi2IXO5CLhMWa2ENgN6AgcAVydYfn/kHRJeMPKbxKvXCvrnHNRKdWLhInq7Ancb2bvKvPR9M0IAvouJOWDDpedcy42SvUi4ZuSngPWAS6Q1I7Mx5MPAtY1s8W5VtA55wohhkPQGQXoYwnmL39qZoskrQocnWH5U4BVgDk51q+slOuTR9Y8/rFiVyHvpt9+cLGr4AqspC4SrmCseN0c/gRYBXhf0nh+ng/ap9k552JFTX9qVt6l6kHfkOKzTMeRL8muOs45Vxz56kBLag28BLQiiLGPm9kl4X0hjwCrAm8CR6Qb/m00QJvZzk2tqJm92NQynHOuEPJ4kfAnYBcz+y7MQfSKpGeAM4G/m9kjkm4jGD6+NVVBaafZSVpJ0kWS7giX15e0Vya1lLS/pI8kLZC0UNK3khZmsq9zzhVSvqbZWeC7cLFF+EqMOjwerh8CDExXp0zmQd8LLAa2DZe/BK7IYD+Aa4F9zKyDmbU3s3Zm1j7DfZ1zrmAqpIxfkgZJmpD0GpRclqRKSRMJJkg8D3wCzDezpeEmMwiSyKWUySyO9czsEEm/BwhncmT6t0CtmXl6Uedc7GUzi8PM7gDuSPF5HbClpFWAJ4GNcqlTJgF6saQ2BF10JK1H0oyMNCZIepQgJ0fyLI4nsq2oc85FKYr7VMxsvqQXgN7AKpKqwl50N4LRiJQyCdCXAKOAGkkPAtsBR2VYv/bAIoLbxJfVGfAA7ZyLlXzl4pC0OrAkDM5tgH7ANcALwIEEMzmOBJ5KV1baAG1mz0t6C9iG4Lbv08M8z2mZWaY3tDjnXFHlsQPdFRgiqZLgOt9jZjY8fID2I5KuAN4myPSZUqbpRvsA2xP0flsQjKk0StK5ZnatpJvDfX7GzE7L8LjOOVcQ+ZpmZ2aTCLJ+Nlz/KbBVNmVlkm50MNAdeDhcdbykXc0s1aOrEhcGJ2RTGeecK5YY3umdUQ96F2BjM0tcJBwCvJtqBzN7Ovw6pMk1dM65AiipXBxJPgbWBKaFyzXhurTCwfLzgE2A1on1ZubpRp1zsVJS6UYlPU0wftwOmCrpjXB5a+CNDMt/EHgUGEDwhO8jgblNqbBzzkUhhh3olD3o6/NQ/qpmdrek08O8HC+Gme2ccy5WSqoHnadER0vCr7MkDQBmAp3yUK5zzuVV/MJzZrM4tgFuBjYGWgKVwPcZ5tS4QlIH4KywjPbAGblX1znnolEZwzGOTJIl3QL8HvgIaAMcB/wrw/K/MbMFZjbFzHY2s57A17lVNf/GvvwS+wzYnb369+PuOxu9rb7klFO7KiRGX9KPB07bHoBb/7g1r17Znxcv352bjv4tVZXx+6XKRjmdq2Sl2C4FSZAyehVKJgEaM/sYqDSzOjO7F+ifYfk3Z7iu4Orq6rjqyssZfNtdPDlsBKNGDueTjzOanBJr5dauQf3W58OZyzPUPv7aNLa9cBR9Ln6W1i0qOXyHdYtYu6Ypt3OVUKrtiuNTvTMJ0IsktQQmSrpW0p/S7Sept6SzgNUlnZn0upRgiKTopkyeRE3NWnSrqaFFy5b033MAY14YXexqNVk5tatrxzbsunlXHnz5s2XrRk+evez92599TdeObYpRtbwop3OVrFTblU260YLVKYNtjgi3OwX4nmAe9P5p9mkJtAU6AqsBHcL9zwbuy7Wy+TSntpYuXbssW+5cXU1tbW0Ra5Qf5dSuK363JZcPnUS9/SJbAFWV4qDea/G/KbNXsGdpKKdzlaxU21WSPWgzm2ZmP5rZQjO7zMzOBK5Ks8+LBEn9l5rZn4GJBJmc1gD+0Nh+yUmwS2XcykWj3+Zd+erbn5g07ZsVfn7N4T0Z9+FcXv8oo7xdzqUVxzHoTJMlNdQ73QZmVhcOjUBwo8pQM1uQqnHJSbB/XPrLJEv51Lm6mtmzlve+5tTWUl1dHeUhC6Jc2rVV99XYfYtf0XezrrRuUUHb1i0YfNzWnHTX65y9zyas1q4VR91X2qleyuVcNVSq7aqM4TzojC4SNsH7kr4FdgXaSTqKYCZI0W3aYzOmT/+cGTO+YMnixYwaOYI+O5f+Hejl0q4rn5jMlucMp9d5Ixh0+2u88v4cTrrrdQ7bYR123rQLx9/+GisY+Sgp5XKuGirVdlUo81ehpLrV+zeNfUSQcjQT7xBktlsC7Bke781sKhiVqqoqLrjwYk4cdBz19XUM3O8Aundfv9jVarJybVfCdUf0ZMa8RYz8c/ALP+KtL7nh6feKXKvclOu5KtV2xXAaNLJGuiHhY1oaZWY7py1cOhl40Mzmh8sdgd+b2eB0+0Y9xOHyZ83jHyt2FfJu+u0HF7sKLgutq5p+I+BZT3+Qccy5Ye8NCxLOU93qnTYAZ+AUYH9J1WbWg2AGyEVA2gDtnHOFFMcedNRj0DXABSzPyfEunovDORdDcZxml+ssjkx9SzD3ua2kvsDxwMLUuzjnXOFVNcNZHBOByQQ3q5wIfENMLhI651yyOPag0wZoBQ6XdHG4vKakTB98eDLBA2dbEcyd3pQgcb9zzsVKqd7qPZgguP4+XP6WNNnsJCUu648EOgOfEPSe2wOjcqqpc85FKI496EzGoLc2s99IehvAzL5JukOwMaeHX5cCezf4bHiWdXTOucjFcRZHJgF6iaRKgucRJh4EW59mnw6StiXIvdEzaX17oGsuFXXOuSjFMWF/JgH6n8CTQGdJVwIHEsxlTmVDYC+CB84m96C/BebnUE/nnItUDONz+gBtZg9KehPoS3Cb90Azm5pmt18BvQjmPyf3oNsBY3Osq3PORUYxfCphJs8kXBNYBDydvM7MpqfY7SHgGeBG4GOgG0Gi/iXA4qZU2DnnolCSPWhgBMH4s4DWwDrABwRT5hpjZva5pG7AF8DzpB+3ds65oinJAG1mmyUvh1nuTkqz20MEY9BbENyksk9ykcCQ7KrpnHPRKmQi/kxlfau3mb0laes02+wVvr0OeNXMRuZSOeecK5TKqO+rzkEmY9BnJi1WAL8BZqbZJ5FL+k/ABZKWEMyJFsHwR9vcquucc9Eo5B2CmcqkB90u6f1SgjHp/6TZ54bw6xtANcHY9TSCW77T3eTinHMFV3Jj0OENKu3M7OxsCk3kkpZ0K/AesIuZ7Rwm7H8u18q6eCrH5PYdf3tKsasQiW/G31LsKsRWDDvQjefikFRlZnXAdk0of2uCGR9dJD0OHIb3oJ1zMVSBMn6lIqlG0guS3pP0rqTTw/WdJD0v6aPwa8f0dWrcG+HXiZKGSTpC0v6JV4Zt7kZww8pcgqRLidu/nXMuVvKYLGkpcJaZbQJsA5wsaRPgfGC0ma0PjA6XU8pkDLo1MA/YheXzoQ14IoN9fwBWAVYiuBOxJ/BdBvs551xBVeVpENrMZgGzwvffSppK0DHdF9gp3GwIMAY4L2WdUnzWOZzBMYXlgXlZHTKs6zzgZmCDcP9TgGsy3Nc55wommzFoSYOAQUmr7jCzO1aw3drAr4HXgeoweAPMJphAkVKqAF0JtIUVDrhkGqDPAe4FPg2X1waOznBf55wrmGym2YXB+BcBOZmktgQz3s4ws4XJN8KYmUlKG0dTBehZZnZ5hvVtzFjgdoLhjfnAs8C4JpbpnHN5l89ZHJJaEATnB80sMRxcK6mrmc2S1BWYk66cVBcJ81Hd+whyd/yVYKhjXeD+PJTrnHN5VZHFKxUFXeW7galmdmPSR8OAI8P3RwJPpatTqh5033Q7Z6BHeCUz4QVJ7+WhXOecy6s83km4HXAEMFnSxHDdn4GrgcckHUtw417aGwgaDdBm9nUeKvqWpG3M7DWAMIfHhDyU65xzeZWvAG1mr9D4CERWHd+skyVlqSfwqqRE7ug1gQ8kTSYYJ9884uM751xGYngjYeQBun/E5TvnXF7E8VbvSAO0mU2LsnznnMuXssgH7Zxz5SiG6aA9QDvnHJRuPmjnnCt7PsThnHMx5UMczjkXU96DjpmxL7/ENVdfSX1dPfsdcBDH/nFQ+p1KgLcr3t4fcRnffv8TdfX1LK2rZ/vDruWqMway5449WLykjs9mfMWgSx5gwXc/FLuqOSvFcxW/8BzPXn1B1NXVcdWVlzP4trt4ctgIRo0czicff1zsajWZt6s09B/0D7b53dVsf9i1AIx+7X16HnQVWx3yNz6aNodzjtmtyDXMXameq0op41ehNNsAPWXyJGpq1qJbTQ0tWrak/54DGPPC6GJXq8m8XaVp9GvvU1dXD8Abkz9jjepVilyj3JXqucrjE1XyJtIALekXzzNc0bpimFNbS5euXZYtd66upra2tog1yg9vV/yZGU8PPoWxD57LMfv/8tfhD/v25tmxpZtTrFTPlbL4VyhRj0HfDPwmg3XONRt9j/47M+cuYPWObRl+2yl88Plsxr71CQDnHrs7dXX1PDJyfJFr2fzE8BphNAFaUm+CB8SuHj42K6E9wZNaGttv2WNkbhl8e6QXFjpXVzN71uxly3Nqa6muTvsEmtjzdsXfzLkLAJj7zXcM+98kfrvp2ox96xMO33tr9tyxB3sc/88i17BpSvVcpXtadzFENcTRkuBxWVVAu6TXQuDAxnYyszvMrJeZ9Yr6qu+mPTZj+vTPmTHjC5YsXsyokSPos/MukR6zELxd8bZS65a0XanVsve79t6Idz+ZSb9tN+bMo3blwDNu54cflxS5lk1TqucqjmPQkfSgzexFSa8Am5vZZVEco6mqqqq44MKLOXHQcdTX1zFwvwPo3n39Ylerybxd8dZ51XY8euMfAaiqrOTRZybw/KtTmfLUJbRqWcXwW08B4IweV4oAABI+SURBVI3Jn3PalY8Us6o5K9VzFcdbvWWW6fNfcyhcGmdmvXPZ98elGT+Y1rm86/jbU4pdhUh8M/6WYlchEq2rmj4+Mfr9rzKOOX03Wq0g0Tzqi4QTJQ0DhgLfJ1YmPUTROedioZCzMzIVdYBuDcwDkgegDPAA7ZyLlRiOcESesP/oKMt3zrl8iWMPOuobVbpJelLSnPD1H0ndojymc87lokKZvwpWp4jLvxcYBvwqfD0drnPOuVipkDJ+FaxOEZe/upnda2ZLw9e/gdUjPqZzzmVNWbwKJeoAPU/S4ZIqw9fhBBcNnXMuVppjD/oY4GBgdvg6EPALh8652IljDzrqWRzTgH2iPIZzzuVF/CZxRD6LY11JT0uaG87ieErSulEe0znnctEchzgeAh4DuhLM4hgKPBzxMZ1zLmtxHOKIOkCvZGb3J83ieIDg7kLnnIuXGEboqG/1fkbS+cAjBLd4HwKMlNQJwMy+jvj4zjmXkTjeSRh1gD44/Ho8LMtOJ+B34bKPRzvnYiGOuTiiHuI4D9jCzNYhuIPwHeAAM1vHzDw4O+diI4YjHJEH6IvMbKGk7Qky2t0F3BrxMZ1zLmuSMn4VStQBui78OgC408xGEDwOyznnYiWfj7ySdE84tXhK0rpOkp6X9FH4tWPaciJ+ospw4EugH8GTvH8A3jCzLdLt+/3iCCtWJJWFTIPl3Aps8Kdhxa5CJKbfvE+Tf7nemf5txjFnizXbpTyepB2B74D7zKxHuO5a4GszuzqcPNHRzM5LVU7UPeiDgWeB3c1sPtAJOCfiYzrnXPbyOAhtZi8BDWep7QsMCd8PAQamKyfqW70XkfT0FDObBcyK8pjOOZeLbKbZSRoEDEpadYeZ3ZFmt+owBkKQm6g63XGinmbnnHMlIZtrf2EwTheQU+1vktIOqUQ9xOGccyUhnxcJG1ErqWtwLHUF5qTbwQO0c84RDHFk+i9Hw4Ajw/dHAk+l28EDtHPOkfdpdg8D44ANJc2QdCxwNdBP0kfAruFySj4G7Zxz5PcOQTP7fSMf9c2mHA/QzjkHsUzY7wHaOeegoIn4M+UB2jnniGUH2gO0c84BsYzQHqCdc47mmbDfOedKQgyHoD1AO+ccxHKEwwO0c84BBU3EnykP0M45hw9xOOdcbMUwPjfvXByX/uXP9O2zLQftt3exq5JXY19+iX0G7M5e/ftx9505Z0SMnXJsVzm1qUIw8tw+3Hv8VgDUrLoST521Ay9d3Jd/Hd2TFpVxDIFJYvjU2GYdoPfedz9uufXOYlcjr+rq6rjqyssZfNtdPDlsBKNGDueTjz8udrWarBzbVW5tOmandfm49ttlyxfsszF3vfAJO14+mgWLlnBI77WKWLv0CpDNLmvNOkD37PVbOnToUOxq5NWUyZOoqVmLbjU1tGjZkv57DmDMC6OLXa0mK8d2lVObuqzSmr6bVvPIuOnL1m27wWqMnBg8QOTx179g9827FKt6GSlAPuisNesAXY7m1NbSpevyX4TO1dXU1tYWsUb5UY7tKqc2Xbp/D6566j3q64OHhHRcuSULf1hKXbg8a/4PdOnQuphVTKtCmb8KVqeoCpa0f/h48QWSFkr6VtLCNPsMkjRB0oR77irt8Tjnmou+m1bz1Xc/MfmLBcWuShPFbxA6ylkc1wJ7m9nUTHdIfs7X94st40egu+U6V1cze9bsZctzamuprk77bMrYK8d2lUubeq3biX49urDzJtW0alFBu9ZVXHpAD9q3qaKyQtTVG11XacPsBT8Wu6opxXGaXZRDHLXZBGeXH5v22Izp0z9nxowvWLJ4MaNGjqDPzrsUu1pNVo7tKpc2XfP0VLa++Hm2u/T/OOXeN3n1w684/b63GPfRPPbcsisAB25dw3OTZ6cpqbji13+OoActaf/w7QRJjwL/BX5KfG5mT+T7mLm64NwzeXP8eObP/4b+fftwwsmnMnD/A4tdrSapqqriggsv5sRBx1FfX8fA/Q6ge/f1i12tJivHdpVjm5L97an3uOXonpyz18a8O2MBjyZdQIyjOPagZXkeSZB0b4qPzcyOyaScchziqCzk1QXnVmCDPw0rdhUiMf3mfZr8yzV74ZKMY06X9i0K8suc9x60mR2d7zKdcy5qcew+RTmLY4ikVZKWO0q6J6rjOedcU8RxHnSUszg2N7P5iQUz+0bSryM8nnPO5SyOCfujnMVRIaljYkFSJzw5k3MurmI4jSPKgHkDME7S0HD5IODKCI/nnHM5i1//OcIAbWb3SZoAJCZ27m9m70V1POeca4qKGM6zizoXRyfgezO7BZgraZ2Ij+ecczmJ40XCKGdxXAKcB1wQrmoBPBDV8ZxzrtxEOQa9H/Br4C0AM5spqV2Ex3POuZzFcIQj0gC92MxMkgFIWjnCYznnXJM0t2l2j0m6HVhF0h+B/wPK6/ElzrmyEccx6Ch70KsDjwMLgQ2Bi4FdIzyec87lrLkNcfQzs/OA5xMrJN1AcOHQOediJY5DHFGkGz0ROAlYV9KkpI/aAWPzfTznnMuH5tKDfgh4BvgbcH7S+m/N7OsIjuecc02Wz/gsqT/wD6ASuMvMrs6lnCjSjS4AFgC/z3fZzjkXmTxFaEmVwL+AfsAMYLykYbncSe3Ji5xzjrze6r0V8LGZfQog6RFgX6B8AvTKLQs3IiRpUPjA2rJSju0qxzZB4do1/eZ9oj7EMqV2rlpXZd6HljQIGJS06o6ktq4BfJH02Qxg61zqFHUujlIxKP0mJakc21WObYLybFc5tgkAM7vDzHolvSL5j8gDtHPO5deXQE3ScrdwXdY8QDvnXH6NB9aXtI6klsDvgJye1hvbMegCK5lxsiyVY7vKsU1Qnu0qxzalZWZLJZ0CPEswze4eM3s3l7JklvGTxp1zzhWQD3E451xMeYB2zrmYKrkALWltSVOaWMavJD2erzo1oR4DJW2Sw347Sdo2g+32kXR+uu2iIGkVSSc1sYwxknqF70eGZf6s3Licy0LI9LwXoB45n1tJ/5Z0YL7rVK5KLkDng5nNNLM4/JAMBLIK0JKqgJ2AtL+oZjYs1xwAebAKQdKsvDCzPc1sfsNyY3QuI5XNeS+AvJ5bl4KZldQLWBt4H3gQmEqQc3ol4HNgtXCbXsCY8H0fYGL4epsgq97awJTw86OAJ4BRwEfAtUnH2g0YR/DYrqFA23D91QS3bU4Crg/XHQRMAaYR5MCeCNxOcBX3O+BK4B3gNaCa4Bfta+CzcNv1wtco4E3gZWCjsOx/A7cBr4d1nU0wr3IisAOwd/jZ2wQPRqhOatstSWX8E3gV+BQ4MFy/E/Ai8FS4/mrgMOANYDKwXrjd6sB/CKYQjQe2C9dfCtwDjAn3Py1c/wjwQ1jH69Kcu75h3SeHZbUKtx8D9Arffw6s1rDcBueyErg+PA+TgFMbO18F+DldGRgRnvMpwCFhG64N2/kG0D3p+/K/sH6jgTUzOe9F/B1seA7OCX8mJgGXJW33h3DdO8D9qX4O/dXI97rYFcjhh2NtwJICxD3A2TQeoJ9O2rYtwdTC5F/qo8IflA5Aa4IAWxMGg5eAlcPtziN46MCqwAcsnwGzSvh1MsF/Bk8n1WNw+ENqwN7humuBi5J+WA9MattoYP3w/dbA/5K2Gw5UhsuXAmcn7dcxqT7HATcktS05QA8l+KtpE4JcARAE6PlAV6BVGAAuCz87HbgpfP8QsH34fk1galJdXg33XQ2YR/CA4GXf4zTn7iKC22I3CNfdB5wRvh/DLwP0z8ptcC5PJAj6VeFyp8bOVwF+Tg8A7kxa7hC24cJw+Q/A8KSf0SPD98cA/83kvBf5dzDxPd+NYDqdwp+t4cCOwKbAhyz/XeiU6ufQXyt+leo86C/MLJFb+gHgtBTbjgVulPQg8ISZzdAv03yMtiALH5LeA9Yi+DNuE2BsuH1Lgt70AuBH4G5Jwwl+IBPHuR3oArwgqQ5oA8wBFidt9yZBlqufkdSWoFc9NKl+rZI2GWpmdY20sRvwqKSuYT0/a2S7/5pZPfCepOqk9ePNbFZYj0+A58L1k4Gdw/e7Apsk1a19WGeAEWb2E/CTpDkEfyE0puG5+wvwmZl9GK4bApwM3JSijMbsCtxmZksBzOzrcGhgRecrapOBGyRdQxCIXw6/dw+Hnz8M/D183xvYP3x/P8F/4gmpznsc7Ba+3g6X2wLrA1sQ1P0rCM5F0j6N/Ry6Bko1QDecvG3AUpaPqbde9oHZ1ZJGAHsSBNvdCX5hk/2U9L6O4Psi4Hkz+0XaVElbEfxZfiBwCrCLmZ0g6XpgG4KA2dPM5oXbn21h9yGp/IYqgPlmtmUjbf6+kfUANwM3mtkwSTsR9LRWJLmdamR9fdJyfVJdK4BtzOxn37sw6Kzo+9eYhuduPkEvNxIW3DTwi/MV1fGSjvuhpN8Q/NxdIWl04qPkzTIoKtV5jwMBfzOz23+2Ujo1xT6N/Ry6Bkr1IuGaknqH7w8FXiH487FnuO6AxIaS1jOzyWZ2DcE42UYZHuM1YDtJ3cNyVpa0Qdhr7GBmI4E/EfQUkLQewZ/s1cA3QI2kTpLWSnGMbwnGxDGzhcBnkg4Ky5OkLdLtF+rA8nv9j8ywfdl6Dlj2Syepsf9IEhrWMaHhuZsArJ34PgNHEIyJZ1suBI9XOz7sNRN+/1d4vqIm6VfAIjN7gGCc9jfhR4ckfR0Xvn+V4HZgCMb/X26k2FRtL6TkejwLHJP4a0rSGpI6E4ypHyRp1XB9p6LUtMSVaoD+ADhZ0lSC8ddbgcuAf0iaQNCLSzhD0pTw8VtLCJ72kpaZzSUYw3043HccQXBvBwwP170CnBnuch3wKMGwRDeCsdTnCcZ2G/MIcI6kt8MAfxhwrKR3gHcJcsiuyNPAfpImStqBoMc8VNKbwFeZtC8HpwG9JE0Kh4FOSLVx+NfD2PB7f13SRw3P3d+BownqP5mg135bDuUC3AVMByaF38NDafx8RW0z4A1JE4FLgCvC9R3DupxO8B8GBP/xHR2uPyL8bEUanveiSD4HBMN1DwHjwvP3ONDOglubrwReDM/FjcWqbynzW71dwUham2A8tkeRq1IUkj4nuOgZ1X+irsyUag/aOefKnvegnXMuprwH7ZxzMeUB2jnnYsoDtHPOxZQHaPczkurCaVxTJA2VtFITylqWuUzSXaky9+WaqU3S55JWy3R9I2UcJemWfBzXuXzyAO0a+sHMtgynwi2mwXznxE0g2TKz48zsvRSb7EQ8MrU5FxseoF0qLwPdw97ty5KGEeRPqJR0naTx4Y0rx8Oyux9vkfSBpP8DOicK0s9zO/eX9JakdySNDudHnwD8KXEThqTVJf0nPMZ4SduF+64q6TlJ70q6iyxuFZa0laRx4Y1Br0raMOnjmrCOH0m6JGmfwyW9EdbrdkmVDcpcWdKIsC1TJB2Cc3lSqrk4XMTCnvIeBOlPIbhVuYeZfSZpELDAzH4rqRXBXWXPAb8GNiRIMlVNkOLzngblrg7cCewYltUpTGp0G/CdmV0fbvcQ8Hcze0XSmgS3FG9McFfeK2Z2uaQBwLFZNOt9gjSdSyXtClzF8rQAWwE9gEXAeAX5W74nuCV7OzNbImkwwd2e9yWV2R+YaWYDwnp3yKI+zqXkAdo11Ca8PRmCHvTdBEMPb5hZIkvebsDmWv5kjA4EGcx2BB4Os6/NlPS/FZS/DfBSoqwGWc6SNZY9b0fCzG9mNkLSN1m0rQMwRNL6BImKWiR99nxScqsngO0JEnD1JAjYsDw7YbJfZK3Loj7OpeQB2jX0Q8OMemFwSs6qJoJk+M822G7PPNYjVfa8XP0VeMHM9guHVcYkfbaiDIkChpjZBY0VuKKsdWZ2eVMq6VyCj0G7XDwLnCipBYCCLH8rEzzg4JBwjLory3NJJ3sN2FHSOuG+iSxnDTO1NZY97yWCJEhI2oMg4VKmkrP+HdXgs34Kst+1IXgU2ViCBygcqCA7WyI73s+yE6rxrHXONZn3oF0u7iJ4qsZbCrq0cwmC2pMEuZbfI8gqN67hjmY2NxzDfkJSBcGQQT+CTG2PS9qXIDCfBvxLQYa3KoLAfAJB1sKHJb1LkKZzeop6TpJUH75/jCAR/hBJFxE8jirZGwSP9OoGPGBmEwDCbZ8L67qE4GEC05L22wy4LjzOEoKnujiXF56LwznnYsqHOJxzLqY8QDvnXEx5gHbOuZjyAO2cczHlAdo552LKA7RzzsWUB2jnnIup/weost75zioHBAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}