t5_pawraphrase_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "t5_finetuning_seb_force_old_versions",
      "provenance": [],
      "collapsed_sections": [
        "brPOSAkjNP5t",
        "8L5hidiq1cZU"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "232ee61a8cec4ba083124d7524a0b566": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3cdd303ec2da4f9ba95ce66e44f8516d",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_095a70181fa74fef864b525413c68e54",
              "IPY_MODEL_7fdc45c7d8f0448e9e96fdc78e5af289"
            ]
          }
        },
        "3cdd303ec2da4f9ba95ce66e44f8516d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "095a70181fa74fef864b525413c68e54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_743d7ec434334555a1d6725c9fb346ad",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 791656,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 791656,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3be7af6d37034a9e9a033a837b1664e7"
          }
        },
        "7fdc45c7d8f0448e9e96fdc78e5af289": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e397a49231f245d6ab71436f3ad836f1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 792k/792k [00:00&lt;00:00, 1.25MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ea73bfb751f2476c9763398106093816"
          }
        },
        "743d7ec434334555a1d6725c9fb346ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3be7af6d37034a9e9a033a837b1664e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e397a49231f245d6ab71436f3ad836f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ea73bfb751f2476c9763398106093816": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a84f455cb38e4aa0b7979d28a0751095": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0e38913807814c048260a847a9b6dd9f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a49a2978a0f840248a210f8b588c27ec",
              "IPY_MODEL_c35a1df6a73248f19e88dd26fa52eae3"
            ]
          }
        },
        "0e38913807814c048260a847a9b6dd9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a49a2978a0f840248a210f8b588c27ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_239b3414ed3646c1837af5696c9bc196",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1197,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1197,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_147d9621249a4c4ea9dd1e846530b5c8"
          }
        },
        "c35a1df6a73248f19e88dd26fa52eae3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f0a63314cf1540c2af212bf02c481274",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.20k/1.20k [00:00&lt;00:00, 10.9kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b07e9da30db742218680e65d9ba3c72a"
          }
        },
        "239b3414ed3646c1837af5696c9bc196": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "147d9621249a4c4ea9dd1e846530b5c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0a63314cf1540c2af212bf02c481274": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b07e9da30db742218680e65d9ba3c72a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f1e25485282c43dab9232f560109d87e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_845da3d470a446878d7e3c6f6b0ad659",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_51929b2804cf4dae88c9f8b4a2d89109",
              "IPY_MODEL_66b26fdb20c14ebe8b7c4b17fca18537"
            ]
          }
        },
        "845da3d470a446878d7e3c6f6b0ad659": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "51929b2804cf4dae88c9f8b4a2d89109": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d5145ade82824c86948712452f27bea1",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 242065649,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 242065649,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8b98edeaf56048c3a45e9784e907fef3"
          }
        },
        "66b26fdb20c14ebe8b7c4b17fca18537": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_98b497fa096a48f9b848ab0e4430f149",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 242M/242M [00:16&lt;00:00, 15.0MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e3226284ee36434796c7f0ea882e78d2"
          }
        },
        "d5145ade82824c86948712452f27bea1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8b98edeaf56048c3a45e9784e907fef3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "98b497fa096a48f9b848ab0e4430f149": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e3226284ee36434796c7f0ea882e78d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "12a43958a9734db58fb3ab498bf0fe20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4c1c9e78cf184965a2f2ace2c7b5fd9b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2856aa98722040efa538b978551c8e07",
              "IPY_MODEL_41e524d17c9748fd9c5bbc988419d9b2"
            ]
          }
        },
        "4c1c9e78cf184965a2f2ace2c7b5fd9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "2856aa98722040efa538b978551c8e07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_999bfdeac5934e09ae2ee680c429afc6",
            "_dom_classes": [],
            "description": "Validation sanity check: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_165d45d5ca224c7abf38363b33aac183"
          }
        },
        "41e524d17c9748fd9c5bbc988419d9b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ca9d710858084a7dbf675ecf5e747f23",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [00:00&lt;00:00,  5.33it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1c4c2d6f5bbe434aa0e59c73da83e98e"
          }
        },
        "999bfdeac5934e09ae2ee680c429afc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "165d45d5ca224c7abf38363b33aac183": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca9d710858084a7dbf675ecf5e747f23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1c4c2d6f5bbe434aa0e59c73da83e98e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c8f080e05fee4bec91047f367eb0be2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4deb129bc7f14e6cacee62a87e303d5c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ddea067b851845e2931a7d6994f0bafe",
              "IPY_MODEL_fae1bdb5f04a4cbba6b9d21dfd1eaee0"
            ]
          }
        },
        "4deb129bc7f14e6cacee62a87e303d5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": "row wrap",
            "width": "100%",
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": "inline-flex",
            "left": null
          }
        },
        "ddea067b851845e2931a7d6994f0bafe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_57a50a77f9374bcd9acc78715a3eb430",
            "_dom_classes": [],
            "description": "Epoch 1:  43%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 884,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 378,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f51e7ec096ed437d96513c2e5816b861"
          }
        },
        "fae1bdb5f04a4cbba6b9d21dfd1eaee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_416f2baf6d434b3abe9115025b1bcc89",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 378/884 [01:38&lt;02:11,  3.86it/s, loss=0.045, v_num=0]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_378ee109bd8c4619b177f548035e7e3d"
          }
        },
        "57a50a77f9374bcd9acc78715a3eb430": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f51e7ec096ed437d96513c2e5816b861": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": "2",
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "416f2baf6d434b3abe9115025b1bcc89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "378ee109bd8c4619b177f548035e7e3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BJXx37984OXa"
      },
      "source": [
        "# Set up the Training Infrastructure"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tASNTo34N4Ij"
      },
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "naCXT0gVMp3R"
      },
      "source": [
        "!mkdir data"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6x65MnugJplf"
      },
      "source": [
        " !curl -L https://www.dropbox.com/s/xgvawan1vpuels9/train_light.csv?dl=1 -o data/train.csv -s\n",
        " !curl -L https://www.dropbox.com/s/fhon3pf1gf3dog6/val.csv?dl=1 -o data/val.csv -s\n",
        " !curl -L https://www.dropbox.com/s/yz5abappvpinxyr/test.csv?dl=1 -o data/test.csv -s\n",
        " !curl -L https://www.dropbox.com/s/1n76cka49r5ctje/train.csv?dl=1 -o data/train_long.csv -s"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SDVQ04fGRb1v",
        "outputId": "0eaa9614-bee9-4d4f-8e3e-fd58ee5807c0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 180
        }
      },
      "source": [
        "!pip install torch==1.4.0 -q\n",
        "!pip install transformers==2.9.0 -q\n",
        "!pip install pytorch_lightning==0.7.5 -q"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 753.4MB 21kB/s \n",
            "\u001b[31mERROR: torchvision 0.7.0+cu101 has requirement torch==1.6.0, but you'll have torch 1.4.0 which is incompatible.\u001b[0m\n",
            "\u001b[K     |████████████████████████████████| 645kB 9.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 24.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 45.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 56.7MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 235kB 8.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 829kB 15.1MB/s \n",
            "\u001b[?25h  Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HS8mNXq6bdxq",
        "cellView": "form",
        "outputId": "864db9a8-c39c-462e-90b4-da9a686b2827",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82
        }
      },
      "source": [
        "# @title Output\n",
        "import argparse\n",
        "import glob\n",
        "import os\n",
        "import json\n",
        "import time\n",
        "import logging\n",
        "import random\n",
        "import re\n",
        "from itertools import chain\n",
        "from string import punctuation\n",
        "\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.tokenize import sent_tokenize\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import pytorch_lightning as pl\n",
        "\n",
        "\n",
        "from transformers import (\n",
        "    AdamW,\n",
        "    T5ForConditionalGeneration,\n",
        "    T5Tokenizer,\n",
        "    get_linear_schedule_with_warmup\n",
        ")"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:PyTorch version 1.4.0 available.\n",
            "INFO:transformers.file_utils:TensorFlow version 2.3.0 available.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IswYuhWaz7QJ"
      },
      "source": [
        "def set_seed(seed):\n",
        "  random.seed(seed)\n",
        "  np.random.seed(seed)\n",
        "  torch.manual_seed(seed)\n",
        "  if torch.cuda.is_available():\n",
        "    torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "set_seed(42)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0dt4BJL0MgsD"
      },
      "source": [
        "!mkdir output"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RKNr7fgzcKpZ"
      },
      "source": [
        "## Model\n",
        "\n",
        "We'll be using [pytorch-lightning](https://github.com/PytorchLightning/pytorch-lightning) for training. Most of the below code is adapted from [HugginFace](https://github.com/huggingface/transformers/blob/master/examples/lightning_base.py)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a4hjvsBJ5Zk5"
      },
      "source": [
        "Definition of hyperparameters and other."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "urduopvizqTq"
      },
      "source": [
        "args_dict = dict(\n",
        "    data_dir=\"data\", # path for data files\n",
        "    output_dir=\"output\", # path to save the checkpoints\n",
        "    model_name_or_path='t5-small',\n",
        "    tokenizer_name_or_path='t5-small',\n",
        "    max_seq_length=512,\n",
        "    learning_rate=3e-4,\n",
        "    weight_decay=0.0,\n",
        "    adam_epsilon=1e-8,\n",
        "    warmup_steps=0,\n",
        "    train_batch_size=8,\n",
        "    eval_batch_size=8,\n",
        "    num_train_epochs=2,\n",
        "    gradient_accumulation_steps=16,\n",
        "    n_gpu=1,\n",
        "    # early_stop_callback=False,\n",
        "    fp_16=False, # if you want to enable 16-bit training then install apex and set this to true\n",
        "    opt_level='O1', # you can find out more on optimisation levels here https://nvidia.github.io/apex/amp.html#opt-levels-and-properties\n",
        "    max_grad_norm=1.0, # if you enable 16-bit training then set this to a sensible value, 0.5 is a good default\n",
        "    seed=42,\n",
        ")"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "B7uVNBtXST5X"
      },
      "source": [
        "# @title class T5FineTuner(pl.LightningModule)\n",
        "class T5FineTuner(pl.LightningModule):\n",
        "  def __init__(self, hparams):\n",
        "    super(T5FineTuner, self).__init__()\n",
        "    self.hparams = hparams\n",
        "    \n",
        "    self.model = T5ForConditionalGeneration.from_pretrained(hparams.model_name_or_path)\n",
        "    self.tokenizer = T5Tokenizer.from_pretrained(hparams.tokenizer_name_or_path)\n",
        "  \n",
        "  def is_logger(self):\n",
        "    return True\n",
        "  \n",
        "  def forward(\n",
        "      self, input_ids, attention_mask=None, decoder_input_ids=None, decoder_attention_mask=None, lm_labels=None\n",
        "  ):\n",
        "    return self.model(\n",
        "        input_ids,\n",
        "        attention_mask=attention_mask,\n",
        "        decoder_input_ids=decoder_input_ids,\n",
        "        decoder_attention_mask=decoder_attention_mask,\n",
        "        lm_labels=lm_labels,\n",
        "    )\n",
        "\n",
        "  def _step(self, batch):\n",
        "    lm_labels = batch[\"target_ids\"]\n",
        "    lm_labels[lm_labels[:, :] == self.tokenizer.pad_token_id] = -100\n",
        "\n",
        "    outputs = self(\n",
        "        input_ids=batch[\"source_ids\"],\n",
        "        attention_mask=batch[\"source_mask\"],\n",
        "        lm_labels=lm_labels,\n",
        "        decoder_attention_mask=batch['target_mask']\n",
        "    )\n",
        "\n",
        "    loss = outputs[0]\n",
        "\n",
        "    return loss\n",
        "\n",
        "  def training_step(self, batch, batch_idx):\n",
        "    loss = self._step(batch)\n",
        "\n",
        "    tensorboard_logs = {\"train_loss\": loss}\n",
        "    return {\"loss\": loss, \"log\": tensorboard_logs}\n",
        "  \n",
        "  def training_epoch_end(self, outputs):\n",
        "    avg_train_loss = torch.stack([x[\"loss\"] for x in outputs]).mean()\n",
        "    tensorboard_logs = {\"avg_train_loss\": avg_train_loss}\n",
        "    return {\"avg_train_loss\": avg_train_loss, \"log\": tensorboard_logs, 'progress_bar': tensorboard_logs}\n",
        "\n",
        "  def validation_step(self, batch, batch_idx):\n",
        "    loss = self._step(batch)\n",
        "    return {\"val_loss\": loss}\n",
        "  \n",
        "  def validation_epoch_end(self, outputs):\n",
        "    avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n",
        "    tensorboard_logs = {\"val_loss\": avg_loss}\n",
        "    return {\"avg_val_loss\": avg_loss, \"log\": tensorboard_logs, 'progress_bar': tensorboard_logs}\n",
        "\n",
        "  def configure_optimizers(self):\n",
        "    \"Prepare optimizer and schedule (linear warmup and decay)\"\n",
        "\n",
        "    model = self.model\n",
        "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
        "    optimizer_grouped_parameters = [\n",
        "        {\n",
        "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
        "            \"weight_decay\": self.hparams.weight_decay,\n",
        "        },\n",
        "        {\n",
        "            \"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n",
        "            \"weight_decay\": 0.0,\n",
        "        },\n",
        "    ]\n",
        "    optimizer = AdamW(optimizer_grouped_parameters, lr=self.hparams.learning_rate, eps=self.hparams.adam_epsilon)\n",
        "    self.opt = optimizer\n",
        "    return [optimizer]\n",
        "  \n",
        "  def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None,using_native_amp=None):\n",
        "    if self.trainer.use_tpu:\n",
        "      xm.optimizer_step(optimizer)\n",
        "    else:\n",
        "      optimizer.step()\n",
        "    optimizer.zero_grad()\n",
        "    self.lr_scheduler.step()\n",
        "  \n",
        "  def get_tqdm_dict(self):\n",
        "    tqdm_dict = {\"loss\": \"{:.3f}\".format(self.trainer.avg_loss), \"lr\": self.lr_scheduler.get_last_lr()[-1]}\n",
        "\n",
        "    return tqdm_dict\n",
        "\n",
        "  def train_dataloader(self):\n",
        "    train_dataset = get_dataset(tokenizer=self.tokenizer, type_path=\"train\", args=self.hparams)\n",
        "    dataloader = DataLoader(train_dataset, batch_size=self.hparams.train_batch_size, drop_last=True, shuffle=True, num_workers=4)\n",
        "    t_total = (\n",
        "        (len(dataloader.dataset) // (self.hparams.train_batch_size * max(1, self.hparams.n_gpu)))\n",
        "        // self.hparams.gradient_accumulation_steps\n",
        "        * float(self.hparams.num_train_epochs)\n",
        "    )\n",
        "    scheduler = get_linear_schedule_with_warmup(\n",
        "        self.opt, num_warmup_steps=self.hparams.warmup_steps, num_training_steps=t_total\n",
        "    )\n",
        "    self.lr_scheduler = scheduler\n",
        "    return dataloader\n",
        "\n",
        "  def val_dataloader(self):\n",
        "    val_dataset = get_dataset(tokenizer=self.tokenizer, type_path=\"val\", args=self.hparams)\n",
        "    return DataLoader(val_dataset, batch_size=self.hparams.eval_batch_size, num_workers=4)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "oh1R5C-GwMqx"
      },
      "source": [
        "# @title logger\n",
        "logger = logging.getLogger(__name__)\n",
        "\n",
        "class LoggingCallback(pl.Callback):\n",
        "  def on_validation_end(self, trainer, pl_module):\n",
        "    logger.info(\"***** Validation results *****\")\n",
        "    if pl_module.is_logger():\n",
        "      metrics = trainer.callback_metrics\n",
        "      # Log results\n",
        "      for key in sorted(metrics):\n",
        "        if key not in [\"log\", \"progress_bar\"]:\n",
        "          logger.info(\"{} = {}\\n\".format(key, str(metrics[key])))\n",
        "\n",
        "  def on_test_end(self, trainer, pl_module):\n",
        "    logger.info(\"***** Test results *****\")\n",
        "\n",
        "    if pl_module.is_logger():\n",
        "      metrics = trainer.callback_metrics\n",
        "\n",
        "      # Log and save results to file\n",
        "      output_test_results_file = os.path.join(pl_module.hparams.output_dir, \"test_results.txt\")\n",
        "      with open(output_test_results_file, \"w\") as writer:\n",
        "        for key in sorted(metrics):\n",
        "          if key not in [\"log\", \"progress_bar\"]:\n",
        "            logger.info(\"{} = {}\\n\".format(key, str(metrics[key])))\n",
        "            writer.write(\"{} = {}\\n\".format(key, str(metrics[key])))"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vfhlYUUV2NIh"
      },
      "source": [
        "## Paraphrase Generation finetuned on PAWS Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IENAXVsDtLOY",
        "outputId": "8b02241e-564e-4316-e93c-86f13213706b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115
        }
      },
      "source": [
        "train_data = 'data/train.csv'\n",
        "val_data = 'data/val.csv'\n",
        "train = pd.read_csv(train_data)\n",
        "train = train[:25]\n",
        "print(train.head())"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                           sentence1                                          sentence2\n",
            "0  They were there to enjoy us and they were ther...  They were there for us to enjoy and they were ...\n",
            "1  After the end of the war in June 1902 , Higgin...  In August , after the end of the war in June 1...\n",
            "2  From the merger of the Four Rivers Council and...  Shawnee Trails Council was formed from the mer...\n",
            "3  The group toured extensively and became famous...  The group toured extensively and was famous in...\n",
            "4  Kathy and her husband Pete Beale ( Peter Dean ...  Kathy and her husband Peter Dean ( Pete Beale ...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qdEgCwL7cIyi"
      },
      "source": [
        "### Prepare Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "McQC1FotigqA",
        "outputId": "cd3a37ef-0d4f-4b4f-8248-2a2f1c43aee9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182,
          "referenced_widgets": [
            "232ee61a8cec4ba083124d7524a0b566",
            "3cdd303ec2da4f9ba95ce66e44f8516d",
            "095a70181fa74fef864b525413c68e54",
            "7fdc45c7d8f0448e9e96fdc78e5af289",
            "743d7ec434334555a1d6725c9fb346ad",
            "3be7af6d37034a9e9a033a837b1664e7",
            "e397a49231f245d6ab71436f3ad836f1",
            "ea73bfb751f2476c9763398106093816"
          ]
        }
      },
      "source": [
        "tokenizer = T5Tokenizer.from_pretrained('t5-base')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140232797761096 acquired on /root/.cache/torch/transformers/68f1b8dbca4350743bb54b8c4169fd38cbabaad564f85a9239337a8d0342af9f.9995af32582a1a7062cb3173c118cb7b4636fa03feb967340f20fc37406f021f.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmpkbjvos_m\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "232ee61a8cec4ba083124d7524a0b566",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=791656.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model in cache at /root/.cache/torch/transformers/68f1b8dbca4350743bb54b8c4169fd38cbabaad564f85a9239337a8d0342af9f.9995af32582a1a7062cb3173c118cb7b4636fa03feb967340f20fc37406f021f\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/68f1b8dbca4350743bb54b8c4169fd38cbabaad564f85a9239337a8d0342af9f.9995af32582a1a7062cb3173c118cb7b4636fa03feb967340f20fc37406f021f\n",
            "INFO:filelock:Lock 140232797761096 released on /root/.cache/torch/transformers/68f1b8dbca4350743bb54b8c4169fd38cbabaad564f85a9239337a8d0342af9f.9995af32582a1a7062cb3173c118cb7b4636fa03feb967340f20fc37406f021f.lock\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model from cache at /root/.cache/torch/transformers/68f1b8dbca4350743bb54b8c4169fd38cbabaad564f85a9239337a8d0342af9f.9995af32582a1a7062cb3173c118cb7b4636fa03feb967340f20fc37406f021f\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "QcHi90l11U_D"
      },
      "source": [
        "# @title ParaphraseDataset(Dataset)\n",
        "class ParaphraseDataset(Dataset):\n",
        "    def __init__(self, tokenizer, data_dir, type_path, max_len=256, truncation=True):\n",
        "        self.path = os.path.join(data_dir, type_path + '.csv')\n",
        "\n",
        "        self.source_column = \"sentence1\"\n",
        "        self.target_column = \"sentence2\"\n",
        "        self.data = pd.read_csv(self.path)\n",
        "\n",
        "        self.max_len = max_len\n",
        "        self.tokenizer = tokenizer\n",
        "        self.inputs = []\n",
        "        self.targets = []\n",
        "\n",
        "        self._build()\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.inputs)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        source_ids = self.inputs[index][\"input_ids\"].squeeze()\n",
        "        target_ids = self.targets[index][\"input_ids\"].squeeze()\n",
        "\n",
        "        src_mask = self.inputs[index][\"attention_mask\"].squeeze()  # might need to squeeze\n",
        "        target_mask = self.targets[index][\"attention_mask\"].squeeze()  # might need to squeeze\n",
        "\n",
        "        return {\"source_ids\": source_ids, \"source_mask\": src_mask, \"target_ids\": target_ids, \"target_mask\": target_mask}\n",
        "\n",
        "    def _build(self):\n",
        "        for idx in range(len(self.data)):\n",
        "            input_, target = self.data.loc[idx, self.source_column], self.data.loc[idx, self.target_column]\n",
        "\n",
        "            input_ = \"paraphrase: \"+ input_ + ' </s>'\n",
        "            target = target + \" </s>\"\n",
        "\n",
        "            # tokenize inputs\n",
        "            tokenized_inputs = self.tokenizer.batch_encode_plus(\n",
        "                [input_], max_length=self.max_len, pad_to_max_length=True, return_tensors=\"pt\"\n",
        "            )\n",
        "            # tokenize targets\n",
        "            tokenized_targets = self.tokenizer.batch_encode_plus(\n",
        "                [target], max_length=self.max_len, pad_to_max_length=True, return_tensors=\"pt\"\n",
        "            )\n",
        "\n",
        "            self.inputs.append(tokenized_inputs)\n",
        "            self.targets.append(tokenized_targets)\n",
        "    "
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gsnsKY6jemsr",
        "outputId": "2d81138d-290d-4adf-d51c-bf885f08e26d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "dataset = ParaphraseDataset(tokenizer, data_dir='data', type_path='val')\n",
        "len(dataset)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3536"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7g1gz05ccAzg",
        "outputId": "27b8a2c7-0595-4517-e733-69c083af00b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "data = dataset[100]\n",
        "print(tokenizer.decode(data['source_ids']))\n",
        "print(tokenizer.decode(data['target_ids']))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "paraphrase: It was credited to Lennon - McCartney, but John Lennon often stated that he wrote it.\n",
            "It was credited to Lennon -- McCartney, but John Lennon often stated that he wrote it.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W4cfw8bMcNdA"
      },
      "source": [
        "### Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aTvkv4rzhPjy"
      },
      "source": [
        "args = argparse.Namespace(**args_dict)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r5ngAP4OXFqZ"
      },
      "source": [
        "checkpoint_callback = pl.callbacks.ModelCheckpoint(\n",
        "    filepath=args.output_dir, prefix=\"checkpoint\", monitor=\"val_loss\", mode=\"min\", save_top_k=5\n",
        ")\n",
        "\n",
        "train_params = dict(\n",
        "    accumulate_grad_batches=args.gradient_accumulation_steps,\n",
        "    gpus=args.n_gpu,\n",
        "    max_epochs=args.num_train_epochs,\n",
        "    # early_stop_callback=False,\n",
        "    precision=32,\n",
        "    # amp_level=args.opt_level,\n",
        "    gradient_clip_val=args.max_grad_norm,\n",
        "    checkpoint_callback=checkpoint_callback,\n",
        "    callbacks=[LoggingCallback()],\n",
        ")"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2h2aGPgp0vOf"
      },
      "source": [
        "def get_dataset(tokenizer, type_path, args):\n",
        "  return ParaphraseDataset(tokenizer=tokenizer, data_dir=args.data_dir, type_path=type_path,  max_len=args.max_seq_length)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4IOQpawZA9XC"
      },
      "source": [
        "**Initialize model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kJsz3a4SilAF",
        "outputId": "21148e5e-0746-452a-f335-907c1a3db319",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "a84f455cb38e4aa0b7979d28a0751095",
            "0e38913807814c048260a847a9b6dd9f",
            "a49a2978a0f840248a210f8b588c27ec",
            "c35a1df6a73248f19e88dd26fa52eae3",
            "239b3414ed3646c1837af5696c9bc196",
            "147d9621249a4c4ea9dd1e846530b5c8",
            "f0a63314cf1540c2af212bf02c481274",
            "b07e9da30db742218680e65d9ba3c72a",
            "f1e25485282c43dab9232f560109d87e",
            "845da3d470a446878d7e3c6f6b0ad659",
            "51929b2804cf4dae88c9f8b4a2d89109",
            "66b26fdb20c14ebe8b7c4b17fca18537",
            "d5145ade82824c86948712452f27bea1",
            "8b98edeaf56048c3a45e9784e907fef3",
            "98b497fa096a48f9b848ab0e4430f149",
            "e3226284ee36434796c7f0ea882e78d2"
          ]
        }
      },
      "source": [
        "model = T5FineTuner(args)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:filelock:Lock 140232797551528 acquired on /root/.cache/torch/transformers/26561bc9e840d8945f475d0d4c4b9df32025eadd79894b867b570cb1d09e67a9.3817cc1260a6b941b17af62b4f2a942b9825f209d8e2eed99e79e96f85f59aab.lock\n",
            "INFO:transformers.file_utils:https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmp0xitbpot\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a84f455cb38e4aa0b7979d28a0751095",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1197.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json in cache at /root/.cache/torch/transformers/26561bc9e840d8945f475d0d4c4b9df32025eadd79894b867b570cb1d09e67a9.3817cc1260a6b941b17af62b4f2a942b9825f209d8e2eed99e79e96f85f59aab\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/26561bc9e840d8945f475d0d4c4b9df32025eadd79894b867b570cb1d09e67a9.3817cc1260a6b941b17af62b4f2a942b9825f209d8e2eed99e79e96f85f59aab\n",
            "INFO:filelock:Lock 140232797551528 released on /root/.cache/torch/transformers/26561bc9e840d8945f475d0d4c4b9df32025eadd79894b867b570cb1d09e67a9.3817cc1260a6b941b17af62b4f2a942b9825f209d8e2eed99e79e96f85f59aab.lock\n",
            "INFO:transformers.configuration_utils:loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json from cache at /root/.cache/torch/transformers/26561bc9e840d8945f475d0d4c4b9df32025eadd79894b867b570cb1d09e67a9.3817cc1260a6b941b17af62b4f2a942b9825f209d8e2eed99e79e96f85f59aab\n",
            "INFO:transformers.configuration_utils:Model config T5Config {\n",
            "  \"architectures\": [\n",
            "    \"T5WithLMHeadModel\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "INFO:filelock:Lock 140232797551528 acquired on /root/.cache/torch/transformers/9b662cba85524bef76fff5eb77d767407ac36f3fe492869331c011efd2b3a082.388aab7f5c8ed273dc71eb98334d76a3caf1b3280b476c1c77fba861c65445f3.lock\n",
            "INFO:transformers.file_utils:https://cdn.huggingface.co/t5-small-pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/torch/transformers/tmp49meyfj3\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f1e25485282c43dab9232f560109d87e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=242065649.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.file_utils:storing https://cdn.huggingface.co/t5-small-pytorch_model.bin in cache at /root/.cache/torch/transformers/9b662cba85524bef76fff5eb77d767407ac36f3fe492869331c011efd2b3a082.388aab7f5c8ed273dc71eb98334d76a3caf1b3280b476c1c77fba861c65445f3\n",
            "INFO:transformers.file_utils:creating metadata file for /root/.cache/torch/transformers/9b662cba85524bef76fff5eb77d767407ac36f3fe492869331c011efd2b3a082.388aab7f5c8ed273dc71eb98334d76a3caf1b3280b476c1c77fba861c65445f3\n",
            "INFO:filelock:Lock 140232797551528 released on /root/.cache/torch/transformers/9b662cba85524bef76fff5eb77d767407ac36f3fe492869331c011efd2b3a082.388aab7f5c8ed273dc71eb98334d76a3caf1b3280b476c1c77fba861c65445f3.lock\n",
            "INFO:transformers.modeling_utils:loading weights file https://cdn.huggingface.co/t5-small-pytorch_model.bin from cache at /root/.cache/torch/transformers/9b662cba85524bef76fff5eb77d767407ac36f3fe492869331c011efd2b3a082.388aab7f5c8ed273dc71eb98334d76a3caf1b3280b476c1c77fba861c65445f3\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:transformers.modeling_utils:Weights of T5ForConditionalGeneration not initialized from pretrained model: ['encoder.embed_tokens.weight', 'decoder.embed_tokens.weight', 'lm_head.weight']\n",
            "INFO:transformers.tokenization_utils:loading file https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model from cache at /root/.cache/torch/transformers/68f1b8dbca4350743bb54b8c4169fd38cbabaad564f85a9239337a8d0342af9f.9995af32582a1a7062cb3173c118cb7b4636fa03feb967340f20fc37406f021f\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RSJytKv1BFyc"
      },
      "source": [
        "**Initialize trainer**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PxO8OTA3irbw",
        "outputId": "e622fcea-dd3e-48ee-f055-059bf501ac41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "trainer = pl.Trainer(**train_params)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:lightning:GPU available: True, used: True\n",
            "INFO:lightning:CUDA_VISIBLE_DEVICES: [0]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wo7cSSvFGEhe"
      },
      "source": [
        "**start fine-tuning**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVGd6imfizLP",
        "outputId": "01181088-7f7d-4e3e-a613-31db581e5067",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "12a43958a9734db58fb3ab498bf0fe20",
            "4c1c9e78cf184965a2f2ace2c7b5fd9b",
            "2856aa98722040efa538b978551c8e07",
            "41e524d17c9748fd9c5bbc988419d9b2",
            "999bfdeac5934e09ae2ee680c429afc6",
            "165d45d5ca224c7abf38363b33aac183",
            "ca9d710858084a7dbf675ecf5e747f23",
            "1c4c2d6f5bbe434aa0e59c73da83e98e",
            "c8f080e05fee4bec91047f367eb0be2d",
            "4deb129bc7f14e6cacee62a87e303d5c",
            "ddea067b851845e2931a7d6994f0bafe",
            "fae1bdb5f04a4cbba6b9d21dfd1eaee0",
            "57a50a77f9374bcd9acc78715a3eb430",
            "f51e7ec096ed437d96513c2e5816b861",
            "416f2baf6d434b3abe9115025b1bcc89",
            "378ee109bd8c4619b177f548035e7e3d"
          ]
        }
      },
      "source": [
        "trainer.fit(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:lightning:\n",
            "    | Name                                                                  | Type                       | Params\n",
            "-----------------------------------------------------------------------------------------------------------------\n",
            "0   | model                                                                 | T5ForConditionalGeneration | 60 M  \n",
            "1   | model.shared                                                          | Embedding                  | 16 M  \n",
            "2   | model.encoder                                                         | T5Stack                    | 35 M  \n",
            "3   | model.encoder.block                                                   | ModuleList                 | 18 M  \n",
            "4   | model.encoder.block.0                                                 | T5Block                    | 3 M   \n",
            "5   | model.encoder.block.0.layer                                           | ModuleList                 | 3 M   \n",
            "6   | model.encoder.block.0.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "7   | model.encoder.block.0.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "8   | model.encoder.block.0.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "9   | model.encoder.block.0.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "10  | model.encoder.block.0.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "11  | model.encoder.block.0.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "12  | model.encoder.block.0.layer.0.SelfAttention.relative_attention_bias   | Embedding                  | 256   \n",
            "13  | model.encoder.block.0.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "14  | model.encoder.block.0.layer.0.dropout                                 | Dropout                    | 0     \n",
            "15  | model.encoder.block.0.layer.1                                         | T5LayerFF                  | 2 M   \n",
            "16  | model.encoder.block.0.layer.1.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "17  | model.encoder.block.0.layer.1.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "18  | model.encoder.block.0.layer.1.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "19  | model.encoder.block.0.layer.1.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "20  | model.encoder.block.0.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "21  | model.encoder.block.0.layer.1.dropout                                 | Dropout                    | 0     \n",
            "22  | model.encoder.block.1                                                 | T5Block                    | 3 M   \n",
            "23  | model.encoder.block.1.layer                                           | ModuleList                 | 3 M   \n",
            "24  | model.encoder.block.1.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "25  | model.encoder.block.1.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "26  | model.encoder.block.1.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "27  | model.encoder.block.1.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "28  | model.encoder.block.1.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "29  | model.encoder.block.1.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "30  | model.encoder.block.1.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "31  | model.encoder.block.1.layer.0.dropout                                 | Dropout                    | 0     \n",
            "32  | model.encoder.block.1.layer.1                                         | T5LayerFF                  | 2 M   \n",
            "33  | model.encoder.block.1.layer.1.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "34  | model.encoder.block.1.layer.1.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "35  | model.encoder.block.1.layer.1.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "36  | model.encoder.block.1.layer.1.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "37  | model.encoder.block.1.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "38  | model.encoder.block.1.layer.1.dropout                                 | Dropout                    | 0     \n",
            "39  | model.encoder.block.2                                                 | T5Block                    | 3 M   \n",
            "40  | model.encoder.block.2.layer                                           | ModuleList                 | 3 M   \n",
            "41  | model.encoder.block.2.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "42  | model.encoder.block.2.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "43  | model.encoder.block.2.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "44  | model.encoder.block.2.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "45  | model.encoder.block.2.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "46  | model.encoder.block.2.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "47  | model.encoder.block.2.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "48  | model.encoder.block.2.layer.0.dropout                                 | Dropout                    | 0     \n",
            "49  | model.encoder.block.2.layer.1                                         | T5LayerFF                  | 2 M   \n",
            "50  | model.encoder.block.2.layer.1.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "51  | model.encoder.block.2.layer.1.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "52  | model.encoder.block.2.layer.1.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "53  | model.encoder.block.2.layer.1.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "54  | model.encoder.block.2.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "55  | model.encoder.block.2.layer.1.dropout                                 | Dropout                    | 0     \n",
            "56  | model.encoder.block.3                                                 | T5Block                    | 3 M   \n",
            "57  | model.encoder.block.3.layer                                           | ModuleList                 | 3 M   \n",
            "58  | model.encoder.block.3.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "59  | model.encoder.block.3.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "60  | model.encoder.block.3.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "61  | model.encoder.block.3.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "62  | model.encoder.block.3.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "63  | model.encoder.block.3.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "64  | model.encoder.block.3.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "65  | model.encoder.block.3.layer.0.dropout                                 | Dropout                    | 0     \n",
            "66  | model.encoder.block.3.layer.1                                         | T5LayerFF                  | 2 M   \n",
            "67  | model.encoder.block.3.layer.1.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "68  | model.encoder.block.3.layer.1.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "69  | model.encoder.block.3.layer.1.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "70  | model.encoder.block.3.layer.1.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "71  | model.encoder.block.3.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "72  | model.encoder.block.3.layer.1.dropout                                 | Dropout                    | 0     \n",
            "73  | model.encoder.block.4                                                 | T5Block                    | 3 M   \n",
            "74  | model.encoder.block.4.layer                                           | ModuleList                 | 3 M   \n",
            "75  | model.encoder.block.4.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "76  | model.encoder.block.4.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "77  | model.encoder.block.4.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "78  | model.encoder.block.4.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "79  | model.encoder.block.4.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "80  | model.encoder.block.4.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "81  | model.encoder.block.4.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "82  | model.encoder.block.4.layer.0.dropout                                 | Dropout                    | 0     \n",
            "83  | model.encoder.block.4.layer.1                                         | T5LayerFF                  | 2 M   \n",
            "84  | model.encoder.block.4.layer.1.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "85  | model.encoder.block.4.layer.1.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "86  | model.encoder.block.4.layer.1.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "87  | model.encoder.block.4.layer.1.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "88  | model.encoder.block.4.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "89  | model.encoder.block.4.layer.1.dropout                                 | Dropout                    | 0     \n",
            "90  | model.encoder.block.5                                                 | T5Block                    | 3 M   \n",
            "91  | model.encoder.block.5.layer                                           | ModuleList                 | 3 M   \n",
            "92  | model.encoder.block.5.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "93  | model.encoder.block.5.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "94  | model.encoder.block.5.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "95  | model.encoder.block.5.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "96  | model.encoder.block.5.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "97  | model.encoder.block.5.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "98  | model.encoder.block.5.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "99  | model.encoder.block.5.layer.0.dropout                                 | Dropout                    | 0     \n",
            "100 | model.encoder.block.5.layer.1                                         | T5LayerFF                  | 2 M   \n",
            "101 | model.encoder.block.5.layer.1.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "102 | model.encoder.block.5.layer.1.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "103 | model.encoder.block.5.layer.1.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "104 | model.encoder.block.5.layer.1.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "105 | model.encoder.block.5.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "106 | model.encoder.block.5.layer.1.dropout                                 | Dropout                    | 0     \n",
            "107 | model.encoder.final_layer_norm                                        | T5LayerNorm                | 512   \n",
            "108 | model.encoder.dropout                                                 | Dropout                    | 0     \n",
            "109 | model.decoder                                                         | T5Stack                    | 41 M  \n",
            "110 | model.decoder.block                                                   | ModuleList                 | 25 M  \n",
            "111 | model.decoder.block.0                                                 | T5Block                    | 4 M   \n",
            "112 | model.decoder.block.0.layer                                           | ModuleList                 | 4 M   \n",
            "113 | model.decoder.block.0.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "114 | model.decoder.block.0.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "115 | model.decoder.block.0.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "116 | model.decoder.block.0.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "117 | model.decoder.block.0.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "118 | model.decoder.block.0.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "119 | model.decoder.block.0.layer.0.SelfAttention.relative_attention_bias   | Embedding                  | 256   \n",
            "120 | model.decoder.block.0.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "121 | model.decoder.block.0.layer.0.dropout                                 | Dropout                    | 0     \n",
            "122 | model.decoder.block.0.layer.1                                         | T5LayerCrossAttention      | 1 M   \n",
            "123 | model.decoder.block.0.layer.1.EncDecAttention                         | T5Attention                | 1 M   \n",
            "124 | model.decoder.block.0.layer.1.EncDecAttention.q                       | Linear                     | 262 K \n",
            "125 | model.decoder.block.0.layer.1.EncDecAttention.k                       | Linear                     | 262 K \n",
            "126 | model.decoder.block.0.layer.1.EncDecAttention.v                       | Linear                     | 262 K \n",
            "127 | model.decoder.block.0.layer.1.EncDecAttention.o                       | Linear                     | 262 K \n",
            "128 | model.decoder.block.0.layer.1.EncDecAttention.relative_attention_bias | Embedding                  | 256   \n",
            "129 | model.decoder.block.0.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "130 | model.decoder.block.0.layer.1.dropout                                 | Dropout                    | 0     \n",
            "131 | model.decoder.block.0.layer.2                                         | T5LayerFF                  | 2 M   \n",
            "132 | model.decoder.block.0.layer.2.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "133 | model.decoder.block.0.layer.2.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "134 | model.decoder.block.0.layer.2.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "135 | model.decoder.block.0.layer.2.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "136 | model.decoder.block.0.layer.2.layer_norm                              | T5LayerNorm                | 512   \n",
            "137 | model.decoder.block.0.layer.2.dropout                                 | Dropout                    | 0     \n",
            "138 | model.decoder.block.1                                                 | T5Block                    | 4 M   \n",
            "139 | model.decoder.block.1.layer                                           | ModuleList                 | 4 M   \n",
            "140 | model.decoder.block.1.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "141 | model.decoder.block.1.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "142 | model.decoder.block.1.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "143 | model.decoder.block.1.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "144 | model.decoder.block.1.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "145 | model.decoder.block.1.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "146 | model.decoder.block.1.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "147 | model.decoder.block.1.layer.0.dropout                                 | Dropout                    | 0     \n",
            "148 | model.decoder.block.1.layer.1                                         | T5LayerCrossAttention      | 1 M   \n",
            "149 | model.decoder.block.1.layer.1.EncDecAttention                         | T5Attention                | 1 M   \n",
            "150 | model.decoder.block.1.layer.1.EncDecAttention.q                       | Linear                     | 262 K \n",
            "151 | model.decoder.block.1.layer.1.EncDecAttention.k                       | Linear                     | 262 K \n",
            "152 | model.decoder.block.1.layer.1.EncDecAttention.v                       | Linear                     | 262 K \n",
            "153 | model.decoder.block.1.layer.1.EncDecAttention.o                       | Linear                     | 262 K \n",
            "154 | model.decoder.block.1.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "155 | model.decoder.block.1.layer.1.dropout                                 | Dropout                    | 0     \n",
            "156 | model.decoder.block.1.layer.2                                         | T5LayerFF                  | 2 M   \n",
            "157 | model.decoder.block.1.layer.2.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "158 | model.decoder.block.1.layer.2.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "159 | model.decoder.block.1.layer.2.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "160 | model.decoder.block.1.layer.2.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "161 | model.decoder.block.1.layer.2.layer_norm                              | T5LayerNorm                | 512   \n",
            "162 | model.decoder.block.1.layer.2.dropout                                 | Dropout                    | 0     \n",
            "163 | model.decoder.block.2                                                 | T5Block                    | 4 M   \n",
            "164 | model.decoder.block.2.layer                                           | ModuleList                 | 4 M   \n",
            "165 | model.decoder.block.2.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "166 | model.decoder.block.2.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "167 | model.decoder.block.2.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "168 | model.decoder.block.2.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "169 | model.decoder.block.2.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "170 | model.decoder.block.2.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "171 | model.decoder.block.2.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "172 | model.decoder.block.2.layer.0.dropout                                 | Dropout                    | 0     \n",
            "173 | model.decoder.block.2.layer.1                                         | T5LayerCrossAttention      | 1 M   \n",
            "174 | model.decoder.block.2.layer.1.EncDecAttention                         | T5Attention                | 1 M   \n",
            "175 | model.decoder.block.2.layer.1.EncDecAttention.q                       | Linear                     | 262 K \n",
            "176 | model.decoder.block.2.layer.1.EncDecAttention.k                       | Linear                     | 262 K \n",
            "177 | model.decoder.block.2.layer.1.EncDecAttention.v                       | Linear                     | 262 K \n",
            "178 | model.decoder.block.2.layer.1.EncDecAttention.o                       | Linear                     | 262 K \n",
            "179 | model.decoder.block.2.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "180 | model.decoder.block.2.layer.1.dropout                                 | Dropout                    | 0     \n",
            "181 | model.decoder.block.2.layer.2                                         | T5LayerFF                  | 2 M   \n",
            "182 | model.decoder.block.2.layer.2.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "183 | model.decoder.block.2.layer.2.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "184 | model.decoder.block.2.layer.2.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "185 | model.decoder.block.2.layer.2.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "186 | model.decoder.block.2.layer.2.layer_norm                              | T5LayerNorm                | 512   \n",
            "187 | model.decoder.block.2.layer.2.dropout                                 | Dropout                    | 0     \n",
            "188 | model.decoder.block.3                                                 | T5Block                    | 4 M   \n",
            "189 | model.decoder.block.3.layer                                           | ModuleList                 | 4 M   \n",
            "190 | model.decoder.block.3.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "191 | model.decoder.block.3.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "192 | model.decoder.block.3.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "193 | model.decoder.block.3.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "194 | model.decoder.block.3.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "195 | model.decoder.block.3.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "196 | model.decoder.block.3.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "197 | model.decoder.block.3.layer.0.dropout                                 | Dropout                    | 0     \n",
            "198 | model.decoder.block.3.layer.1                                         | T5LayerCrossAttention      | 1 M   \n",
            "199 | model.decoder.block.3.layer.1.EncDecAttention                         | T5Attention                | 1 M   \n",
            "200 | model.decoder.block.3.layer.1.EncDecAttention.q                       | Linear                     | 262 K \n",
            "201 | model.decoder.block.3.layer.1.EncDecAttention.k                       | Linear                     | 262 K \n",
            "202 | model.decoder.block.3.layer.1.EncDecAttention.v                       | Linear                     | 262 K \n",
            "203 | model.decoder.block.3.layer.1.EncDecAttention.o                       | Linear                     | 262 K \n",
            "204 | model.decoder.block.3.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "205 | model.decoder.block.3.layer.1.dropout                                 | Dropout                    | 0     \n",
            "206 | model.decoder.block.3.layer.2                                         | T5LayerFF                  | 2 M   \n",
            "207 | model.decoder.block.3.layer.2.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "208 | model.decoder.block.3.layer.2.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "209 | model.decoder.block.3.layer.2.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "210 | model.decoder.block.3.layer.2.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "211 | model.decoder.block.3.layer.2.layer_norm                              | T5LayerNorm                | 512   \n",
            "212 | model.decoder.block.3.layer.2.dropout                                 | Dropout                    | 0     \n",
            "213 | model.decoder.block.4                                                 | T5Block                    | 4 M   \n",
            "214 | model.decoder.block.4.layer                                           | ModuleList                 | 4 M   \n",
            "215 | model.decoder.block.4.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "216 | model.decoder.block.4.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "217 | model.decoder.block.4.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "218 | model.decoder.block.4.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "219 | model.decoder.block.4.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "220 | model.decoder.block.4.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "221 | model.decoder.block.4.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "222 | model.decoder.block.4.layer.0.dropout                                 | Dropout                    | 0     \n",
            "223 | model.decoder.block.4.layer.1                                         | T5LayerCrossAttention      | 1 M   \n",
            "224 | model.decoder.block.4.layer.1.EncDecAttention                         | T5Attention                | 1 M   \n",
            "225 | model.decoder.block.4.layer.1.EncDecAttention.q                       | Linear                     | 262 K \n",
            "226 | model.decoder.block.4.layer.1.EncDecAttention.k                       | Linear                     | 262 K \n",
            "227 | model.decoder.block.4.layer.1.EncDecAttention.v                       | Linear                     | 262 K \n",
            "228 | model.decoder.block.4.layer.1.EncDecAttention.o                       | Linear                     | 262 K \n",
            "229 | model.decoder.block.4.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "230 | model.decoder.block.4.layer.1.dropout                                 | Dropout                    | 0     \n",
            "231 | model.decoder.block.4.layer.2                                         | T5LayerFF                  | 2 M   \n",
            "232 | model.decoder.block.4.layer.2.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "233 | model.decoder.block.4.layer.2.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "234 | model.decoder.block.4.layer.2.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "235 | model.decoder.block.4.layer.2.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "236 | model.decoder.block.4.layer.2.layer_norm                              | T5LayerNorm                | 512   \n",
            "237 | model.decoder.block.4.layer.2.dropout                                 | Dropout                    | 0     \n",
            "238 | model.decoder.block.5                                                 | T5Block                    | 4 M   \n",
            "239 | model.decoder.block.5.layer                                           | ModuleList                 | 4 M   \n",
            "240 | model.decoder.block.5.layer.0                                         | T5LayerSelfAttention       | 1 M   \n",
            "241 | model.decoder.block.5.layer.0.SelfAttention                           | T5Attention                | 1 M   \n",
            "242 | model.decoder.block.5.layer.0.SelfAttention.q                         | Linear                     | 262 K \n",
            "243 | model.decoder.block.5.layer.0.SelfAttention.k                         | Linear                     | 262 K \n",
            "244 | model.decoder.block.5.layer.0.SelfAttention.v                         | Linear                     | 262 K \n",
            "245 | model.decoder.block.5.layer.0.SelfAttention.o                         | Linear                     | 262 K \n",
            "246 | model.decoder.block.5.layer.0.layer_norm                              | T5LayerNorm                | 512   \n",
            "247 | model.decoder.block.5.layer.0.dropout                                 | Dropout                    | 0     \n",
            "248 | model.decoder.block.5.layer.1                                         | T5LayerCrossAttention      | 1 M   \n",
            "249 | model.decoder.block.5.layer.1.EncDecAttention                         | T5Attention                | 1 M   \n",
            "250 | model.decoder.block.5.layer.1.EncDecAttention.q                       | Linear                     | 262 K \n",
            "251 | model.decoder.block.5.layer.1.EncDecAttention.k                       | Linear                     | 262 K \n",
            "252 | model.decoder.block.5.layer.1.EncDecAttention.v                       | Linear                     | 262 K \n",
            "253 | model.decoder.block.5.layer.1.EncDecAttention.o                       | Linear                     | 262 K \n",
            "254 | model.decoder.block.5.layer.1.layer_norm                              | T5LayerNorm                | 512   \n",
            "255 | model.decoder.block.5.layer.1.dropout                                 | Dropout                    | 0     \n",
            "256 | model.decoder.block.5.layer.2                                         | T5LayerFF                  | 2 M   \n",
            "257 | model.decoder.block.5.layer.2.DenseReluDense                          | T5DenseReluDense           | 2 M   \n",
            "258 | model.decoder.block.5.layer.2.DenseReluDense.wi                       | Linear                     | 1 M   \n",
            "259 | model.decoder.block.5.layer.2.DenseReluDense.wo                       | Linear                     | 1 M   \n",
            "260 | model.decoder.block.5.layer.2.DenseReluDense.dropout                  | Dropout                    | 0     \n",
            "261 | model.decoder.block.5.layer.2.layer_norm                              | T5LayerNorm                | 512   \n",
            "262 | model.decoder.block.5.layer.2.dropout                                 | Dropout                    | 0     \n",
            "263 | model.decoder.final_layer_norm                                        | T5LayerNorm                | 512   \n",
            "264 | model.decoder.dropout                                                 | Dropout                    | 0     \n",
            "265 | model.lm_head                                                         | Linear                     | 16 M  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "12a43958a9734db58fb3ab498bf0fe20",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c8f080e05fee4bec91047f367eb0be2d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-obOz6v70iB"
      },
      "source": [
        "!mkdir t5_base_paraphrase"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OQBJcrrWi2vC"
      },
      "source": [
        "## save the model this way so next time you can load it using T5ForConditionalGeneration.from_pretrained\n",
        "model.model.save_pretrained('t5_base_paraphrase/')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fwqqmAQwN7S_"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XhjELPOk7-cz"
      },
      "source": [
        "!cp -r t5_base_paraphrase drive/My\\ Drive/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLPV-uY04Auh"
      },
      "source": [
        "print('Saved model.')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}