Transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of transformer.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "s_qNSzzyaCbD"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5b60abf9c0e4499989c41e850fe59021": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_57d1d64fadde4a8f82147aba1b2a63ee",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8b26a02905074a8f8a00f02d5b666ea5",
              "IPY_MODEL_37305fbbe6e94ec4bf163f0e83421fba"
            ]
          }
        },
        "57d1d64fadde4a8f82147aba1b2a63ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8b26a02905074a8f8a00f02d5b666ea5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_56d7c15bf27746bd9765ae9eb737314f",
            "_dom_classes": [],
            "description": "Dl Completed...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b094a86a2d0461cb331c014d547ff5f"
          }
        },
        "37305fbbe6e94ec4bf163f0e83421fba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_641274a360f9479f9b3b4591d6f9fa28",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:10&lt;00:00, 10.20s/ url]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b00e0ce392034636b18ecbe1e90f3fad"
          }
        },
        "56d7c15bf27746bd9765ae9eb737314f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b094a86a2d0461cb331c014d547ff5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "641274a360f9479f9b3b4591d6f9fa28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b00e0ce392034636b18ecbe1e90f3fad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e8988239e934faaa9bdecf76ce68ebe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_892228a0ed4248c0bb5e20a6f38e7153",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ac10b872e2ae489a9a613d909090af92",
              "IPY_MODEL_2cbb22c7d38d4cbb9cd5603ccfa8b88f"
            ]
          }
        },
        "892228a0ed4248c0bb5e20a6f38e7153": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ac10b872e2ae489a9a613d909090af92": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a4c9a8a959184a52b39632125357ed88",
            "_dom_classes": [],
            "description": "Dl Size...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_68728995204446ffbfaaf595c5c3f7b1"
          }
        },
        "2cbb22c7d38d4cbb9cd5603ccfa8b88f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_0d913d2abf4b4201b40875d20ed7ca13",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 124/124 [00:10&lt;00:00, 12.22 MiB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_66cd0e2b8cb5489e89a97388caad68b8"
          }
        },
        "a4c9a8a959184a52b39632125357ed88": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "68728995204446ffbfaaf595c5c3f7b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d913d2abf4b4201b40875d20ed7ca13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "66cd0e2b8cb5489e89a97388caad68b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0a747599110e42629b9bd1e2be1f0b8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_936cca48e2944a9199d4173a9f9e2194",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b68e1963cf654ec7b93b305a9dcd2b90",
              "IPY_MODEL_ebc6fa617c1349c3ab9c2615009d9f29"
            ]
          }
        },
        "936cca48e2944a9199d4173a9f9e2194": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b68e1963cf654ec7b93b305a9dcd2b90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a73476be8ed44042b24829cec8b77831",
            "_dom_classes": [],
            "description": "Extraction completed...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_59ed3a3f7ecd49e287022030d9ca63da"
          }
        },
        "ebc6fa617c1349c3ab9c2615009d9f29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_06980653638a4ed3afda6752828f2f42",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:10&lt;00:00, 10.11s/ file]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2904933c6da543d4a34cb2b3fe3c83bb"
          }
        },
        "a73476be8ed44042b24829cec8b77831": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "59ed3a3f7ecd49e287022030d9ca63da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "06980653638a4ed3afda6752828f2f42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2904933c6da543d4a34cb2b3fe3c83bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "40125e3c3bed4b60a1212e94ef02ba1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_16ffda9c9abb46e89f36b0335b17469f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5e09407e32174daf9624645765d9a2d9",
              "IPY_MODEL_2aad3e01cb0444649611e8e8d956c925"
            ]
          }
        },
        "16ffda9c9abb46e89f36b0335b17469f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e09407e32174daf9624645765d9a2d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_69f6f81101b44d16a3253a638722bf27",
            "_dom_classes": [],
            "description": "Generating splits...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 3,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 3,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2c7ab2a846f04f9ba0c309bda06b09cf"
          }
        },
        "2aad3e01cb0444649611e8e8d956c925": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_cc80b6d7c59749dbba988a5da25cfef2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3/3 [00:11&lt;00:00,  5.57s/ splits]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7971baf42cd54841ad96b3f5a91aa0d7"
          }
        },
        "69f6f81101b44d16a3253a638722bf27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2c7ab2a846f04f9ba0c309bda06b09cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "cc80b6d7c59749dbba988a5da25cfef2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7971baf42cd54841ad96b3f5a91aa0d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b244214e2eee4223a591aa643a1a28d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3d1e3cc5ec0a42c0949c12880fa73808",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1f28dd8d86664deb855cf4256b04cb72",
              "IPY_MODEL_76991de3ac054540a1b8a9c584fc0a58"
            ]
          }
        },
        "3d1e3cc5ec0a42c0949c12880fa73808": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1f28dd8d86664deb855cf4256b04cb72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bf01a764298944c48ccd3c3bef7446ab",
            "_dom_classes": [],
            "description": "Generating train examples...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 51785,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 51785,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ec27db42c2ce4fa38acb3b6ea4db8811"
          }
        },
        "76991de3ac054540a1b8a9c584fc0a58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c94634a59b3c46a3975a649e54f9401c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 51785/51785 [00:10&lt;00:00, 5896.85 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7cd4a0851a65416a95688c0867c52015"
          }
        },
        "bf01a764298944c48ccd3c3bef7446ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ec27db42c2ce4fa38acb3b6ea4db8811": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c94634a59b3c46a3975a649e54f9401c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7cd4a0851a65416a95688c0867c52015": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "79205fd698d240f3aa389aecbc09b675": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_22a5b49cb3ff4b37b07257e05dfe1566",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2829b35524304712ba4b594b2dece8cf",
              "IPY_MODEL_40b105d3223e4a8981b6132ce5201155"
            ]
          }
        },
        "22a5b49cb3ff4b37b07257e05dfe1566": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2829b35524304712ba4b594b2dece8cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5c4490e2f336405b89f4b4ef8baaeafe",
            "_dom_classes": [],
            "description": "Shuffling ted_hrlr_translate-train.tfrecord...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 51785,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 51785,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ac23aabb298049f9859a4929b40fbdc3"
          }
        },
        "40b105d3223e4a8981b6132ce5201155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6f9acfe519004eac9ddb8b3c88caa75e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 51785/51785 [00:00&lt;00:00, 16919.41 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e055a21b74eb4c10979361f6eb07ee42"
          }
        },
        "5c4490e2f336405b89f4b4ef8baaeafe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ac23aabb298049f9859a4929b40fbdc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f9acfe519004eac9ddb8b3c88caa75e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e055a21b74eb4c10979361f6eb07ee42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "31d4b67b974c4d9c957f9ff6b390025c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_424a4334315c4c1a8bf8128cdcca6140",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7d81bdbd6953446099673062680c67f0",
              "IPY_MODEL_031654e054ab4b5aacc732dd06661cbe"
            ]
          }
        },
        "424a4334315c4c1a8bf8128cdcca6140": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d81bdbd6953446099673062680c67f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0499e83b4ec7470fa5ae69f8ce416719",
            "_dom_classes": [],
            "description": "Generating validation examples...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1193,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1193,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2f121988b33e4e869b45f29958d8b928"
          }
        },
        "031654e054ab4b5aacc732dd06661cbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d6167913c6f04461aae9aa53efef118f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1193/1193 [00:00&lt;00:00, 2961.05 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e825a514d69b4166b6892a7c7af56af0"
          }
        },
        "0499e83b4ec7470fa5ae69f8ce416719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2f121988b33e4e869b45f29958d8b928": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d6167913c6f04461aae9aa53efef118f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e825a514d69b4166b6892a7c7af56af0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7c992164de204993a7d967f2ccd10f44": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_910ca05795ba4cfebcab67a432c10f0f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a28f693750aa47ed89647739c5f0e270",
              "IPY_MODEL_880ee75ba9b94ca2b2d5d7866a698011"
            ]
          }
        },
        "910ca05795ba4cfebcab67a432c10f0f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a28f693750aa47ed89647739c5f0e270": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3123d3a5e78f4e0f9d72a4c315508304",
            "_dom_classes": [],
            "description": "Shuffling ted_hrlr_translate-validation.tfrecord...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1193,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1193,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b707695d81844963b6c4efb35a12de86"
          }
        },
        "880ee75ba9b94ca2b2d5d7866a698011": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dcccbf6d2a6f406092cee2d9bef9ebdf",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1193/1193 [00:00&lt;00:00, 34094.92 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f6e89caf78e48f98daf0da1a1cb13ab"
          }
        },
        "3123d3a5e78f4e0f9d72a4c315508304": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b707695d81844963b6c4efb35a12de86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dcccbf6d2a6f406092cee2d9bef9ebdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f6e89caf78e48f98daf0da1a1cb13ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b1c9247d3582444db79239ad9ebcfbcc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_92273eb703de46b9bac2e648fd12a5cc",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_757c08d314ff4d73861eef146b095ef4",
              "IPY_MODEL_5f050f9a08504f98b0519d49f59e47d6"
            ]
          }
        },
        "92273eb703de46b9bac2e648fd12a5cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "757c08d314ff4d73861eef146b095ef4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6791956cbe8a46d29ea5b09da6cc3687",
            "_dom_classes": [],
            "description": "Generating test examples...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1803,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1803,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_67fc5100b0de4721b516e744b8275918"
          }
        },
        "5f050f9a08504f98b0519d49f59e47d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d88919b31a9941babf3d9fa084acc337",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1803/1803 [00:00&lt;00:00, 3230.79 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_378cf12b2620493183410125b78fce46"
          }
        },
        "6791956cbe8a46d29ea5b09da6cc3687": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "67fc5100b0de4721b516e744b8275918": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d88919b31a9941babf3d9fa084acc337": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "378cf12b2620493183410125b78fce46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "34df074c29a5473e93554ec8d41257a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d2d6a2d1dc7e442683b7ff6ab5e6a0b0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_956a78a0145e4871a0b4c9b8c363b2a1",
              "IPY_MODEL_286a4f08514b41678756834da3580857"
            ]
          }
        },
        "d2d6a2d1dc7e442683b7ff6ab5e6a0b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "956a78a0145e4871a0b4c9b8c363b2a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_da1c4073f5224b5c8db8dd056710eb5c",
            "_dom_classes": [],
            "description": "Shuffling ted_hrlr_translate-test.tfrecord...: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "",
            "max": 1803,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1803,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6b4399fe23d54fbca25d0a70e68f9294"
          }
        },
        "286a4f08514b41678756834da3580857": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_271b4c503a5c4f67a22257443e889596",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1803/1803 [00:00&lt;00:00, 56384.39 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5aeccb95e95f4caf8589e1e44f55dd4c"
          }
        },
        "da1c4073f5224b5c8db8dd056710eb5c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6b4399fe23d54fbca25d0a70e68f9294": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "271b4c503a5c4f67a22257443e889596": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5aeccb95e95f4caf8589e1e44f55dd4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/UtpalMattoo/NLP-Attention-Transformers/blob/master/Transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s_qNSzzyaCbD"
      },
      "source": [
        "##### Copyright 2019 The TensorFlow Authors."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "cellView": "form",
        "id": "jmjh290raIky"
      },
      "source": [
        "#@title Licensed under the Apache License, Version 2.0 (the \"License\");\n",
        "# you may not use this file except in compliance with the License.\n",
        "# You may obtain a copy of the License at\n",
        "#\n",
        "# https://www.apache.org/licenses/LICENSE-2.0\n",
        "#\n",
        "# Unless required by applicable law or agreed to in writing, software\n",
        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
        "# See the License for the specific language governing permissions and\n",
        "# limitations under the License."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J0Qjg6vuaHNt"
      },
      "source": [
        "# Transformer model for language understanding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOpGoE2T-YXS"
      },
      "source": [
        "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/transformer\">\n",
        "    <img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />\n",
        "    View on TensorFlow.org</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/transformer.ipynb\">\n",
        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />\n",
        "    Run in Google Colab</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/transformer.ipynb\">\n",
        "    <img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />\n",
        "    View source on GitHub</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/transformer.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
        "  </td>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M-f8TnGpE_ex"
      },
      "source": [
        "This tutorial trains a <a href=\"https://arxiv.org/abs/1706.03762\" class=\"external\">Transformer model</a> to translate Portuguese to English. This is an advanced example that assumes knowledge of [text generation](text_generation.ipynb) and [attention](nmt_with_attention.ipynb).\n",
        "\n",
        "The core idea behind the Transformer model is *self-attention*—the ability to attend to different positions of the input sequence to compute a representation of that sequence. Transformer creates stacks of self-attention layers and is explained below in the sections *Scaled dot product attention* and *Multi-head attention*.\n",
        "\n",
        "A transformer model handles variable-sized input using stacks of self-attention layers instead of [RNNs](text_classification_rnn.ipynb) or [CNNs](../images/intro_to_cnns.ipynb). This general architecture has a number of advantages:\n",
        "\n",
        "* It make no assumptions about the temporal/spatial relationships across the data. This is ideal for processing a set of objects (for example, [StarCraft units](https://deepmind.com/blog/alphastar-mastering-real-time-strategy-game-starcraft-ii/#block-8)).\n",
        "* Layer outputs can be calculated in parallel, instead of a series like an RNN.\n",
        "* Distant items can affect each other's output without passing through many RNN-steps, or convolution layers (see [Scene Memory Transformer](https://arxiv.org/pdf/1903.03878.pdf) for example).\n",
        "* It can learn long-range dependencies. This is a challenge in many sequence tasks.\n",
        "\n",
        "The downsides of this architecture are:\n",
        "\n",
        "* For a time-series, the output for a time-step is calculated from the *entire history* instead of only the inputs and current hidden-state. This _may_ be less efficient.   \n",
        "* If the input *does* have a  temporal/spatial relationship, like text, some positional encoding must be added or the model will effectively see a bag of words. \n",
        "\n",
        "After training the model in this notebook, you will be able to input a Portuguese sentence and return the English translation.\n",
        "\n",
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/attention_map_portuguese.png\" width=\"800\" alt=\"Attention heatmap\">"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XFG0NDRu5mYQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1ab67038-d822-43a7-a9b3-c65afa359af0"
      },
      "source": [
        "!pip install -q tfds-nightly\n",
        "\n",
        "# Pin matplotlib version to 3.2.2 since in the latest version\n",
        "# transformer.ipynb fails with the following error:\n",
        "# https://stackoverflow.com/questions/62953704/valueerror-the-number-of-fixedlocator-locations-5-usually-from-a-call-to-set\n",
        "!pip install matplotlib==3.2.2"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 3.7MB 12.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib==3.2.2 in /usr/local/lib/python3.6/dist-packages (3.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (1.3.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (1.19.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (2.4.7)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib==3.2.2) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dKSOOalvbhhs"
      },
      "source": [
        "# tf.data: Build TensorFlow input pipelines\n",
        "# https://www.tensorflow.org/guide/data\n",
        "# https://www.tensorflow.org/api_docs/python/tf/data/experimental\n"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjJJyJTZYebt"
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "import tensorflow as tf\n",
        "\n",
        "import time\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fd1NWMxjfsDd"
      },
      "source": [
        "## Setup input pipeline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t4_Qt8W1hJE_"
      },
      "source": [
        "Use [TFDS](https://www.tensorflow.org/datasets) to load the [Portugese-English translation dataset](https://github.com/neulab/word-embeddings-for-nmt) from the [TED Talks Open Translation Project](https://www.ted.com/participate/translate).\n",
        "\n",
        "This dataset contains approximately 50000 training examples, 1100 validation examples, and 2000 test examples."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8q9t4FmN96eN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 249,
          "referenced_widgets": [
            "5b60abf9c0e4499989c41e850fe59021",
            "57d1d64fadde4a8f82147aba1b2a63ee",
            "8b26a02905074a8f8a00f02d5b666ea5",
            "37305fbbe6e94ec4bf163f0e83421fba",
            "56d7c15bf27746bd9765ae9eb737314f",
            "5b094a86a2d0461cb331c014d547ff5f",
            "641274a360f9479f9b3b4591d6f9fa28",
            "b00e0ce392034636b18ecbe1e90f3fad",
            "9e8988239e934faaa9bdecf76ce68ebe",
            "892228a0ed4248c0bb5e20a6f38e7153",
            "ac10b872e2ae489a9a613d909090af92",
            "2cbb22c7d38d4cbb9cd5603ccfa8b88f",
            "a4c9a8a959184a52b39632125357ed88",
            "68728995204446ffbfaaf595c5c3f7b1",
            "0d913d2abf4b4201b40875d20ed7ca13",
            "66cd0e2b8cb5489e89a97388caad68b8",
            "0a747599110e42629b9bd1e2be1f0b8f",
            "936cca48e2944a9199d4173a9f9e2194",
            "b68e1963cf654ec7b93b305a9dcd2b90",
            "ebc6fa617c1349c3ab9c2615009d9f29",
            "a73476be8ed44042b24829cec8b77831",
            "59ed3a3f7ecd49e287022030d9ca63da",
            "06980653638a4ed3afda6752828f2f42",
            "2904933c6da543d4a34cb2b3fe3c83bb",
            "40125e3c3bed4b60a1212e94ef02ba1a",
            "16ffda9c9abb46e89f36b0335b17469f",
            "5e09407e32174daf9624645765d9a2d9",
            "2aad3e01cb0444649611e8e8d956c925",
            "69f6f81101b44d16a3253a638722bf27",
            "2c7ab2a846f04f9ba0c309bda06b09cf",
            "cc80b6d7c59749dbba988a5da25cfef2",
            "7971baf42cd54841ad96b3f5a91aa0d7",
            "b244214e2eee4223a591aa643a1a28d1",
            "3d1e3cc5ec0a42c0949c12880fa73808",
            "1f28dd8d86664deb855cf4256b04cb72",
            "76991de3ac054540a1b8a9c584fc0a58",
            "bf01a764298944c48ccd3c3bef7446ab",
            "ec27db42c2ce4fa38acb3b6ea4db8811",
            "c94634a59b3c46a3975a649e54f9401c",
            "7cd4a0851a65416a95688c0867c52015",
            "79205fd698d240f3aa389aecbc09b675",
            "22a5b49cb3ff4b37b07257e05dfe1566",
            "2829b35524304712ba4b594b2dece8cf",
            "40b105d3223e4a8981b6132ce5201155",
            "5c4490e2f336405b89f4b4ef8baaeafe",
            "ac23aabb298049f9859a4929b40fbdc3",
            "6f9acfe519004eac9ddb8b3c88caa75e",
            "e055a21b74eb4c10979361f6eb07ee42",
            "31d4b67b974c4d9c957f9ff6b390025c",
            "424a4334315c4c1a8bf8128cdcca6140",
            "7d81bdbd6953446099673062680c67f0",
            "031654e054ab4b5aacc732dd06661cbe",
            "0499e83b4ec7470fa5ae69f8ce416719",
            "2f121988b33e4e869b45f29958d8b928",
            "d6167913c6f04461aae9aa53efef118f",
            "e825a514d69b4166b6892a7c7af56af0",
            "7c992164de204993a7d967f2ccd10f44",
            "910ca05795ba4cfebcab67a432c10f0f",
            "a28f693750aa47ed89647739c5f0e270",
            "880ee75ba9b94ca2b2d5d7866a698011",
            "3123d3a5e78f4e0f9d72a4c315508304",
            "b707695d81844963b6c4efb35a12de86",
            "dcccbf6d2a6f406092cee2d9bef9ebdf",
            "6f6e89caf78e48f98daf0da1a1cb13ab",
            "b1c9247d3582444db79239ad9ebcfbcc",
            "92273eb703de46b9bac2e648fd12a5cc",
            "757c08d314ff4d73861eef146b095ef4",
            "5f050f9a08504f98b0519d49f59e47d6",
            "6791956cbe8a46d29ea5b09da6cc3687",
            "67fc5100b0de4721b516e744b8275918",
            "d88919b31a9941babf3d9fa084acc337",
            "378cf12b2620493183410125b78fce46",
            "34df074c29a5473e93554ec8d41257a9",
            "d2d6a2d1dc7e442683b7ff6ab5e6a0b0",
            "956a78a0145e4871a0b4c9b8c363b2a1",
            "286a4f08514b41678756834da3580857",
            "da1c4073f5224b5c8db8dd056710eb5c",
            "6b4399fe23d54fbca25d0a70e68f9294",
            "271b4c503a5c4f67a22257443e889596",
            "5aeccb95e95f4caf8589e1e44f55dd4c"
          ]
        },
        "outputId": "6e858ecd-cb5c-4a9c-a76e-24fe2572e9fc"
      },
      "source": [
        "examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True,\n",
        "                               as_supervised=True)\n",
        "train_examples, val_examples = examples['train'], examples['validation']"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[1mDownloading and preparing dataset 124.94 MiB (download: 124.94 MiB, generated: Unknown size, total: 124.94 MiB) to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0...\u001b[0m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5b60abf9c0e4499989c41e850fe59021",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Dl Completed...', max=1.0, style=Progre…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9e8988239e934faaa9bdecf76ce68ebe",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Dl Size...', max=1.0, style=ProgressSty…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0a747599110e42629b9bd1e2be1f0b8f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Extraction completed...', max=1.0, styl…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "40125e3c3bed4b60a1212e94ef02ba1a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Generating splits...', max=3.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b244214e2eee4223a591aa643a1a28d1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Generating train examples...', max=51785.0, style=Progres…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "79205fd698d240f3aa389aecbc09b675",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Shuffling ted_hrlr_translate-train.tfrecord...', max=5178…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "31d4b67b974c4d9c957f9ff6b390025c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Generating validation examples...', max=1193.0, style=Pro…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7c992164de204993a7d967f2ccd10f44",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Shuffling ted_hrlr_translate-validation.tfrecord...', max…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b1c9247d3582444db79239ad9ebcfbcc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Generating test examples...', max=1803.0, style=ProgressS…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "34df074c29a5473e93554ec8d41257a9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Shuffling ted_hrlr_translate-test.tfrecord...', max=1803.…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\r\u001b[1mDataset ted_hrlr_translate downloaded and prepared to /root/tensorflow_datasets/ted_hrlr_translate/pt_to_en/1.0.0. Subsequent calls will reuse this data.\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RCEKotqosGfq"
      },
      "source": [
        "Create a custom subwords tokenizer from the training dataset. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVBg5Q8tBk5z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cc6c2854-9cde-4774-c271-eda92f6a0e0e"
      },
      "source": [
        "# https://www.tensorflow.org/datasets/api_docs/python/tfds/deprecated/text/SubwordTextEncoder#build_from_corpus\n",
        "# https://colab.research.google.com/github/tensorflow/examples/blob/master/courses/udacity_intro_to_tensorflow_for_deep_learning/l10c01_nlp_lstms_with_reviews_subwords_dataset.ipynb#scrollTo=33AthPiALFZK\n",
        "# SubwordTextEncoder.build_from_corpus() will create a tokenizer for us. You could also use this \n",
        "# functionality to get subwords from a much larger corpus of text as well\n",
        "\n",
        "tokenizer_en = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
        "    (en.numpy() for pt, en in train_examples), target_vocab_size=2**13)\n",
        "\n",
        "tokenizer_pt = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
        "    (pt.numpy() for pt, en in train_examples), target_vocab_size=2**13)\n",
        "\n",
        "print (type(tokenizer_en))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'tensorflow_datasets.core.deprecated.text.subword_text_encoder.SubwordTextEncoder'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sV3rRQsMBues",
        "outputId": "1f88de7d-697d-4041-beea-dd277922a010"
      },
      "source": [
        "i=11\n",
        "j=0\n",
        "for data in train_examples:\n",
        "  if (j < i):\n",
        "    a, b = data\n",
        "    print (f\"a is: {a} and \\n b is: {b}\\n\") #a and b are byte strings\n",
        "    j+=1"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "a is: b'e quando melhoramos a procura , tiramos a \\xc3\\xbanica vantagem da impress\\xc3\\xa3o , que \\xc3\\xa9 a serendipidade .' and \n",
            " b is: b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .'\n",
            "\n",
            "a is: b'mas e se estes fatores fossem ativos ?' and \n",
            " b is: b'but what if it were active ?'\n",
            "\n",
            "a is: b'mas eles n\\xc3\\xa3o tinham a curiosidade de me testar .' and \n",
            " b is: b\"but they did n't test for curiosity .\"\n",
            "\n",
            "a is: b'e esta rebeldia consciente \\xc3\\xa9 a raz\\xc3\\xa3o pela qual eu , como agn\\xc3\\xb3stica , posso ainda ter f\\xc3\\xa9 .' and \n",
            " b is: b'and this conscious defiance is why i , as an agnostic , can still have faith .'\n",
            "\n",
            "a is: b\"`` `` '' podem usar tudo sobre a mesa no meu corpo . ''\" and \n",
            " b is: b'you can use everything on the table on me .'\n",
            "\n",
            "a is: b\"`` eu escrevo muito acerca do `` '' teatro de seguran\\xc3\\xa7a '' '' , que s\\xc3\\xa3o produtos que fazem as pessoas sentirem-se seguras mas que , na realidade , n\\xc3\\xa3o fazem nada . ''\" and \n",
            " b is: b\"`` i write a lot about `` '' security theater , '' '' which are products that make people feel secure , but do n't actually do anything . ''\"\n",
            "\n",
            "a is: b'colocaram-no bem no fundo duma mina de ferro no minnesota , nos \\xc3\\xbaltimos dois dias anunciaram os resultados mais sens\\xc3\\xadveis at\\xc3\\xa9 agora .' and \n",
            " b is: b\"and they 've put it deep down in an iron mine in minnesota , ok , deep under the ground , and in fact , in the last couple of days announced the most sensitive results so far .\"\n",
            "\n",
            "a is: b'algumas pessoas t\\xc3\\xaam medo de que n\\xc3\\xa3o gostem delas .' and \n",
            " b is: b'see , some people might fear girls not liking them back .'\n",
            "\n",
            "a is: b'n\\xc3\\xa3o , o que nos aconteceu , chris , \\xc3\\xa9 que o poder , o pre\\xc3\\xa7o est\\xc3\\xa1 fixado fora da margem .' and \n",
            " b is: b\"no , what happened to us , chris , is that power , it 's priced off the margin .\"\n",
            "\n",
            "a is: b'de volta \\xc3\\xa0 minha pergunta : porque \\xc3\\xa9 que fiquei ?' and \n",
            " b is: b'back to my question : why did i stay ?'\n",
            "\n",
            "a is: b\"`` quando descobri que havia uma s\\xc3\\xa9rie de ind\\xc3\\xbastrias sem prote\\xc3\\xa7\\xc3\\xa3o de direitos de autor , pensei : `` '' qual ser\\xc3\\xa1 a l\\xc3\\xb3gica subjacente ? ''\" and \n",
            " b is: b\"`` when i found out that there are a whole bunch of industries that did n't have copyright protection , i thought , `` '' what exactly is the underlying logic ? ''\"\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DYWukNFkGQN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3d7a4a4f-6b4a-4391-ae79-13d6f420ff28"
      },
      "source": [
        "sample_string = 'did they eat fish and chips ?'\n",
        "\n",
        "tokenized_string = tokenizer_en.encode(sample_string)\n",
        "print ('Tokenized string is {}'.format(tokenized_string))\n",
        "\n",
        "original_string = tokenizer_en.decode(tokenized_string)\n",
        "print ('The original string: {}'.format(original_string))\n",
        "\n",
        "assert original_string == sample_string"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tokenized string is [98, 25, 1037, 1903, 4, 7283, 33]\n",
            "The original string: did they eat fish and chips ?\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9KJWJjrsZ4Y"
      },
      "source": [
        "The tokenizer encodes the string by breaking it into subwords if the word is not in its dictionary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf2ntBxjkqK6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "335758f2-80f4-434c-bdf6-312cc30345b4"
      },
      "source": [
        "for ts in tokenized_string:\n",
        "  print ('{} ----> {}'.format(ts, tokenizer_en.decode([ts])))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "98 ----> did \n",
            "25 ----> they \n",
            "1037 ----> eat \n",
            "1903 ----> fish \n",
            "4 ----> and \n",
            "7283 ----> chips\n",
            "33 ---->  ?\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcRp7VcQ5m6g"
      },
      "source": [
        "BUFFER_SIZE = 20000\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kGi4PoVakxdc"
      },
      "source": [
        "Add a start and end token to the input and target. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZwnPr4R055s"
      },
      "source": [
        "def encode(lang1, lang2):\n",
        "\n",
        "  lst1 = [tokenizer_pt.vocab_size]\n",
        "  lst2 = [tokenizer_en.vocab_size]\n",
        "\n",
        "  print (f\"Protugese source sentence - = {lang1.numpy()}\")\n",
        "  lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(\n",
        "      lang1.numpy()) + [tokenizer_pt.vocab_size+1]\n",
        "  print (f\"Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = {lang1}\\n\")\n",
        "  \n",
        "  print (f\"English source sentence - = {lang2.numpy()}\")\n",
        "  lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(\n",
        "      lang2.numpy()) + [tokenizer_en.vocab_size+1]\n",
        "  print (f\"English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = {lang2}\\n\")\n",
        "  \n",
        "  return lang1, lang2"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tx1sFbR-9fRs"
      },
      "source": [
        "You want to use `Dataset.map` to apply this function to each element of the dataset.  `Dataset.map` runs in graph mode.\n",
        "\n",
        "* Graph tensors do not have a value. \n",
        "* In graph mode you can only use TensorFlow Ops and functions. \n",
        "\n",
        "So you can't `.map` this function directly: You need to wrap it in a `tf.py_function`. The `tf.py_function` will pass regular tensors (with a value and a `.numpy()` method to access it), to the wrapped python function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mah1cS-P70Iz"
      },
      "source": [
        "def tf_encode(pt, en):\n",
        "  result_pt, result_en = tf.py_function(encode, [pt, en], [tf.int64, tf.int64])\n",
        "  result_pt.set_shape([None])\n",
        "  result_en.set_shape([None])\n",
        "\n",
        "  return result_pt, result_en"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6JrGp5Gek6Ql"
      },
      "source": [
        "Note: To keep this example small and relatively fast, drop examples with a length of over 40 tokens."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2QEgbjntk6Yf"
      },
      "source": [
        "MAX_LENGTH = 40"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c081xPGv1CPI"
      },
      "source": [
        "def filter_max_length(x, y, max_length=MAX_LENGTH):\n",
        "  return tf.logical_and(tf.size(x) <= max_length,\n",
        "                        tf.size(y) <= max_length)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9mk9AZdZ5bcS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7b1ada57-1e39-4be5-e1c4-26ba76ae459f"
      },
      "source": [
        "# https://www.tensorflow.org/api_docs/python/tf/data/Dataset\n",
        "# https://www.tensorflow.org/tutorials/load_data/numpy\n",
        "# https://stackoverflow.com/questions/62436302/accessing-data-in-tensorflow-prefetchdataset\n",
        "\n",
        "\n",
        "print (f\"number of training examples: {len(train_examples)}\")\n",
        "print (f\"type of training examples: {type(train_examples)}\\n\") # <class 'tensorflow.python.data.ops.dataset_ops.PrefetchDataset'>\n",
        "lst = list(train_examples) # convert to list and then to tf.data.Dataset.from_tensor_slices\n",
        "tf_slices_train_examples = tf.data.Dataset.from_tensor_slices(lst)\n",
        "#Finally, access tensor elements in tensor of type tf.data.Dataset.from_tensor_slices\n",
        "for example_no, source_target in enumerate(tf_slices_train_examples.take(3)): \n",
        "  print (f\"example_no is: {example_no} and source_target is: {source_target}\\n\")\n",
        "\n",
        "train_dataset = train_examples.map(tf_encode)\n",
        "# train_dataset = tf_slices_train_examples.map(tf_encode) - Fix: TO DO - not really needed\n",
        "print (f\"number of training examples: {len(train_dataset)} - type of training examples: {type(train_dataset)}\\n\")\n",
        "\n",
        "train_dataset = train_dataset.filter(filter_max_length)\n",
        "# cache the dataset to memory to get a speedup while reading from it.\n",
        "train_dataset = train_dataset.cache()\n",
        "train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(BATCH_SIZE)\n",
        "train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "val_dataset = val_examples.map(tf_encode)\n",
        "val_dataset = val_dataset.filter(filter_max_length).padded_batch(BATCH_SIZE)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "number of training examples: 51785\n",
            "type of training examples: <class 'tensorflow.python.data.ops.dataset_ops.PrefetchDataset'>\n",
            "\n",
            "example_no is: 0 and source_target is: [b'e quando melhoramos a procura , tiramos a \\xc3\\xbanica vantagem da impress\\xc3\\xa3o , que \\xc3\\xa9 a serendipidade .'\n",
            " b'and when you improve searchability , you actually take away the one advantage of print , which is serendipity .']\n",
            "\n",
            "example_no is: 1 and source_target is: [b'mas e se estes fatores fossem ativos ?' b'but what if it were active ?']\n",
            "\n",
            "example_no is: 2 and source_target is: [b'mas eles n\\xc3\\xa3o tinham a curiosidade de me testar .'\n",
            " b\"but they did n't test for curiosity .\"]\n",
            "\n",
            "number of training examples: 51785 - type of training examples: <class 'tensorflow.python.data.ops.dataset_ops.MapDataset'>\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fXvfYVfQr2n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b88571e-123a-4b77-cf8f-e095b147b730"
      },
      "source": [
        "pt_batch, en_batch = next(iter(val_dataset))\n",
        "pt_batch, en_batch"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Protugese source sentence - = b'tinham comido peixe com batatas fritas ?'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 342, 3032, 15, 5917, 20, 4347, 1189, 17, 1784, 1412, 29, 8215]\n",
            "\n",
            "English source sentence - = b'did they eat fish and chips ?'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 98, 25, 1037, 1903, 4, 7283, 33, 8088]\n",
            "\n",
            "Protugese source sentence - = b'estava sempre preocupado em ser apanhado e enviado de volta .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 95, 198, 6850, 7990, 14, 44, 6732, 7, 6, 2211, 15, 4, 555, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i was always worried about being caught and sent back .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 20, 260, 5796, 35, 183, 2955, 4, 1565, 380, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'escolhi um com o tom de pele de uma lagosta com um escald\\xc3\\xa3o .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 4479, 7990, 9, 20, 7, 5611, 4, 4416, 4, 10, 4622, 2878, 20, 9, 1348, 2682, 97, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i chose one with the skin color of a lobster when sunburnt .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 5453, 47, 28, 3, 3176, 1960, 6, 7, 3730, 1560, 7863, 59, 1148, 3254, 7947, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'mas eu penso que isto \\xc3\\xa9 claramente falso .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 25, 27, 278, 5, 42, 8, 3850, 3308, 8069, 2, 8215]\n",
            "\n",
            "English source sentence - = b'but i think this is quite clearly untrue .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 23, 12, 84, 16, 13, 367, 1917, 1923, 1071, 7932, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'temos medido o nosso progresso muito rigorosamente .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 62, 1181, 15, 7, 122, 3173, 7990, 33, 3264, 2745, 5868, 585, 2, 8215]\n",
            "\n",
            "English source sentence - = b'we have measured our progress very rigorously .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 14, 24, 5938, 43, 5067, 68, 4573, 7524, 171, 167, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e , na minha opini\\xc3\\xa3o , \\xc3\\xa9 uma cura n\\xc3\\xa3o s\\xc3\\xb3 para mim , mas para toda a gente .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 8059, 1, 24, 69, 7276, 1, 8, 10, 6346, 13, 105, 11, 297, 1, 25, 11, 166, 3, 2002, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"and from what i feel , it 's a cure for me , but for us all .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 48, 29, 12, 3483, 1, 19, 8, 9, 7, 3373, 21, 131, 1, 23, 21, 93, 287, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'\\xc3\\xa9 um trabalho em desenvolvimento de uma hist\\xc3\\xb3ria pessoal para uma hist\\xc3\\xb3ria global .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 8, 9, 259, 14, 1248, 4, 10, 249, 3067, 11, 10, 249, 927, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"it 's a work in progress from a personal story to a global history .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 19, 8, 9, 7, 187, 11, 5067, 48, 7, 791, 360, 5, 7, 350, 843, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'quero dizer , \\xc3\\xa9 uma proposta perdedora .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 167, 210, 1, 8, 10, 6844, 7990, 4073, 4310, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"i mean , it 's just a losing proposition .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 333, 1, 19, 8, 9, 54, 7, 4613, 5891, 4052, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'ent\\xc3\\xa3o , como \\xc3\\xa9 que temos estas conversas mais facilmente e mais vezes ?'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 119, 1, 21, 8, 5, 62, 128, 6350, 26, 6984, 6, 26, 413, 29, 8215]\n",
            "\n",
            "English source sentence - = b'so , how do we have these conversations more easily and more often ?'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 169, 1, 61, 30, 14, 24, 58, 6414, 67, 3489, 4, 67, 2685, 33, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e emitam certificados falsos .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 5490, 90, 4877, 36, 6299, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and issue rogue certificates .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 2039, 6573, 4994, 7760, 7293, 7946, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b\"`` ent\\xc3\\xa3o e se n\\xc3\\xb3s come\\xc3\\xa7\\xc3\\xa1ssemos a usar a teoria dos cinco sentidos em todos os nossos `` '' designs '' '' ? ''\"\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 32, 77, 6, 16, 64, 1236, 6762, 1317, 3, 258, 3, 1379, 36, 460, 1161, 17, 14, 63, 12, 7289, 8073, 51, 1036, 8073, 6119, 8215]\n",
            "\n",
            "English source sentence - = b'so what if we started using the five senses theory in all of our designs ?'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 18, 29, 44, 14, 198, 283, 3, 341, 920, 9, 1650, 11, 39, 6, 43, 1132, 7946, 33, 8088]\n",
            "\n",
            "Protugese source sentence - = b'tba \\xe2\\x80\\x94 para anunciar .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 8074, 866, 58, 11, 5104, 281, 2, 8215]\n",
            "\n",
            "English source sentence - = b'tba , to be announced .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 7947, 1225, 1, 5, 34, 7808, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'a estrutura tem apenas cinco metros de altura e sete andares .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 3, 1035, 70, 80, 460, 2037, 4, 760, 6, 1160, 2945, 181, 2, 8215]\n",
            "\n",
            "English source sentence - = b'the building is only five meters tall with seven floors .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 3, 440, 13, 122, 341, 2473, 3278, 7863, 28, 860, 2776, 7946, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'ok , se existe algo mais estranho penso que tem que ser a energia negra .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 368, 1, 16, 399, 121, 26, 4127, 278, 5, 70, 5, 44, 3, 423, 2300, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"okay , if there 's anything stranger i think it has to be dark energy .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 326, 1, 44, 79, 8, 9, 544, 3672, 7863, 12, 84, 17, 86, 5, 34, 1046, 856, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'trabalhei para a guarda costeira , fui embaixador da boa vontade da onu .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 5144, 11, 3, 1388, 7990, 1235, 1399, 366, 1, 385, 3827, 3000, 4705, 19, 521, 3044, 19, 602, 8075, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i worked for the coastguard , was made a u.n. goodwill ambassador .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 675, 21, 3, 3247, 4255, 1, 20, 182, 7, 7948, 7877, 7941, 302, 483, 97, 2389, 3390, 3034, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'infelizmente , no meu pa\\xc3\\xads , n\\xc3\\xa3o havia dormit\\xc3\\xb3rio para raparigas , portanto fui aceite na escola de medicina mas n\\xc3\\xa3o pude ir para l\\xc3\\xa1 .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 1516, 1, 22, 73, 641, 1, 13, 324, 536, 1511, 2866, 7990, 11, 3267, 8073, 1, 584, 385, 5109, 7990, 24, 548, 4, 3191, 25, 13, 2461, 7990, 205, 11, 410, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"unfortunately , in my country , there was n't a dormitory for girls , so i was accepted in medical school , but i could not go there .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 1711, 1, 11, 32, 513, 1, 64, 20, 7941, 7870, 26, 7, 3034, 1280, 4414, 21, 3747, 1, 18, 12, 20, 5784, 11, 1201, 425, 1, 23, 12, 88, 36, 120, 79, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'podem ver por que \\xc3\\xa9 que odeio o chris .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 74, 79, 23, 5, 8, 5, 1546, 586, 7, 7, 993, 1587, 2, 8215]\n",
            "\n",
            "English source sentence - = b'you can see why i hate chris .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 15, 31, 80, 154, 12, 3606, 715, 1883, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'gostaria de falar sobre o porqu\\xc3\\xaa de isto acontecer , de ser uma calamidade para a sociedade , o que podemos fazer e , para terminar , o porqu\\xc3\\xaa de esta altura ser fant\\xc3\\xa1stica para se ser professor de matem\\xc3\\xa1tica .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 732, 4, 182, 60, 7, 2981, 4, 42, 488, 1, 4, 44, 10, 663, 1436, 516, 11, 3, 1483, 1, 7, 5, 76, 55, 8059, 1, 11, 2185, 8072, 1, 7, 2981, 4, 54, 760, 44, 1961, 7990, 11, 16, 44, 1506, 4, 1306, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"so , i 'd like to talk about why that is , why that 's such a calamity for society , what we can do about it and , to close with , why this is an amazing time to be a math teacher .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 169, 1, 7936, 8, 49, 53, 5, 209, 35, 154, 10, 156, 1, 154, 40, 8, 9, 320, 7, 1010, 2802, 733, 21, 919, 1, 29, 14, 31, 30, 35, 17, 359, 1, 5, 839, 705, 1, 154, 16, 13, 45, 550, 130, 5, 34, 7, 2125, 3073, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e foi um acontecimento incr\\xc3\\xadvel e uma aprendizagem incr\\xc3\\xadvel .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 53, 9, 3868, 7990, 1627, 6, 10, 1666, 1917, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and it was an amazing occurrence and an amazing understanding .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 17, 20, 45, 550, 7535, 1544, 4, 45, 550, 6933, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'n\\xc3\\xa3o havia arte visual , era tudo um audit\\xc3\\xb3rio .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 13, 324, 705, 3753, 1, 56, 123, 9, 5093, 5136, 2, 8215]\n",
            "\n",
            "English source sentence - = b'there was no visual art , everything was auditory .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 64, 20, 110, 1403, 831, 1, 308, 20, 7791, 5005, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e isso significa que pessoas , como muitos de n\\xc3\\xb3s , que temos dirigido organiza\\xc3\\xa7\\xc3\\xb5es , e que \\xc3\\xa0 nossa maneira tent\\xc3\\xa1mos encontrar as melhores pessoas que pudemos , na maioria , fracass\\xc3\\xa1mos em conseguir obter o melhor delas .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 52, 303, 5, 225, 1, 21, 199, 4, 243, 1, 5, 62, 3213, 15, 5673, 7916, 1, 6, 5, 48, 98, 409, 7183, 7990, 361, 18, 579, 45, 5, 2461, 193, 1, 24, 5950, 8055, 1, 6290, 1088, 285, 14, 836, 942, 7, 222, 1662, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and it means that people like many of us , who have run organizations , and gone out of our way to try to find the very best people we can , mostly fail to get the best out of them .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 17, 292, 10, 57, 53, 138, 6, 171, 1, 82, 24, 527, 2523, 7946, 1, 4, 1237, 77, 6, 43, 123, 5, 340, 5, 189, 3, 68, 332, 57, 14, 641, 1, 2411, 6362, 5, 85, 3, 332, 77, 6, 161, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'( mota a acelerar ) [ som ] [ toque ] [ vis\\xc3\\xa3o ] [ cheiro ] [ sabor ] jl : e \\xc3\\xa9 assim que o gr\\xc3\\xa1fico dos cinco sentidos funciona .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 67, 3286, 3, 3, 5326, 65, 1331, 1538, 722, 1331, 4918, 722, 1331, 3488, 722, 1331, 4875, 722, 1331, 6813, 722, 8064, 8066, 39, 6, 8, 192, 5, 7, 2690, 36, 460, 1161, 17, 637, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"( motorcyle revving ) [ sound ] [ touch ] [ sight ] [ smell ] [ taste ] jl : and that 's how the five senses graph works .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 94, 2353, 779, 615, 460, 7949, 4757, 92, 1620, 1512, 524, 1620, 6515, 524, 1620, 2456, 524, 1620, 3827, 524, 1620, 6530, 524, 7937, 7939, 56, 4, 40, 8, 9, 61, 3, 341, 920, 9, 4648, 805, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'por exemplo , factorizar express\\xc3\\xb5es quadr\\xc3\\xa1ticas com coeficientes maiores que um .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 23, 251, 1, 5988, 2412, 2841, 145, 6193, 4908, 17, 20, 3595, 2369, 1349, 876, 5, 424, 2, 8215]\n",
            "\n",
            "English source sentence - = b'for example , factoring quadratics with leading coefficients greater than one .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 21, 269, 1, 1549, 46, 6593, 2800, 9, 28, 2534, 759, 5689, 9, 1944, 112, 227, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'nuna , j\\xc3\\xa1 fazem 10 anos que n\\xc3\\xa3o te vejo .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 963, 367, 1, 106, 422, 349, 115, 5, 13, 333, 4911, 8069, 2, 8215]\n",
            "\n",
            "English source sentence - = b'nuna , it has been already 10 years that i haven\\xe2\\x80\\x99t seen you .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 1433, 612, 1, 17, 86, 106, 400, 343, 148, 10, 12, 651, 7941, 5794, 26, 436, 51, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e isso significava que ela sabia que tinha apenas uma oportunidade para recolher os seus dados .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 52, 7202, 7990, 5, 126, 683, 5, 103, 80, 10, 982, 11, 4758, 12, 162, 740, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and that meant she knew she only had one shot at collecting her data .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 10, 1386, 142, 473, 142, 122, 70, 47, 3554, 38, 7745, 7863, 176, 493, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e resultou .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 3380, 5703, 8075, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and , it worked .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 359, 1, 17, 5494, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'ele bateu com a cabe\\xc3\\xa7a .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 61, 3033, 91, 20, 3, 1238, 2, 8215]\n",
            "\n",
            "English source sentence - = b'he hit his head .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 65, 1110, 128, 853, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'n\\xc3\\xa3o era que eu n\\xc3\\xa3o tentasse .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 13, 56, 5, 27, 13, 1060, 751, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"it 's not because he did n't try .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 19, 8, 9, 36, 66, 65, 98, 7941, 7870, 26, 4998, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'eu sou um dos seus gestos de ensinar .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 27, 227, 9, 36, 162, 4281, 12, 4, 3209, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i am one of her teaching gestures .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 313, 47, 6, 176, 1458, 3017, 2893, 7946, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'( aplausos )'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 67, 107, 173, 8215]\n",
            "\n",
            "English source sentence - = b'( applause )'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 94, 136, 192, 8088]\n",
            "\n",
            "Protugese source sentence - = b'tinham consumido bebidas com corantes ?'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 342, 7053, 7, 7092, 3680, 20, 2378, 999, 29, 8215]\n",
            "\n",
            "English source sentence - = b'had they consumed colored drinks ?'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 70, 25, 6418, 49, 1209, 55, 4278, 7946, 33, 8088]\n",
            "\n",
            "Protugese source sentence - = b'lembro-me das minhas primeiras aulas de desenho na escola como um conjunto de sentimentos contradit\\xc3\\xb3rios .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 1417, 8003, 38, 43, 529, 3064, 2782, 4, 6024, 24, 548, 21, 9, 863, 4, 4389, 879, 7021, 2866, 8073, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i remember my first drawing lessons in school as a bunch of contradictory feelings .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 466, 32, 124, 5694, 5956, 11, 377, 37, 7, 2153, 6, 4698, 1954, 5005, 7863, 2706, 7946, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'ou seja , o impacto \\xc3\\xa9 gerado pelas pessoas , o que consomem na sua riqueza , e como isso \\xc3\\xa9 produzido .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 35, 1017, 1, 7, 1031, 8, 1811, 15, 726, 225, 1, 7, 5, 4669, 2752, 7990, 24, 86, 6819, 935, 1, 6, 21, 52, 8, 6847, 8069, 2, 8215]\n",
            "\n",
            "English source sentence - = b'that is , impact is generated by people , what they consume in their affluence , and how it is produced .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 10, 156, 1, 1153, 13, 6727, 7863, 60, 181, 1, 29, 25, 6418, 7863, 11, 69, 2320, 1202, 1448, 1, 4, 61, 17, 13, 2514, 7931, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'sem piada nenhuma .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 136, 3783, 7990, 4247, 8055, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and without the humor .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 294, 3, 1990, 331, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'por isso , juntei tudo .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 23, 88, 1, 3424, 304, 482, 2, 8215]\n",
            "\n",
            "English source sentence - = b'so i set up everything .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 18, 12, 398, 87, 1036, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'o que quer que aconte\\xc3\\xa7a , n\\xc3\\xa3o desistam .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 7, 5, 282, 5, 5323, 1, 13, 7852, 200, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"whatever happens , do n't give up .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 1013, 1494, 1, 30, 7941, 7870, 26, 221, 217, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'alguns s\\xc3\\xa3o mais sobre assuntos sociais , outros mais sobre assuntos econ\\xc3\\xb3micos .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 159, 34, 26, 60, 2057, 17, 1586, 1, 242, 26, 60, 2057, 17, 5494, 2, 8215]\n",
            "\n",
            "English source sentence - = b'some games are more about social issues , some are more about economic issues .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 89, 4653, 7863, 22, 67, 35, 280, 3331, 1, 89, 22, 67, 35, 574, 3331, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e na realidade , se olharem atentamente poder\\xc3\\xa3o ver an\\xc3\\xa9is \\xc3\\xa0 volta deste aglomerado .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 24, 540, 1, 16, 1688, 3139, 7581, 2132, 79, 1298, 2952, 48, 356, 364, 1712, 7722, 527, 120, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"and , in fact , if you look very carefully you 'll be able to see rings around this cluster .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 359, 1, 11, 259, 1, 44, 15, 129, 68, 4115, 51, 8, 164, 34, 180, 5, 80, 2026, 9, 165, 16, 3780, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'hawken cumpriu neg\\xc3\\xb3cios e ind\\xc3\\xbastria , liderando a humanidade para longe do abismo porque , com o continuado decl\\xc3\\xadnio n\\xc3\\xa3o verificado da biosfera , uma pessoa muito querida est\\xc3\\xa1 em risco aqui \\xe2\\x80\\x94 e , francamente , um risco inaceit\\xc3\\xa1vel .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 591, 4542, 2048, 6035, 91, 3188, 6, 1843, 5115, 601, 1, 6596, 132, 3, 3300, 11, 1343, 15, 1583, 2208, 659, 1, 20, 7, 1753, 15, 1434, 2257, 2355, 13, 4546, 439, 19, 7483, 1078, 8055, 1, 10, 311, 33, 5888, 19, 49, 14, 1131, 276, 58, 8059, 1, 6970, 147, 1, 9, 1131, 4633, 2922, 597, 2, 8215]\n",
            "\n",
            "English source sentence - = b'hawken fulfilled business and industry , leading humankind away from the abyss because , with continued unchecked decline of the biosphere , a very dear person is at risk here \\xe2\\x80\\x94 frankly , an unacceptable risk .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 954, 4148, 664, 6728, 55, 576, 4, 2130, 1, 2534, 1219, 166, 378, 48, 3, 897, 2014, 9, 1499, 1, 28, 2866, 49, 3661, 243, 1862, 49, 6404, 7863, 6, 3, 913, 6992, 1, 7, 68, 3040, 216, 374, 13, 38, 1089, 137, 42, 6351, 1, 45, 1743, 6088, 2427, 2980, 2169, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'seria , \\xc3\\xa9 claro , uma linha horizontal no topo .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 5157, 1, 8, 347, 1, 10, 966, 5015, 505, 22, 6459, 2, 8215]\n",
            "\n",
            "English source sentence - = b'it would , of course , be a horizontal line along the top .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 17, 4981, 1, 6, 278, 1, 34, 7, 4250, 2900, 7863, 757, 754, 3, 1457, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'apenas queria dizer que tenho saudades tuas e que te amo , e , por favor , volta para mim e mantem-te viva .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 80, 397, 224, 5, 164, 471, 6141, 5270, 6776, 17, 6, 5, 333, 4011, 1, 8059, 1, 23, 2146, 1, 356, 11, 472, 6, 3667, 8067, 8003, 333, 2589, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i just wanted to say that i miss you , and i love you , and please come back to me and stay alive .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 54, 267, 5, 258, 10, 12, 6651, 51, 1, 4, 12, 295, 51, 1, 4, 1814, 172, 158, 5, 81, 4, 877, 2196, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'a minha fam\\xc3\\xadlia comia uma refei\\xc3\\xa7\\xc3\\xa3o por dia , \\xc3\\xa0 noite .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 3, 69, 702, 3032, 3, 10, 7632, 68, 23, 196, 1, 48, 1104, 2, 8215]\n",
            "\n",
            "English source sentence - = b'my family ate one meal per day , at night .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 32, 541, 659, 47, 5608, 7863, 494, 224, 1, 38, 698, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e isso tornou-se no t\\xc3\\xadtulo de um artigo da fortune .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 52, 1224, 8003, 16, 22, 6452, 7990, 4, 9, 6728, 19, 5472, 812, 8059, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and that became the headline of a fortune article .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 10, 431, 3, 7642, 3599, 6, 7, 5401, 104, 4322, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'no entanto , aqui est\\xc3\\xa1 o rel\\xc3\\xb3gio do chris .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 22, 758, 1, 84, 49, 7, 2805, 6124, 7, 15, 993, 1587, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"however , here 's chris 's clock .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 1252, 1, 137, 8, 9, 715, 1883, 8, 9, 4934, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'o que aconteceu \\xc3\\xa9 o mesmo que num acelerador de part\\xc3\\xadculas por\\xc3\\xa9m a muito grande , grande escala .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 7, 5, 710, 8, 7, 78, 5, 100, 5832, 360, 4, 2744, 614, 5341, 3, 33, 1345, 1, 148, 1347, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"what 's happened is a particle accelerator at a huge , huge scale .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 196, 8, 9, 521, 13, 7, 5588, 3398, 5957, 6162, 38, 7, 4639, 1, 630, 1212, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e estas s\\xc3\\xa3o as quest\\xc3\\xb5es com que temos de nos preocupar durante os pr\\xc3\\xb3ximos 50 anos .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 128, 34, 18, 1213, 20, 5, 62, 4, 31, 2811, 7990, 183, 12, 1724, 729, 102, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and these are the questions that we have to worry about for the next 50 years .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 58, 22, 3, 739, 10, 14, 24, 5, 2392, 35, 21, 3, 244, 665, 228, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'a energia renov\\xc3\\xa1vel \\xc3\\xa9 27 % do nosso total , a caminhar para os 100 % .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 3, 423, 4754, 184, 8, 8008, 8013, 233, 15, 122, 2659, 1, 3, 6711, 7990, 11, 12, 4363, 1421, 8215]\n",
            "\n",
            "English source sentence - = b'renewable energy is 27 percent of our total , going for 100 percent .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 7457, 482, 13, 4335, 7863, 193, 6, 43, 6161, 312, 1, 73, 21, 643, 636, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'\\xc3\\xa9 uma coisa maravilhosa passar esse testemunho .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 8, 10, 133, 2256, 7990, 636, 180, 2121, 5204, 595, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"it 's an amazing feeling to have that lineage .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 19, 8, 9, 45, 550, 1022, 5, 24, 10, 660, 686, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b\"`` `` '' a forma como o custo do que desperdi\\xc3\\xa7o , o que \\xc3\\xa9 perdido , `` '' se alguma vez me esquecer de que tu `` um dia vir\\xc3\\xa1s viver aqui tamb\\xc3\\xa9m . '' bem , todos os dias da minha vida desde ent\\xc3\\xa3o , `` '' a crian\\xc3\\xa7a de amanh\\xc3\\xa3 '' '' tem falado para mim com uma mensagem simples , mas profunda , que eu pretendi partilhar convosco . ''\"\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 149, 3, 96, 21, 7, 1496, 15, 5, 3218, 2769, 8069, 1, 7, 5, 8, 4072, 177, 16, 287, 134, 38, 6658, 7990, 4, 5, 795, 777, 9, 231, 2063, 1761, 7990, 500, 84, 640, 503, 104, 1, 63, 12, 473, 19, 69, 143, 316, 119, 177, 3, 1169, 4, 5824, 108, 70, 1919, 15, 11, 472, 20, 10, 1305, 506, 1, 25, 2979, 1, 5, 27, 5175, 203, 873, 6349, 37, 8215]\n",
            "\n",
            "English source sentence - = b\"`` the way the cost of what i squander , what is lost , if ever i forget that you will someday come and live here too . '' '' well , every day of my life since , `` '' tomorrow 's child '' '' has spoken to me with one simple but profound message , which i presume to share with you . ''\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 62, 3, 123, 3, 867, 6, 29, 12, 7946, 1932, 7545, 7945, 1, 29, 13, 6664, 1, 44, 276, 12, 2369, 10, 15, 97, 1879, 251, 172, 4, 271, 145, 463, 139, 133, 1, 146, 251, 6, 32, 219, 3828, 116, 3414, 8, 9, 1038, 286, 86, 5838, 212, 5, 81, 28, 47, 476, 23, 4012, 3723, 1, 91, 12, 3443, 7372, 7863, 5, 533, 28, 51, 75, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e a boa vontade do mercado \\xc3\\xa9 espantosa .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 3, 521, 3044, 15, 1548, 8, 5486, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and the goodwill of the marketplace is astonishing .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 3, 483, 97, 6, 3, 1436, 306, 13, 7315, 117, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'\\xc3\\xa9 uma cole\\xc3\\xa7\\xc3\\xa3o de rostos .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 8, 10, 2711, 68, 4, 5163, 8073, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"it 's a collection of faces .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 19, 8, 9, 7, 5448, 6, 7209, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'emiss\\xc3\\xb5es l\\xc3\\xadquidas de gases de efeito de estufa desceram 82 % em tonelagem absoluta .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 5755, 6584, 18, 4, 1881, 129, 4, 1745, 4, 5742, 803, 7990, 7857, 90, 8014, 8008, 233, 14, 6785, 318, 756, 6113, 8055, 2, 8215]\n",
            "\n",
            "English source sentence - = b'net greenhouse gas emissions down 82 percent in absolute tonnage .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 2841, 1638, 1282, 2182, 7231, 213, 7887, 898, 193, 11, 6128, 1404, 612, 1415, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b\"`` pensei que o espa\\xc3\\xa7o principal para mostrar o meu trabalho era a internet porque eu queria um conceito aberto que convidasse todos a clicar no bot\\xc3\\xa3o `` '' partilhar '' '' , tanto no computador como no c\\xc3\\xa9rebro . ''\"\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 32, 1384, 5, 7, 427, 1591, 11, 621, 7, 73, 259, 56, 3, 763, 47, 27, 397, 9, 1851, 2949, 7990, 5, 3843, 465, 63, 3, 3731, 127, 22, 6378, 51, 3071, 8072, 268, 524, 22, 889, 21, 22, 581, 37, 8215]\n",
            "\n",
            "English source sentence - = b'i thought that the main space to show my work was the internet because i want an open concept that invites everybody to push the share button in both the computer and their brain .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 293, 10, 3, 1437, 404, 5, 261, 32, 187, 20, 3, 662, 66, 12, 102, 45, 495, 2564, 10, 4441, 175, 620, 5, 2278, 3, 533, 3934, 7863, 11, 395, 3, 554, 4, 69, 475, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'estava t\\xc3\\xa3o habituado a essa pessoa que n\\xc3\\xa3o queria parar .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 95, 131, 2836, 15, 3, 185, 311, 5, 13, 397, 2610, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i was so used to that person that i didn\\xe2\\x80\\x99t want to stop .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 20, 18, 237, 5, 10, 374, 10, 12, 809, 7941, 5794, 26, 102, 5, 1537, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e penso que existem quatro tipos principais , e o primeiro tipo que quero descrever \\xc3\\xa9 analisar o muito grande .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 278, 5, 355, 421, 667, 5392, 171, 1, 6, 7, 270, 154, 5, 167, 5061, 8, 4181, 7, 33, 1345, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and i think there are four major types , and the first type that i want to describe is probing the very big .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 12, 84, 64, 22, 358, 1074, 4355, 7946, 1, 4, 3, 124, 847, 10, 12, 102, 5, 2635, 13, 951, 7782, 7863, 3, 68, 1833, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'ora , alice teve dificuldades em obter financiamento para a sua pesquisa .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 443, 1, 415, 2712, 653, 7846, 14, 942, 6973, 816, 11, 3, 86, 1509, 2, 8215]\n",
            "\n",
            "English source sentence - = b'now , alice had trouble getting funding for her research .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 111, 1, 958, 588, 70, 6510, 318, 5661, 7863, 21, 176, 1679, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'para onde est\\xc3\\xa1s a olhar ?'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 11, 85, 2267, 3, 4971, 29, 8215]\n",
            "\n",
            "English source sentence - = b'what are you looking at ?'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 29, 22, 15, 281, 830, 33, 8088]\n",
            "\n",
            "Protugese source sentence - = b'alguns jogos s\\xc3\\xa3o mais sobre assuntos de guerra .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 159, 7734, 34, 26, 60, 2057, 17, 4, 2042, 2, 8215]\n",
            "\n",
            "English source sentence - = b'some games are more about warfare .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 89, 4653, 7863, 22, 67, 35, 561, 1000, 7932, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'eu era um jovem , e j\\xc3\\xa1 tinha entrado e desistido da faculdade algumas vezes .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 27, 56, 9, 3804, 1, 6, 106, 103, 5488, 15, 6, 7852, 676, 19, 3310, 7990, 217, 413, 2, 8215]\n",
            "\n",
            "English source sentence - = b'i was a young man , and i had been dropping in and out of college .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 20, 7, 366, 406, 1, 4, 12, 70, 106, 7239, 46, 11, 4, 77, 6, 2953, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'o google era meu amigo .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 7, 1880, 56, 73, 4703, 2, 8215]\n",
            "\n",
            "English source sentence - = b'google was my friend .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 1344, 20, 32, 1445, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'e a pequena caixa indicadora faz as luzes brilhar .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 6, 3, 463, 1611, 7752, 1289, 280, 18, 5951, 3229, 8072, 2, 8215]\n",
            "\n",
            "English source sentence - = b'and the indicator box makes the lights flash .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 4, 3, 7617, 50, 1497, 344, 3, 3589, 6022, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b't\\xc3\\xaam infraestruturas caseiras e uma vida urbana vibrante .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 87, 3427, 17, 1479, 7829, 6, 10, 143, 5604, 3, 5592, 336, 2, 8215]\n",
            "\n",
            "English source sentence - = b'they have home-brewed infrastructure and vibrant urban life .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 25, 24, 410, 7876, 2253, 2104, 7863, 5130, 4, 1259, 6843, 26, 1922, 199, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'acho que , pelo menos um deles , ter\\xc3\\xa1 algo de interessante a dizer .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 291, 99, 1, 135, 220, 9, 915, 1, 3368, 121, 4, 789, 3, 210, 2, 8215]\n",
            "\n",
            "English source sentence - = b'my guess is at least one of them has something interesting to say .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 32, 1610, 13, 38, 777, 47, 6, 100, 86, 127, 462, 5, 252, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'estava a entrar em bancarrota no outono passado porque algu\\xc3\\xa9m os hackou .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 95, 3, 1276, 14, 1330, 1634, 404, 22, 3530, 602, 7, 1145, 47, 284, 12, 591, 1852, 290, 2, 8215]\n",
            "\n",
            "English source sentence - = b'it was running into bankruptcy last fall because they were hacked into .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 17, 20, 1262, 99, 1919, 1071, 1783, 2378, 214, 1475, 66, 25, 72, 3476, 55, 2178, 7942, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'eu vendo um produto a um mercado que n\\xc3\\xa3o o quer , mas \\xc3\\xa9 for\\xc3\\xa7ado por lei a compr\\xc3\\xa1-lo .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 27, 1300, 15, 9, 2810, 3, 9, 1548, 5, 13, 7, 2130, 1, 25, 8, 7377, 7, 23, 1780, 3, 4868, 178, 8003, 232, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"i sell a product to a market that does n't want it , but is forced by law to buy it .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 2509, 7, 2031, 5, 7, 1603, 10, 149, 7941, 7870, 26, 102, 19, 1, 23, 13, 2626, 60, 1366, 5, 1079, 19, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'mas , atualmente , vivemos num mundo muito audioc\\xc3\\xaantrico .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 195, 1, 2112, 1, 1673, 100, 111, 33, 5093, 2081, 6760, 3057, 2, 8215]\n",
            "\n",
            "English source sentence - = b'however , in this day and age , we live in a very audio-centric world .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 1252, 1, 11, 16, 251, 4, 686, 1, 14, 271, 11, 7, 68, 6453, 7876, 6833, 263, 150, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b\"`` apresento-vos `` '' a crian\\xc3\\xa7a de amanh\\xc3\\xa3 '' '' . `` `` sem um nome , um rosto invis\\xc3\\xadvel e sem saber o teu tempo ou lugar , `` '' crian\\xc3\\xa7a de amanh\\xc3\\xa3 , ainda antes da nascen\\xc3\\xa7a , `` '' conheci-te pela primeira vez na manh\\xc3\\xa3 da passada ter\\xc3\\xa7a-feira . ''\"\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 32, 4893, 8069, 8003, 780, 51, 3, 1169, 4, 5824, 315, 299, 136, 9, 1570, 1, 9, 5376, 3425, 7990, 6, 136, 479, 7, 2798, 163, 35, 965, 177, 1169, 4, 5824, 1, 130, 269, 19, 4084, 2989, 177, 5074, 8003, 333, 139, 252, 134, 24, 5702, 19, 4593, 7990, 440, 635, 8003, 3012, 37, 8215]\n",
            "\n",
            "English source sentence - = b\"`` please meet `` '' tomorrow 's child . '' '' `` `` without a name , an unseen face , and knowing not your time or place , tomorrow 's child , though yet unborn , i met you first last tuesday morn . ''\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 62, 1814, 5935, 159, 3414, 8, 9, 1038, 4142, 62, 294, 7, 1232, 1, 45, 1924, 1765, 212, 1077, 1, 4, 1894, 36, 76, 130, 50, 510, 1, 3414, 8, 9, 1038, 1, 721, 716, 823, 2194, 1, 12, 761, 15, 124, 214, 1139, 239, 251, 1411, 7941, 75, 8088]\n",
            "\n",
            "Protugese source sentence - = b'levei a escola a s\\xc3\\xa9rio e pela primeira vez , em toda a minha vida , eu recebi um pr\\xc3\\xa9mio acad\\xc3\\xa9mico por excel\\xc3\\xaancia e e recebi uma men\\xc3\\xa7\\xc3\\xa3o honrosa , no primeiro semestre do ensino secund\\xc3\\xa1rio .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 1777, 203, 3, 548, 3, 1831, 7990, 6, 139, 252, 363, 1, 14, 166, 3, 69, 218, 1, 27, 5643, 9, 4233, 7136, 7, 23, 2045, 4258, 1147, 7990, 6, 6, 5643, 10, 3287, 68, 4818, 3898, 1, 22, 270, 1208, 2046, 7990, 15, 1221, 3258, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"i took school seriously , and for the first time ever in my life , i received an academic award for excellence , and made dean 's list from the first semester in high school .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 12, 314, 377, 3555, 1, 4, 21, 3, 124, 130, 276, 11, 32, 199, 1, 12, 3096, 45, 5216, 7863, 5468, 7863, 21, 7220, 386, 1, 4, 182, 3040, 7941, 8, 9, 1986, 48, 3, 124, 5032, 6757, 216, 11, 437, 425, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'portanto os governos ocidentais est\\xc3\\xa3o a fornecer ferramentas aos governos totalitaristas para fazer isto contra os seus cidad\\xc3\\xa3os .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 584, 12, 2142, 3184, 3118, 213, 71, 3, 5473, 7990, 1093, 140, 2142, 2659, 6926, 2414, 11, 55, 42, 403, 12, 162, 4874, 8073, 2, 8215]\n",
            "\n",
            "English source sentence - = b'so western governments are providing totalitarian governments with tools to do this against their own citizens .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 18, 2059, 1991, 22, 4008, 6161, 958, 4365, 864, 1991, 28, 915, 5, 30, 16, 487, 69, 205, 3639, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'este mercado \\xc3\\xa9 doido , minha gente .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 59, 1548, 8, 120, 1430, 1, 69, 2002, 2, 8215]\n",
            "\n",
            "English source sentence - = b\"this market 's crazy , everybody .\"\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 16, 1436, 8, 9, 2379, 1, 1581, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'quando n\\xc3\\xa3o conseguia adormecer devido ao intenso frio , ou \\xc3\\xa0s dores da fome , eu esperava que , na manh\\xc3\\xa3 seguinte , a minha irm\\xc3\\xa3 voltasse para me acordar com a minha comida preferida .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 40, 13, 1296, 2718, 3796, 383, 791, 41, 3808, 2396, 2764, 1, 35, 165, 478, 19, 2634, 1, 27, 5484, 7990, 99, 1, 24, 5702, 1229, 1, 3, 69, 5968, 555, 465, 11, 38, 2513, 30, 20, 3, 69, 1125, 6530, 2, 8215]\n",
            "\n",
            "English source sentence - = b'when i could not fall asleep from bitter cold or hunger pains , i hoped that , the next morning , my sister would come back to wake me up with my favorite food .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 59, 12, 88, 36, 1475, 886, 6294, 702, 48, 946, 1557, 2721, 50, 5988, 235, 2119, 7946, 1, 12, 1236, 49, 40, 1, 3, 244, 2354, 1, 32, 4558, 71, 172, 158, 5, 4987, 81, 87, 28, 32, 1474, 694, 2, 8088]\n",
            "\n",
            "Protugese source sentence - = b'voc\\xc3\\xaas s\\xc3\\xa3o os meus altifalantes e amplificam o som .'\n",
            "Protugese tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for Portugese = [8214, 118, 34, 12, 266, 351, 706, 1919, 468, 6, 7935, 1275, 7, 1538, 2, 8215]\n",
            "\n",
            "English source sentence - = b'you people are like my loudspeakers , and amplify sound .'\n",
            "English tokenized vector - beginning and end of vector is vocab_size and vocab_size+1 for English = [8087, 15, 57, 22, 53, 32, 2036, 5269, 1, 4, 6465, 1127, 3344, 1512, 2, 8088]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(64, 38), dtype=int64, numpy=\n",
              " array([[8214,  342, 3032, ...,    0,    0,    0],\n",
              "        [8214,   95,  198, ...,    0,    0,    0],\n",
              "        [8214, 4479, 7990, ...,    0,    0,    0],\n",
              "        ...,\n",
              "        [8214,  584,   12, ...,    0,    0,    0],\n",
              "        [8214,   59, 1548, ...,    0,    0,    0],\n",
              "        [8214,  118,   34, ...,    0,    0,    0]])>,\n",
              " <tf.Tensor: shape=(64, 40), dtype=int64, numpy=\n",
              " array([[8087,   98,   25, ...,    0,    0,    0],\n",
              "        [8087,   12,   20, ...,    0,    0,    0],\n",
              "        [8087,   12, 5453, ...,    0,    0,    0],\n",
              "        ...,\n",
              "        [8087,   18, 2059, ...,    0,    0,    0],\n",
              "        [8087,   16, 1436, ...,    0,    0,    0],\n",
              "        [8087,   15,   57, ...,    0,    0,    0]])>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nBQuibYA4n0n"
      },
      "source": [
        "## Positional encoding\n",
        "\n",
        "Since this model doesn't contain any recurrence or convolution, positional encoding is added to give the model some information about the relative position of the words in the sentence. \n",
        "\n",
        "The positional encoding vector is added to the embedding vector. Embeddings represent a token in a d-dimensional space where tokens with similar meaning will be closer to each other. But the embeddings do not encode the relative position of words in a sentence. So after adding the positional encoding, words will be closer to each other based on the *similarity of their meaning and their position in the sentence*, in the d-dimensional space.\n",
        "\n",
        "See the notebook on [positional encoding](https://github.com/tensorflow/examples/blob/master/community/en/position_encoding.ipynb) to learn more about it. The formula for calculating the positional encoding is as follows:\n",
        "\n",
        "$$\\Large{PE_{(pos, 2i)} = sin(pos / 10000^{2i / d_{model}})} $$\n",
        "$$\\Large{PE_{(pos, 2i+1)} = cos(pos / 10000^{2i / d_{model}})} $$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WhIOZjMNKujn"
      },
      "source": [
        "def get_angles(pos, i, d_model):\n",
        "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "  return pos * angle_rates"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Rz82wEs5biZ"
      },
      "source": [
        "def positional_encoding(position, d_model):\n",
        "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                          np.arange(d_model)[np.newaxis, :],\n",
        "                          d_model)\n",
        "  \n",
        "  # apply sin to even indices in the array; 2i\n",
        "  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "  \n",
        "  # apply cos to odd indices in the array; 2i+1\n",
        "  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    \n",
        "  pos_encoding = angle_rads[np.newaxis, ...]\n",
        "    \n",
        "  return tf.cast(pos_encoding, dtype=tf.float32)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kLCla68EloE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "13e93caf-684b-4242-d0eb-f05189da9252"
      },
      "source": [
        "pos_encoding = positional_encoding(50, 512)\n",
        "print (pos_encoding.shape)\n",
        "\n",
        "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n",
        "plt.xlabel('Depth')\n",
        "plt.xlim((0, 512))\n",
        "plt.ylabel('Position')\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 50, 512)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd5xU1fmHn/femdneKywsvSpSRBCxYe8aE1s0lhhNYokaE2OKJjHFmKIxicaoMZpmjwb8YbCAoiDFQkfaUndh2b47u9PunfP7494ZZpeFHWAXWTzP53Oc2++ZdThz5/ue9/uKUgqNRqPRfD4wPusOaDQajebgoQd9jUaj+RyhB32NRqP5HKEHfY1Go/kcoQd9jUaj+RyhB32NRqP5HNGjg76IbBKR5SKyREQ+dLfli8ibIrLOfc3ryT5oNBrNZ4WIPCUiO0VkxR72i4j8QUTWi8gyEZmQsO8ad5xcJyLXdFefDsaT/jSl1Dil1ER3/W7gbaXUMOBtd12j0WgOR54GztrL/rOBYW67EfgzOA/HwI+BycAk4Mfd9YD8Wcg7FwLPuMvPABd9Bn3QaDSaHkcpNReo38shFwJ/Vw4LgFwR6QOcCbyplKpXSjUAb7L3L4+k8XTHRfaCAt4QEQX8RSn1OFCilNru7t8BlHR2oojciPPNR0Z62tFtKp1xowawZPVmxo0sZ+snKxlwxCCWbGkiIy+Hfi3baWgMUTr+CJat2443LZ0jCk2UbbGmxUNbQx1FfUsoU01Ubawh1RAKRw5kY6vQsLMO0+ujsCiXmuo6VDRKVkE+QwrSCG2toL4ugK0gN91LZnkJbd5sNlW3UJKfTkEKWDU7aN3ZQosVBSDdNMjITSGlqAA7LYfKT1biEyEjxSQ1Nw1vXh7R1CxawjYNrWHaAhZWKIgdCUPUZsKQIqKtzYRb2oi0hgmHo4SiClspooAApoBHhIKyXKxACCtoYYdswtEoVpT4sbF8awPIPmIkYVsRtqKELZuwFSVqK6dFo6io7TZneWypD/F4EdMLhokyTOcVIarAVqCU0681FdsRERBBcF8NY9e6YSBiICJ4U0yUApRCuddw1kE5/yGWKa5UlMysVEQEAQwR3NsgCIbg7HO3VVXWxd+1ci7Q4RO5a33IoD5I7PPm/kfctfbrDqvXb0vmMw/AkcP6d7pdZPdty9dsSfq6AEeNLO/82p1sW/pp8tcet4frdsaSfbiuc+0B+3Dtzclfd1T76y5ZvRkVqKtVShUlfZEOGNn9FFYwqWNVoG4lkHjw4+44lyxlwNaE9W3utj1tP2B6etA/XilVKSLFwJsi8mniTqWUcr8QdsP9wz0OcPRRR6jl5mTmzXuUnCk3Mff9R/hOxigeeekJCm6ZxXFfOof7Z/+MV2as43vz5tH3gvspPeJoFl6fgd1Ux0nvFvDRi//i8nvv4P7wa/z0K08wPNPHtS89wVcWpfLKI0+TWTqQa288lz8/9G8iwVZOuPpSXrrySDbe9hX+9c/lNEWiXDyqlOP++B2Wlp3C1Q+9x51XjOXqwSa1j/2ChX+ay5yaNgAm5KQy+fxhDLnxGlqOPJsfZo+mb4qHKQNzGHHRUfT90pdoHXkK725u4rkPt7JsWTU7N6zFv2MTVtDPohdvpG3hG1S+u4SqxZVs3tLMprYI9WGbcFRhCuR4TQp9JtfcdSG1yzZQt6aWhopGKv1hakI2DRGbgB3Fdv+6PkM4+6U32NIUZFNtK5vrWqmqa6O1OURbU4hgW5hQSyPhtiasgB8r2Mq875RjFpRi5hVDRi7RlCyiablEzBTaIlFaI1EClqI5ZHHK5T/B9PowPD4MjxfD48NMScP0+OLLhseHx+el37ACrHAUK2JjRWxsK4oViRK1oth2FNuKErWj2JZF1Aoz5eQR+DwGPo/pvJoGKR7D3da+3fPjp1FR2/kMuV9ezrLzGnVfAR792w8wBEwRDBFMw/lS6bguAgbC0Rfc1e5ae2PGGw8Buwb52E9qcTcYCSP0gGm3dnm9RN5+90+dDvBGJxuLT7gl6eu++/4j7dY7u0eM/Kk3J31dgPfnPZr0sbnH3ZT0sfM6XDdnyk1Elvwt+W+NzrCCeEZckNShkSV/CyZI172CHpV3lFKV7utO4BUcbara/fmC+7qzJ/ug0Wg0+4QIYphJtW6gEkj8WdjP3ban7QdMjw36IpIhIlmxZeAMYAUwHYhFoq8B/ttTfdBoNJp9R9xfrF23bmA6cLU7i+dYoMmVv2cBZ4hInhvAPcPddsD0pLxTArzi/pz1AP9WSv1PRBYDL4jI9cBm4NIe7INGo9HsG+6TfvdcSp4FTgYKRWQbzowcL4BS6jFgJnAOsB5oA65z99WLyM+Axe6l7lNK7S0gnDQ9NugrpSqAsZ1srwNO3ZdrrdoZZsp3r+adkZOZcuvDLDjmRC4dU8yl851v2unn53HbTav50S/O5ew/LyTYVMvf7ziBOWefQeTl11g281eUTzmPB84czNsjniVgK07/+hQWpo7m3ddfRkVtRp94DC/NXENbXRUDjjufu04bTvTNJ1k6fS01IZsJuamMunQi1rhz+ef/1jF+fB/OGFpAdNG/2fTmSpY3hQhHFf3TvAwemkfZieNg2GRW1QTI9BgMyvBSPKaYwolHoMrHsKU5zEdbG9m4rZnm2gaCDdVYQT8A4YqVNK7dSuPGBhq3+6kJ2fitKOGoI9D7DCHDNMj3mbRW1tK2009bbYCmoIXfitJqO8fG9HxTnNYQiNDQFqauNUydP0woYBEOWIRDFpFgG3Y4gB0KELXCqKiNkZ6FkZqB+NKIelJRvnSUJ4WwpZyAcFQRtqOErChixn7yGohhYnh9GO5PYMPjQwwT0+NBRLBj2r0dRUWdQLKKKqLKeVVKEY2quHZuGoJpGM6riLveSUuIkqpodO+fT9u9dpJ6/q7rdq3n7wudBXb3h870fDmAi3dTt3olAojZPYO+UuqKLvYroNMAiVLqKeCpbulIAj0dyNVoNJrehQhGNz3pH4roQV+j0Wg60F3yzqGIHvQ1Go0mkW7U9A9F9KCv0Wg0CQiC4fF+1t3oMXqFy2aopZG3TwnyxrZmZp9r8srqGo5dOJfXHnmSn//set468csck5dG7bW/ZOFzLzDp0ksYPe8RXlldw+2PfICybe65/hhqH7idmZXNnN0/mz7f/infe3EZtWsXU3zEVO654AgqP55DRlF/zjltKMemN7Ly8ddY3BAkx2swYUoZRRdfyVsbG3l38VYun9ifstaNVL4+m7UraqgOWaSZwuhsH/2OG0jG5FPYYeQyb3M9JSke+g/MpXTiUFLHTKHBV8CS7S18vLmB+u0ttO7cQri1CQDTl0Zw0wYa11fRvK2ZmpBNs+UkWoETxM30GOR43UDujjr81a201QdoikTjAV87IfPUFMFnCPXBCDubQ9T5QwQDEcKBCOGQ5SRIhQJYYSeIG7Ui2JEwRkY2RkYWUV8aUW8aypNCROEEcKMKy4a2iE3QisaDtrEgriQGcc1YMFcwPQYqihOwjSpsO+oGbVU8OUu5QVwVtVG2HQ/U+kwnASuWmNUxkGuItAu07i0xCzoPfu6JfYmJxu6XTGLW/vB5DrIeFA7uPP2Djn7S12g0mg701gE9GfSgr9FoNImIdNuUzUMRPehrNBpNAsLh/aTfKzT9/uV9+OVxt3DvHy/jwYnX893vnsQxP3yTPuNP4/rqV3m1ooEvT/8pF/98NukFfXntm5N57uZ/MjwzhY3vT+eocy/gqvwaZjz8Hvk+kxPvv4RnNipWvv0evowcTj/rSE5Or8UOBxg8eQq3nTCQxmf/xIJ52/BbUY7NT2PUV6ZRlX8kT83fRNXqT5k2MIfA3FfY+NYG1vrD2AoGpvvoP6GUPtOOxRpwNB9VtTBn9U6GZnrpM6EPOePGEelzBOsbgny4uYGqrU207NxO2N9A1AojhokvI4eGtVtp2txMfV0gnpiVaJwWS8xKz0+jZbuftroA9WGbpohN0NXbExOzfIaQZhrU+8PUt4ZpjCVmhWwiIQsr4McOB4hGXD0/lpyVkYXyZaC86eBNJepJIRhLzLIVQStK0IrSFrHbGa2JYWIkJGUZHh+GIZimgWka8cQs24qiosS1/Li2H9P0bUfXjxmtmYbgSdDw2+n6Ipiu2N3diVkSv27yiVl70vM7O+ZA0YlZ3YwYmB5fUq03op/0NRqNJhE5vJ/09aCv0Wg0CQh6nr5Go9F8rjicB/1eoelnN1RSmurh0eFfBWDZNQ+wbs4rzP7VOTx85aNcfWI5D1sT2Dx/Bt++8xKqbr+SxQ1Brrj3LLL7DefpGybxyc3fZWlTkAtPGUjrOXfwu2eX4q/exMBjp/Gj04ay/c+/pWDoBL55/ijKKz9g6ZPvs7olRP80L0d+YRS+067m1TU1rPxkO83b1pK27j02/PcDlm1poj5sk+8zGVmaQf+TR+MbP431zYp31tVSWdFA3zHFlE4ejWf0sVSGTD7e3szSTfXUV/sJNOyIz9H3pGWSklNI4/pqmrc1syMYm6O/y2gt0+Po+TmpHjJK0mmtbqWlKURTJEowqgjYu4zZYuf4DCHVkPgc/VDAIhSIEAlZRIJB7LAzR9925+nHtHRJy0L50lDeFKLeNEJWNK7nh21FW8SmLRIlZEfjRmsx47W4nu/O2TdMA8NjIIYQtZyCKUopp2BKB6O1mOFbrHVptObO0TcMiev5Xc3Rj7EnPb8jyc6t70r3j13nUNXzP2sOia7refoajUbzeULLOxqNRvO5QUQwvL1zZk4y6EFfo9FoEtGGaxqNRvP5Qg/6nzE7qv1ct+NDsi/4Lf5Fj1N4+5+ZdsP1tN56Ga12lAmvv865F/2aISdfxN19qvjB00v44sgCgl/9BZcPqqB87mP89K2NHJOXyvgHf8K1M1azaf4scspHccsXj6Rs9f8x/YkFjPvp9Vx1ZCEbbr+D9ysaMEU4bkQ+A666lCWBLJ599xNq1nxE1Aqzc8YrVMzdwtZABJ8hDM/0UT61H3knnExj7hDeX1XDh2tqaKisos/EgWSMm0IgfzArNjUxf10ttZUttNZsIdTS4CRCeXz40rNJLyij8aMmqlvCNER2BXFNgUyPQbYbyM0oySCzJIP6dQ3Uh50ErsTqWtA+iJtmGtS3hmhpDRMKRggHLCIha5fRmpuYFU0IoCqfY7KmvOlYGITtqNucIG7IjhKybKdyVoLBmtkhiGt6DEyP4QRKPUY8Ecu2VNx4LZaYlWi01i6Q2yExq12ClpuYFauctbcgbiwxCzoP2MZITMza3yBudxutHQx6QRcPCkZv+J+1n/SK2TsajUZzsBARxEiuJXm9s0RkjYisF5G7O9n/kIgscdtaEWlM2Gcn7JveHe+vVzzpazQazcHENLvneVhETOAR4HRgG7BYRKYrpVbFjlFK3ZFw/K3A+IRLBJRS47qlMy76SV+j0WgSEbrzSX8SsF4pVaGUCgPPARfu5fgrgGe74V3skV4x6JcUpDH+NyspO/pUTp0lmClpvH4aPPLcKr7zyBWc9Nv5WMFWXv3+yfzvzG/hM4RTXvw1lz22kAdPKWbmLc8QjirO+e6pvCUjePOVeQCMPX0K143MYNn9TzC3to2fnTsa+78Pseg/q9kRtJiQm8qY646nbex5PLFgMxs/WUNbXRVpeaWsn7GUpU0hAraib6qHYaMK6X/aRBg5lU92tPLGyh1Ub2nEX72JoinjiQ4az4aGEIs2N7BhUyNN1bUEG6qxgn4AfBk5pOWVkpWfSeN2PzuCVjuNPs004kZrOfmpZBank1GaS1PQoikSpdWO7ma0lmi2lukR6mJGawGLcMgiEmzDDgewQ4F4QlQ0sisxKupNR/nSiXpTCcWSsqKKsB0l5BqtxV5jGr5htE/OMj0eTNNJynKSs5wCKlHb1fLdBK1YYlaing+OTm6K7LFwSiypynATtPZGop4Pe07Miun5iexr0tDe/mElXutA/gEebkZrh0RiFjGXzW4b9MuArQnr29xtu99XZAAwCJidsDlVRD4UkQUictF+vqV2aHlHo9Fo2tH1A0QChSLyYcL640qpx/fzxpcDLymlEp9OBiilKkVkMDBbRJYrpTbs5/UBPehrNBpNe1x5J0lqlVIT97K/EuifsN7P3dYZlwM3J25QSlW6rxUi8g6O3n9Ag36vkHc0Go3mYNKN8s5iYJiIDBIRH87AvtssHBEZCeQBHyRsyxORFHe5EJgKrOp47r7SKwb9QMkA1r/7GssfPIf5f3+G6X+8gScnX88XRxbw+sRv8skrz3LFLVeR8cidzNjWzFdvOY7H/UNY8t//sP62G3hrZytfOLoPObf/jrv//hH1FUspn3Q6D3/xKBqf+BlvvbsFgPHhtXz0+5ksbghSmuph4lmDyf3i1/jPp7W898EWGjatwPD4yB86gZVr6tkRtMjxGozJT2PAqSNJn3IOmyMZvL22hg3r62jcupZgUw2+o05kB9ks3NbEB+tqqdvhzNGPG62lOkZrGYWl5BalUxmwaLaiuxVDz/eZFKZ4yCjOILNvFpllRdSHbVrt6G5Ga6Y4Wn6qYZDpcVpba5hwIOKarYWxAv7diqHH9HzANVtL21U0pZ3R2q4WiNi7jNU8Pqd53VdXzzdNI15IJT5P325vvJZospbYErV8Xwdt30iYo29K8kZrKmp3abR2IHP0d11jz3P0u1vP780cKno+OH0xPZJU6wqllAXcAswCVgMvKKVWish9InJBwqGXA88ppVTCtlHAhyKyFJgD/Cpx1s/+ouUdjUaj6UB3OpUqpWYCMztsu7fD+k86OW8+MKbbOuKiB32NRqNJQNzZYIcretDXaDSaDuxDILfXoQd9jUaj6cDhPOj3ikDups07+MEv7+CdkZOZctXVFPzyBja1RThpwSxu+dE/KJ9yHo9NtPjLA7O5cEAOafc8xs8eeh1fRg7PvrCKsTmpHPfYvdw+41PWzH6d7H7Duenyoxi+ZTYLfvc2m9oiTC1Io+Kh3/LOsp0AnDgsn2E3fJlV0pen3t7AjpUfYYcDZPcbzpCjSlnrD2EKDM/0MWjaAIpPO5Xm4tG8u6me91ZUU7NxG221VaioTaB4BMuqW3l/XQ07tzXTvH0TwaZaolYYw+MjJSuP9IIycoszGNgni1rXQM1WToJVmilkewyKUkwyStLJ6ptJZlkRGWVFNEU6C+I656S6AeBMj5CZ4iHYFiHkGq3Fgrh2KIAdDmJ3qFYFoLzpRMRDyIoSdKtmBSNR2iK7ErOCdpRA2HYTsXY3WosFb02PgWE6CVq2pZwA7h6M1oB2/ejMaC1eTUuIJ2bFfpJ3FlRNTMzaW3WrRKO1jtv2xL4EcXsyYHmwKmbtwxz23ok47zGZ1hvRT/oajUaTgOA8nByu6EFfo9FoEpHD21pZD/oajUbTgd5cXL4resVvGG96Fjetfpw3tjUz+2x46ImPufuxK5n6+48JNdXy+k9OZ+bx12GKcMbMh7nojx9Qu3YxJ15+Pn4rysU/OJ0308Yz/fl3UVGbo88+gW8ekcnSn/6Rt3a20j/Ny+TrjuH9Z5dT5Rqtjb3xJAITv8DDcyuo+PhTWmu2kpZXSv8jR/OVKQMI2Ir+aV5GjSlmwNmTYcwpfLi9ldeWbWf7xnr81Zuwgn4Mj4/1DSHmVdSxtqKBhsodnRqtZRfmUFSSyVH9c3czWsv2mHGjtaw+mWSU5pJZVoSnqMxNzGpvtBYrnhIzWsvxmqRkpxAOWE5iVoLRmh0O7ma0FiNmtBZMMFqLJWTFjNYCYafF9fwORmuGx4gbrcUKqezJaC2xD3HTtw7JWfEkLdOI6/hew3C0/Q7/UGOJWXvS87syWjOkezX4Q9Vo7bPmUOu6Y7iWXOuN9Hi3RcQUkU9E5DV3fZCILHQLCjzvpiZrNBrNoUFsckASrTdyML6rbsNJP47xAPCQUmoo0ABcfxD6oNFoNEkiGKaRVOuN9GivRaQfcC7wpLsuwCnAS+4hzwDd4hGt0Wg03YHoJ/0D4vfAXUDUXS8AGl0TIth7QYEb3eIBH5akBPnx7S/z40ev4MFJN3LlsWX8feR1LH31OW77/vXIfdfz2vYWvn7vmfxqe1+W/PclBp94IS9cPZ7Lpw3E880H+O6Ti6mvWMrgqWfxyCVHUfPwPcycsxlT4LSp/eh/07dZ3BCgf5qXY78wguxLbuLZFTt5f95mGjatwPSlUTRyImccW87ZQ/PJ95mMK81k0FljSD3ufNYHU5m5qpoNa+to3PIpwaYaxDBJyyvhg62NLFhXS21Vs1sMvR5wjNZS80rIKu5DfkkGR5blMKIoczejtaIUk6J0LxnFGWT1yyGrvISU0lI8peW7zdGPafkZpmOyluM18aV7Scn2EQpECAcCWAE/kaDfNVoL72a0FsOZn++YrIUsRUtod6M1f9CiLWzv1WjN9LivrsafrNFarEh7MkZrhtHecG1PRmuJdGW0Ftvccd5+IgdqtNYdWvzB1PO7e276oabnx+jOGrmHGj026IvIecBOpdRH+3O+UupxpdREpdTEwoKCbu6dRqPRdI4InScDdtJ6Iz05ZXMqcIGInAOkAtnAw0CuiHjcp/29FRTQaDSaz4TeOqAnQ4896Sulvq+U6qeUGojjFT1bKXUlji/0l9zDrgH+21N90Gg0mn1FSO4pv7d+MXwWyVnfA54TkZ8DnwB//Qz6oNFoNJ0iAj5tw3BgKKXeAd5xlyuASftyfu2KNVwyYTyPDLkWHy8x7H9vcM75P2bMeZdyT9rH3P3YYq48toz6a+/nwev/SGbpQJ6643gqv3M1E5/8Pef/awnr332NgqET+PG1R9Nv8T958Y9zqQpanN8vm3Hfv475dj98hnDyhFKG3vwN5vmzeGrWx2xf/gF2OEDh8GMYO7GMqyb0o7B6CUdmpzDkjCEUnXEONblDeXNFNfOX76B240ba6hyjtZSsfDJLBvHWqmp2bG6keXsFwaZaJzjpSyM1p5CMonLySjIZ0T+XMWU5DM1PZ6ZrtJbpMcjzmhSlmGT1zSS7n1MtK6NvMZ6Scsgp3i2I6zN2Ga3leA3SfCapeamk5aUSCkR2VcuKhB2jtYgTzO0skOtUyooSsnavltWakJgViNjtgrimxzFYixmtiUg8Scs0jd2M1qJWGGW3D+LCLtO1dklZHiMevPUmJGglGmAlBnH3x2gt8QFuf4zW4ud2YrTW3UHcZO/fPdf7nARxxTH5O1zRNgwajUaTgHB4a/p60NdoNJpEpPfq9clw+ApXGo1Gsx84T/pGUi2p64mcJSJrXOuZuzvZf62I1IjIErd9LWHfNSKyzm3XdMf76xVP+raCAf97g7PPvRv/oscZcudrZBT1Z/73pvBY30mMykph8uuvMOae2bTVVXHXfbcy7qO/8eu/fkz2ZZnMf+lFfBk5XPrlk/hi9k7evftvzKsLMDYnlWO/dyY14y7m3r9/zJ3FmYy/40K29p/KAy8tZ+OHHxNsqiGrzxAGTxjJ144byHCjjtrpLzDi2DL6n38q1uhTmLuugekfVbK9Yif+6k3Y4QCe1EwySwZSWF5CxYZ6mqoqaaurwg4HEMPEl5FDRlE5uUUZlPXN4qj+OYwszKA0w/lfkqjn5xRnkN0vi+zyYrLKSzBLyjGKy7GzSnYzWktzk7IyPQbZXpM0V89PzUvFCvixwwH3tfPCKYmELOUYrlnRBD0/SshyCqfEErNiRVQMj29X0ZSY2ZopcX3fMATTI9hWlKgdxbaseOGUzhKzYrQzXBPBa+zS8WNGa6bs/pN8b3q+Eytob7S2p8IpHXX+feWzKJxyqOv5hzrd9aQvIibwCHA6TjLqYhGZrpRa1eHQ55VSt3Q4Nx/4MTARUMBH7rkNB9In/aSv0Wg0CRiyKwO8q5YEk4D1SqkKpVQYeA64MMmunAm8qZSqdwf6N4Gz9utNJaAHfY1Go+mAM0Os6wYUxuxi3HZjh0uVAVsT1vdkPfNFEVkmIi+JSP99PHef6BXyjkaj0RwspBOpcC/UKqUmHuAtZwDPKqVCIvJ1HCPKUw7wmnukVzzplx4xmClff4qyo0/l1FlC9fK5zHjoauYddzpVwQjXvnYfZz79KRvfn87kyy/lx8P8vHDjX2mK2Pzu0bcJNFQz/vyzeeDMway46/vMXF1LaaqH0686isxrf8QvZm9g9XufcPStJ8I5t/D79zax7L3VNG9bS2pOEf2OGs9XTh7MtPJMwrP/xbpXP2bYxVMwJp3P4u1tvLqkki1ramnasopQSz2Gx0d6YV9y+5UzeEg+dZW1+Ks3EWltApzCKekFfckpKaS4LJsJA/IYXZRJv2wfmaF6txC6o+cX5KWS2TeTrH55ZJWX4CsbgLfvQKKZhYR8WUCini9kmM78/ByvQUqOj1RXz0/Ny8AK+okEdhmtdVY4JYYYJkHbKYjuD1v43fn4bREbf8japedHbAJhC8Prw/R4HMvZ2Jx8j7Sbr2+YjmVtYuGUvRmtxfT+uOFawrz8jkZrsXn6nRVO2RPJFE7ZV6O1xOt0PP9gGa31hoknh3qIoBszciuB/gnru1nPKKXqlFIhd/VJ4Ohkz90fesWgr9FoNAeLWHJWMi0JFgPD3OJRPhxLmunt7yd9ElYvYFf9kVnAGSKSJyJ5wBnutgNCyzsajUaTgCDdZsOglLJE5BacwdoEnlJKrRSR+4APlVLTgW+JyAWABdQD17rn1ovIz3C+OADuU0rVH2if9KCv0Wg0Ceyjpt8lSqmZwMwO2+5NWP4+8P09nPsU8FS3dQY96Gs0Gk07Dncbhl6h6a+qiRBqqWf5g+cw/+/PcNd9t5J7/w28sHwn3/75ufyqbSwf/OvfDD7xQv73jUm8c9FNLKgPcMVpg6j5dAHDpl3I3645mtoHbue/M9ZhK8U5J5Uz6Hv38MTyembOXEl9xVIKr/8uTy3Zzsy31lO7djGmL43i0ZM5/6RBfGFkITL/BdY+P5dlK2rIOOWLrLeyeWlpFctX7KR+4yoCDdXxalm5/YfTd1Ae00YV01K1vl21rLSCvmSX9qOgTyYTBuQxpk82g3JTyTdCeOo3k+MmZRWle8nul0VOeS7ZA/uQ2r8/3r4DsbNLsTOLaBIaYMMAACAASURBVAg6wcTOqmWlZ6eQmusmZuWmkZKbRSToJGfFjNb2FsQVw+y0WlZrePcgbrxylplotLaHJC2P0a5aVmIwubMgbqLhWmK1rFiClje23Q3odkZniVm7vee9VMsyhHa2a10FcROvGWNPQdz9HVt0taweRBdR0Wg0ms8PMT/9wxU96Gs0Gk0H9KCv0Wg0nxOMw7yISq94Z8HmRt548nbeGTmZKVddzV31L/H7v3zINy4ewfIv3Mvv7n+G/MFj+e8Pp7Huq1/kheU7uWhwHhOe+jOlY6fx+69PpuTNh5nx8HtUBS3OGZbP+J/dzuv+Yv784gqql8/Fm5HD67WpPDljNVVL56KiNoXDj+H44wdyzdH9yN80j00vzGDF+1tZ6w9RmTWE6aurmbekiup1a2it2RovnJLdbwSlA/I45YgSpvTLI9BQHS+ckpZXQlbJAArLshkzMJ+x/XIYUZhOn3QDT90mwhUrKfSZlKZ6HD2/XzbZg/qQUV6Gt89AVF5folnFNISiNAbteOGUXXq+QUaap53RWlpBDqkF2dihAFErstfCKTE9XwwzruO3hHcVTvEHLcdsLWThD0bihmsxvd7jNXcZrCUUTolr/YbssXBKRz0/hs9j4DWMPRZOMY32RVS6MlqLv9e9FE5J1PP3dH6y6MIpuzjk9XzQmr5Go9F8nhDivjqHJXrQ12g0mg4czlbSetDXaDSaBAT2OP33cKBXDPr9+pdi3Hwpb2xrZvbZ8INxT3HB0Hzyn3iZs274K2IYPHLPRWQ8ciePvbiaY/PTOO2l+/nJ0ig//OaJnLhzDv93x7MsbQpyWnEGxz9wDSv7nshPn1zEpkWzEcOk/JhpPDB9FZsWzSfS2kT+4LGMnjKMW08YzODWdVQ+9yxrZqxlRXOIgK14fV0dMxZuZfvazfh3bCJqhfFm5JBdNpzSgUUcN7qY4wfmM6IghagVxvD4SM0pJLN0EPl9shhWnsuEAbkcUZxJWaYXb916rI0raF2/ztHzy7LIHZhD9qBSsgf2wdN3EFLYDyurhCbLoCFos70lFDdZi+n5OameuMlaemE6qa6en1qQ48zR30vhlEQ9XwwTf9jGH7Z2Ga0FnTn6LSErPj8/ELaxIraj5Scaq8U0/IQ5+x7Xgzym50e7KOISL4yeYK5miOA1pV3hlMTlZPV82F3P78x8DZxBwBDZJz2/swfFjnp+d8/RP9T1/F6D+1k7XOkVg75Go9EcLATwJlkKsTeiB32NRqNJQMs7Go1G83nCnRJ8uKIHfY1Go0kgFsM5XOkVwlVu03aeeG0dP370Ch6cdCOjslI4+eM5TPvBLJqrNvDDe67l9KVP8JcHZtM31ctlf/smf7dH85fHXuOGwmre/doDzKpuZUJuKqf97EJ2Tv0qtz23hLXvvoMV8FM6dhpXnTeST+d+QFtdFVl9hjDs2KP49qnDGOupofbFv7HqhSUsbgjQFIlSlGLy3Aeb2fppJY1bV2MF/XhSM8nuM4SSwWVMGF3MtGGFHFmcTtrONYhhOkHckkEUluUzeEAukwfnM7Ykm/5ZXlIbt2BvWU1g/ac0rttKfkkGuQOyyRlYQs6QMrz9hmKWDsLO6UOrpNIQsqlqCVHZEiQtIYib53OSstIL00kvSCO1ICsexPXk5mO71bJiAdTOiAVxTa+PlpBTMavFNVmLG62F7fhrOGxjW9HdjdViCVkep1qWJ6GYdMekrD0ZrcXaLnM1I14lKzFRa1flrF3vI1mTtcTlWBC3XXD3wD66e/wH1v1B1+6+XvcPer1pHHWM/bpuvRH9pK/RaDQJiPtAcbiiB32NRqNJ4HCXd/Sgr9FoNB3ordJNMvSK3zDbd7TwvTtP4JEh1wJw1dKXmPTzeWxb/D+uuuOrfMuez59u/AemCDc8+CXeGX4Z9z70Jk1bVvPBNXfy6po6hmf6OP+uU7Gv+BG3vLyc5W/OJdCwg+LRU7ng7BHcPLkfLds3kFHUn6HHTuL2s0Ywrcii5dW/svKfC1lU1UJNyCbHazAhN5WNK7bTuGkFkdYmTF8amaUDKR4ymDGjijljZDHj+2SS07SR8Ip5pOYUkVkyiIL+xfQfkMtxwwoZ3yebgbk+MlqrUVtXE1y7goa1W2lYV0Pe4FxyBhWTM7QMX7/BePoOxs4ppc2TSV3AZkdLmO0tIbY1BMj2GOT7TPJ9pmOuVphGemEaaYVZpBbkkF6chzcvDyO7YK96fmJSlun1xZOz2iVlBS38IYuWYCSu51sRGysSxfAYeLztTdc8XiNeWCWm56d4jF2Ga13o+TES9XyPaSRo+Lv0fK+5yy8lGT0/fm3pWs83RPZLj+7uwil7vE8vGKB604OzsMvAr6uW1PVEzhKRNSKyXkTu7mT/t0VklYgsE5G3RWRAwj5bRJa4bXrHc/cH/aSv0Wg0iXRjjVwRMYFHgNOBbcBiEZmulFqVcNgnwESlVJuIfBP4NXCZuy+glBrXLZ1x6RVP+hqNRnOwcDT95FoSTALWK6UqlFJh4DngwsQDlFJzlFJt7uoCoF83vp3d0IO+RqPRJBCzYUimAYUi8mFCu7HD5cqArQnr29xte+J64PWE9VT3ugtE5KLueH+9Qt4pzkvl/S/fz8+/eT/+RY9z/N93sHrWS5z5zRt4dMROnjjplzREbG7/6dmsO+u7fPO+N9i5ah5DTr6I5/9wG31TvVx80xRybv8dX395BR/MeBd/9SYKhx/DGeeO5funDCZl9pOk5ZUyePIUbjp3JOcNSCX48u9Z/vRcPljfQFXQItPj6PnDTh1IQ8VSgk01GB6fq+cPZ+TIQs46ooRj+mZR2FaFtWIetQs+JqNoInllpfQtz2XqsEIm9MlhcG4K2cFaZNsqguuXUf/pZurX7KBhYyNDzhpB3vD+pA4Ygrd8OFZOXwIpedS2Wezwh6lsDrKloY3NdW0c53Xm6KfnO1p+emE6aQWZpBXlOXp+bi5GbjFmXlGXhdANjw8xY8te/GHLLZayS88PhJ0iKoGgFdfzrYjtmKolzM83TInr+Wk+M67n+zxmUvPzIWa4Fu1Uz/cmLDtF0WWfTNFU1G5XCB32rOfvD8nq+QecB9ADWvnhPHMlKQT2YcZmrVJqYrfcVuQqYCJwUsLmAUqpShEZDMwWkeVKqQ0Hcp8ee9IXkVQRWSQiS0VkpYj81N0+SEQWukGN50XE11N90Gg0mn0lNmWzmwK5lUD/hPV+7rb29xQ5DfghcIFSKhTbrpSqdF8rgHeA8fv9xlx6Ut4JAacopcYC44CzRORY4AHgIaXUUKAB5+eMRqPRHCKIa+fddUuCxcAw92HXB1wOtJuFIyLjgb/gDPg7E7bniUiKu1wITAUSA8D7RY8N+srB76563aaAU4CX3O3PAN2iU2k0Gk130J1P+kopC7gFmAWsBl5QSq0UkftE5AL3sN8AmcCLHaZmjgI+FJGlwBzgVx1m/ewXParpu9OVPgKG4kxb2gA0un8I2EtQww2I3AjQJz21J7up0Wg0cRwbhu6LayilZgIzO2y7N2H5tD2cNx8Y020dcenR2TtKKdudY9oPZ+rSyH0493Gl1ESl1MSMQcP5xrd+R9nRp3LqLOGjF//F1Guu5b+nmvzzlNtY6w9x850nUX/t/VzxqzlUfTSLAcedz+O3TiXfZ3LZV8fT554/cOf/rWHWy3Np3raW/MFjOfncifz4jGHkfvAvPv71iww69nhuPG8Ul4/Mxfq/R1n+1znMX1HD1kCETI/B2JwURp48gMEXTyPQsCMhiDuSEaOLuHBsX47rn0NJuBpr+VxqP1hM1cIK8vv3p+9AJ4h7dFkOQ/NTyY00INtWEVr7CfUrNtKwZjsNFY3U1AbIG15O6kAniGvn9CWUXkBdwGJna5itTQG2NAbYXNfGtvo28n0mmXmppBemkVGSQUZxFmlFeaQV5OAryMfMK8bMKcDIyidqhXf7O3cM4poeH4bHi+Hx4Q9ZNLVF2gVxW4IWoYSkLCtiE7Wi8cpZHp+JYUo8QSsxKcvnMXdVzkoyiAvEq2btKYjrjVXP6uTTvKeKXLAriBuroBX/m7ivsSe5A4lrfpZB3P25/uc+iOsiklzrjRyU2TtKqUYRmQNMAXJFxOM+7Xca1NBoNJrPEuOAv5IPXXpy9k6RiOS6y2k4GWmrcbSpL7mHXQP8t6f6oNFoNPuKoJ/095c+wDOurm/gBDBeE5FVwHMi8nOc9OO/9mAfNBqNZp/pDX5G+0uPDfpKqWV0MqfUnW86aV+uVbFpB+VfnsryB88hZ8pNTLnqat66KJt/TbyCpU1BvnX78bTd/jAX/3w2Wz54jfIp5/GXO45n4qrnKL12HP3vf5w7Z23mlWffoXHTCnIHHslJ50/h/nNHUfzh83x8/z95+6PtfP23o7lmTCH2jD+w5JFZzF9Szaa2CGmmMDYnhTEnlTPskml4T/gShudXZJYOpGjoaIaNLuKicWVMLc+lT6SG6Iq51M5bQNXCDVSvqKH0wlxOHFHElAF5jCpMp8BqwKhcRXjtJ9Qt20Dd6krq1jWwc2crO4IWaUOG4Rs4EjuvP6GMonhS1pamIFsaA1TUtLK5tpXmxiCZealkFGc4mr6r56cX55FSXOjo+XnFGDmFRNNydvu77k3PN7y+dnq+PxiJ6/nhkIUViRK1nGZFoqSmezvV89N8Zjs932ca+6Tnq6jtGq5Jl3p+Rz16b3p+jEQ935A96/n785NY6/m9lF78FJ8MSX2WReRiEVknIk0i0iwiLSLS3NOd02g0moONdO88/UOOZJ/0fw2cr5Ra3ZOd0Wg0mkMBLe9AtR7wNRrN54XDeMxPetD/UESeB17FsVcAQCn1nx7plUaj0XxG6HKJDtlAG3BGwjYFHJRB35OWyeqHz2XOyMlMufVh5nwhk78f7QRxb7vzRNru+CMX3vc2m+fPoHzKefz1zhOZtPLfTP/aY1y0YT63u0Hc+oql5A8ey0nnT+E3F4x2gri/eIa3FlVRFbS466giojP+wCd/nMl7n+yIB3En5KZy1CkDGXbZqXhPvJRPrZx4EHf0mBIuGlfGiQNy6WvVEF3+DjXvzady/nqqV9SwpiXMtFHFuwVxQ6sWdRrErQ3b8SBuMKOIGjeIu6khsFsQt605REZxRrukrD0FcTsGcvcWxDVT0jA9vi6DuLEELduKJh3E9XmMfQriAkkHcRM1Vh3E1RwIh/GYn9ygr5S6rqc7otFoNIcKh3OhkWRn7/QTkVdEZKfbXhaRHq3uotFoNJ8F4pZLTKb1RpL9Qvsbjh1oX7fNcLdpNBrNYYfOyIUipVTiIP+0iNzeEx3qjCP7ZfH6oInMrW1j9tnw5NFXsdYf5jv3nEHN1x7gC/e+QeXimQw+8UL+ceeJHPHBY7x089+ZVxfg9RkV/N/zb9O0ZTUFQydw5heO4xdnjyB/3tMs/sW/eXtJNTuCFgPTvURe+g2fPPIG7y3fZbI2ITeVMacNZOhlp2OeeBmrgxm8sLSKkmFHcORRJXxhfBnH98+hNLQda+kcat5fyLb569mxqpb1/gjVIYvzB+YzojCNgnCdUylr1SJql22gblUV9evrqakNUBmwaIjY+K0oVv4AQukF1LRZVDY7Jmsb651KWYl6fltLqJ2en9GnYJfJWkEpkl1INDXL0fRTsuJ/z2T0/JjhWmd6fsxkLabn23Y0aT0/xWPsk57vVLhKTs+PafHJ6Pmw90pZHfV82c9/4V3p+d39sNhLx6FDCkHLOwB1InKViJhuuwqo68mOaTQazWeFiCTVeiPJDvpfBS4FdgDbcQzTdHBXo9Ecfri/AJNpvZFkZ+9sBi7o8kCNRqPp5QhODYfDlb0O+iJyl1Lq1yLyR5x5+e1QSn2rx3qWQP3yNSww+vDjR6/gwUk30mzZfP+hL/LJmXdx3d2vUr1iLqPO/BLP33E8pf/9Ff/83n/4uDHItKJ0vvn31/BXb6J49FQuuvgY7jtjKKn/+xMLfvkys1fXUhOyGZLh48QpZSz+7UzeX1dPVdAix2twTF4aR5wzhEGXnYcx5WKWNpk8+8lW3ltSxYQJfbjILZpS1LqFyCezqX5vEVULNlL1aR3r/WGqQxYBWzG6KJ3cQDVsWU5g9cdxPb9ufQM76wPsCNpxPT8cVbSl5lPbalHZHGJLU5CNda1xPd/fGKS1OUTAHyLU0kxmn5wEPb8AI7cYM6/I0fPTclBpOdgpmbRFHK08Uc83vT532YvpS8Pw+jA9PmfZ46OxLUwgbBMIWu2Kplhhm6itsN25+nasiIqr5ScWTUnzmfjM2LrT9kXPB9rp+V5zl37fUc83jeT1fOhcz+8uLT/x+jG0nt976K3STTJ0Je/ErBc+xCl72LFpNBrNYYWTkdt98o6InCUia0RkvYjc3cn+FBF53t2/UEQGJuz7vrt9jYic2R3vb69P+kqpGe5im1LqxQ4dvaQ7OqDRaDSHGt31nO/WE3kEp4jUNmCxiEzvUOD8eqBBKTVURC4HHgAuE5HRwOXAEThT5d8SkeFKqc5/uiZJsoHc7ye5TaPRaHo5jlyYTEuCScB6pVSFUioMPAdc2OGYC4Fn3OWXgFPF0ZcuBJ5TSoWUUhuB9exjLZLO6ErTPxs4BygTkT8k7MoGrAO9uUaj0Rxy7FviVaGIfJiw/rhS6vGE9TJga8L6NmByh2vEj1FKWSLSBBS42xd0OLcs6Z7tga5m71Th6PkX0F7DbwHuONCbJ0s4qvjpa3fzO+/J+HiJH7xwG8/2vYi77/oHzdvWcvQlV/LKzcdi//7bPPGbOWxoDXN+v2xOeeRrXP3T5ZQdcw7XXTKGu44vJ/iPnzH3gdd5a0sTfivKkdkpTJ02gFHf+BK/uOgBakI2RSkmk/PTGXnxKPp/6ULUpIt4v6qN5z7ezKIlVVSvq+C+yy5hUlkWuXVrCX30Ftvnfkjlgi1sq2hkvT9MbdgmHFWYAnn+rUQ3LqVt5RLqVlZQu6qahopGdjQG2RHclZRlu6Hy6jYniLupMcDmujYqavxU1QfiQdxgW5hQSzORtibSRxeQUVqAtyBmslYEmQVxkzXbm05rOEpbJLorIcswMb1OAlZipSyPG8CNJWn5gxbhsN1pENcK29i2k5wVtaP43ACuz2OQ7jPbJWUlBnF9HqNdEHdX0HZXEDcx8BqN2kkHcTt78tpTEDdGskHcfQ266iBu70WUQrr43CRQq5Sa2JP96W660vSXAktF5F9KKf1kr9FoPheIinbXpSqB/gnr/dxtnR2zTUQ8QA5O8msy5+4ze9X0ReQFd/ETEVmW0JaLyLIDvblGo9EceihQ0eRa1ywGhonIIBHx4QRmp3c4Zjpwjbv8JWC2Ukq52y93Z/cMAoYBiw703XUl79zmvp53oDfSaDSaXoPaLS1pPy+jLBG5BZgFmMBTSqmVInIf8KFSajrwV+AfIrIeqMf5YsA97gVgFU4M9eYDnbkDXcs7293FWiCglIqKyHBgJPD6gd48WfocMYgrq8Yy8y8P41/0ON9dV8hf73qMqBXmrK9fy/OXj6Li1iv497Mr8VtRvjypL1P+cBeLS05g6EnzuevKcXy5XLHz17fzwaPvM7e2DYCpBWkcc9EIhtxwLY2jzqAm9Ev6p3mZNCCbkV8cR58vXkLr8JOYXdHIcx9uZcWyanZu+BT/jk2cNCCH1K0f0brgTSrnLqFqURUbtzWzNWBRn6Dn53hN7E8X0rJiKXUrNlK3ppaGikYq/WFqQk5SVsDepef7DKGiPsCWpiCbalupqPFT3RCgtTlEW1OINn+ISGsT4bYmrICfzLIivIUlGG7RFDJyiabmEE3NJmKm0Ba2aY1EaYuoPer5iSZrZoqj63t8XkIhCyscK5Ziu8lYTgGVRD3ftqwELd/Yq57vSzRcS9DzOyZkRRM0Va9pYAhd6vkdJf1k9PyuDNYOVHvv7HSt5x/iKJXsU3ySl1MzgZkdtt2bsBwEOp0Cr5T6BfCLbusMyU/ZnAukikgZ8AbwFeDp7uyIRqPRHCqIiibVeiPJDvqilGoDLgYeVUpdgpMwoNFoNIcZCqJWcq0XkqyfvojIFOBKnOwxcPQpjUajObxQdKu8c6iR7KB/O04G7itucGEwMKfnutWe1XU2y/74F8qnnMeps4QF//4TmaUDueP2i7l7sJ/5p5/Di4uqyPeZfPWSUYz+9QP8uyaPX/1hPo/ePIUTqGDt93/JnJc+ZWlTkByvwQmFGYz96iTKrvs6FdmjeHreZkZlpTDxqGJGXDqJvPOvZHvOcP63cifPLdrKplU7qa9YQVtdFVErjG/V29TPm03l+6vY/tEO1te2URW0aIrY2MrR5nO8Bn1TvdQvXEjdik3Ur2+gbnMTlQGnAHpTxNH+E/X8TI/B2rpWKna2srmulbqGAG3NIWd+fmuQcEs9kaAfK+DHDgfxFg/BLCjFzCuOF0tRaTkE8dAWjtIaiRKworSErLihWuLcfEe/T2uv57vmaZGQHZ+P72j6Kl5AJabpq6hN1ArH9fw0n6ddwZSYjm8aspumD13r+cq22+n5Hefqwy4930hQt7vS82PnQXJ6/v74b/X03PzO7qHpDhREP+eDvlLqXeBdEckUkUylVAVwUBw2NRqN5mDTW/X6ZEi2MPoYEfkEWAmsEpGPRERr+hqN5vCk++bpH3IkK+/8Bfi2UmoOgIicDDwBHNdD/dJoNJrPBqUgeRuGXkeyg35GbMAHUEq9IyIZPdQnjUaj+Uw5nOWdZAf9ChG5B/iHu34VUNEzXdqdQFMDU+/4Cm/cOoWcKTdRPuU8Hv/2CUz59AVeOfZR3trZyticVC78zjTyvvMQ3521nhdfeovqFXM59pxaFv7yad78oJKqoEXfVA8nH1XM2BtPIf2CG5nXksHjs9awaOE2nj99IMMvPxXvSZfyqZ3Pyx9V8vribVSuraJpy2oCDTsASMnKp/q16VR+sI4dS3eypsWpkuW3nA9KmikU+jyUpXnoU5DGjoXrqFvXwM6drXGDtaaIUyULnNJssSButsdkZWUzm2tbaW4M0tYcoq0lRKjVT6S1qV0Q1woF8JSUY+QUxg3WoilZtFmKtohNqxUlEInSFLRoClm7BXHjAVy3apbHl4JhGnh8Jh6viZVgtma7wdt2iVlW2AnkRsJOANdNyOoYxI0308AQ2WuVrFgQV9kJyVmGsdeErFgAVyS5AG7i/boK4u5vAaVkgrgHUp1JB3B7ku5NzjrU2JfC6EXAf4CXgUJ3m0aj0Rx+fF41fRFJBb4BDAWWA3cqpSIHo2MajUbzmdDNNgyHGl3JO88AEeA94GxgFM6cfY1GozksET7fmv5opdQYABH5K91g67k/9O1XypwzIrw5cjLH3fYHXr3hGBp+8nUefHQBVcEIXxiWz0l/upmKoy7hy39eyLJZ7+Kv3kRO+Sjeuu4h5uzwE7CjTMhNZcpZgxl+w+WEj72Ef6+u5al3lrPh4w00bF7B6N/ciD3+XGZvbuaFTzbw4ZLtVK9bR3PVBqygH8PjIzWnkOx+I1g341G2bmpkY2ukXcGUTI9BSYqj5xeXZZE/LJ+qRVVUNYfYEbRpttoXTDEF0kyDTI9Bntck32cws6oZf1OQQEuYgD9EqKUxbrBmh4NY4QDRSJioFUby+2CnuQZrnjTawlEClqI1EqU1bNMUsmgKRvCHbUxf6u56foLBmmkaeLwmhsfA4zUIh6w9GqzFtPxYclaaz9yjwZppCD7TwGsIhiF7LZgC7fV8FbW71PPjunySQneinr+3gimJknuyOmhnHG56/gF0vZegwD58Z+909VmOSzn7WkRFRPqLyBwRWSUiK0XkNnd7voi8KSLr3Ne8/ei3RqPR9AwxG4bDVNPvatAfKyLNbmsBjooti0hzF+daODGA0cCxwM1udfe7gbeVUsOAt911jUajOWQ4nF02u/LT329TNdeLf7u73CIiq3GK+l4InOwe9gzwDvC9/b2PRqPRdC+f70ButyAiA4HxwEKgJKE4yw6gZA/n3AjcCFCWk8kDU26mNmzx9pmKd487mZdX7KQkxcOtXx3HkJ//jqc2e3jwF7PZsuhNAMqnnMcl545kxnmPkO8zOa08j6OuO5aSq77OurTBPP7mBt6ct5mqFUvwV29CRW22Dz+TmUt28MKCLWz5tIb6imW01VWhojae1EwyivuTXz6M0oG5rHilnq2BSFyf9xlCvs+kJMVDeZaPvMG5FIwoJG94f96bVRE3WAvYuyry7Jqbb5Dj6vk5WSk01rQS8IfjBmvhtibsUAAr2IrtavkxPdzOKkKlZBFQJoEEg7XGgIU/7MzP94csmkNWgn7fucGax2vi8Rlxbb+1ObTb3PyYhh/T8+OavtfcTc+Pmax5DQNTnGIoXkO6NFiLL7vbvYaxW/HzRD3fkOR05o5z+JMxWDuUtPx9v3/33uvw1/ITOIwH/QP5TCeFiGTizO2/XSnVThJy60B2WpdMKfW4UmqiUmpiQUZaT3dTo9FoHGI2DMm0XkiPDvoi4sUZ8P+llPqPu7laRPq4+/sAO3uyDxqNRrNvKJQVSaodCMlMahGRcSLygTsZZpmIXJaw72kR2SgiS9w2Lpn79tigL87v2L8Cq5VSDybsSqz8fg3w357qg0aj0ewzioP1pJ/MpJY24Gql1BHAWcDvRSQ3Yf93lVLj3LYkmZv2pKY/FaeW7nIRiXXmB8CvgBdE5HpgM3BpD/ZBo9Fo9gmFahdb6kG6nNSilFqbsFwlIjtxLHEa9/emPTboK6XeZ895JKfuy7Wqqpooys3n5t9fwoOTbmRDa5jz+2Vzyh+vo3Lq1zj7+aUsmfU+zdvWktVnCKOnHcePLjiCU7ObeCw7hanTBjDqG19CnXw1L66p4y+vLmHDks3Urf+YSGsTntRMcvoN51dzNrDgkyp2rN1A8/YNRFqbEMMkvaAvWX2GUjywlKFD8jl5ZDGr/eF4QlaO14gbrJX2LrxafAAAH7lJREFUySR/WB75w/uSN2oAKYNGsjUwY48JWdkeg3yfSWGKh/TCNDJKMmhpCBBqaSbS1kS4tWm3hKzEgKSVlk9rJEpbvEKWTUvYoilo4Q/bNIUi+IMWTW0RvKmZTnKWG8TtLCErFtA1PYZbLWvPCVnxQG7UjlfO2lNCViyY6zGNpBKyEukqIatj1azO6MyIbV8SsvY1APtZBnG7O4ALn7cgLvtSOatQRD5MWH9cKfV4kucmNaklhohMAnzAhoTNvxCRe3F/KSilQl3d9KDM3tFoNJrewz756dcqpSbuaaeIvAWUdrLrh+3uqJQSkU4ntbjX+f/2zjw8jrvM85+3qrullmTrlixbjuX4NgkJORxCBiYkgQSWHJsNIYFhmF0yHpb7AYYkZGFgnp1nAzObsCwsYG52MjAQyEOAgElCjuUIwUnsxI7t2PER35Zlqa2jpe7q+u0f9etWtdwttXxIavf7eZ56uupX1VX1s1tvV3/fq4OgyvF7jMmFFt1J8GURA9YQ/Er4x4luWI2+oihKGGNO2kk7eipzVbF9InJIRDqMMQfGC2oRkdnAL4G7jDFPhc6d/ZUwIiLfAT5Ryj2d9pBNRVGU8sLkpMuJlpNkwqAWEYkBDwDfN8bcP2ZfNgpSgBuAjaVctCye9FsbqvnPm3/JFzZliHE/n/zo6+j8zD3cs76fb3z6N+x75mGcSIyz33A9775uBR+4pJPqJ7/H+i/fzzs++1Yab17NizKXr/5iK0/+4RUOvPgcg917AKhr76J50bmc/ao2fvXrLfTteoFk7yGMnyFaW8+s9i4aOrvoWNjIZctauWxhE69qq2WDb4i7QmPUZU51hPn1VTQtaaJpcTONKxZQt3gx0a4V+M0LSKRH9cG4K8TdUS2/KeYyq76KurZaalri1HXMZqjnUF6BtbEJWWF6hzM5PT/bLGXAavqDqUDL7xtKMzDi4cbieQlZkZgbaPqhhKywtp/T9EPNUsIJWX7owx8Pafqu1fCjblAkbVTXl5zePFFCVng76oY0/AIJWWGNfywT/WGeai2/EOOdo9QicaWiCVmngGz0zumnYFCLiFwEvM8Yc5sdewPQLCJ/Y9/3NzZS5z4RaSXwna4nKIM/IWVh9BVFUaYOMxlH7olfxZgeCgS1GGPWAbfZ9X8F/rXI+684keuq0VcURQljmKqQzWlBjb6iKEoek4reKTvKwuh7nQu54J4tbH/iIQaeXsMj7krefs+zvPTEI6QGE7Qufy2XvelcPveW5SzpeZadd/w3nvnRRp46muQT9z3Ivc8f4P4nnmb3hhdJ7H2JTCpJdX0rDV3nMH/5PK56zVzetqKdN3zv38ikkrixODXNc5nduYw5XY2cv7SF1y9q5oK5s+maHSV6aOtocbWaCM0L6mlZ1kzD0k4ali0k2rUc6ViE19BJbyb4J445QtwVat1RLb+xNkq8pYa6thpq22uJtzVSO6eJ5K8O5hqfF9PyxXERx6VveDQuP6vn948EWv7AcLA+MJymf9gjWls/Goefa4DuWB1/jL4fcfBS6eD6mfy4fONnyGS37RNRVtPPavhR2wQ96gY6ftQRXKvplxKbH94eW1xt7Bgcr42X4mQbq+ePp+WfqPZeTM9XLX8Gcwqjd2YiZWH0FUVRpg590lcURakcpi56Z1pQo68oihLCYHJ9nM9E1OgriqKE0Sf96eflXQeJPvZzOi9+M1euFZ5f+zUGDu2i/qwVXHTjdXz22pVcVnWIQ1/7e379zT/y+8ODHE1lmFMd4Z3f/jM71u/i6M4NpAcTRGvraew6h7nLz+ay8+dy7TlzuKijltmHX8T4mZwDt21BG0sXNfGXy9q4pLOeRY1VxHt34a/bwLGN6zmvvoq2ebOChCxbXC3WtRx33lIyjZ0cc2roHvTYe2yIukhQXK3RdsdqjEWoba+hpqWG2rYaatrqqe1oJt7aSLS1ndRPthcsrpZFHBcnEkMclwMDI/Tbzlj9qVEHbl8yzcBwmqFUhoFhj1QqQ6wqkpeAlUvOirq4EcFxHWKhJKvMSLJgcbWcQzcTSs6KugWLq2U7ZjkiufVSHbhZ3FBnq3BxtTzHLqPOzFIzJUtJyKo0By5UuBMXAkduOjXdd3HaKAujryiKMnVMTXLWdKFGX1EUZSwq7yiKolQIxpyKYmozlrIw+pHqWm7/7x/ljr/sov7S9zOrYxGrbnk3d12/kqsaBjj6/c/x6Dd/z+9eSdA9kqG1yuVtHbNYfuMK/vmBn+UapTQvvoA5Sxdx8XkdXHduB5d2zqLh6DZG1j7MzifX0br8GlrOamfpkmYuX97GqnkNLGqMUde/D/+55xjc8jxHnt/OkRcPsez18/MapbidgZafcOvoTnrsOzbIrr4ku3uGmFsdOa5RSlbLDxKymok2t+A2tuE2tuIl10+o5bvRoBnKgf6RoMBaMk1iKEjCGhjx6B9O57R8L53BS/vE4tHjGqVEos5xWn5VxCEei5BJJSfU8rP3WRVxxtXys4labhHdvdgfmfEzE2r5MNpgZbJ/rKVq+Scrc6uWX15o9I6iKEqlYAwmo0ZfURSlIjDG4Ke96b6N04YafUVRlDAGfdKfbs6ZP5sPb/sWj//db3jdR77EZ65dyRviRzj8nc/yyDf/wP87MMDRVKDlX9s5mxU3nUPnTTfgX3At5orbaVl6MR1LF3Kpjcu/eG4d9Ue2MPLrR9j5xDr2/3kvu1/u5fX3fDwXl7+woYraxCv4zz3HwOZAy+/Z2s3RbUfZf2yEm794C1WLVubi8vucGrqHPPYeG+SVRJId3YPs7hlk75EhPj6r6jgtPxeX39yC2zwHt7ENahvx4/UFi6uN1fKdSBQnEuOV3qGgsNqwRyKZyovLT48Een4m4+OlMlTXRseNyw+am9tt1zmuUUohLT+rfVa7TtG4fEeCWPtsg/Pw/MbT8rNk/QDjafkw+TZw2eOnU8s/kfOfDj1fyUeNvqIoSoVgjMHXevqKoiiVw5kcvaON0RVFUcLY6J1SlpNBRJpE5GER2WZfG4sclxGR9XZ5MDS+UET+JCLbReTfbRP1CVGjryiKEiIbvVPKcpLcATxqjFkCPGq3C5E0xpxvl+tC458H7jXGLAZ6gfeWctGykHeOPr+Fz3y4l5gjPHq1YecXP8BPbGesZMbQVRPlqmXNLL/5QtpvfAd981fx0x29/PC+Dbzm+htynbHOba0muvNPDPzoEbY+sYH9zxxkx/5+9iTTHE1l+MzVy3KdsdK/f4beTRvp2bSTni099O7oY89Qmu4Rj2OeT/yKt5Np6KQ7E6F7yGN3Xz97Ekl2WgfuwZ4hBo+NMHhshDnnt+V1xoq3NRJpbMVpbCPSPAe/pgG/ahZ+vJ6UE3xZZztjiePiRGM41pmbdeC6VXHcSIy9vclcZ6yBYS9IxEr5NiHLOnI9g+/51M6uzuuMFY+5VFknbtiBmx3zUslccbRCDtzR9aDgWrYz1miXrHwHbuDcHb8oWsGktCKF1cY6cIsVOStGMQduobNMNrnqdDhwlanDnxpH7vXA5Xb9e8DjwO2lvFGCD+8VwDtD7/8s8NWJ3qtP+oqiKGFsyGaJ8k6LiKwLLasncaV2Y8wBu34QaC9yXLU991MicoMdawb6jDHZnxt7gXmlXLQsnvQVRVGmjMll5B4xxlxUbKeIPALMKbDrrvxLGiMipshpFhhj9onI2cBvReQFIFHqDY5Fjb6iKEoIw6mL3jHGXFVsn4gcEpEOY8wBEekADhc5xz77ukNEHgdeA/wEaBCRiH3a7wT2lXJPZWH0U77h7Rd0cP7qN3LPqtW8PJgi7grn1Vdz3uvns+zWNxK9/Ba2mWa+s+kgDz34FHu27CPxymbW/+hOOv0e/I0/58h3fs++P27j4IbDbB9IsX/YY8AL/nPjrrD44FOkHn+G/S9s58jGPfRs66Wne5B9SY/edIZE2iflB1/Ge6rP4lBPml19A+w6OsSO7kH2Hh0i0TfM4LFhkv0pkv39pAcTdFyymJq2RqpamnKJWE59C368Hq96Fn51PUOeYSjlk/Q8q93HENfFDen4TjRGJBYPNP1YHCcaY/eRQUZCRdW80HrG88lkfHz7Wl0bJZan5Y/q+NlCa7HQ4qdTebp99g8hPAbg+xmqIzYhy2r5UcfJ0/HDun6pxdayuFntfgIt/2R197FvP9VF0lTHLxOMwU9NSRmGB4H3AHfb15+NPcBG9AwZY0ZEpAW4DPiC/WXwGHAT8MNi7y+EavqKoihhDPi+X9JyktwNvElEtgFX2W1E5CIR+aY9ZgWwTkQ2AI8BdxtjXrT7bgc+JiLbCTT+b5Vy0bJ40lcURZkqDFNTZdMY0wNcWWB8HXCbXf8DcG6R9+8AVk32umr0FUVRwhjy+jifaZSF0e9YuYCFax/mK+v3E+N+brmwgxU3X0Trje/icOu5/OTlXn7wwCu8vGkDR17exGD3HnwvhRuL0/Djf2Lr717gwLMH2X5gkP3DQUx+xkDMEVqrXNqrIpxVE2XbF7/MkS099O5KsC/p5WLykxmfjPWrxxwh7gq/3HaEHYeDmPwjvUkG+oYZGkgxPJgi1X+U1FACLzlAJjVM8yUX5hqk+DUN+NX1ZKpnkXJiDKZ9hgY9kmlDYiRN/0iGaLwuF5PvVlkNP6Tju7F4rhHKscRIwZj8bKE14xsynofvpWiui+Vi8uNRN0/Hdx3J0/OjTlBwDY6PyYdAx89iMhmqIk7BmPzwdrgRSvhc4xE0UTm+qFohHf9E6pCVGpM/2RyAia6hzGSMlmE4EUTk2yJyWEQ2hsZKSjtWFEWZNiYXp192nE5H7neBa8aMlZp2rCiKMi0YY8ikvJKWcuS0GX1jzJPA0THD1xOkC2Nfb0BRFGVGYaykOfFSjky1pl9q2jE2nXk1wFkdRQ9TFEU5tWjnrNPDBGnHGGPWAGsAauctNZes/haJvS8x8PQa+uav4uEdvfzw8T1s3fQo3dtfZPDwHjKpJG4sTm3rfGZ3LqP9rAZ+/KkP5gqqZUyQ6FMfzTpvIzQvqKdlWTMNSzv52b88VtR5WxcRal2HpphLU8zl63/YnSuoVsh5640k8b0guSn6qr/Cr64nbQuqDaZ9hoZ9kuk0/SmPxLBHYsRjIOXRP+IRq2vMFVQr5Lx1XYdIzCUSdRhIJHPO20wmSMjyM37OeWsymdx9NNVV5RVUG7u4tlhatvOV76WD/4siztvcejg5q4jzNlw0bSIH7tj92eSs8Zy3J/KTNexgPZOct9pY6yQxYDJFTVPZM9VGv6S0Y0VRlOnCYKaqyua0MNUZudm0Y5hE2rCiKMqUYcD4pqSlHDltT/oi8gOCWtEtIrIX+AeCNOMfich7gd3Azafr+oqiKCeCMZBJaXLWpDHG3Fpk13FpxxOR7Osl1n+UeRdeyZVrhd2bH6Jv1wsM9ewPNPPaembNXUTTWYuY09XApUtbuezsZs5tq+V/fHrYJmFFmFsdYV5djKYljTQtbqZpxQLqliwm1rUcv6WLDZ/+Ve6acVeIuw6zIw71UZfWKpdZ9VXUNMepa69l16b9pAcTeTp+Jp3K6edhXbq/cRGDaUMy6TOUTuVp+AMjHsdGPBJDthHKiEe8cU4uKSsSdYnEsjq+bYASdXEiDpGow+FXEqNavr12tlCa8QM937frbbOqRvV7m4wVdRyiruT0fMexr7Yw2ng6fpiaqJtXEC2s44/q7lJUbx5P5xeR0SYqofc7Y46ZLMcVXBvnHKe6+JpzioV31fFPIcaopq8oilJJ+Gr0FUVRKgQN2VQURakcDOCXqZO2FNToK4qihDFGHbnTzZx57fz0Gx/hvPYa6i99P24sTryxnbkXXk37WQ28emkLr1/cwsXzZtM1O0r00FbSL/2C/l9v5Or2WpoX1NO0uJGmFWfRsGwh0a7lSMciMg2d9GYidA957O4dpj7q5CVgNdZGibfUUNdWQ217LfG2RmpaG6jpaKb3GxvyErDGOiLFcXPLlp5hEsOB4zYxEiRgJYbSDAwH6wPD1ok77OGlM9S1tOQlYDmhpCw3IoFz13bA2r1pb14CVnbJZLczo9UxW2dXHZeAFXUDp23UyXa9Gl3PpFO5+UzU7SrqOHkJWOGKmnnjRd4/Hq712I7nuD1RR2sx5606bisXo8lZiqIoFYQafUVRlEpCM3IVRVEqhynKyC2lv4iIvFFE1oeWYRG5we77rojsDO07v5TrlsWTfluym6qP3MJvnznI6z72v/OSrzoiw7gHNpPa8hhHf76FbVv2cGRLDz37B9iX9Fj9bx/OJV95DfPoHvLoHvTY1TvE7p2j3a96e4f5dFdDLvmqpq2OmrZGauY0EW9twmlsI9I8B6ehFT9ez/C/fD7vHsMavhMNOl05kShOJMYfXunNS77KavhDVsP30j5eKpPrdlXfXJNLvsoWWcsmVVVFHOKxSLDtOjzVfzSXfJXV8MNdroIleGppqo7mJV+5Y9YdIdD83dHkrCyFNPjwWMTNT75yZFS/DydtFTvXeDjka+/HJVVN6myh941zzuOOneS5T7WGH0b1/NOLYcri9LP9Re4WkTvs9u1592LMY8D5EHxJANuB34QO+XtjzP2TuWhZGH1FUZQpwxj8qYneuZ6gVA0E/UUeZ4zRH8NNwK+MMUMnc1GVdxRFUUIYEzzpl7KcJCX3F7HcAvxgzNg/icjzInKviFSVclF90lcURRnDJLpitYjIutD2GtsLBAAReQSYU+B9d+Vdb4L+IrYU/bnA2tDwnQRfFjGC3iO3A/840Q2XhdHft7ePr+99ibgrPHq1YWTzQxz94VZ6Nu/l5S09dB8c4OBwhiMpjwHPJxn6Bt746neysy/J7q1D7Ojeyu4jgyT6hhk8NkyyP8Xw4FCucNpFH76SeHsrbmMrbmNbTr/34/X4VbMY8IXBtGEo7eNEYgX1eycaIxILiqU5kRhuVZzHNh8uqt97qaD5SbgJyooL5hKLONTEXGIRN6ffZzX9cOOT1GACOF6/D+v6EDRAaYxH8/T7qOPkmp0Uan4ykaYfJuZkG5zk6/fZn5KFGqCUiht609i3n0w8fbH3qmRe4ZhJPcUfMcZcVPxU5qpi+0RkMv1FbgYeMMakQ+fO/koYEZHvAJ8o5YZV3lEURQlj4/RLWU6SyfQXuZUx0o79okCCJ6obgI2lXLQsnvQVRVGmCsOUFVwr2F9ERC4C3meMuc1udwHzgSfGvP8+EWkl+HG6HnhfKRdVo68oihLGGDKp02/0jTE9FOgvYoxZB9wW2t4FzCtw3BUncl01+oqiKCGMAd9oGYZppXV2FZ/8L6+jaUUX96xaTW86w4Dnk7IZca4EjsS6iMPc6ihNMYfWqgg1TXFu+z9/ZKh/hJHBgcBhO5jAGx7E91J4I8lcdymAWX91N5nq2QykfQbTPknPJ5n2SRz1SIz0MzBiO16NeMyau8g6cGO4sbh14FaFulq5ueJor+zoJWMdtV4qgzEm1+lqbJcr42c4Z96KvO5WuSVUJC3qOLgC3vAgkO+whcJdrhrj0YIO27GF0UpJojq+4Fr2HPkO22KdriaDUNjpeiLdssaet1S0YFplkVGjryiKUhkY4Ayut6ZGX1EUZSz6pK8oilIh+IacdHwmUhZG3z/rbJ766y+w8+gQMe5nUW2UppjLrOYaalri1LbXUts2i5o5zdS0NRJrbsJt7sBtbGXrbT/NafZhcsXRbAKVG4lx/84UiZGDgXZvC6QlkmmSKY/+YY9kKMFqzrKVOc0+2/DEce22bXCSTaZ6cu3zeZp9oQJp4WSq5R2zcpp9xA1eAy1/dD1bLC3rlxhLobHZVZE8zT5bIG1sg5Osfj2ZwmgRV4o2OTnZhiTumBOc6gYnwTlVs1dGUXlHURSlQjAYlXcURVEqBXXkKoqiVBhq9KeZbbsP8bcf+p/46RQDT6+B2sagCFr1bNKROENpn6Rn6Ev77EtlSIx4JIbTDKQyxBvbjyuE5lYFr5FYNNDjbWz9vQ++aIuh5RdA8zM+Gc8L9HgbV3/1tRfkYufHFkHLxdi7DlFHeOj7O/MKoYW18kJx9UuaascthBZuVpJJJUv6NzR+hrpYoLqPLYIGhePqJ0OsBN39ROPqww1ZTiWT0fFVo68cjNHoHUVRlIrBoNE7iqIoFYNq+oqiKBWGyjuKoigVQqDpT/ddnD7Kwui7sWraVl6GG3G4cq3gpY7ipbttolSGjGfwPT/Xjcr4hozn4Xsp3vzO/2Cdqy7xqJvXfWpsQbNP/8N3Q0lSfsHuU1luvfA6HOE4R2shx+tw4kjufaUkPJ1VH7S6LKX71GQSqGqjwZkK+SRPNuEp6uaf4FT6Pd3T5EVV56xSDH3SVxRFqRAMMCUtVKYJNfqKoighDEajdxRFUSqFIHpHjf60cs6CJn7/pbcBUH/p+yf13u9+7e0lH/ux7j0lH3vZ/FklH1uo4Nt4tNWenv+WmuiJtjGZmMjpqIJmUe1dmVLOcEfu6bMC4yAi14jIVhHZLiJ3TMc9KIqiFCL7pF/KcjKIyNtFZJOI+LYZerHjCtpLEVkoIn+y4/8uIrFSrjvlRl9EXOArwFuAlcCtIrJyqu9DURSlGBlT2nKSbARuBJ4sdsAE9vLzwL3GmMVAL/DeUi46HU/6q4DtxpgdxpgU8EPg+mm4D0VRlOPwCcowlLKcDMaYzcaYrRMcVtBeShC/fQVwvz3ue8ANpVxXzBQ7LETkJuAaY8xtdvvdwCXGmA+OOW41sNpunkPwrXim0AIcmfCo8uFMmw+ceXOqpPksMMa0nuiJReTX9vylUA0Mh7bXGGPWTPJ6jwOfMMasK7CvoL0EPgs8ZZ/yEZH5wK+MMedMdL0Z68i1/3BrAERknTGmqOZVbuh8Zj5n2px0PqVjjLnmVJ1LRB4B5hTYdZcx5men6jqTYTqM/j5gfmi7044piqKcURhjrjrJUxSzlz1Ag4hEjDEek7Cj06Hp/xlYYj3PMeAW4MFpuA9FUZSZTkF7aQJd/jHgJnvce4CSfjlMudG330ofBNYCm4EfGWM2TfC2SWlkZYDOZ+Zzps1J5zPDEJH/KCJ7gUuBX4rIWjs+V0Qeggnt5e3Ax0RkO9AMfKuk6061I1dRFEWZPqYlOUtRFEWZHtToK4qiVBAz2uiXa7kGEfm2iBwWkY2hsSYReVhEttnXRjsuIvIlO8fnReSC6bvzwojIfBF5TERetGnjH7HjZTknEakWkadFZIOdz+fseMG0dhGpstvb7f6u6bz/YoiIKyLPicgv7Ha5z2eXiLwgIutFZJ0dK8vP3Exixhr9Mi/X8F1gbKzvHcCjxpglwKN2G4L5LbHLauCrU3SPk8EDPm6MWQm8FviA/b8o1zmNAFcYY84DzgeuEZHXUjyt/b1Arx2/1x43E/kIgbMvS7nPB+CNxpjzQzH55fqZmzkYY2bkQuDRXhvavhO4c7rvaxL33wVsDG1vBTrsegew1a5/Hbi10HEzdSEIDXvTmTAnoAZ4liDL8QgQseO5zx9B5MSldj1ij5Ppvvcx8+gkMIJXAL8gaF5WtvOx97YLaBkzVvafueleZuyTPjAPCNc63mvHypV2Y8wBu34QaLfrZTVPKwW8BvgTZTwnK4WsBw4DDwMvA30mCJGD/HvOzcfuTxCEyM0kvgh8ktGmT82U93wgKHj5GxF5xpZlgTL+zM0UZmwZhjMZY4wRkbKLlRWROuAnwEeNMcckVOi+3OZkjMkA54tIA/AAsHyab+mEEZG3AYeNMc+IyOXTfT+nkL8wxuwTkTbgYRHZEt5Zbp+5mcJMftI/08o1HBKRDgD7etiOl8U8RSRKYPDvM8b81A6X9ZwAjDF9BJmNl2LT2u2u8D3n5mP31xOkwc8ULgOuE5FdBFUYrwD+F+U7HwCMMfvs62GCL+ZVnAGfuelmJhv9M61cw4MEqdKQnzL9IPDXNvrgtUAi9PN1RiDBI/23gM3GmHtCu8pyTiLSap/wEZE4gX9iM8XT2sPzvAn4rbHC8UzAGHOnMabTGNNF8HfyW2PMuyjT+QCISK2IzMquA28mqLRblp+5GcV0OxXGW4C3Ai8R6K13Tff9TOK+fwAcANIE2uJ7CTTTR4FtwCNAkz1WCKKUXgZeAC6a7vsvMJ+/INBXnwfW2+Wt5Ton4NXAc3Y+G4HP2PGzgaeB7cCPgSo7Xm23t9v9Z0/3HMaZ2+XAL8p9PvbeN9hlU/bvv1w/czNp0TIMiqIoFcRMlncURVGUU4wafUVRlApCjb6iKEoFoUZfURSlglCjryiKUkGo0VemHRHJ2EqKm2zly4+LyAl/NkXkU6H1LglVO1WUSkeNvjITSJqgkuKrCBKl3gL8w0mc71MTH6IolYkafWVGYYKU+9XAB212pSsi/ywif7Z10v8OQEQuF5EnReSXEvRc+JqIOCJyNxC3vxzus6d1ReQb9pfEb2wWrqJUJGr0lRmHMWYH4AJtBNnMCWPMxcDFwN+KyEJ76CrgQwT9FhYBNxpj7mD0l8O77HFLgK/YXxJ9wH+autkoysxCjb4y03kzQU2V9QTlnJsJjDjA08aYHSaomPkDgnIRhdhpjFlv158h6HWgKBWJllZWZhwicjaQIaigKMCHjDFrxxxzOUE9oDDFaoqMhNYzgMo7SsWiT/rKjEJEWoGvAV82QWGotcB/taWdEZGltuoiwCpbhdUB3gH8zo6ns8cripKPPukrM4G4lW+iBP14/y+QLeH8TQI55llb4rkbuMHu+zPwZWAxQRnhB+z4GuB5EXkWuGsqJqAo5YJW2VTKEivvfMIY87bpvhdFKSdU3lEURakg9ElfURSlgtAnfUVRlApCjb6iKEoFoUZfURSlglCjryiKUkGo0VcURakg/j+7fyjNRp+DjgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a_b4ou4TYqUN"
      },
      "source": [
        "## Masking"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s42Uydjkv0hF"
      },
      "source": [
        "Mask all the pad tokens in the batch of sequence. It ensures that the model does not treat padding as the input. The mask indicates where pad value `0` is present: it outputs a `1` at those locations, and a `0` otherwise."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2i8-e1s8ti9"
      },
      "source": [
        "def create_padding_mask(seq):\n",
        "  print (seq)\n",
        "  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  print (seq)\n",
        "  # add extra dimensions to add the padding\n",
        "  # to the attention logits.\n",
        "  # why add a new axis when a padding mask can is returned by tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A7BYeBCNvi7n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ab6f09de-8eb6-47cf-f088-345e3b7dda8a"
      },
      "source": [
        "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n",
        "create_padding_mask(x)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[[7 6 0 0 1]\n",
            " [1 2 3 0 0]\n",
            " [0 0 0 4 5]], shape=(3, 5), dtype=int32)\n",
            "tf.Tensor(\n",
            "[[0. 0. 1. 1. 0.]\n",
            " [0. 0. 0. 1. 1.]\n",
            " [1. 1. 1. 0. 0.]], shape=(3, 5), dtype=float32)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3, 1, 1, 5), dtype=float32, numpy=\n",
              "array([[[[0., 0., 1., 1., 0.]]],\n",
              "\n",
              "\n",
              "       [[[0., 0., 0., 1., 1.]]],\n",
              "\n",
              "\n",
              "       [[[1., 1., 1., 0., 0.]]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0hzukDBgVom"
      },
      "source": [
        "The look-ahead mask is used to mask the future tokens in a sequence. In other words, the mask indicates which entries should not be used.\n",
        "\n",
        "This means that to predict the third word, only the first and second word will be used. Similarly to predict the fourth word, only the first, second and the third word will be used and so on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dVxS8OPI9uI0"
      },
      "source": [
        "def create_look_ahead_mask(size):\n",
        "\n",
        "  # tf.matrix_band_part(input, -1, 0) ==> Lower triangular part.\n",
        "  # https://www.tensorflow.org/api_docs/python/tf/linalg/band_part\n",
        "  \n",
        "  print (tf.linalg.band_part(tf.ones((size, size)), -1, 0))\n",
        " \n",
        "  # switch to the uppper triangular part\n",
        "  mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
        "  return mask  # (seq_len, seq_len)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxKGuXxaBeeE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3b1164fe-f6ae-42f9-b3d6-e1e006424581"
      },
      "source": [
        "x = tf.random.uniform((1, 3))\n",
        "print (x)\n",
        "temp = create_look_ahead_mask(x.shape[1])\n",
        "temp"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor([[0.82464564 0.16693962 0.7969377 ]], shape=(1, 3), dtype=float32)\n",
            "tf.Tensor(\n",
            "[[1. 0. 0.]\n",
            " [1. 1. 0.]\n",
            " [1. 1. 1.]], shape=(3, 3), dtype=float32)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3, 3), dtype=float32, numpy=\n",
              "array([[0., 1., 1.],\n",
              "       [0., 0., 1.],\n",
              "       [0., 0., 0.]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xluDl5cXYy4y"
      },
      "source": [
        "## Scaled dot product attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vsxEE_-Wa1gF"
      },
      "source": [
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/scaled_attention.png\" width=\"500\" alt=\"scaled_dot_product_attention\">\n",
        "\n",
        "The attention function used by the transformer takes three inputs: Q (query), K (key), V (value). The equation used to calculate the attention weights is:\n",
        "\n",
        "$$\\Large{Attention(Q, K, V) = softmax_k(\\frac{QK^T}{\\sqrt{d_k}}) V} $$\n",
        "\n",
        "The dot-product attention is scaled by a factor of square root of the depth. This is done because for large values of depth, the dot product grows large in magnitude pushing the softmax function where it has small gradients resulting in a very hard softmax. \n",
        "\n",
        "For example, consider that `Q` and `K` have a mean of 0 and variance of 1. Their matrix multiplication will have a mean of 0 and variance of `dk`. Hence, *square root of `dk`* is used for scaling (and not any other number) because the matmul of `Q` and `K` should have a mean of 0 and variance of 1, and you get a gentler softmax.\n",
        "\n",
        "The mask is multiplied with -1e9 (close to negative infinity). This is done because the mask is summed with the scaled matrix multiplication of Q and K and is applied immediately before a softmax. The goal is to zero out these cells, and large negative inputs to softmax are near zero in the output."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LazzUq3bJ5SH"
      },
      "source": [
        "\"\"\"\n",
        "\n",
        "https://blog.demofox.org/2014/02/20/soft-maximum-vs-hard-maximum/\n",
        "https://www.johndcook.com/blog/2010/01/20/how-to-compute-the-soft-maximum/\n",
        "https://deepai.org/machine-learning-glossary-and-terms/softmax-layer\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "\n",
        "  \"\"\"Calculate the attention weights.\n",
        "  q, k, v must have matching leading dimensions.\n",
        "  k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
        "  The mask has different shapes depending on its type(padding or look ahead) \n",
        "  but it must be broadcastable for addition.\n",
        "  \n",
        "  Args:\n",
        "    q: query shape == (..., seq_len_q, depth)\n",
        "    k: key shape == (..., seq_len_k, depth)\n",
        "    v: value shape == (..., seq_len_v, depth_v)\n",
        "    mask: Float tensor with shape broadcastable \n",
        "          to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "    \n",
        "  Returns:\n",
        "    output, attention_weights\n",
        "  \"\"\"\n",
        "\n",
        "# https://www.tensorflow.org/api_docs/python/tf/linalg/matmul\n",
        "  matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n",
        "  \n",
        "  # scale matmul_qk\n",
        "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "  scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "\n",
        "  # add the mask to the scaled tensor.\n",
        "  if mask is not None:\n",
        "    print (\"Applying mask\\n\")\n",
        "    scaled_attention_logits += (mask * -1e9)  \n",
        "\n",
        "  # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "  # add up to 1.\n",
        "  attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n",
        "  # print (attention_weights)\n",
        "  output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "  return output, attention_weights"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FiqETnhCkoXh"
      },
      "source": [
        "As the softmax normalization is done on K, its values decide the amount of importance given to Q.\n",
        "\n",
        "The output represents the multiplication of the attention weights and the V (value) vector. This ensures that the words you want to focus on are kept as-is and the irrelevant words are flushed out."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n90YjClyInFy"
      },
      "source": [
        "def print_out(q, k, v):\n",
        "  temp_out, temp_attn = scaled_dot_product_attention(\n",
        "      q, k, v, None)\n",
        "  print ('Attention weights are:')\n",
        "  print (temp_attn)\n",
        "  print ('Output is:')\n",
        "  print (temp_out)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yAzUAf2DPlNt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "be74940a-718d-49fe-ec14-7daa5a341459"
      },
      "source": [
        "np.set_printoptions(suppress=True)\n",
        "\n",
        "# temp_k and temp_v are both the same - but in this test input - only the  number of words (4) is \n",
        "# the same in both cases. The dimensions (second member in shape vector), is different for both \n",
        "# temp_k and temp_v. Should both temp_k and temp_v not be identical in size and content?\n",
        "\n",
        "temp_k = tf.constant([[10,0,0],\n",
        "                      [0,10,0],\n",
        "                      [0,0,10],\n",
        "                      [0,0,10]], dtype=tf.float32)  # (4, 3)\n",
        "\n",
        "temp_v = tf.constant([[   1,0],\n",
        "                      [  10,0],\n",
        "                      [ 100,5],\n",
        "                      [1000,6]], dtype=tf.float32)  # (4, 2)\n",
        "\n",
        "# This `query` aligns with the second `key`,\n",
        "# so the second `value` is returned.\n",
        "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zg6k-fGhgXra",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dcc47849-ba23-4ee2-bd58-132d5aadada0"
      },
      "source": [
        "# This query aligns with a repeated key (third and fourth), \n",
        "# so all associated values get averaged.\n",
        "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UAq3YOzUgXhb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dfef2acc-b29a-4ca5-c7ef-13d0eaff3afd"
      },
      "source": [
        "# This query aligns equally with the first and second key, \n",
        "# so their values get averaged.\n",
        "temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aOz-4_XIhaTP"
      },
      "source": [
        "Pass all the queries together."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6dlU8Tm-hYrF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2140d903-6d58-418a-c26d-cac57b133411"
      },
      "source": [
        "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor(\n",
            "[[0.  0.  0.5 0.5]\n",
            " [0.  1.  0.  0. ]\n",
            " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor(\n",
            "[[550.    5.5]\n",
            " [ 10.    0. ]\n",
            " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kmzGPEy64qmA"
      },
      "source": [
        "## Multi-head attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fz5BMC8Kaoqo"
      },
      "source": [
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/multi_head_attention.png\" width=\"500\" alt=\"multi-head attention\">\n",
        "\n",
        "\n",
        "Multi-head attention consists of four parts:\n",
        "*    Linear layers and split into heads.\n",
        "*    Scaled dot-product attention.\n",
        "*    Concatenation of heads.\n",
        "*    Final linear layer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JPmbr6F1C-v_"
      },
      "source": [
        "Each multi-head attention block gets three inputs; Q (query), K (key), V (value). These are put through linear (Dense) layers and split up into multiple heads. \n",
        "\n",
        "The `scaled_dot_product_attention` defined above is applied to each head (broadcasted for efficiency). An appropriate mask must be used in the attention step.  The attention output for each head is then concatenated (using `tf.transpose`, and `tf.reshape`) and put through a final `Dense` layer.\n",
        "\n",
        "Instead of one single attention head, Q, K, and V are split into multiple heads because it allows the model to jointly attend to information at different positions from different representational spaces. After the split each head has a reduced dimensionality, so the total computation cost is the same as a single head attention with full dimensionality."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BSV3PPKsYecw"
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "    \n",
        "    assert d_model % self.num_heads == 0 #perfectly divides. Remainder is 0\n",
        "    \n",
        "    self.depth = d_model // self.num_heads # dimensionality of each head. Quotient is the number of vector elements representing each word in head\n",
        "    \n",
        "    self.wq = tf.keras.layers.Dense(d_model) # before splitting into heads, wq has d_model neurons/units or elements (after splitting has self.depth elements in each head)\n",
        "    self.wk = tf.keras.layers.Dense(d_model) # before splitting into heads, wk has d_model neurons/units or elements (after splitting has self.depth elements in each head)\n",
        "    self.wv = tf.keras.layers.Dense(d_model) # before splitting into heads, wv has d_model neurons/units or elements (after splitting has self.depth elements in each head)\n",
        "    \n",
        "    self.dense = tf.keras.layers.Dense(d_model)\n",
        "        \n",
        "  def split_heads(self, x, batch_size):\n",
        "    \"\"\"Split the last dimension into (num_heads, depth).\n",
        "    Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "    \"\"\"\n",
        "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "    return tf.transpose(x, perm=[0, 2, 1, 3])\n",
        "    \n",
        "  def call(self, v, k, q, mask):\n",
        "    print (tf.shape(q))\n",
        "    batch_size = tf.shape(q)[0]\n",
        "    print (tf.shape(q)[0])\n",
        "    \n",
        "    q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
        "    print (f\"\\nBefore splitting q, q shape is: {q.shape}\")\n",
        "    k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
        "    print (f\"Before splitting k, k shape is: {k.shape}\")\n",
        "    v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
        "    print (f\"Before splitting v, v shape is: {v.shape}\")\n",
        "\n",
        "    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "    print (f\"\\nAfter splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): {q.shape}\")\n",
        "    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "    print (f\"After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): {k.shape}\")\n",
        "    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "    print (f\"After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): {v.shape}\")\n",
        "    \n",
        "    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "    scaled_attention, attention_weights = scaled_dot_product_attention(\n",
        "        q, k, v, mask)\n",
        "    \n",
        "    print (f\"\\nScaled_weights - depth is still 64 - the chosen dimension per head: {scaled_attention.shape}\")\n",
        "    print (f\"Attention_weights - see how the depth changed; one scalar weight per word. {attention_weights.shape[-1]} for last dimension or axis = -1: {attention_weights.shape}\")\n",
        "\n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
        "    \n",
        "    print (f\"\\nScaled attention right before cocatenating heads back together : {scaled_attention.shape}\")\n",
        "    \n",
        "    concat_attention = tf.reshape(scaled_attention, \n",
        "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "    \n",
        "    print (f\"\\nAfter concatenating the heads back together : {concat_attention.shape}\")\n",
        "    \n",
        "    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "        \n",
        "    return output, attention_weights"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0D8FJue5lDyZ"
      },
      "source": [
        "Create a `MultiHeadAttention` layer to try out. At each location in the sequence, `y`, the `MultiHeadAttention` runs all 8 attention heads across all other locations in the sequence, returning a new vector of the same length at each location."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hu94p-_-2_BX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a48feb0b-4077-42e4-a6c7-461b99e2177e"
      },
      "source": [
        "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n",
        "y = tf.random.uniform((1, 60, 512))  # (batch_size, encoder_sequence, d_model)\n",
        "print (y.shape)\n",
        "\n",
        "out, attn = temp_mha(y, k=y, q=y, mask=None)\n",
        "\n",
        "out.shape, attn.shape  "
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 60, 512)\n",
            "tf.Tensor([  1  60 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(1, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (1, 60, 512)\n",
            "Before splitting k, k shape is: (1, 60, 512)\n",
            "Before splitting v, v shape is: (1, 60, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (1, 8, 60, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (1, 8, 60, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (1, 8, 60, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (1, 8, 60, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 60 for last dimension or axis = -1: (1, 8, 60, 60)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (1, 60, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (1, 60, 512)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RdDqGayx67vv"
      },
      "source": [
        "## Point wise feed forward network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBqzJXGfHK3X"
      },
      "source": [
        "Point wise feed forward network consists of two fully-connected layers with a ReLU activation in between."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ET7xLt0yCT6Z"
      },
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "  print (f\"d_model: {d_model}; dff: {dff}\")\n",
        "  return tf.keras.Sequential([\n",
        "      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff) # layer 1 - \n",
        "      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model) #layer 2\n",
        "  ])"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mytb1lPyOHLB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "983fb2bf-ebdd-44b3-ef1d-1707d29b5289"
      },
      "source": [
        "sample_ffn = point_wise_feed_forward_network(512, 2048)\n",
        "print (type(sample_ffn))\n",
        "sample_ffn(tf.random.uniform((64, 50, 512))).shape"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "d_model: 512; dff: 2048\n",
            "<class 'tensorflow.python.keras.engine.sequential.Sequential'>\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 50, 512])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7e7hKcxn6-zd"
      },
      "source": [
        "## Encoder and decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yScbC0MUH8dS"
      },
      "source": [
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/transformer.png\" width=\"600\" alt=\"transformer\">"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MfYJG-Kvgwy2"
      },
      "source": [
        "The transformer model follows the same general pattern as a standard [sequence to sequence with attention model](nmt_with_attention.ipynb). \n",
        "\n",
        "* The input sentence is passed through `N` encoder layers that generates an output for each word/token in the sequence.\n",
        "* The decoder attends on the encoder's output and its own input (self-attention) to predict the next word. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QFv-FNYUmvpn"
      },
      "source": [
        "### Encoder layer\n",
        "\n",
        "Each encoder layer consists of sublayers:\n",
        "\n",
        "1.   Multi-head attention (with padding mask) \n",
        "2.    Point wise feed forward networks. \n",
        "\n",
        "Each of these sublayers has a residual connection around it followed by a layer normalization. Residual connections help in avoiding the vanishing gradient problem in deep networks.\n",
        "\n",
        "The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis. There are N encoder layers in the transformer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncyS-Ms3i2x_"
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "\n",
        "    super(EncoderLayer, self).__init__()\n",
        "    self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)    \n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "  def call(self, x, training, mask):\n",
        "     \n",
        "    print (f\"x.shape: {x.shape}; training: {training}; mask: {mask}\")\n",
        "    attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model); x acts as the query, values and keys inputs\n",
        "    print (f\"attn_output.shape (after multi-headed attention results are concatenated): {attn_output.shape}\")\n",
        "    attn_output = self.dropout1(attn_output, training=training) # use dropout on attn_output\n",
        "    print (f\"attn_output.shape (after dropout on multi-headed attention results - same dims but are they marked in-active): {attn_output.shape}\")\n",
        "\n",
        "    # https://www.tensorflow.org/addons/tutorials/layers_normalizations\n",
        "    # https://arxiv.org/pdf/1803.08494.pdf    \n",
        "    \n",
        "    out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)    \n",
        "    ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
        "    ffn_output = self.dropout2(ffn_output, training=training)\n",
        "    out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
        "    \n",
        "    return out2"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzZRXdO0mI48",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e34281f0-3dc7-4a57-c335-0043f6d8a5ee"
      },
      "source": [
        "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n",
        "\n",
        "sample_encoder_layer_output = sample_encoder_layer(\n",
        "    tf.random.uniform((64, 43, 512)), False, None)\n",
        "\n",
        "sample_encoder_layer_output.shape  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "d_model: 512; dff: 2048\n",
            "x.shape: (64, 43, 512); training: False; mask: None\n",
            "tf.Tensor([ 64  43 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 43, 512)\n",
            "Before splitting k, k shape is: (64, 43, 512)\n",
            "Before splitting v, v shape is: (64, 43, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 43, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 43, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 43, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 43, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 43 for last dimension or axis = -1: (64, 8, 43, 43)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 43, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 43, 512)\n",
            "attn_output.shape (after multi-headed attention results are concatenated): (64, 43, 512)\n",
            "attn_output.shape (after dropout on multi-headed attention results - same dims but are they marked in-active): (64, 43, 512)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 43, 512])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6LO_48Owmx_o"
      },
      "source": [
        "### Decoder layer\n",
        "\n",
        "Each decoder layer consists of sublayers:\n",
        "\n",
        "1.   Masked multi-head attention (with look ahead mask and padding mask)\n",
        "2.   Multi-head attention (with padding mask). V (value) and K (key) receive the *encoder output* as inputs. Q (query) receives the *output from the masked multi-head attention sublayer.*\n",
        "3.   Point wise feed forward networks\n",
        "\n",
        "Each of these sublayers has a residual connection around it followed by a layer normalization. The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis.\n",
        "\n",
        "There are N decoder layers in the transformer.\n",
        "\n",
        "As Q receives the output from decoder's first attention block, and K receives the encoder output, the attention weights represent the importance given to the decoder's input based on the encoder's output. In other words, the decoder predicts the next word by looking at the encoder output and self-attending to its own output. See the demonstration above in the scaled dot product attention section."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9SoX0-vd1hue"
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(DecoderLayer, self).__init__()\n",
        "\n",
        "    self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "    self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        " \n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout3 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "    \n",
        "  def call(self, x, enc_output, training, \n",
        "           look_ahead_mask, padding_mask):\n",
        "    # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
        "\n",
        "    # mha takes mask as input (step 1: look_ahead_mask)\n",
        "    attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn1 = self.dropout1(attn1, training=training)\n",
        "    out1 = self.layernorm1(attn1 + x)\n",
        "    \n",
        "    # mha takes mask as input (step 2: padding mask)\n",
        "    attn2, attn_weights_block2 = self.mha2(\n",
        "        enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn2 = self.dropout2(attn2, training=training)\n",
        "    out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
        "    \n",
        "    ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
        "    ffn_output = self.dropout3(ffn_output, training=training)\n",
        "    out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
        "    \n",
        "    return out3, attn_weights_block1, attn_weights_block2"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ne2Bqx8k71l0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a1d7a00-4e6e-4564-e6e2-715a3734be1d"
      },
      "source": [
        "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n",
        "\n",
        "sample_decoder_layer_output, _, _ = sample_decoder_layer(\n",
        "    tf.random.uniform((64, 50, 512)), sample_encoder_layer_output, \n",
        "    False, None, None) # both look-ahead and padding mask passed-in are None/null\n",
        "\n",
        "sample_decoder_layer_output.shape  # (batch_size, target_seq_len, d_model)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "d_model: 512; dff: 2048\n",
            "tf.Tensor([ 64  50 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 50, 512)\n",
            "Before splitting k, k shape is: (64, 50, 512)\n",
            "Before splitting v, v shape is: (64, 50, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 50, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 50, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 50, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 50, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 50 for last dimension or axis = -1: (64, 8, 50, 50)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 50, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 50, 512)\n",
            "tf.Tensor([ 64  50 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 50, 512)\n",
            "Before splitting k, k shape is: (64, 43, 512)\n",
            "Before splitting v, v shape is: (64, 43, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 50, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 43, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 43, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 50, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 43 for last dimension or axis = -1: (64, 8, 50, 43)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 50, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 50, 512)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 50, 512])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SE1H51Ajm0q1"
      },
      "source": [
        "### Encoder\n",
        "\n",
        "The `Encoder` consists of:\n",
        "1.   Input Embedding\n",
        "2.   Positional Encoding\n",
        "3.   N encoder layers\n",
        "\n",
        "The input is put through an embedding which is summed with the positional encoding. The output of this summation is the input to the encoder layers. The output of the encoder is the input to the decoder."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpEox7gJ8FCI"
      },
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Encoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "    \n",
        "    self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, \n",
        "                                            self.d_model)\n",
        "    \n",
        "    # list with elements as initialized encoders   \n",
        "    self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n",
        "                       for _ in range(num_layers)]\n",
        "    \n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "        \n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    \n",
        "    # adding embedding and position encoding.\n",
        "    print (f\"Input shape BEFORE initialzing embedding {x.shape}\")\n",
        "    x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n",
        "    print (f\"Input shape AFTER initialzing embedding {x.shape}. self.embedding knows  x is (batch size, sequence length) and dimensionality of word is d_model\")\n",
        "    \n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32)) # Why? Is this element-wise multiplication a scaling technique \n",
        "    print (f\"*************x = {x.shape}; d_model = {self.d_model}; \")    \n",
        "    x += self.pos_encoding[:, :seq_len, :]  # Adding the positional encoding to the word embedding\n",
        "    \n",
        "    # Dropout right after the positional encoding? Ask.\n",
        "    x = self.dropout(x, training=training)\n",
        "\n",
        "    # you just need the encoder output from the final layer - so you can overwrite the output from layer 1 by the output from layer 2\n",
        "    for i in range(self.num_layers): \n",
        "      x = self.enc_layers[i](x, training, mask)\n",
        "\n",
        "    return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": 80,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QG9nueFQKXx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "034cc354-d8c7-4217-ce9a-a5fb95ce282a"
      },
      "source": [
        "sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8, \n",
        "                         dff=2048, input_vocab_size=8500,\n",
        "                         maximum_position_encoding=10000)\n",
        "\n",
        "temp_input = tf.random.uniform((64, 62), dtype=tf.int64, minval=0, maxval=200)\n",
        "\n",
        "sample_encoder_output = sample_encoder(temp_input, training=False, mask=None)\n",
        "\n",
        "print (sample_encoder_output.shape)  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "d_model: 512; dff: 2048\n",
            "d_model: 512; dff: 2048\n",
            "Input shape BEFORE initialzing embedding (64, 62)\n",
            "Input shape AFTER initialzing embedding (64, 62, 512). self.embedding knows  x is (batch size, sequence length) and dimensionality of word is d_model\n",
            "*************x = (64, 62, 512); d_model = 512; \n",
            "2\n",
            "x.shape: (64, 62, 512); training: False; mask: None\n",
            "tf.Tensor([ 64  62 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 62, 512)\n",
            "Before splitting k, k shape is: (64, 62, 512)\n",
            "Before splitting v, v shape is: (64, 62, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 62, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 62 for last dimension or axis = -1: (64, 8, 62, 62)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 62, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 62, 512)\n",
            "attn_output.shape (after multi-headed attention results are concatenated): (64, 62, 512)\n",
            "attn_output.shape (after dropout on multi-headed attention results - same dims but are they marked in-active): (64, 62, 512)\n",
            "x.shape: (64, 62, 512); training: False; mask: None\n",
            "tf.Tensor([ 64  62 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 62, 512)\n",
            "Before splitting k, k shape is: (64, 62, 512)\n",
            "Before splitting v, v shape is: (64, 62, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 62, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 62 for last dimension or axis = -1: (64, 8, 62, 62)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 62, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 62, 512)\n",
            "attn_output.shape (after multi-headed attention results are concatenated): (64, 62, 512)\n",
            "attn_output.shape (after dropout on multi-headed attention results - same dims but are they marked in-active): (64, 62, 512)\n",
            "<class 'tensorflow.python.framework.ops.EagerTensor'>\n",
            "(64, 62, 512)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p-uO6ls8m2O5"
      },
      "source": [
        "### Decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZtT7PKzrXkNr"
      },
      "source": [
        " The `Decoder` consists of:\n",
        "1.   Output Embedding\n",
        "2.   Positional Encoding\n",
        "3.   N decoder layers\n",
        "\n",
        "The target is put through an embedding which is summed with the positional encoding. The output of this summation is the input to the decoder layers. The output of the decoder is the input to the final linear layer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5_d5-PLQXwY"
      },
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Decoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "    \n",
        "    self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "    \n",
        "    self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n",
        "                       for _ in range(num_layers)]\n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "  def call(self, x, enc_output, training, \n",
        "           look_ahead_mask, padding_mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    attention_weights = {}\n",
        "    \n",
        "    x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
        "    print (f\"Multplying each member of the input matrix of shape {x.shape} by sqrt(d_model) which is sqrt of {self.d_model}\")\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32)) # Is this element-wise multiplication some kind of scaling technique\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "    \n",
        "    x = self.dropout(x, training=training)\n",
        "\n",
        "    for i in range(self.num_layers):\n",
        "      x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
        "                                             look_ahead_mask, padding_mask)\n",
        "      print (f\"*********************Decoder Layer:{i} x:{x.shape}; block1:{block1.shape} block2:{block2.shape}****************************\")\n",
        "\n",
        "      attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
        "      attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
        "    \n",
        "    # x.shape == (batch_size, target_seq_len, d_model)\n",
        "    return x, attention_weights"
      ],
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1jXoAMRZyvu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "076d3f7a-eac2-4eac-d215-a79ee45ea7fa"
      },
      "source": [
        "sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8, \n",
        "                         dff=2048, target_vocab_size=8000,\n",
        "                         maximum_position_encoding=5000)\n",
        "temp_input = tf.random.uniform((64, 26), dtype=tf.int64, minval=0, maxval=200)\n",
        "\n",
        "output, attn = sample_decoder(temp_input, \n",
        "                              enc_output=sample_encoder_output, \n",
        "                              training=False,\n",
        "                              look_ahead_mask=None, \n",
        "                              padding_mask=None)\n",
        "\n",
        "output.shape, attn['decoder_layer2_block2'].shape"
      ],
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "d_model: 512; dff: 2048\n",
            "d_model: 512; dff: 2048\n",
            "Multplying each member of the input matrix of shape (64, 26, 512) by sqrt(d_model) which is sqrt of 512\n",
            "tf.Tensor([ 64  26 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 26, 512)\n",
            "Before splitting k, k shape is: (64, 26, 512)\n",
            "Before splitting v, v shape is: (64, 26, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 26, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 26 for last dimension or axis = -1: (64, 8, 26, 26)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 26, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 26, 512)\n",
            "tf.Tensor([ 64  26 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 26, 512)\n",
            "Before splitting k, k shape is: (64, 62, 512)\n",
            "Before splitting v, v shape is: (64, 62, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 26, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 62 for last dimension or axis = -1: (64, 8, 26, 62)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 26, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 26, 512)\n",
            "*********************Decoder Layer:0 x:(64, 26, 512); block1:(64, 8, 26, 26) block2:(64, 8, 26, 62)****************************\n",
            "tf.Tensor([ 64  26 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 26, 512)\n",
            "Before splitting k, k shape is: (64, 26, 512)\n",
            "Before splitting v, v shape is: (64, 26, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 26, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 26 for last dimension or axis = -1: (64, 8, 26, 26)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 26, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 26, 512)\n",
            "tf.Tensor([ 64  26 512], shape=(3,), dtype=int32)\n",
            "tf.Tensor(64, shape=(), dtype=int32)\n",
            "\n",
            "Before splitting q, q shape is: (64, 26, 512)\n",
            "Before splitting k, k shape is: (64, 62, 512)\n",
            "Before splitting v, v shape is: (64, 62, 512)\n",
            "\n",
            "After splitting q, q shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 26, 64)\n",
            "After splitting k, k shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "After splitting v, v shape is (batch_size, num_heads, seq_len_q, depth): (64, 8, 62, 64)\n",
            "\n",
            "Scaled_weights - depth is still 64 - the chosen dimension per head: (64, 8, 26, 64)\n",
            "Attention_weights - see how the depth changed; one scalar weight per word. 62 for last dimension or axis = -1: (64, 8, 26, 62)\n",
            "\n",
            "Scaled attention right before cocatenating heads back together : (64, 26, 8, 64)\n",
            "\n",
            "After concatenating the heads back together : (64, 26, 512)\n",
            "*********************Decoder Layer:1 x:(64, 26, 512); block1:(64, 8, 26, 26) block2:(64, 8, 26, 62)****************************\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([64, 26, 512]), TensorShape([64, 8, 26, 62]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 93
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y54xnJnuYgJ7"
      },
      "source": [
        "## Create the Transformer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uERO1y54cOKq"
      },
      "source": [
        "Transformer consists of the encoder, decoder and a final linear layer. The output of the decoder is the input to the linear layer and its output is returned."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PED3bIpOYkBu"
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n",
        "               target_vocab_size, pe_input, pe_target, rate=0.1):\n",
        "    super(Transformer, self).__init__()\n",
        "\n",
        "    self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n",
        "                           input_vocab_size, pe_input, rate)\n",
        "\n",
        "    self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n",
        "                           target_vocab_size, pe_target, rate)\n",
        "\n",
        "    self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
        "    \n",
        "  def call(self, inp, tar, training, enc_padding_mask, \n",
        "           look_ahead_mask, dec_padding_mask):\n",
        "\n",
        "    enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n",
        "    \n",
        "    # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
        "    dec_output, attention_weights = self.decoder(\n",
        "        tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
        "    \n",
        "    final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
        "    \n",
        "    return final_output, attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tJ4fbQcIkHW1"
      },
      "source": [
        "sample_transformer = Transformer(\n",
        "    num_layers=2, d_model=512, num_heads=8, dff=2048, \n",
        "    input_vocab_size=8500, target_vocab_size=8000, \n",
        "    pe_input=10000, pe_target=6000)\n",
        "\n",
        "temp_input = tf.random.uniform((64, 38), dtype=tf.int64, minval=0, maxval=200)\n",
        "temp_target = tf.random.uniform((64, 36), dtype=tf.int64, minval=0, maxval=200)\n",
        "\n",
        "fn_out, _ = sample_transformer(temp_input, temp_target, training=False, \n",
        "                               enc_padding_mask=None, \n",
        "                               look_ahead_mask=None,\n",
        "                               dec_padding_mask=None)\n",
        "\n",
        "fn_out.shape  # (batch_size, tar_seq_len, target_vocab_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wsINyf1VEQLC"
      },
      "source": [
        "## Set hyperparameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zVjWCxFNcgbt"
      },
      "source": [
        "To keep this example small and relatively fast, the values for *num_layers, d_model, and dff* have been reduced. \n",
        "\n",
        "The values used in the base model of transformer were; *num_layers=6*, *d_model = 512*, *dff = 2048*. See the [paper](https://arxiv.org/abs/1706.03762) for all the other versions of the transformer.\n",
        "\n",
        "Note: By changing the values below, you can get the model that achieved state of the art on many tasks."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lnJn5SLA2ahP"
      },
      "source": [
        "num_layers = 4\n",
        "d_model = 128\n",
        "dff = 512\n",
        "num_heads = 8\n",
        "\n",
        "input_vocab_size = tokenizer_pt.vocab_size + 2\n",
        "target_vocab_size = tokenizer_en.vocab_size + 2\n",
        "dropout_rate = 0.1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xYEGhEOtzn5W"
      },
      "source": [
        "## Optimizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GOmWW--yP3zx"
      },
      "source": [
        "Use the Adam optimizer with a custom learning rate scheduler according to the formula in the [paper](https://arxiv.org/abs/1706.03762).\n",
        "\n",
        "$$\\Large{lrate = d_{model}^{-0.5} * min(step{\\_}num^{-0.5}, step{\\_}num * warmup{\\_}steps^{-1.5})}$$\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYQdOO1axwEI"
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "    \n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "    \n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "    \n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7r4scdulztRx"
      },
      "source": [
        "learning_rate = CustomSchedule(d_model)\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
        "                                     epsilon=1e-9)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f33ZCgvHpPdG"
      },
      "source": [
        "temp_learning_rate_schedule = CustomSchedule(d_model)\n",
        "\n",
        "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
        "plt.ylabel(\"Learning Rate\")\n",
        "plt.xlabel(\"Train Step\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YgkDE7hzo8r5"
      },
      "source": [
        "## Loss and metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oxGJtoDuYIHL"
      },
      "source": [
        "Since the target sequences are padded, it is important to apply a padding mask when calculating the loss."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlhsJMm0TW_B"
      },
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67oqVHiT0Eiu"
      },
      "source": [
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "  \n",
        "  return tf.reduce_sum(loss_)/tf.reduce_sum(mask)\n",
        "\n",
        "\n",
        "def accuracy_function(real, pred):\n",
        "  accuracies = tf.equal(real, tf.argmax(pred, axis=2))\n",
        "  \n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  accuracies = tf.math.logical_and(mask, accuracies)\n",
        "\n",
        "  accuracies = tf.cast(accuracies, dtype=tf.float32)\n",
        "  mask = tf.cast(mask, dtype=tf.float32)\n",
        "  return tf.reduce_sum(accuracies)/tf.reduce_sum(mask)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phlyxMnm-Tpx"
      },
      "source": [
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.Mean(name='train_accuracy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aeHumfr7zmMa"
      },
      "source": [
        "## Training and checkpointing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UiysUa--4tOU"
      },
      "source": [
        "transformer = Transformer(num_layers, d_model, num_heads, dff,\n",
        "                          input_vocab_size, target_vocab_size, \n",
        "                          pe_input=input_vocab_size, \n",
        "                          pe_target=target_vocab_size,\n",
        "                          rate=dropout_rate)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZOJUSB1T8GjM"
      },
      "source": [
        "def create_masks(inp, tar):\n",
        "  # Encoder padding mask\n",
        "  enc_padding_mask = create_padding_mask(inp)\n",
        "  \n",
        "  # Used in the 2nd attention block in the decoder.\n",
        "  # This padding mask is used to mask the encoder outputs.\n",
        "  dec_padding_mask = create_padding_mask(inp)\n",
        "  \n",
        "  # Used in the 1st attention block in the decoder.\n",
        "  # It is used to pad and mask future tokens in the input received by \n",
        "  # the decoder.\n",
        "  look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
        "  dec_target_padding_mask = create_padding_mask(tar)\n",
        "  combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
        "  \n",
        "  return enc_padding_mask, combined_mask, dec_padding_mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fzuf06YZp66w"
      },
      "source": [
        "Create the checkpoint path and the checkpoint manager. This will be used to save checkpoints every `n` epochs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hNhuYfllndLZ"
      },
      "source": [
        "checkpoint_path = \"./checkpoints/train\"\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "# if a checkpoint exists, restore the latest checkpoint.\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "  print ('Latest checkpoint restored!!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Di_Yaa1gf9r"
      },
      "source": [
        "The target is divided into tar_inp and tar_real. tar_inp is passed as an input to the decoder. `tar_real` is that same input shifted by 1: At each location in `tar_input`, `tar_real` contains the  next token that should be predicted.\n",
        "\n",
        "For example, `sentence` = \"SOS A lion in the jungle is sleeping EOS\"\n",
        "\n",
        "`tar_inp` =  \"SOS A lion in the jungle is sleeping\"\n",
        "\n",
        "`tar_real` = \"A lion in the jungle is sleeping EOS\"\n",
        "\n",
        "The transformer is an auto-regressive model: it makes predictions one part at a time, and uses its output so far to decide what to do next. \n",
        "\n",
        "During training this example uses teacher-forcing (like in the [text generation tutorial](./text_generation.ipynb)). Teacher forcing is passing the true output to the next time step regardless of what the model predicts at the current time step.\n",
        "\n",
        "As the transformer predicts each word, *self-attention* allows it to look at the previous words in the input sequence to better predict the next word.\n",
        "\n",
        "To prevent the model from peeking at the expected output the model uses a look-ahead mask."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LKpoA6q1sJFj"
      },
      "source": [
        "EPOCHS = 20"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJwmp9OE29oj"
      },
      "source": [
        "# The @tf.function trace-compiles train_step into a TF graph for faster\n",
        "# execution. The function specializes to the precise shape of the argument\n",
        "# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n",
        "# batch sizes (the last batch is smaller), use input_signature to specify\n",
        "# more generic shapes.\n",
        "\n",
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]\n",
        "\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "  \n",
        "  enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
        "  \n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions, _ = transformer(inp, tar_inp, \n",
        "                                 True, \n",
        "                                 enc_padding_mask, \n",
        "                                 combined_mask, \n",
        "                                 dec_padding_mask)\n",
        "    loss = loss_function(tar_real, predictions)\n",
        "\n",
        "  gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
        "  optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "  \n",
        "  train_loss(loss)\n",
        "  train_accuracy(accuracy_function(tar_real, predictions))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qM2PDWGDJ_8V"
      },
      "source": [
        "Portuguese is used as the input language and English is the target language."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bbvmaKNiznHZ"
      },
      "source": [
        "for epoch in range(EPOCHS):\n",
        "  start = time.time()\n",
        "  \n",
        "  train_loss.reset_states()\n",
        "  train_accuracy.reset_states()\n",
        "  \n",
        "  # inp -> portuguese, tar -> english\n",
        "  for (batch, (inp, tar)) in enumerate(train_dataset):\n",
        "    train_step(inp, tar)\n",
        "    \n",
        "    if batch % 50 == 0:\n",
        "      print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
        "          epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
        "      \n",
        "  if (epoch + 1) % 5 == 0:\n",
        "    ckpt_save_path = ckpt_manager.save()\n",
        "    print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n",
        "                                                         ckpt_save_path))\n",
        "    \n",
        "  print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, \n",
        "                                                train_loss.result(), \n",
        "                                                train_accuracy.result()))\n",
        "\n",
        "  print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QfcsSWswSdGV"
      },
      "source": [
        "## Evaluate"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y6APsFrgImLW"
      },
      "source": [
        "The following steps are used for evaluation:\n",
        "\n",
        "* Encode the input sentence using the Portuguese tokenizer (`tokenizer_pt`). Moreover, add the start and end token so the input is equivalent to what the model is trained with. This is the encoder input.\n",
        "* The decoder input is the `start token == tokenizer_en.vocab_size`.\n",
        "* Calculate the padding masks and the look ahead masks.\n",
        "* The `decoder` then outputs the predictions by looking at the `encoder output` and its own output (self-attention).\n",
        "* Select the last word and calculate the argmax of that.\n",
        "* Concatentate the predicted word to the decoder input as pass it to the decoder.\n",
        "* In this approach, the decoder predicts the next word based on the previous words it predicted.\n",
        "\n",
        "Note: The model used here has less capacity to keep the example relatively faster so the predictions maybe less right. To reproduce the results in the paper, use the entire dataset and base transformer model or transformer XL, by changing the hyperparameters above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5buvMlnvyrFm"
      },
      "source": [
        "def evaluate(inp_sentence):\n",
        "  start_token = [tokenizer_pt.vocab_size]\n",
        "  end_token = [tokenizer_pt.vocab_size + 1]\n",
        "  \n",
        "  # inp sentence is portuguese, hence adding the start and end token\n",
        "  inp_sentence = start_token + tokenizer_pt.encode(inp_sentence) + end_token\n",
        "  encoder_input = tf.expand_dims(inp_sentence, 0)\n",
        "  \n",
        "  # as the target is english, the first word to the transformer should be the\n",
        "  # english start token.\n",
        "  decoder_input = [tokenizer_en.vocab_size]\n",
        "  output = tf.expand_dims(decoder_input, 0)\n",
        "    \n",
        "  for i in range(MAX_LENGTH):\n",
        "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n",
        "        encoder_input, output)\n",
        "  \n",
        "    # predictions.shape == (batch_size, seq_len, vocab_size)\n",
        "    predictions, attention_weights = transformer(encoder_input, \n",
        "                                                 output,\n",
        "                                                 False,\n",
        "                                                 enc_padding_mask,\n",
        "                                                 combined_mask,\n",
        "                                                 dec_padding_mask)\n",
        "    \n",
        "    # select the last word from the seq_len dimension\n",
        "    predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n",
        "\n",
        "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
        "    \n",
        "    # return the result if the predicted_id is equal to the end token\n",
        "    if predicted_id == tokenizer_en.vocab_size+1:\n",
        "      return tf.squeeze(output, axis=0), attention_weights\n",
        "    \n",
        "    # concatentate the predicted_id to the output which is given to the decoder\n",
        "    # as its input.\n",
        "    output = tf.concat([output, predicted_id], axis=-1)\n",
        "\n",
        "  return tf.squeeze(output, axis=0), attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CN-BV43FMBej"
      },
      "source": [
        "def plot_attention_weights(attention, sentence, result, layer):\n",
        "  fig = plt.figure(figsize=(16, 8))\n",
        "  \n",
        "  sentence = tokenizer_pt.encode(sentence)\n",
        "  \n",
        "  attention = tf.squeeze(attention[layer], axis=0)\n",
        "  \n",
        "  for head in range(attention.shape[0]):\n",
        "    ax = fig.add_subplot(2, 4, head+1)\n",
        "    \n",
        "    # plot the attention weights\n",
        "    ax.matshow(attention[head][:-1, :], cmap='viridis')\n",
        "\n",
        "    fontdict = {'fontsize': 10}\n",
        "    \n",
        "    ax.set_xticks(range(len(sentence)+2))\n",
        "    ax.set_yticks(range(len(result)))\n",
        "    \n",
        "    ax.set_ylim(len(result)-1.5, -0.5)\n",
        "        \n",
        "    ax.set_xticklabels(\n",
        "        ['<start>']+[tokenizer_pt.decode([i]) for i in sentence]+['<end>'], \n",
        "        fontdict=fontdict, rotation=90)\n",
        "    \n",
        "    ax.set_yticklabels([tokenizer_en.decode([i]) for i in result \n",
        "                        if i < tokenizer_en.vocab_size], \n",
        "                       fontdict=fontdict)\n",
        "    \n",
        "    ax.set_xlabel('Head {}'.format(head+1))\n",
        "  \n",
        "  plt.tight_layout()\n",
        "  plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lU2_yG_vBGza"
      },
      "source": [
        "def translate(sentence, plot=''):\n",
        "  result, attention_weights = evaluate(sentence)\n",
        "  \n",
        "  predicted_sentence = tokenizer_en.decode([i for i in result \n",
        "                                            if i < tokenizer_en.vocab_size])  \n",
        "\n",
        "  print('Input: {}'.format(sentence))\n",
        "  print('Predicted translation: {}'.format(predicted_sentence))\n",
        "  \n",
        "  if plot:\n",
        "    plot_attention_weights(attention_weights, sentence, result, plot)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsxrAlvFG8SZ"
      },
      "source": [
        "translate(\"este é um problema que temos que resolver.\")\n",
        "print (\"Real translation: this is a problem we have to solve .\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7EH5y_aqI4t1"
      },
      "source": [
        "translate(\"os meus vizinhos ouviram sobre esta ideia.\")\n",
        "print (\"Real translation: and my neighboring homes heard about this idea .\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-hVCTSUMlkb"
      },
      "source": [
        "translate(\"vou então muito rapidamente partilhar convosco algumas histórias de algumas coisas mágicas que aconteceram.\")\n",
        "print (\"Real translation: so i 'll just share with you some stories very quickly of some magical things that have happened .\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_1MxkSZvz0jX"
      },
      "source": [
        "You can pass different layers and attention blocks of the decoder to the `plot` parameter."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t-kFyiOLH0xg"
      },
      "source": [
        "translate(\"este é o primeiro livro que eu fiz.\", plot='decoder_layer4_block2')\n",
        "print (\"Real translation: this is the first book i've ever done.\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RqQ1fIsLwkGE"
      },
      "source": [
        "## Summary\n",
        "\n",
        "In this tutorial, you learned about positional encoding, multi-head attention, the importance of masking and how to create a transformer.\n",
        "\n",
        "Try using a different dataset to train the transformer. You can also create the base transformer or transformer XL by changing the hyperparameters above. You can also use the layers defined here to create [BERT](https://arxiv.org/abs/1810.04805) and train state of the art models. Futhermore, you can implement beam search to get better predictions."
      ]
    }
  ]
}