annotated_transformer_ipynb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "annotated_transformer.ipynb의 사본",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Fintecuriosity11/Unstructured-data-analysis/blob/master/annotated_transformer_ipynb.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l-hYvmAnT0J6",
        "outputId": "2c894391-0894-4dad-e84c-78117f3ca605"
      },
      "source": [
        "# 차트에 한글이 나오게끔 한글 폰트 설치\n",
        "!sudo apt-get install -y fonts-nanum\n",
        "!sudo fc-cache -fv\n",
        "!rm ~/.cache/matplotlib -rf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  fonts-nanum\n",
            "0 upgraded, 1 newly installed, 0 to remove and 37 not upgraded.\n",
            "Need to get 9,604 kB of archives.\n",
            "After this operation, 29.5 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 fonts-nanum all 20170925-1 [9,604 kB]\n",
            "Fetched 9,604 kB in 2s (5,592 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 1.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package fonts-nanum.\n",
            "(Reading database ... 155219 files and directories currently installed.)\n",
            "Preparing to unpack .../fonts-nanum_20170925-1_all.deb ...\n",
            "Unpacking fonts-nanum (20170925-1) ...\n",
            "Setting up fonts-nanum (20170925-1) ...\n",
            "Processing triggers for fontconfig (2.12.6-0ubuntu2) ...\n",
            "/usr/share/fonts: caching, new cache contents: 0 fonts, 1 dirs\n",
            "/usr/share/fonts/truetype: caching, new cache contents: 0 fonts, 3 dirs\n",
            "/usr/share/fonts/truetype/humor-sans: caching, new cache contents: 1 fonts, 0 dirs\n",
            "/usr/share/fonts/truetype/liberation: caching, new cache contents: 16 fonts, 0 dirs\n",
            "/usr/share/fonts/truetype/nanum: caching, new cache contents: 10 fonts, 0 dirs\n",
            "/usr/local/share/fonts: caching, new cache contents: 0 fonts, 0 dirs\n",
            "/root/.local/share/fonts: skipping, no such directory\n",
            "/root/.fonts: skipping, no such directory\n",
            "/var/cache/fontconfig: cleaning cache directory\n",
            "/root/.cache/fontconfig: not cleaning non-existent cache directory\n",
            "/root/.fontconfig: not cleaning non-existent cache directory\n",
            "fc-cache: succeeded\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GUziLQk-tiwn"
      },
      "source": [
        "폰트 설치가 끝났으면 런타임을 재시작해서 폰트가 적용되게 하고 다음 셀을 실행하세요."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l18LPLlFzaOU"
      },
      "source": [
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import math, copy, time\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# plt.style.use('dark_background')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 828
        },
        "id": "MkMwlax4UAS-",
        "outputId": "d3c0b720-d283-4044-e96d-3c9ffcb8cb0a"
      },
      "source": [
        "from matplotlib import font_manager, rc\n",
        "\n",
        "# 그래프에 한글 나오는지 확인\n",
        "font_fname = '/usr/share/fonts/truetype/nanum/NanumGothic.ttf'     # A font of your choice\n",
        "font_name = font_manager.FontProperties(fname=font_fname).get_name()\n",
        "rc('font', family=font_name)\n",
        "\n",
        "plt.plot(range(50), range(50), 'r')\n",
        "plt.title('가격 추이')\n",
        "plt.ylabel('가격')\n",
        "plt.xlabel('시간(분)')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "findfont: Font family ['NanumGothic'] not found. Falling back to DejaVu Sans.\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 44032 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 44201 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 52628 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 51060 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "findfont: Font family ['NanumGothic'] not found. Falling back to DejaVu Sans.\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 49884 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 44036 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 48516 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 49884 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 44036 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 48516 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 44032 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 44201 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 52628 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 51060 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAEWCAYAAABhffzLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAezElEQVR4nO3deZRU1bXH8e8W5SmCiog4C4km2iKDtiNgFEVJNIH3llM0iQMKzlOiYoxGk+jDARUVZFYMDiCIzJPMAoKNzCCCBEeQNoKCA+N+f5zrk2A3Nt1961bV/X3WYlXdW9Vd+y6Lzfacc/cxd0dERNJjl6QDEBGRzFLiFxFJGSV+EZGUUeIXEUkZJX4RkZRR4hcRSRklfhGRlNk16QBEkmRmrYDbS3hpBNAXeLGE11a6+wXb/Z6qwPiSPsPdm5pZN+CYEl6+0d1n71zUIhWjxC9pdyBwn7u//t0JM6sOPA1UAya6+1+2/QEzG1DC79kFWOHuvyvlvbXcvel2r90A7F3xSxDZORrqERFJGSV+EZGUUeIXEUkZJX4RkZRR4hcRSRklfhGRlFHiFxFJGSV+EZGU0Q1cItDRzNZsc1wFeC96/nsza7rd+2uV8ntamNnE7c59d7du7RJeOxi4eidjFakw09aLIiLpoqEeEZGUUeIXEUmZnBjj32+//bxu3bpJhyEiklNmzZr1mbvX3v58TiT+unXrUlRUlHQYIiI5xczeL+m8hnpERFIm1orfzFYA64AtwGZ3LzSzfYF+QF1gBXChu68p7XeIiEjlykTFf4a7N3L3wui4PTDO3Y8ExkXHIiKSIUkM9bQC+kTP+wCtE4hBRCS14k78Dowxs1lm1jY6V8fdV0bPVwF1SvpBM2trZkVmVlRcXBxzmCIi6RH3qp6m7v6xme0PjDWzd7Z90d3dzEq8ddjduwPdAQoLC3V7sYhIJYm14nf3j6PH1cAg4ETgUzM7ECB6XB1nDCIi8p9iS/xmtqeZ1fjuOXA2sAAYAlwWve0yYHBcMYiI5KwPPoBbboHNmyv9V8dZ8dcB3jCzucBMYLi7jwI6ELoYLgXOio5FRARg61bo0gWOOQZ69IA5cyr9I2Ib43f35UDDEs7/Gzgzrs8VEclZS5bAVVfBG29AixbQvTvE0K5Gd+6KiCRt0ybo0AEaNoQFC+DZZ2H06FiSPuRIrx4Rkbw1Zw5ceSXMng3/8z/QuTMccECsH6mKX0QkCd9+C3ffDYWF8MknMGAADBwYe9IHVfwiIpk3dSq0aRPG9C+/HDp2hH33zdjHq+IXEcmU9evhppugWbNQ8Y8eHcbzM5j0QYlfRCQzxoyB+vXh6afhhhvCJO7ZZycSihK/iEicPv8crrgCzjkHdt8dpkyBJ5+E6tUTC0mJX0QkLgMHQkEB/POfYSJ3zhxo0iTpqDS5KyJS6VatCsM5AwdC48YwahQ0apR0VP9PFb+ISGVxhz59QpU/bFi4KWvmzKxK+qCKX0SkcqxYAe3ahUncpk2hZ0/4+c+TjqpEqvhFRCpi61Z46qmwYmfatHDn7aRJWZv0QRW/iEj5LV4cmqpNmwYtW0K3bnDYYUlH9aNU8YuI7KxNm+CBB8LY/TvvwPPPw4gROZH0QRW/iMjOefvt0FRt7ly48MKwJr9OiVuHZy1V/CIiZfHNN9C+PZx4Inz6KQwaBP365VzSB1X8IiI/bsqUMJb/7ruhudojj0DNmklHVW6q+EVESrNuHVx/PZx2WhjXHzs2LNPM4aQPSvwiIiUbOTLse/vMM2HT8/nz4ayzko6qUijxi4hs69//hj/8AX71K6hRI/TOf/xx2HPPpCOrNEr8IiIQ2i288kpot/DSS3DPPWEFzymnJB1ZpdPkrojIypVw3XXw2mtw/PFhLL9Bg6Sjio0qfhFJL3fo3RuOPjp00Hz4YXjzzbxO+qCKX0TS6l//grZt4fXXw6qdnj3hyCOTjiojVPGLSLps2QKdOoWmajNmhFU7EyakJumDKn4RSZNFi8KNWNOnh1U7XbvCoYcmHVXGqeIXkfy3cSP8/e9hN6x334W+fcNGKSlM+qCKX0TyXVFRaLMwbx5cfHEY5tl//6SjSpQqfhHJT19/DbffDiedBJ99BoMHh/X5KU/6oIpfRPLRxIlw9dWwbFl4fOQR2HvvpKPKGqr4RSR/fPEFXHMNnHFG2BJx3Djo3l1JfzuxJ34zq2Jms81sWHRcz8xmmNkyM+tnZlXjjkFEUmD48NBUrUcP+OMfQ1O15s2TjiorZaLivxlYvM3xQ8Dj7n4EsAZok4EYRCRfFRfDpZfCeeeFdsnTp8Ojj0K1aklHlrViTfxmdghwLtAzOjagOTAgeksfoHWcMYhInnKHl18OTdVeeQXuuw9mzQo7ZMkOxT25+wRwB1AjOq4FrHX3zdHxR8DBMccgIvnm44/h2mth6NCQ6Hv1CnfiSpnEVvGb2XnAanefVc6fb2tmRWZWVFxcXMnRiUhOcg9j+AUFocdOx44wbZqS/k6Ks+JvAvzGzH4F7A7sBXQC9jGzXaOq/xDg45J+2N27A90BCgsLPcY4RSQXvPdeWJo5YUJYtdOjB/z0p0lHlZNiq/jd/S53P8Td6wIXA+Pd/VJgAnB+9LbLgMFxxSAieWDLFnjsMTj22DCG36NHWKappF9uSazjvxO4zcyWEcb8eyUQg4jkggUL4NRTw/LMs876vsmaWdKR5bSM3Lnr7hOBidHz5YCm3UWkdBs3woMPhj977x1aLVx0kRJ+JVHLBhHJLjNmhKZqCxeG9flPPAH77Zd0VHlFLRtEJDt89RXcdlvY3PyLL0Lb5L59lfRjoIpfRJI3fnxYsbN8eei189BDsNdeSUeVt1Txi0hy1q4NCf/MM2GXXcJSzWeeUdKPmRK/iCRjyJDQVK1379A3f+5cOP30pKNKBSV+Ecms1avDTlitWkGtWmEy9+GH1VQtg5T4RSQz3OGFF0K7hUGDwh64RUVQWJh0ZKmjyV0Rid+HH4amasOHw8knh6ZqBQVJR5VaqvhFJD5bt0LXrmEsf8KEsCb/jTeU9BOmil9E4rF0aVixM2lSaLfQvTvUq5d0VIIqfhGpbJs3h83NGzSAOXPCsM6YMUr6WUQVv4hUnnnzQruFoiJo3Ro6d4aDDko6KtmOKn4RqbgNG+Dee+H44+GDD6B/f3j1VSX9LKWKX0QqZvr0UOUvXgx/+EPonV+rVtJRyQ6o4heR8vnqK7j1VmjSBNavhxEjoE8fJf0coIpfRHbe66+HFTsrVsB110GHDlCjRtJRSRmp4heRsluzJgzrtGgBVavC5MlhAldJP6co8YtI2QwaFG686tMH2rcPTdWaNUs6KikHDfWIyI6tWgU33ggDBkCjRqHtwnHHJR2VVIAqfhEpmTs8/3yo8ocOhQcegJkzlfTzgCp+Efmh99+Hdu1g9Gg49dRw9+1RRyUdlVQSVfwi8r2tW8Nkbf36oZnaU0/BlClK+nlGFb+IBEuWwFVXhYR/zjnQrRscfnjSUUkMVPGLpN2mTWEdfsOGsHAhPPccjByppJ/HVPGLpNns2WFd/uzZcP75YWjngAOSjkpipopfJI2+/Rb+/Gc44QT45BMYOBBeeUVJPyVU8YukzdSpocpfsgSuuAI6doSaNZOOSjJIFb9IWqxbF27EatYsVPyjR0Pv3kr6KaTEL5IGo0eHJZqdO4fkv2ABnH120lFJQpT4RfLZ55/D5ZdDy5ZQrVpYk9+pE1SvnnRkkiAlfpF8NXBgaLfQty/cfXdYudOkSdJRSRbQ5K5Ivlm5Em64IWx92LgxjBoVmquJRGKr+M1sdzObaWZzzWyhmd0fna9nZjPMbJmZ9TOzqnHFIJIq7uHmq4KC0EGzQ4fQVE1JX7YT51DPBqC5uzcEGgEtzexk4CHgcXc/AlgDtIkxBpF0WLEitFm44go49liYNw/uvBN21f/Uyw/Flvg9WB8d7hb9caA5MCA63wdoHVcMInlvyxZ48smwYmf6dOjSBSZOhJ/9LOnIJIvFOrlrZlXMbA6wGhgLvAesdffN0Vs+Ag4u5WfbmlmRmRUVFxfHGaZIblq8GE47DW6+OTwuXAjXXgu7aM2G7Fis3xB33+LujYBDgBOBMvd2dffu7l7o7oW1a9eOLUaRnLNpU9gUpVEjeOcd+Oc/w5j+YYclHZnkiIwMALr7WjObAJwC7GNmu0ZV/yHAx5mIQSQvzJoV2i3MnQsXXhiaqu2/f9JRSY6Jc1VPbTPbJ3q+B9ACWAxMAM6P3nYZMDiuGETyxjffhA3OTzoJVq8OG5/366ekL+USZ8V/INDHzKoQ/oHp7+7DzGwR8LKZ/QOYDfSKMQaR3Dd5ctggZenSUO0/+ijss0/SUUkOiy3xu/s8oHEJ55cTxvtFZEe+/BLuuius1KlXD15/Hc48M+moJA9o+l8kG40cGZZoPvMM3HILzJ+vpC+VRnd3iGSTf/8bbr01rNQpKIBp0+Dkk5OOSvKMKn6RbOAO/fvD0UfDSy/BPffA228r6UssVPGLJO2TT+D66+G11+D448NYfoMGSUcleUwVv0hS3KFXrzCkM2oUPPwwvPmmkr7EThW/SBKWL4e2bWHcuNBuoWdPOPLIpKOSlFDFL5JJW7bA44+HDpozZ4ZVOxMmKOlLRqniF8mUhQvDDVgzZsC554akf+ihSUclKaSKXyRuGzfC3/4WdsNatgxeeAGGDlXSl8So4heJ01tvhSp//nz47W/DRufqNisJU8UvEoevv4bbbw/r8D//HIYMgRdfVNKXrKCKX6SyTZoUmqotWwbt2sFDD8Heeycdlcj/U8UvUlm++AKuuQZOPz2s0R8/Hrp2VdKXrFOmit/M7v2Rt6x2966VEI9Ibho+PFT3K1fCn/4E998P1aolHZVIico61HMycDFgpbzeB1Dil/QpLg7dM198MXTTfPVVOFFdxyW7lTXxb3H3L0t70cy8kuIRyQ3u8PLLcNNNYYjn/vvDDllVqyYdmciPKmvi/7HErsQv6fHRR3DttTBsWKjue/UK1b5Ijihr4t/NzPYq5TUDqlRSPCLZa+vW0FPn9tth0ybo2BFuvhmq6OsvuaWsif9N4JZSXjNgZOWEI5Klli2Dq6+GiRPhjDOgRw/46U+TjkqkXMqa+E9Ck7uSRlu2wBNPhI1RdtstJPw2bcBK+6sgkv00uStSmgUL4MorQ9uFX/86NFU7+OCkoxKpsLLewKXJXUmPjRvhvvvguONgxYqwemfwYCV9yRua3BXZ1syZocpfuBAuuSQ0Vdtvv6SjEqlUOzu5W9rA5qjKCUckIV99BffeG8bzDzooLNU899ykoxKJRZkSv7vfH3cgIokZPz6s2Fm+PKzP79AB9irtf3BFcp+atEl6rV0bEv6ZZ4a1+BMnQpcuSvqS95T4JZ2GDIFjjoHeveGOO2DuXPjFL5KOSiQjlPglXVavhosvhlatwqTtjBmhX/4eeyQdmUjGKPFLOriHvW4LCmDQIPjHP6CoCAoLk45MJOO0A5fkvw8/DBukjBgBp5wSmqodfXTSUYkkRhW/5K+tW8PdtsccEyZun3gCpkxR0pfUiy3xm9mhZjbBzBaZ2UIzuzk6v6+ZjTWzpdFjzbhikBRbujQ0U7vuOjjppNB+QZ00RYB4K/7NwB/dvYCwg9f1ZlYAtAfGufuRwLjoWKRybN4MDz8MDRqElTq9esGYMVCvXtKRiWSN2Mb43X0lsDJ6vs7MFgMHA62A06O39QEmAnfGFYekyNy5oXPmrFnQujV07hzuwhWR/5CRMX4zqws0BmYAdaJ/FABWAXVK+Zm2ZlZkZkXFxcWZCFNy1YYNoW1yYWGYyO3fP+x9q6QvUqLYE7+ZVQcGArds39rZ3Z1SOnu6e3d3L3T3wtq1a8cdpuSqadOgceOwPPOSS2DRIrjgAvXLF9mBWBO/me1GSPovuPur0elPzezA6PUDgdVxxiB5av36MFnbtGlosDZyJPTpA7VqJR2ZSNaLc1WPAb2Axe7+2DYvDQEui55fBgyOKwbJU2PHwrHHwpNPhlU7CxZAy5ZJRyWSM+Ks+JsAvweam9mc6M+vgA5ACzNbCpwVHYv8uDVrQq/8s8+GqlVh8mR4+mmoUSPpyERySpyret6g9P79Z8b1uZKnBg0K1X1xMdx1V+idv/vuSUclkpPUskGy26pVcOONMGAANGoU2i40bpx0VCI5TS0bJDu5h8naggIYOhQefDBsi6ikL1Jhqvgl+7z/PrRrB6NHQ5Mm0LMnHHVU0lGJ5A1V/JI9tm4Nd9vWrw9vvAFPPRUmcJX0RSqVKn7JDkuWhHYLU6fCOedAt25w+OFJRyWSl1TxS7I2bYL//V9o2DDcdfvcc+FmLCV9kdio4pfkzJ4dqvzZs+H888PQzgEHJB2VSN5TxS+Z9+238Oc/wwknwMqVMHAgvPKKkr5Ihqjil8yaOjVU+UuWwBVXQMeOUFN78Yhkkip+yYz168ONWM2ahTbKo0dD795K+iIJUOKX+I0eHfa97dw5JP/580O/HRFJhBK/xOfzz+Hyy0PnzGrVwtr8Tp2gevWkIxNJNSV+iceAAXD00fDCC3D33WHlzqmnJh2ViKDJXalsK1fCDTeErQ+POy4M8zRqlHRUIrINVfxSOdzh2WdDU7URI+Chh2DGDCV9kSykil8qbsUKaNs27IzVrFloqvaznyUdlYiUQhW/lN+WLWH7w/r1Yfp06NIFJk5U0hfJcqr4pXwWL4arroJp0+CXv4SuXeGww5KOSkTKQBW/7JxNm+CBB8LY/ZIl0LcvDB+upC+SQ1TxS9nNmhU2O583Dy66KAzz7L9/0lGJyE5SxS8/7ptv4M474aST4LPP4LXX4OWXlfRFcpQqftmxyZPDWP7SpeHxkUdgn32SjkpEKkAVv5Tsyy/huuvgF78Iq3fGjYMePZT0RfKAEr/80IgRoalat25w221hTL9586SjEpFKosQv3/vsM/jd7+Dcc2GvvcJSzY4dYc89k45MRCqREr+Edgv9+oV2C/36wV//Cm+/HSZzRSTvaHI37T75BK69FoYMgcLCMJZ/7LFJRyUiMVLFn1buoadOQQGMGRNW60yfrqQvkgKq+NPovfdCU7Xx48OqnZ494Ygjko5KRDJEFX+abNkCjz0Wqvq33gqrdsaPV9IXSRlV/GmxYAG0aQMzZ4ZVO127wiGHJB2ViCQgtorfzHqb2WozW7DNuX3NbKyZLY0ea8b1+RLZuBHuvz/shrV8edgKcehQJX2RFItzqOc5oOV259oD49z9SGBcdCxxeestOP54uO8+uOACWLQILrkEzJKOTEQSFFvid/fJwOfbnW4F9Ime9wFax/X5qfb11/CnP8HJJ8OaNWGp5gsvQO3aSUcmIlkg02P8ddx9ZfR8FVAnw5+f/yZODM3U3nsP2rULe9/uvXfSUYlIFklsVY+7O+ClvW5mbc2syMyKiouLMxhZjvrii5DozzgjHI8fHyZwlfRFZDuZTvyfmtmBANHj6tLe6O7d3b3Q3Qtra4hix4YNC03VevYMQzzz5n3/D4CIyHYynfiHAJdFzy8DBmf48/NLcXGYrP31r6FmzXDn7SOPQLVqSUcmIlkszuWcLwHTgZ+b2Udm1gboALQws6XAWdGx7Cx3eOml0G5hwICwamfWLDjxxKQjE5EcENvkrrv/tpSXzozrM1Pho49CU7Vhw0L3zJ49oX79pKMSkRyilg25YuvW0GKhoCB00HzsMZg6VUlfRHaaWjbkgqVL4eqrYdKksBNWjx7wk58kHZWI5ChV/Nls82Z49FFo0ADmzAnDOq+/rqQvIhWiij9bzZsXmqoVFUGrVtClCxx0UNJRiUgeUMWfbTZsgHvvDT123n8/bIU4aJCSvohUGlX82eTNN0OVv2gR/P738PjjUKtW0lGJSJ5RxZ8NvvoKbr0VTj0V1q2D4cPh+eeV9EUkFqr4kzZuXFix869/hfX5HTrAXnslHZWI5DFV/ElZuzZ00TzrLNh117BUs0sXJX0RiZ0SfxIGDw43Yj37LNxxB8ydC6edlnRUIpISSvyZ9OmncNFF0Lp12BRlxozQL3+PPZKOTERSRIk/E9yhb99Q5b/2Gvz972F9fmFh0pGJSAppcjduH3wA11wDI0fCKaeEu28LCpKOSkRSTBV/XLZuhWeeCRukTJoEnTrBlClK+iKSOFX8cXj33bBiZ8qUsGqne3eoVy/pqEREAFX8lWvzZnj4YWjYEObPh969YcwYJX0RySqq+CvL3Llw5ZXw9tvw3/8NnTvDgQcmHZWIyA+o4q+oDRvgnnvCCp2PPoJXXoGBA5X0RSRrqeKviGnTwlj+4sVqqiYiOUMVf3msXw+33AJNm4YGayNHqqmaiOQMVfw7a+xYaNsWVqyAG26ABx+EGjWSjkpEpMxU8ZfVmjVh8vbss+G//iss1XzqKSV9Eck5SvxlMWhQuPHq+eehffuw/23TpklHJSJSLhrq2ZFVq+DGG2HAAGjUCEaMgMaNk45KRKRCVPGXxB369AlV/tChYRx/5kwlfRHJC6r4t7diBbRrF+64bdIkNFU76qikoxIRqTSq+L+zdSs8/TTUrw9Tp4aJ28mTlfRFJO+o4gd4551wI9bUqXDOOdCtGxx+eNJRiYjEIt0V/6ZNYfy+YUNYtCiM648cqaQvInktvRX/7NlhXf6cOXDBBWFop06dpKMSEYld+ir+b76Bu+6CE04IyzVffRX691fSF5HUSFfF/8Yb0KZN2CjliiugY0eoWTPpqEREMiqRit/MWprZEjNbZmbtY//AdetCX51mzWDjxrBUs3dvJX0RSaWMJ34zqwJ0Bn4JFAC/NbP4NqIdNSos0ezSBW66KeyM1aJFbB8nIpLtkqj4TwSWuftyd98IvAy0iuWT2rWDX/4SqlULwzydOkH16rF8lIhIrkgi8R8MfLjN8UfRuf9gZm3NrMjMioqLi8v3SUccAX/5S1i5c+qp5fsdIiJ5Jmsnd929O9AdoLCw0Mv1S26/vTJDEhHJC0lU/B8Dh25zfEh0TkREMiCJxP8WcKSZ1TOzqsDFwJAE4hARSaWMD/W4+2YzuwEYDVQBerv7wkzHISKSVomM8bv7CGBEEp8tIpJ26WvZICKSckr8IiIpo8QvIpIySvwiIilj7uW7NyqTzKwYeL+cP74f8FklhpMrdN3pktbrhvRee1mu+3B3r739yZxI/BVhZkXuXph0HJmm606XtF43pPfaK3LdGuoREUkZJX4RkZRJQ+LvnnQACdF1p0tarxvSe+3lvu68H+MXEZH/lIaKX0REtqHELyKSMnmd+DO+qXtCzKy3ma02swXbnNvXzMaa2dLoMe92ljezQ81sgpktMrOFZnZzdD6vr93MdjezmWY2N7ru+6Pz9cxsRvR97xe1Pc87ZlbFzGab2bDoOO+v28xWmNl8M5tjZkXRuXJ/z/M28Wd8U/dkPQe03O5ce2Ccux8JjIuO881m4I/uXgCcDFwf/TfO92vfADR394ZAI6ClmZ0MPAQ87u5HAGuANgnGGKebgcXbHKflus9w90bbrN0v9/c8bxM/mdzUPWHuPhn4fLvTrYA+0fM+QOuMBpUB7r7S3d+Onq8jJIODyfNr92B9dLhb9MeB5sCA6HzeXTeAmR0CnAv0jI6NFFx3Kcr9Pc/nxF+mTd3zWB13Xxk9XwXUSTKYuJlZXaAxMIMUXHs03DEHWA2MBd4D1rr75ugt+fp9fwK4A9gaHdciHdftwBgzm2VmbaNz5f6eZ+1m61J53N3NLG/X7ZpZdWAgcIu7fxmKwCBfr93dtwCNzGwfYBBwVMIhxc7MzgNWu/ssMzs96XgyrKm7f2xm+wNjzeydbV/c2e95Plf8ad/U/VMzOxAgelydcDyxMLPdCEn/BXd/NTqdimsHcPe1wATgFGAfM/uumMvH73sT4DdmtoIwdNsc6ET+Xzfu/nH0uJrwD/2JVOB7ns+JP+2bug8BLoueXwYMTjCWWETju72Axe7+2DYv5fW1m1ntqNLHzPYAWhDmNyYA50dvy7vrdve73P0Qd69L+Ps83t0vJc+v28z2NLMa3z0HzgYWUIHveV7fuWtmvyKMCX63qfsDCYcUCzN7CTid0Kb1U+CvwGtAf+AwQkvrC919+wngnGZmTYEpwHy+H/P9M2GcP2+v3cwaECbzqhCKt/7u/jcz+wmhEt4XmA38zt03JBdpfKKhnj+5+3n5ft3R9Q2KDncFXnT3B8ysFuX8nud14hcRkR/K56EeEREpgRK/iEjKKPGLiKSMEr+ISMoo8YuIpIwSv0gFmNkeZjYpagpY2ntqm9moTMYlsiNq2SCyDTO7j9Dp87veL7sCb5Z0zt3vA64EXnX3LWbWAzhwm1+3F+H+kefMbKWZNXH3qRm4DJEdUuIX+aGLo1YIRHfI3lLKOYBLgUui51+5+3nf/RIza0RomwzhhrpLASV+SZyGekTKKWoF8hN3X1GGtxcBzeKNSKRslPhFym8/YG0Z37saOCjGWETKTIlfpPy+AXYv43t3j94vkjglfpFycvc1QBUzK0vy/xmho6JI4pT4RSpmDNC0DO87AxgecywiZaLEL1Ixnfm+J/qO/AboG3MsImWi5Zwi/2k18LyZfdfffxdgVCnncPe3zWxCdAPXl2b22ja/qyrQzcxqA49FQ0MiiVM/fhGRlNFQj4hIyijxi4ikjBK/iEjKKPGLiKSMEr+ISMr8H9KWF1tF4QXxAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UQ5YKII2esk4"
      },
      "source": [
        "# 진짜로(?) 주석 달린 트랜스포머Really annotated transformers<sup>&#8224;</sup>\n",
        "\n",
        "\n",
        "---\n",
        "\n",
        "2021.11.09 조준우 (metamath@gmail.com), https://metamath1.github.io"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qO-X1agw9hwO"
      },
      "source": [
        "본 글의 원문은 \"The Annotated Transfomer\"[[1](https://nlp.seas.harvard.edu/2018/04/03/attention.html)]로 인터넷에서 참고할 수 있는 트랜스포머에 대한 가장 좋은 문서중 하나입니다. 다른 하나는 \"The Illustrated Transformer\"[[2](https://jalammar.github.io/illustrated-transformer/)], [번역문[3](https://nlpinkorean.github.io/illustrated-transformer/)] 인데 이 문서만 읽어봐도 트랜스포머에 대해서 일부분 이해할 수 있어 꼭 읽어보면 좋은 글입니다. 이 글은 [1]에 대해서 더 풍부하게 주석을 달고 [1]에서 언급되지 않은 부분을 조금 더 상세히 풀어 적은 글입니다.\n",
        "\n",
        "[[1](https://nlp.seas.harvard.edu/2018/04/03/attention.html)]은 논문을 그대로 코드로 옮기고 논문 텍스트와 코드를 함께 제시하기 때문에 공신력 면에서 최고지만 앞서 밝혔듯이 설명이 부족하고 함께 진행하는 코드가 top-down 방식으로 기술되어 있어서 글을 읽기가 힘들다는 단점이 있습니다. 저자들이 이렇게 글을 쓴 이유는 트랜스포머를 발표한 논문 \"Attention is all you need\"[[4](https://arxiv.org/abs/1706.03762)]에서 기술하는 순서를 그대로 지켰기 때문입니다. 코드를 이해시킬 필요없이 전체적인 맥락을 설명하는데는 이런 top-down방식이 더 유리하겠지만 글과 코드를 함께 보는 경우는 좀 다를 수 있습니다. 왜냐하면 top-down 방식으로 제시된 코드 조각에는 필연적으로 아래쪽에서 설명될 예정인 의미를 잘 알 수 없는 인자arguement들이 끊임없이 등장하게 되기 때문입니다. 실제로 [[1](https://nlp.seas.harvard.edu/2018/04/03/attention.html)]을 읽어보면 어느 부분에서 '어? 여기 나온 이 인자가 무엇이지?'라는 생각이 들게 되고 문서 뒤로 가서 궁금한 인자가 등장하는 지점과 연결되는 부분을 찾아보고 다시 앞으로 돌아오기를 계속 반복해야 하는데 몇번 하다보면 지쳐서 글 읽기가 싫어지게 되겠죠.\n",
        "\n",
        "이런 이유때문에 코드와 함께 기술된 글이라면 bottom-up 방식으로 작은 부분부터 이해하면서 점진적으로 큰 부분으로 나가는것이 맞다고 생각합니다. 이 글은 [[1](https://nlp.seas.harvard.edu/2018/04/03/attention.html)]의 코드를 bottom-up 방식으로 재구성하여 읽는 이가 문서를 처음부터 순서대로 읽기만해도 트랜스포머에 대해 편안하게 이해할 수 있도록 하는 것을 목적으로 하고 있습니다.\n",
        "\n",
        "----\n",
        "\n",
        "<sup>&#8224;</sup> 이 글의 제목은 트랜스포머가 가장 강력한 성능을 발휘하는 NMT인 구글 번역기로 번역되었습니다.  ;)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pqrfMxGZ72HW"
      },
      "source": [
        "## 트랜스포머를 공부하는 이유\n",
        "\n",
        "트랜스포머는 최근 가장 성공적인 모델임에도 불구하고 CNN, RNN에 대한 사전 지식이 전혀 필요없이 이해할 수 있다는 점에서 입문자분들이 꼭 공부하면 좋을 모델이라고 할 수 있습니다. 트랜스포머를 이해하기 위한 사전 지식은 다음 정도로 정리할 수 있습니다. \n",
        "\n",
        "- 신경망에서 완전 연결층 \n",
        "- 소프트맥스 함수\n",
        "- $\\sin$, $\\cos$ 함수 기초\n",
        "- 데이터를 인코딩해서 다시 디코딩하는 개념\n",
        "- 배치 정규화와 레이어 정규화의 차이\n",
        "- 임베딩 개념\n",
        "- 어텐션 개념\n",
        "\n",
        "앞 네 개 항목은 인공지능에 막 입문한 학습자라도 어느정도 알고 있는 내용이므로 실제로 입문자가 느끼는 추가 학습 부담은 정규화와 임베딩 개념 그리고 가장 핵심인 어텐션이라고 할 수 있습니다. 임베딩에 대해서는 입력을  특정 길이를 가지는 벡터로 바꾼다는 정도만 알아도 전체 내용을 이해하는데 지장이 없습니다. 어텐션에 대해서는 이전 모델인 seq2seq와 그에 대한 어텐션 매커니즘을 알고 있다면 도움이 되지만 몰라도 상관없습니다. \n",
        "\n",
        "이렇게 트랜스포머는 완전 연결층에 대한 기본적인 지식만 가지고 바로 학습할 수 있다는 장점에 더불어 최신 모델들의 근간을 이루는 기본 모델을 공부할 수 있다는 추가 장점도 가지고 있습니다. \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F5_lwTI8gE0G"
      },
      "source": [
        "## 트랜스포머 개요\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1DudF_35VyVpevpsLr7TfVWi6WeC527kc)\n",
        "Figure 1: The Transformer - model architecture.[[4](https://arxiv.org/abs/1706.03762)]\n",
        "\n",
        "트랜스포머에 대한 그림 중 가장 유명한 그림이 바로 위 그림입니다. 이 그림은 처음 보면 아주 복잡하게 보이지만 트랜스포머를 이해하고 보면 간결하게 모델의 핵심을 잘 설명하고 있다는 것을 알게 됩니다. 이 그림을 통해서 먼저 트랜스포머에 대한 전체 맥락을 알아보겠습니다. 트랜스포머가 적용되는 상황은 기계번역, 즉 입력으로 영어 문장이 들어가서 출력으로 한국어 문장이 나오는 상황을 가정하겠습니다. 그러면 위 그림은 다음과 같은 순서로 데이터가 처리되는 것을 이야기하고 있습니다.\n",
        "\n",
        "1. 입력(Input)이 임베딩 층으로 입력되어 문장의 각 단어들이 적당한 벡터로 변환된다.\n",
        "2. 이 변환된 벡터가 위치 인코딩Positional Encoding이라는 것으로 부터 출력된 단어의 위치 정보를 가지는 벡터와 더해진다.\n",
        "3. 2에서 만들어진 벡터가 멀티 헤드 어텐션Multi-Head Attention이라는 층으로 입력되는데 이 때 같은 입력을 키key, 쿼리query, 벨류value로 넣어준다. \n",
        "4. 멀티 헤드 어텐션 층을 거쳐 출력된 벡터가 멀티 헤드 어텐션 층으로 입력되기 전 벡터와 더해지고(ResNet의 스킵커넥션) 레이어 정규화 된다.\n",
        "5. 4의 출력을 Linear 레이어에 입력하여 출력을 만든다. \n",
        "6. 5의 출력과 4의 출력을 더하고 레이어 정규화 한다. (또 한번 스킵커넥션)\n",
        "7. 6에서 출력된 결과를 입력으로 3으로 돌아가 다시 반복한다. 이렇게 N번 반복한다.\n",
        "\n",
        "여기까지가 트랜스포머의 인코더에 해당되는 내용입니다. 다음은 디코더에 대한 내용입니다.\n",
        "\n",
        "1. 입력과 쌍이 되는 정답을 디코더에 입력한다.\n",
        "2. 인코더와 마찬가지로 정답에 대해서 인코더의 1, 2 과정을 거친다.\n",
        "3. 이렇게 만들어진 벡터를 인코더의 3번, 4번 과정과 동일하게 어텐션한다. 단, 이때 마스크를 씌우게 되는데 이 마스크를 씌우는 부분이 트랜스포머를 이해하는 난관 중 하나입니다. 구체적인 세부 사항은 코드와 함께 설명하겠습니다. 여기서는 일단 어텐션 한다고 생각하면 되겠습니다.\n",
        "4. 3의 출력으로 다시 한번 어텐션하는데 이때는 키, 벨류는 인코더의 출력으로 설정하고 쿼리를 앞 3번 과정의 출력으로 설정하여 어텐션 한다. \n",
        "5. 4의 출력을 입력으로 Linear 레이어를 통과하고 스킵커넥션, 레이어 정규화를 적용한다.\n",
        "6. 5의 출력을 입력으로 3으로 돌아가 다시 반복한다. 이렇게 N번 반복한다.\n",
        "\n",
        "마지막으로 디코더의 내용으로 부터 클래스의 확률을 계산하는 부분은\n",
        "\n",
        "1. 디코더에서 6의 출력을 Linear층으로 입력한다.\n",
        "2. 1의 출력을 Softmax층으로 입력하여 클래스에 대한 확률을 계산한다.\n",
        "\n",
        "이고, 이것이 트랜스포머의 가장 핵심적인 내용입니다. 알고 보면 별것 아닌 내용인데 처음 공부하려면 꽤 복잡해 보이기도 한 것이 사실입니다. \n",
        "\n",
        "여기서 주의해야할 점은 트랜스포머에 입력은 타임스탭별로 하니씩 입력되지 않고 모든 타임스탭의 입력이 동시에 입력되고 출력도 마찬가지라는 점입니다. 예를 들어 다음과 같은 입력이 있을 때\n",
        "\n",
        "`<start>` `I` `don't` `know` `what` `you` `mean` `<end>`\n",
        "\n",
        "입력의 일곱개 단어가 모두 한번에 입력되고 여기에 대한 정답이 다음과 같을 때 \n",
        "\n",
        "`<start>` `나는` `니가` `의미하는` `바를` `모르겠다` `<end>`\n",
        "\n",
        "디코더의 입력으로는\n",
        "\n",
        "`<start>` `나는` `니가` `의미하는` `바를` `모르겠다`\n",
        "\n",
        "이 들어가고 출력으로는 \n",
        "\n",
        "`나는` `니가` `의미하는` `바를` `모르겠다` `<end>`\n",
        "\n",
        "가 나오기를 기대하는 것입니다. 물론 RNN을 사용한 seq2seq같은 모델을 아예 모르는 초심자라면 이런 내용에 주의를 할 필요 조차 없습니다. 그냥 입력이 문장 통째로 들어가서 출력이 문장 통째로 나온다고 생각하면 됩니다.\n",
        "\n",
        "이제 코드를 보면서 그림의 순서대로 따라가봅시다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lgX3hWSQRNH3"
      },
      "source": [
        "## Embedding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TWFwxsoy6WJv"
      },
      "source": [
        "트랜스포머에 입력 데이터인 단어를 입력하기 위해서는 단어에 대해 두가지 전처리를 거쳐야 하는데 하나는 임베딩이고 다른 하나는 위치 인코딩입니다. 모델에 단어를 입력할 때 단어를 숫자로 바꿔서 입력하게 되는데 이 때 가급적 목적하는 작업에 도움이 되도록 바꿔야 할 것입니다. 단어-숫자 변환을 최종 작업에 도움이 되는 방향으로 하기 위해 임베딩embedding이라는 층을 사용하게 됩니다. 임베딩 층은 단어를 입력받아 적절한 숫자 벡터로 변환하는 층으로 변환되어 출력되는 벡터의 길이를 코드에서 `d_model`로 표시합니다.\n",
        "\n",
        "임베딩층이 하는 일은 원핫인코딩된 입력벡터를 적절한 밀집벡터로 바꿔주는 것으로 다음처럼 작동합니다.<sup>&#8224;</sup> \n",
        "\n",
        "$$\n",
        "\\mathbf{x}_{\\text{emb}}=\\mathbf{x}W_{\\text{emb}}\n",
        "$$\n",
        "\n",
        "여기서 임베딩 층이 가지는 가중치 행렬 $W_{\\text{emb}}$의 크기는 $(\\text{vocab}, d_{\\text{model}})$이 되고 입력 벡터 $\\mathbf{x}$는 길이가 $(1, \\text{vocab})$인 원핫인코딩된 벡터입니다. 따라서 임베딩 층의 가중치 행렬은 특정 단어에 해당하는 $d_{\\text{model}}$차원의 벡터를 행으로 가지고 있는 룩업 테이블이라고 할 수 있습니다. 다음 그림처럼 단어장에서 단어의 순번을 가지고 변환될 벡터를 찾는 것입니다.\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1ajCVDKg_c8XQyZFUPr3kuAu3bFRnjAcG)\n",
        "\n",
        "그림을 보면 제일 먼저 'prime'이란 단어를 단어장에서 몇번째 단어인지 찾습니다. 그렇게 찾은 단어의 순번 자리만 1이 되고 나머지는 다 0인 전체 단어 개수만큼 요소를 가지는 원핫벡터를 만듭니다. 이 원핫벡터를 $W_{\\text{emb}}$에 행렬곱해서 임베딩 벡터를 얻는 것입니다.<sup>&#8224;&#8224;</sup>  이렇게 임베딩 층을 거치면 입력이 길이 $d_{\\text{model}}$인 벡터로 변환됩니다.\n",
        "\n",
        "---\n",
        "<sup>&#8224;</sup>  [1]에서는 가중치 행렬 $W$의 행수를 입력, 열수를 출력으로 적고 입력 $\\mathbf{x}$를 행벡터로 하여 $W$ 앞에서 곱하는 방식을 쓰고 있습니다. 다른 표현을 예로 들어보면 파이토치에서는 행벡터와 가중치 행렬의 곱을 $\\mathbf{x} W^{T}$로 표현을 합니다. 여기서 $W$는 행수를 출력, 열수를 입력으로 하고 있는 행렬이 됩니다. 개인적으로 가장 선호하는 방식은 가중치 행렬을 파이토치 형태로 쓰고 열 벡터 $\\mathbf{x}$를 뒤에서 곱하는 $W \\mathbf{x}$ 방식입니다.\n",
        "\n",
        "<sup>&#8224;&#8224;</sup>실제로는 이렇게 행렬곱하지 않습니다. 그냥 $W_{\\text{emb}}$에서 'prime'이라는 단어 인덱스에 해당하는 행을 가져옵니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ckIXaJBN0AD0"
      },
      "source": [
        "class Embeddings(nn.Module):\n",
        "    def __init__(self, d_model, vocab):\n",
        "        super(Embeddings, self).__init__()\n",
        "        self.lut = nn.Embedding(vocab, d_model)\n",
        "        self.d_model = d_model\n",
        "\n",
        "    def forward(self, x):\n",
        "        # 임베딩 벡터에 sqrt(d_model)을 곱해서 임베딩 벡터의 요소들 값을 \n",
        "        # 증가 시킴.\n",
        "        # d_model은 512정도 되는 큰 값이이므로 22정도 되는 값이 \n",
        "        # 임베딩 벡터 요소에 곱해짐. \n",
        "        # 곱하는 이유는 뒤에 포지션 벡터를 더할텐데 이 때 포지션 벡터에\n",
        "        # 의해 임베딩 결과가 희석되는 것을 막기 위함\n",
        "        # ref.: https://stackoverflow.com/questions/56930821/why-does-embedding-vector-multiplied-by-a-constant-in-transformer-model\n",
        "        return self.lut(x) * math.sqrt(self.d_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zxe2mijnWm9e"
      },
      "source": [
        "코드는 파이토치 `nn.Embedding`을 그대로 사용합니다. 단 위 코드에서 임베딩 층을 포워드 시킨 결과에 $\\sqrt{d_{\\text{model}}}$을 곱하고 있는 점이 좀 특이합니다. 이렇게 상수를 곱하는 이유는 뒤에 설명할 포지션 벡터가 임베딩 벡터에 더해지는데 이 때 포지션 벡터가 더해지면서 임베딩 벡터의 값이 희석되는 것을 막기 위해 임베딩 벡터의 요소 크기를 상대적으로 크게 하기 위함일 것으로 생각됩니다.[[5](https://stackoverflow.com/questions/56930821/why-does-embedding-vector-multiplied-by-a-constant-in-transformer-model)] 이런 해석은 논문에서 공식적으로 밝히고 있는 것은 아닙니다. 논문에서는 그냥 $\\sqrt{d_{\\text{model}}}$를 곱한다고만 되어 있습니다.\n",
        "\n",
        "이 과정을 거쳐서 길이 $n_{\\text{seq}}$인 입력 문장의 각 단어를 임베딩 벡터로 모두 변환하게 되면 $(n_{\\text{seq}}, d_{\\text{model}})$인 행렬로 변환되게 됩니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QAjuxxn8RUIW"
      },
      "source": [
        "## Positional Encoding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4fdpGrwUsUCt"
      },
      "source": [
        "앞서 이야기한 위치 인코딩입니다. 트랜스포머는 RNN같은 순차적 구조를 사용하지 않고 문장 전체를 한꺼번에 입력하여 문장에 있는 단어간 관계를 특징화하는 모델입니다. RNN은 토큰이 순차적으로 입력되므로 토큰 순서 정보가 자연스럽게 생겨나는 반면 트랜스포머는 토큰의 순서 정보를 인위적으로 만들어 넣어줄 필요가 있습니다. 순서 정보를 인위적으로 만들기위해 논문 저자들은 위치 인코딩이라는 방법을 사용합니다. \n",
        "\n",
        "$$\n",
        "PE_{(pos,2i)}=\\sin \\left( \\frac{pos}{10000^{2i/d_{\\text{model}}}} \\right)\n",
        "$$\n",
        "\n",
        "$$\n",
        "PE_{(pos,2i+1)}=\\cos \\left( \\frac{pos}{10000^{2i/d_{\\text{model}}}} \\right)\n",
        "$$\n",
        "\n",
        "위 식을 사용하여 입력 문장의 위치 정보를 $(n_{\\text{seq}}, d_{\\text{model}})$인 행렬로 인코딩 합니다. 식을 보면 행은 단어의 위치를 나타내는데 단어 위치별 값은 $pos$ 변수에 의해 달라집니다. 열에 대해서는 홀수 열은 $\\cos$함수로 짝수 열은 $\\sin$함수를 사용해서 값을 다르게 계산하는데 열 인덱스 $i$에 대해서 각각 다른 주기(첫 두 열부터 $2 \\pi$ 주기에서 시작해서 마지막 두 열에서는 거의 $10000 \\times 2 \\pi$ 주기를 가짐) 의 삼각함수를 사용하여 모든 열에 대해서 다른 값을 계산하게 됩니다.\n",
        "\n",
        "문장 길이 120에 `d_model=512`인 경우 위치 인코딩을 직접 해보면 다음과 같습니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 214
        },
        "id": "xBewD45hwAnk",
        "outputId": "1ab5ffc2-7948-4d68-c59e-5941f9083e32"
      },
      "source": [
        "seq_length = 120\n",
        "d_model = 512\n",
        "\n",
        "PE = np.zeros((seq_length, d_model))\n",
        "\n",
        "for pos in range(seq_length):\n",
        "    for i in range(d_model//2):\n",
        "        PE[pos, 2*i]   = np.sin( pos / (10000**(2*i/d_model)) )\n",
        "        PE[pos, 2*i+1] = np.cos( pos / (10000**(2*i/d_model)) )\n",
        "\n",
        "plt.figure(figsize=(6,3))\n",
        "plt.imshow(PE, aspect='auto', cmap='gray', interpolation='gaussian')\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAADFCAYAAABevum5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9W4xka3Ym9O243++REXk9VV2nTne7u2UNtgehkdAICwnBCPOArGHQyCBL/QQCgcQYnngAaXgB/DSohY2MNJIZDUiehxFoZPADtjUaG7dv7XNOdV2y8haRkXG/XzcPUd+qFX/tHZfMyKqoOrmkUOSOvfe//4jI+P61v/WttSzbtvFgD/ZgD/Zgn555PvQEHuzBHuzBHux+7AHgH+zBHuzBPlF7APgHe7AHe7BP1B4A/sEe7MEe7BO1B4B/sAd7sAf7RO0B4B/swR7swT5RuxeAtyzr37As6yvLsn5qWdav3cc1HuzBHuzBHmy5WdvWwVuW5QXwNYB/HcA5gH8B4N+zbfsnW73Qgz3Ygz3Ygy21+/Dg/zqAn9q2/cK27RGA3wbwS/dwnQd7sAd7sAdbYr57GPMQwJnaPgfwL5sHWZb1QwA/BIBwOPxzjx49chzMsizouwzLsgBAXlu2bds2LMuSMZy2AcDj8Sxs33ZMAJjNZjKm3uYx5rZt25jNZrAsS+bBbfO6HHMb89j083F7L5zXqvfi9Bk7jcHtZWPoz+cuY+j3pj/j28zjfY2xbEyn737Tz8ft/2eT9/axj+E25jbGWOd7WncMr9eLn/zkJze2befhYvcB8GuZbds/AvAjAPje975n/+Zv/iam0ykAIBAIwOv1yhv0+Xzwer3vfKja+MEDi4CsAWs6ncLj8ciHxev5fD7Z5vUAYDKZYDabwev1wuv1YjKZYDqdyvZ0OsV4PIbH45ExRqORvAcAGI1GmM1m8Pv98Hq9GI1GmEwm8Pv98Pl8mEwmGI1G8Hq9CAQCmM1mGAwGsCwLwWAQADAcDjGbzeRzGQ6HmEwmCAQC8Pv9GI/HGA6H8Pl8CAaDmE6nMkYoFAIADAYDzGYzhEIheDweDAYDTKdTGWM0GmE0Gr0zhsfjQSgUgm3bGAwGsG17YQzOIxAIyBh+vx/BYBCTyUTGCIfD8t5s20Y4HIbH40G/38d0OkUwGITf78dwOMRoNEIgEEAwGMR4PMZgMIDP50MoFMJsNkO/3weAd8YIhULw+XyOYwyHQ3i9XoRCIUynUwyHQxnDsiz0+335fHw+HwaDAcbjses89GccDocBQMYIh8Pwer0Ln08wGMRoNJLviWP0+335fNzGGI/HCIVC8j0Nh0P4/X7HMWazmfy/rDvGOt8T3wvHMD9jjsHPmGMAeOf/xW0Mt++a/8f8rjf5njYZIxgMIhAILIwRDoflM95kjE0/Y6/Xi36/j8lksvYYiUQCP/jBD06X4ex9APwFgGO1ffTmNVcbDof4+uuvMRwO4fF4EI/HEQqFMB6PMZvNEI1G5YPmBxAOh2HbNsbjsXyZlmVhMpnAsiwEAoEFj5kgPJvNMJlM4PV64ff7F7xUvUgAc5DWiwRXVf5NoAcgCxAXFy4SnIPP55PxeZ7P55PjOZ7H45G5c4zZbCaLhJ6DXvj0QsTrc5XnNfU1ODbPMR+2bcPr9S54D/qZDx6vX9PetX5NL67mZ2oeoz0dpzH52fLZ/Nt86H3m+ebf5timub3mdC2341eNte5cTHN6b5tc+7bXvS/7kNf+ULbN93wfAP8vADy1LOsx5sD+twH8nWUntNtt/P7v/754sqlUCtFoFKPRCLZtI5FIIB6PCzhHIhEkk0kAEA8gHo/D5/NhNBrBsizEYjHxbrlKBgIBTKdT8VwJ4Brwgbe3TgRJvUjoOwmCrAYsLhr8oXE/Qcrv9wvwEmj1oqBpDIKwBnB9DhcNLhYEW8uyMJ1OFwCei5sGeoK4BmpzWwM876o0uJuA7/ZYBsAmoOt9nMMywF4F8Obr3Navm6+5PZuLwm1+jLc9z+l9bHKefv5Qtu3r3+f7+dCf1TZs6wBv2/bEsqz/CMD/BcAL4Ddt2/7LZef0+3189dVXmEwmAIBYLCa3MbPZDLFYDIlEQjz2WCyGVCoFj8eD0WiEYDCIdDott31cJEKhEIbDIWzbRjweRyQSwWQywWQyQTgcltti3gXwdnQ8HgtFQmoGmIMkaRXgrVdMANZAZXJn2hslQJveJI8lsHOfBlQN2toT10DLueoxNFi7gbP2xPWi5LTtBtCmx67ft3kXsAzs+Zm4AbVeVM3PUh+nt932mfv1thswOlGCq7xxN8AwjzHjFKvsLguG2zxus39b19nWmJ8CQN/V7oWDt237nwL4p+sePx6P0W63YVlziqXT6SwAl9/vF/Al15pIJOD1eoX3SqfTCAaDAvCZTAaRSEQ48FQqhXg8LgAfiURk0dB3AeS4PR4PotGoLBoAZNHhXYDf74ff75eFyOPxiAdPGoXAqn+0JjgQbM1t7dnrBUTTLubxelHQ2ybAa0A3Xze3TXA2Fwq9zWtrKonH6PesQdkNrFd53+Y+twVBL6KrzjWvq23VPJa9tsqDvu2dwG3truD3AJ4fh32wIKs2v9+Pg4MDWJaF4XCIZrMpXrJt2+h0Ouj1erAsS4Kc9XpdAp0+nw/ValWCPR6PB7VaDdFoVAKjqVQKyWQSs9kM4/EY8XgcqVQKwDwGEAqFJLDCYBFf5yJB/p9jBoNBCeRMJhNZlADIPIC3PDx5dAK+W8DYCaCcPFUNxprK4DaPMY817wJ4vMmFm/SIEwdvevbmOXobWIxVuD2cvHt+Fm7P6ywCbttO5gTOy45b9/XbHndf5vb+tFPyvufzYNuznQD4aDSKX/iFXwAwp2uur6/RarUAzIGy0WiIcsK2bQyHQ3S7XQF4nkdPF5jz+vS4bdtGq9USj30ymSAWi6HdbsPj8QiQ9/v9BZpnNBoJwJMeikQiGI/HmE6nEvjlPnL4AERdA8xBlAsW8NYj14FRk+bhe6WZAOUUiNTHuQUi9bn62eTWqTgyr2EGSE26xYmicfKKnQDZaREz56zvbNzAyVzUnMZf9br5mZvXW+apO9E35nHLbNl1V523DVDW83e6xgMIfzy2MwD/gx/8QIDy+voa9Xpdtm9ublCr1YQe6XQ6aLfb8s9MuRVBlB51r9cTwO/1euh2uwCA6XSKdrst8jJKyEjVUMpGSRcDtePxGKPRCOPxWOSNXEB4LjD/ETBgTC92PB4vvOfpdLqgKyfgk4Onh68Dtm6Ar7fdAF5z7m53AU4UDY/hAqDn4wTKy/j5dc/R72kdYHbbdhpnmbktHMs+62Wgvew1cxHT+91AepMFQlN+m5zvdv1139+ycTc5ZhcXkV2c0yrbCYAPBoM4ODgQ7yyfz6PZbArXfXNzg0qlIrLJer0u25ZlodvtotVqiXKEQKxpntFoJBrS2WyGXq8nnjq1ygzYUg9Orp4ePEGdPD6Bfzabybn84XJsy7JEe8z3x7EI8Jwj6ScCqelFm7SOVtw4AZrbNvCul+vkNeuHvlviedqjdzvPBFcnzn3ZomCCkhPYuvH4Tu9704VBv9/bAtpdvN5VC/m619/0OpuY2xzvwz5GkP2QthMAP5vN0O12YVnzpJF0Oo14PC6Avbe3h1qtJiBZr9dRKpUkkaLVauH6+lq2B4MB2u32Ox49x2PSjvbUmeDAZ5/PJ5TMZDIRD55JRvyboEeah4AzGAyEb+ddAfDWW+d8CEyax+c/Mb18Apj2+IF3s+ncAB9wBlYT2Ny4fhOczUWBz5rmcRpTj7VqMXADfM7DbaxlY5t2G/A0r+H22rrmds1NFpRNwfzBvjm2EwDf6/XwF3/xF/D7/YjFYigWi4hEIgDmyUapVArhcFhojlwuh1QqJbx8u91GIpFAp9MBMAf4SqWCbrcrFEyn08FoNBIg0llh9MCpjCFQTSYTRKNRAWICOxOu6MEDc4Cn0ocBWurUeXegPXiOA7ylQHiHwgAoeXw+eLwGVDNbl/v1Ns3Nq3QDfM7NDciWLRpO+5zGWPdhnmuO4fQezffqdmewbAFYdjex7HrmZ+z2PTjZbUD6NmPuymKwK/P4FG0nAL7b7eLP/uzP4Pf7EY/H0Wg0kMvlJEM1m81Kmr7X60UkEpGgqGVZ6PV6iEQiEpjt9/sIhUJoNBoC8LVaTRYAevBcMEjhsBwAPWUTiLVEkosE/zmZVsykIwI8U7fpwWvw5h2GBnitgyfAkz/n8QQODfgEdFOZo0HfCfCXgTznStPSR32ek+rFDaCXAbz5utNxmhLifs5t2XsxbdlCwPGWnWeev6m37bZguJ2zywB9G7vt5/dgm9lOAHy/38ezZ8/g8/ng9/tRqVRQLBYRCoUQiURweHiIXC4n9VqY+EQvnx5+u92W8SKRCKrVKgBIELVWqwF4G2Ql4NNIxQBv6Q961cBbgGdAVAO0BngqcQjwpHN4rF44LMsSiSIXGQI8A7W8E+D8GDfgPPS4GvT5PvQ2X1sHMJw8ZifOXh/jFlQ1F4fbePL6eKf5LdvvBvpO4LnKAzfPXQe0N1kAVilhtgGG64xx12Pe13v5kLbr898JgCcwTSYTNJtNtNtt3NzcIBgMIhgMolKp4OjoCOFwGMFgEPl8Hul0WsAxFAohn88jFosJ6EUiEaRSKfGMI5GI0C2z2QzValXoGGB+F0EOH4AUpNKBRQ2gBHsAcpcwHA5hWZYkXOkCYyxCRiBmoBaAI8BblrUA8LwrABYzakkn6cWH1+E8eU2+B2A14C8DPnNbg7v+QTtx+vqYVZTJMq9+VVDVSaGyytwA2un6q8Yxx1t17G33m8ean/8m471vsLqv6z3cHby1nQD4UCiEzz77DKPRCPV6HY1GA5VKRTzZVquFZrMp1Eyj0cDh4aFUckskEgulByxrXqeFAdLZbCZJSdxmbRoCbzAYlAQry7IEsOl1kocnsBBMCVyTyUQ8eI5LgKf6BnhbQoAAz7lqgGexMX1HwMqVAEQ9pKtdapDnD90MzGoA0PEAN9NAym3z2YmeWebluy0kTmOZ+835OI3hNk+n4/Q+M8ahn90+G6dxlpl5d2OOt8qc3qMbiH+IJKUH2z3bCYCPRqP43ve+h36/j5ubG7x69UrolsFggKurKwyHQ0QiEfh8Pkl8isfj8Pv9yOVyyOVyAozBYBCxWAw+n0+AmWUFyHMHg0GRMwJAs9kEAClWxho22iOeTqcC1FqxQm+aZYC1Eod3CTyP0klSNB7P2/IGvCbllroEMKWWAKSqpFnOmPEB3tmYFA7nTdPvAXgLDE6gbpqbp20+m143/15H++62fZvXTVWKE73ktgC57TM/H7djliUOmWM4vYdVdt8e6qZ3Lqtee7D3ZzsD8N///vcxGAwk4/Ti4kLKFtRqNVSrVXQ6HcxmMzSbTfT7faRSKfj9ftTrdYxGI8RiMXi9XkSjUSQSCQSDQQk6sv46OXV6/1TiRKNReDwe9Ho9eSa9Q2+Z9aydwJQUE1UzlDwS/DmO5tNNgNe141kygQDPOt/AW4DXiwQBnXcFVNhwgWPAVNNKJuhq0x6+pkNMMwFQUyOrgrr6NTdQ5l2SeY55rvkenBaJVX+vmqv+3Jw+A7fPxcmWfZ7L1E9uts7CtC5Qu81jHVt11/O+bNcWlg81n50AeFZ/tG0bhUIB8XgcuVwO3W4XjUYDr169ws3NjXDXTHJiwbFarYbhcIhMJgO/3y81Z0jZBAIBaexAoNWF+4E5wAMQPT6zXgn4Jj1CUKf0knNjUFUDPMGf79WyLFkgPB7PQlCViw/vLvx+vzTrIMAHAoEFgCdPrxckjkmA57X4o9XcvJvU0slMEHVStbh5zyYtdBsPXl/X6Zx1vP9lf697nNOc3MZze23Z67e1Ta+z6vqr7j7WHed92K4sLrtkOwHw5N4DgQASiQSOjo4QDAbR7XbR7XYRCoVwenqKwWCAbreLWq2GdrstYNbpdDCdTtFqteDz+ZBOp2HbNlKplARhE4mESC0JqrZtC8DTS2ZVy263+w4fzgxYgjspHAI2AAnM6sQlHZAlL0+A13XoqZUnwDNrlgXNdBCXWnsGZcnT04PnmDp2oCkbXQphk0Dsutp62jpUiJs3v8zjd/p72TyWLRjm/mXXMW3Vdc1jVx2zjm3iiW865rbGXXdhWDUXvf1NBem72E4AfK/Xw7NnzyTJqVAoIJvNIpFISFGvUCiEZrOJTqcDr9eLarUqNEe/34dt2+j1evD7/Wg0GqJ193q9iMVi8Hg80gWKHjxrxgOQtnTcZiVKE3z5T6aLjAHvatnpMVOZQw+eAE/QZ1CW1+BcSNHw7oMcPQCJH9CD9/v98llw0eOCwliAuRDxToJUjikPdQJ3J3PzVp0Ak3SZmZFrWe4FuszxnI5ZBspui4DTOOuA0qq5rHP8qmO2DbabmNMi7jSPTwlsP6X3YtpOAHyn08Gf/umfIhaLoVwu47vf/S5yuRw8nnlN9lAohGAwiEajgW63i0gkgpcvX6Lf72M4HKLVauHm5kZkhbVaDdPpFL1eTwKutm0jk8kIR85xCXT04Anw7PNJ8NWKGh181eCnZZWkbDQNQn6dIKdryFuW9U58QPewdPLgdX9XHcy1LEsWFM6HC46WeepsWE3haCUO3wuwuUdvbhPINI/tljzlBMzLOPlNKBKn41dde9k5Ttfi+9Qy22VzWmfftoH/UwG29/k+7nJn8iFsJwB+PB7j8vISgUBAaso8efIE0WgUmUwGmUwG+/v7iMfjomn3+/2oVqvipTebTeHLyZ+z3C/LBhMM2cyapQVI2TBoa9u2ePCkcgjwwNsWeprm4Ov09PV7o5ES0sDKBYcAzzsRth/Udyha0kmun3p9tiNk9qsODuv4gAZ4XcFStyLkfr04aVvm0QPr/+CcjtMePredwHVZANfcNr15k7/fdN7L3suqOx23Z32cuaCu+1nzXP28znmb2Ko7ovdpH/r6H4PtBMAzSDgej1Eul4XqSCaTyOfzsKx5j9VoNCoerG3bSCaT6PV6CAQCeP36tYDwYDBAtVoVySE5bWBOf7AoGB+kSSKRiPw4qHwxa9UQMEmx6KqQmuPmNr1h4O0CoBUtXq/3nUAuqSZ66KR5CPAMvnJO1N1znlTp6EWDwK4LsOmHrknvxNObwGgGZt0AZdWtvRNYuI2jqRzt1a9D3yybl9sdgNvismysVde5D9t1oFtGvz3Y/dqtAd6yrGMA/yuAAgAbwI9s2/51y7IyAP43AI8AvALwy7Zt15eNFQ6HcXBwgF6vh1KphKurK8xmMySTSVxfX8O2bXz22WcCwtlsVrxs6uOn0ynq9brQHIPBAPV6XSSHb+YsyVIAxHumdJGSRBrBmUBOnpugrl+nmSBvLgAEYOBtTRxSNLp5iS5NzECwbh3IBZEAT4qJ49ODNwHepGw0yOvAsam4caNnVoGbE71jgqYGbZPGcaJC7urBr6JWzPk6zd+c+zLbBqjdRkt/17k8gPHHb3fx4CcA/nPbtv8/y7LiAP7Ysqx/BuA/APC7tm3/fcuyfg3ArwH4e8sGisfj+Nmf/Vk0Gg1EIhE8f/4c1WoV3W4X19fXUt+dbff29/dFEsnuTLPZDKVSSboylctlAUDSEAAE4AlwpHts20YikZB/aq/Xi3g8DsC5yJiuMEnPmoDKhYEcOgETwIImnsBpAjzvSgjwvV4PwFsPnvp4DfC8eyEHr2WdDAZzvsAiJ68XJZO64efG44F3OXkzMcpNeqn3828nsDbNBGbtYev9fG0ZiOsxnZ6djll2nja3ejurbNldwqZ3DG7jb/O4j9129X3ex7xuDfC2bV8BuHrzd9uyrL8CcAjglwD8zTeH/RaA38MKgA8Gg3j06BG63a70Sb26uhLp4/Pnz2FZlpQJBoBCoSAKE5YSZoVJttJrtVrizTebTQnaEtBZvZIVJClZ1LSNWS6YAE0QZDcp2nA4FAAHFmkagqhORuJCoPnxwWAgfPp0OsVgMIBlWQLe5Oi50HAOBHjy89qDJ7DrZwL8dDpdaCOoAV4Hg9V3/87fy/Tz69A06wCu0xibeN2rjnPy8s3FQ9sm1IPTtTdZBNa1dTz3XQW4Zfa+gpsf42ezzLbCwVuW9QjAXwPwzwEU3oA/AJQwp3BWGguJJRIJkfaxLV+z2cSrV6/Q6XQQi8UEgHlONBrFwcEBvF6vADy9XIJvr9cT7Tw58GAwKHw3+fpwOCxa+UAgILJGy5rHAcbj8QLtwcYh9FoHg4HUmQfeJkbxH4egrDNPuWBogGccQgM8vXJWruSCw+JmmuPXVA0BXo9Bz13fmWiqRgc7NSfPbVI4Tl78iv+VpR494EzZ6OPdKBrzGLd95h2HeYzbvNe51jrnOcUN3OZnjrEpn/0+AOu+7ig+dVvn7vWudmeAtywrBuB/B/Cf2rbd0l+ebdu2ZVmO78CyrB8C+CEwb+DRaDSkm9PP/MzPwOPxoFarIRwO49mzZ6jX66JqoceZy+UQi8UkexWYe9CxWEza7vX7fQQCAZRKJQlYEpwDgYDspwae2bEM4vIHxRII9Kw1wOsqlKSIqM4hGHNbe8cAFjx1AinpFq3z1+BM3p77B4OB6Pi1Bw+81e9zm0BPVY2OJzh58G++x3fUNLzj4X7TNGgvC8LyGPNvtzsBk95ZBeZOMkw3u82dxjrn3xbQVt2RbMvuUrBs1fe0S2a+z2/CQnMngLcsy485uP9D27b/jzcvly3L2rdt+8qyrH0A107n2rb9IwA/AoB8Pm9/+eWXSKfTeLONzz//HPV6HZlMBtPpFGdnZ8JHv3jxAn6/H81mU4CdXaBIq9CDpdc/Ho9Rr89jvaPRCJ1OB9fX16JI0Z4w6Q8tqSR9E41GFxqFDIdDCYICkAWDdxnUruvMVNIjABY8aP4o6I3zWC4QbgDv5MHzHA3wBHYAC4slg8X8m3cYBGbTg1/2vM6Pe5lH7bQfeNdzva0Hv2r/Mopm1ftwG3+V3QVonK61CcBusgh9EwCR9qm817uoaCwAvwHgr2zb/u/Vrn8C4FcA/P03z7+zaqzBYIDnz58jmUxiOBxKcpLH40EymRQKo9PpoNlsotFo4PT0VAqMAViQOZKymU7njT1isZh47pQ+djodqVhJWSG5ewJfOByWImTAXGJJ8CZ/nUwm5Q4AmNeyoWqHRcvefF4LHjO9X53RqrNPgbcgTLAmOBPgCczsGMUFh9QUqRqOybsAPusqmNTRa6DXPDznCiwHdHPfOrbquFXlDpw8azdqx+3abkC5ao7buCtYZ8x153IfnvNd7kA2nc86n89d5vRNs7t48H8DwN8F8OeWZf34zWv/FebA/o8sy/pVAKcAfnnVQNPpFJVKRbosxWIxnJycSK33p0+fSp/VYDCIfr+PWq0Gy7IkyzMajSKfz4vuPZFIoFAoIBaLIR6Po9/vYzweL2jMuc05RKNR2LYtYBeJRCQJSGehUlYZiUQWZI8ApJQCMPf6u92ugB3BWlMGTpSNVroQtHVWJBObuDhomoh17E2qxgR47blriobUk7kNvAVtndFr8vIGRee4vexHb1IvmyhuTI28Ps/pGrcBn2W0EBf+ZeeaxzuNcRuQXnWXsmw+78MeAPnD2F1UNP8vALdv7Rc3GSsQCIgs8NWrV8In0zve39/HcDhEPB5HIpHAcDhEqVRCq9VCu90WLbwuGZzNZpFMJhEMBhGPx6WrUrvdRrvdxmg0QqPRWNC2s2IlFwC2BKSnyw5TwPwflnQQAZ0eO7d9Pt8CwA+HQwmgAljw6NXn6grwmqNnYJgUDT11xh60B68pGk3V8KGDrWbQlR48gV1z2nqbc+ezG7hvak5A5FbPxjxv1Tir7gCcxll2Dbf963rpbvvfNzje9trva573cZ1PdQHaiUzWWCyGzz77DM1mE69fv8ZPf/pTBAIB5HI5jEYjfP755ygWi/D5fMLJj8djUbBcX19LOQIWKAsGgwgEAgt9XYfDIRqNhiwKuqvSYDBAo9GAZVnSGIT0DLeZAUvTHD1BwtTSs4Y9AAnq6nIBzIzl+QRYenJamknunosVgXk4HEoSFKkXTdWQmzcpGrNImRMXD2BhPqY+Xgdgydvr19exdW/LNzWncfQC5cb9u9E9qyge/dp9UQu3XSQ3Odbpe/tYAPBjmef7sp0A+EgkgidPnqBWq2EymeD09BSvX79Gt9tFr9eT+vCJRALxeByTyUT6tvK5XC7D7/ej1WphOp0iHo8jm80KlZJOp1EsFoXfJ63R7/cxGAxkwWi325JEFI1GJbuVHr1uj8cSCxqgGRMg4LGUMTD3vLvdrtAm1K1zkTHpEAIqAZ77NMCbWnwCPPdr4Ddr1OiHBndd5pjzpFdveuzmg6+b2bBOOvnbBmWBRamhW8Eyp+dV45rbm1Af2wLfde4eVp172zHXtftYwL5J4Py+dP07AfB+vx+Hh4ciUaSXPZvN67Ozlrvf70ckEsHJyQmazSYCgYC07+t0OqhUKuh0OhiNRkgmk7CsOUfv9/sRDoeRzWbh8/kQj8cxHA6Frun3+xiNRuj1epJ0NB6PUa1WpQwA5ZcsbUCZJfl4Ag55fABSqpheut/vR6fTEdAlAGtVDj1nHdw0C4Uxs5fHU4pJz1wDvJZLmh487wqcPHcnD14DPrcJ0MtAfRucsrnPacG4y5jcvy494bSYrDuf2wLZXUF103IH27DbLFIPtj3bCYCniiOdTiMej6PVauHZs2eYTqe4vr7GX/7lXyISiSCTySCXyyGTyeBb3/qW9GcdjUZ4+fIlut2uyBapwkkkEgDmMkomSdFr7XQ6qFar6PV6jl5xrVYTgB8MBohGo9IflU272QkKgLyuOfZkMimAzSYmGpxJM9E0sOpSArrImQZ43gXQg+dioDNbgbcevAZ6PgjwVCvpvzlPE+CBd5uG0Ewah8dqc+Pk1/3BLztOLzLm8bfxZJdRNOvM0e18c2FwWgj5P+C2iNwngL4vcHYb52MrzbuLthMA3+12UalUEIlEsLe3h+9///sSBB0Ohzg/P0cymcTBwQGGwyGi0ShyuRwmkwmSyaSANekQcvmhUEj2RyIRRKNR4eX39vbQbrfh9/sFdKm0oVfMloEa4Jk9S/H0DLsAACAASURBVDmnLm3A8gYEaHrwTIyi/p5KGg3w2lvXxcsIrJq6MbNpA4GAZM8S4NksRAdfNUUzGo2kdyw5eF6HOQBOhcm4DbxbU95MjuKzDrqaoO9E2ZjmBIx6nFWgbYKkW2LPsu1NjtOvLVPVrPPastfdbB3a60PYfcVa3od9THPVthMA3+v18Pr1a6TTacRiMRweHqJer+P6+hrT6RQvX77ExcUFZrMZ+v0+kskkisUiEomEaNer1Squrq7Q7/fR6/UWtrkQUN0SCASkFDEASYyiTJMeLGkcUiC1Wk1AnsodNucgH8+gLACpfUPw9Pv9iMViEmjl+9GBV6pfCJ6arrGst0FYXXKAVIumaLhIaEB34uC5oLnRNMAiBw84g7dT8FXr/fV5pi0DJJOOcQJ283ia5umXLRJORcucts1rOB17mzuEdRacVXZX8HzfAPaxAubHZjsB8IPBAKenp8Krf//738fx8bHw57VaTRQu7XYb8XgcsVhMAqCBQAC1Wg22Pa9b0+/3pRIlg6mpVEo08j6fTzh527al6mOj0VhQqZCTZzZpvV5HNBpFJBLBYDBAJBKR7FmfzydyTyY6eb1ekW/OZvP+rhrgeWfR7Xbf4dcJpgz46oJfPJfePc+hB6/pJk3RmEXITA7e6W9ej4uL3l5WnMxcBJzAniB7Gx7dPMdMhroPLvy2NMmmdheOflvxjlWL7m3G3HROu3gXssp2beHaCYD3eDxoNBpCV7AfKzCXUNbrdXz11VfSdPvLL79EKpVCLpdDPB5HMpnE48ePBaQnkwlevnyJdrsttVzi8TgikQji8Thms3lbvFgsJnTMbDaTRYSgToULMAdIljMeDofo9XoIh8OIxWKyEGlwt6x50gtlmuS24/G4jE+KRittqOahJFLr4oG3nrEptaTnb3LwOsjq8XgWyh5oDt4N7IF3q1/qZzdQ19t67hrQ1wV3J8992bbT+fqhj1235PCyO4Fl1132vMruEqNYd24fu+3q+9iVee0EwLPkL2vOfPnll1JwbG9vD9/5znfQarXQ6XTQaDRwfn6OFy9eCLDFYjHk83l0u12hRBqNBnq9ngRdLy4ukEwmkU6nFzh59nudzWYoFAqyIJDX15mgLJUwHo8RDAYRiUSQSCTEi2ciFMGBvVKZbcts2V6vJ155r9d7h6cPBAICMpqO4bgEdfLfZkVKHYTlucPhUBYA7cET2J08eB5LDl578qYHrzNf3R704t0SofR7NI9ZxU87ee3LzrkLx77s2FULw6pqkGbLwmVxhbvaunc6q+ayDbvLovdg7rYTAB+JRFAsFtFut1Gr1fD1118jnU6L7v3k5ATlchnlchnT6RSlUgmnp6fSmDqZTCKTySCbzUoWbKVSQalUEi+5Uqng6upKAC2RSCAYDApvzjsCUjNcJFiyGIBw8lTZRCIRNJtNCXTSW+eYOvBKIOTCogGeNA4AAXjeATCRyfR8tedMUGbQ1SxRALxtNGJSNJwHPxcuGDrhSgdZCeR8nXkAel4mL89nJ0B3+vuuFMB98c/LaJltUBKr9t/2jmBTIP+Ubdvf2zrX+JC2MwD/6NEjNBoNNJtNXF5e4vnz5ygUCggGgzg5OcGTJ0+k/V6tVkO5XEYwGJTaNfF4XFQyfr8flUpFwLrf76PZbC4AfiKREB6fcsdcLofxeIxutyuUDQGPPHyv15MkplAoJGWM6cGTsmHNGnrx9MrC4TAikYiAKQGe1A5L/2oPnkFTer8EYAZjTYDXlSf1M/n8TTl4SixND14vMpqmMcEdcC45vC2OdRVf7ObdmzSPCeDLOHfzjsPpuqvmDCyvDb/uWLc99kPYrsxvV+Zx37YTAG9ZFvb29hCNRjGZTPAHf/AHuLi4kKbSuVwOe3t7mE7nBcCq1SrOz89Rq9XQ7XYRDAYl05V14h89mneIajabmEwmKJVKUi6YnHwymUQsFhONOqtZhsNhAECtVpOApS4JwH+ORqOBWq0mHjwblugsV3rzABY4eVIy0WgUsVhMtPCUUvKHP5vNFqpTEsSBxQYcugywBni+TorGDeBNWkdTNNqr1wBPb17XrjGBnvM0wV0nRZmAuc7/yyqd+zJ54ibgzDsps/bNOnO9Lfive4zbccsWvG3bNt/Lfc/jm2g7AfAEt0gkgs8//xynp6eo1+sSeD08PMTR0ZHw3Tc3NwLco9EIp6enODg4wPHxsZT4LRaLqNfrCIVCEnxlQHMwGCAWiyGXywkVEYlExAMPBAIA5nXpB4MBer2eeNvaW+52u2i1WsK3B4NBaSlIuaRZq4ZUDr1xKnGoMBkMBrLA0GvmfEx+lkZA1YXDNNcOLFI0XAC0TFI3HdEcPM81PXiTsnHz4IF3qRs9b62scTK9ANxWcbPO69sAqmV3C3r/fdi687/N+9zmHcaDvV/bCYCnpx0KhVAoFPD06VP8+Z//uWSyPnv2DKlUShKHPv/8c1xdXaHRaIgc8vT0VLzweDyOTCaDYrEo9EilUpHEqX6/j1KphL29PQCQQC119Uxeymaz6Ha7Uu+91WpJdcjJZN58pNVqAYAAd7vdFu+dgVjy1KRtgsGgeJjk5DXgE+AtyxL6RycVsWyB5uSdKBuez2dTL+9E0Wg+XgO65uRNbt303s2gK+foFlzV70PbMkBfBtLrBCad7hruws/eBgTvQu2sOx+n41eVNN4kiew+7GNdMHZx3jsB8L1eD+fn50gkEsjn8/jud7+LarWKdruNRqOBr7/+GsViEZlMBuFwGMfHx/jWt76F09NTWJaFq6srvHr1CtFoVMoU7O/vI5fLCU1SqVQkY3Q8HqNWq+Hq6gqWZUkpYh0g9fl8oswJhUIAIBJMAqqmU8jls7k34wVU6gBYCLryR0YFEemK0Wi0APD04BnYBN624SN9YCYhmd438C4Hz3r4GtR1dqymaLQHb3rybt4756IXAQ3kJhevF4JNKYZllI35t9sxq/aZQKe94bv8sFfNYdMFbpvUzKbvaxcBbhP72OfvZDsB8JPJBFdXV+h0OkgkEvjiiy/w+PFjXF9fo9fr4fLyEi9evBAt96NHj/D48WOMx2MEAgHU63XUajVcXFxI0+10Oi0NOwKBAI6Pj9FqtdDr9TCZTMTzZ1p/LBaT+vHkytPpNNrttkgfeQdA5QmrURLQ/X4/Go2GtPdjwxLy6QR4rZUPhUILXaNGo5HUtyGIUmbJf0BWetQ1V7SyhXJIvk7vncFagjZjC5qWMSkaJk85Abz21DXYm9p4U1kDbN7mb5Wt+nGuUzJgHT5+3Xm40TVu495msdlV4/e5rQVn19/vLttOALzf7xee+/nz5zg5ORH6ZDweo1Qq4fLyUppcFItF5PN5tFotBAIB3Nzc4OzsbEHRcnBwIIAdDAZxcHCASqWCer2O6XS6QAuxNHA+n0cikRDuPBaLIZVKCWWTSqWkloxtzzNgR6OR1Iv3er0yJwJ8Op2Gz+cTgPf7/ULHeDweoXaAOdiFw2EBdGC++AWDQeHOAUg1S/7jm0FXevG6C5SpbdfbOrCqAZ7vSVM0JuC7BVndNPE0J/pF6+TdjtkEOFZ54eY+fidO3vo65javTcAc2Lx0wTbvUNa1B9D9OGwnAJ4Bzul0ilevXuH8/Bx7e3tIp9MIBAJ4+fIlWq0WKpUKBoMBjo6OcHBwgGw2i3A4jEajgZubGwmCkrYJBALC2+/v76NcLgtPzUSoZrMpmvZqtbqgdAmHw1LC2OPxIJfLyV0Ag6zUxLObkwZ4FjIjqHs8HrkbYIlhE+DpwRMo6MHrbFbtiTsZwVXXk9EBVXLwWu9OFQ2P5T5zsTCLiy0rV8DzNUVjArSTZ+/G0a9rJn1icsq3WRw2Oe42gHuba20LZG87zgPI777tBMDHYjHs7e2h3++jXC7jJz/5CaLRKCzLwsHBAb744gv8+Mc/Rq/XQ6fTwU9/+lNkMhkB4W9961s4Pz8XCqVUKgknH4vFEI1GkclkcHBwIMDK+jasBV8ul1EqlYQ3J7hHIhGhYLLZLBqNhnDo7XZbwJ7g2G63F2rStFot8ebp/bIYGQAJuhJ0xuOxZPWScgmHw8KdA5AqkAQyXSseeBt0JY0CLII2PXhTOUNAJthTe6+fTQ+eDzdFjZsOfhXIrgrIbmrreLPr8OFOHPyqczeZk14EbzvGXY4z52Hah6gp/2C3tzsDvGVZXgB/BODCtu2/ZVnWYwC/DSAL4I8B/F3btkfLxvD7/djf30en00G5XMbLly9xdHSEWCyGTCaDp0+f4uLiQlrukcaJx+MIh8M4OjrCycmJSBkbjQbOzs6QzWZlIchkMsjn88JRk98n0NVqNWn9x+5NTJwC5kCczWZRq9WEQ282mwLsAKTEsM6QbbfbUuDM4/FInXrSAaRtCIS8e9AgGgqFpI47ACkDbAK8/nFpoAXe1pnXwVeCuubp6bFrOSgBnq9xIVgG7iZNQ+/fBPhlwdcN/w83Ot5NcmqO6TSuWzkBJ9BftXisQ9lsYhzvfengaQ+123fTtuHB/ycA/gpA4s32fwfgf7Bt+7cty/qfAPwqgH+wbIDZbCYNto+Pj/HjH/8YFxcXyOVySKVS2N/fx9HRkYByqVTC2dkZjo+P4fV6kU6ncXx8LO32Go3GQqmCcDiMVColDbt9Ph8KhcJCIhNb/7EuTDQaRTqdlqCo3++XWjb8Z240Guh0OgJMrCnf6/VEvdNut0V6yYCuTlwyAZ4VLzUQc9Ghkf4B3ibi6EAuuXTtResgKsc1AV4DOr17U1WjSxiv8txNr92UTWpzAqS7gJSTd70JnXJXuy/6ZNvzvW+P+0N49A93EW/tTgBvWdYRgH8LwH8L4D+z5p/svwbg77w55LcA/NdYAfD9fh9erxfhcBhPnz7F119/jVKpBNu2EQqFsL+/j88++0zog2fPnuH09FTqzmSzWRwdHUkHplqthmaziXK5jNlsXqaX/Vjj8bgEXUulElqtFmazGTqdDmq1GuLxuEgV8/m89FgNBAJIJBJIp9MCpPV6Ha1WS4KurBtPgPd4PGi1WlKMzOPxYDAYLPDwBH4GYmez2QLA04NnMphlWcLxAxCJJo/XHqGpXiGAA6sBnqCum4jouwBua759mRdvqn6cgq9ur9HuGnQ1zS2YuW5wdlN6RNM8TmMtk3qumqPbNbdpdxlzG/PZtUDwx7CQ3NWD/x8B/BcA4m+2swAatm0zIngO4NDpRMuyfgjgh8A8yNpqteD1enF0dIQnT57g7OxMAqBPnjwRjt7n8+H8/BxXV1dSerdQKCCTyWB/fx/BYBDNZhPNZhPValWud3h4iIODAwlq7u/vo1AoyJfU6XTQarVQq9WkW1OxWJRmHkxaSiaTACB3DvV6XQKobN5NKaVlWWi1WgtUD3X1BBfdEcqyLKk+qWWOLFHMH3YoFMJgMJB5cI40nqs5cA2+PEa35eO23mcCvAZ2TdGY4G5KJk3ZJN8XP3t9nDa9KCxT1riZG0e+Du3idtw6175rnGDT694XD/8h7GOY47q2C+/l1gBvWdbfAnBt2/YfW5b1Nzc937btHwH4EQDEYjH7+voagUAAh4eH+OKLLwScr6+v8erVKzx9+hSJRELA+ezsTCiWq6srfPHFF8hkMvD5fOh0Ori4uMBgMJCa8FdXV0gmk6JkyeVyKBaLAsT1el2qRbIccLValRZ9oVBIGnZQzpfJZHBzcyP0CHvCsjY7g8JaS9/tdheKizHJiWNQoqlpllAoJMlPAKTEMQApRkaAJyjqoCuPIzADWPDetQev9zlx7yZ14+a5a/rmzfft6JlrD9zN078Puw+eehkH/z5snbuAbcxpF4BL267NZ5fsLh783wDwb1uW9W8CCGHOwf86gJRlWb43XvwRgItVA9m2jZubG0QiEdRqNRweHiKXy0mZ3+fPn+Pg4EBa3j169AjlclkAlXx8KBSCZVk4PDxENpvFzc2NFAi7uLhAoVCQzNJ4PI58Pi/0ULVaxc3NDfr9PmazGWq1Gm5ubkRJQ1ANhUICqqlUCslkUn5UpFK0TLHb7UrPVwJ8PB4XgKduH4B48oFAQLxp0lSkowBIFytgDtx+v3+hoJnm2jkG58NtTdlogLcsa4GKMYOs2oM3qRm3omPAYtDXCcDXCbBuU1Wj7TZUyzZsGVXzKXLX7zMG8mBzu3Xo27bt/9K27SPbth8B+NsA/m/btv99AP8PgH/3zWG/AuB3Vo0VCoXQ7/fRbrfx/PlzRCIRHB0diQLm1atXKJVKGI1GCAaDePLkCTKZjGjRX716hXK5DGAOfgzKRiIRWJaFTqeDy8tLXF9fS0s/cux8ZDIZSVjq9/uirSefT807wZZJUKlUSipTas+cANntdtHpdEQjz76v/X5fEqU0uJGXp9QyGAxK8pN+MIFLSzL50HQNgV97yCa1ooOoqx7m8Sa46zuPVVmtTg+aG22zrte9DpgsA5d1y/jelodfdx63Oe6uY+0S6H4I7v5TsvvQwf89AL9tWdZ/A+BPAPzGqhNYwdHr9eL58+f4wQ9+IA2xDw8P8Sd/8ic4PT3F4eEhwuEwisUiHj16hIuLCylz8Pr1a/j9flHMfPbZZ6hWq0KRXF9f4+rqCrZtw+/3I51OiyqGNeZZr2Y6naLT6eDm5kY6QLFSJAEVAFKpFNLptHjRzWYTrVZLPGWqarrdrnjDBHt6weT7gbcePFU7wLsePDl4btPj58ICQGIC+h9b0yXA22bdvAZB27LezXJ108NrcNcU0LLkJyfZ5DKgp5nB1mW2jKpwo2ZWeee6RMOmdhfPn8F4t30cc1n267qSUD7z81kVt1hnvA9lH/r669p9xnSALQG8bdu/B+D33vz9AsBf3+T8UCiEVCoF27ZxcXGBly9f4tGjR4hGozg5OcGXX36JcrkscsNisYjj42NJMqrX6zg/P18A7f39fezt7UmyUavVQrlcFtUKVTWsIb+3t4dEIiGqGGru6/U6bHteL57jWZYlCVHJZFLqvDSbTdTrdempCkDuCPgDa7fb0mTbtm0pD6yDrrpeDa9tAjwzfwnwLOMAvNtT1KQ1CNqckwZpAAseOrfNhxMH7wT4+hp8mMFVPS8TyM1j3FQ0msdf90ezzrFunvwyBc5dgHHVOe8buFbdpXwsQLqrts4d6V1iRTuRyer1epHNZoWPfv78Ofb39+HxeFAsFlEsFlGtVqWOzNOnT1EoFFCtVmFZlqhqisUiptMp9vb2kMlkRCVDdUytVhMFS7PZRKFQQDgclsqRuVxOvNheryfVLIG5TJJJS/Sq2JOVzTRYbti250lMrNg4GAwErMjJEyT7/T6i0egCtUNPnkY6RnP97CM7nU7lroJgyEVIm1bUmFp1Dej04HVSlN7vpqJxUtBo1Y4GcF5Xg7IZgF3Hk78vcwOtbXqx5uK7LeXMxwS4m8z1Y3pfm9hH4cHf1cbjsXjCR0dH+Oqrr1CpVMRb//a3v40//MM/FHC8urrC0dERcrkcgsGgNAgplUoYj8fI5/MLBctYkbLVaiEcDmM2m6FcLiOVSgkdks/nRYpJ/TqzYgnY9XodsVhMEo0CgQDi8TiGwyH8fr/w8QwkkmMnFTKbzYSDJxD2ej2MRiMBw2AwKFUngfmXqwFeUzTkvIPBIPx+v3jg5OJ5vEnPAFgAcA3yvJ03VTWmDn5VqQJTlml68HxeRs84efN64dvUW3d6XZtZL34dDt7p3HXPcdu/zlzveh3g7Zw/NHh+6Ovvkm37s9gJgGdNdQB48uQJvvzyS5E1drtdPH78GF999RUASKmCYrEo/U9PTk6ktsxsNsPl5aVUk5xM5mVxWbCMfPjV1ZXo5kkR7e3tScOOZrMpDUUoY6xWq0in0wiFQgLwkUhEersyW5aVH1nnRgMlpZMEu263i8FgINmsBFN+0QRrUjIa4EnREOAJfvybPLfmy3XQVYOvmfRkUjbUxbOSpRtF4ySb5DVMXbwbB69pGc7V6W/92vsCCU0H0W6bpr9tVc42zW2hWXXONo7Z5vW+6bYTBSSoKBkOhygUCsjlcqjVahIcTafTKBaLUnXx9evXaDQa8Pl8iMViOD4+RiQSQb/fR6vVwsXFBZrNppT8LRQKyOfz0pO03W5Lj1by5eFwGJlMBul0WjxxeuGsOlmtVsWzZ6IRVS7RaBSJRALxeFwKnLG2vPZ8Wcqg1+uh2+1KU3Dq55l8pPlcrajhs9NDV7Ekj6819jQNpGbgU4M8Ne9mtqsTJ++WxWpq4zWAu6lr9BxXgcxtVDW3UbLcVulyW9XMxwBedw0e3/W6m4x535/nrn5fOwHw4/EYjUZDEoIeP34swHp2dgYAODg4QCKRQCwWQ7lcxuXlpdQrPzo6wt7ennDalEQyQJnNZnF4eCgVKqmvr1QqaLfbUh+GxckI9KzayAQo9olloJSVIVkgLJFISCPvWCy2IJskQPJ9aZAnwLvJJkkj8a5Bgzr5eCppeBzlkrrptzYNrssAWyc5Ockq3c51An0N7sseNLd9buC/jrpmlS0D43UXATfly20WiXWook3n92DfHNsJisbr9aJWq8GyLNRqNTx9+lSA/eLiApVKBYVCQbToZ2dnOD09FWlloVDAyckJRqMRBoMBqtUqzs7OpI9rIpHA0dERXr9+LSV+q9UqSqWS1IjJZrPSy5Uc/OXlpVScHI1GqNVqqNfroqKhTp8eczweRyqVkpIK3W4X9Xpdkpy4WPR6PQAQOabeJqXDHzYpGnLszMRlOYPpdCpgz2YZBHjgrfRS861axULjAmTecQCLHLwuYbAM0G/DwdPDX+a9mzy8abdR1dzGE92Uo7/LMeaxm1BCq4K39wn66yqVHuz+bCcAnvQKA6a/8Au/gGw2i9ls3hbvxYsX+P73vy81XaLRKEqlEnK5HIC5d398fIybmxu0221p33d4eCgJSYVCAXt7e7AsSxKZKpUK4vE4vF4vut0uotEo4vF5WZ1cLieBX8oZW60W6vW6VIHs9XoIhULiKdOLb7fbUhqYwVACHZt+W9Y8Y5RePLdJGZGHZ4CXnvhsNhMvnqBLD56cNhc+YBHg9Y9Ng5/Jl2vA1kHaTTl4LZNkvGAZB6/nZb6ulTfa9Hn6tbuC632BkttiosFw2Z2IGzibAd91de+b2CZ3Ew+2G7YzAE8gOjs7w8/93M9hb28PnU4HgUBAatFQt14sFnF5eSmyycFggL29PWSzWXi9XlxdXeH6+ho3NzeYzWbIZrNIJpPY29vDeDxGr9fDzc2N9HK1rLmUMhaLSc2XbDaLdDot5RCAeTC42WyKh9zpdBCPx2Hb84QjZrjqRYPNOuh9z2YzqVWjZZP0iFmwjMBOICTgswsUAd0J4EnlsIetXhy0afB048y58Jh0jG3bC69tqqIxOfhV1I2TvS/vcJPF4i589La5bD3uOtfYxmf5KQRRd2ER29YcdgLgg8EgotEootEoXr58ievra2SzWQBAPp/H8+fPRTYZjUbx6E0Wa6vVEj6dsklSJfV6HdfX1wCATCaDo6Mj5PN5CebyfOrcq9Uq8vk8AoEALMtCJpNBLpdDo9GAZVkSWG00GlL7pdlsSs14JhyZAB+LxTAYDIR+YTCV4MxAKz1jzg9464lxfH7pTh48m4LQw9cySQZa9WLBRCddpIx3DtpjJ8CbvVkJyk7UjJOKRr/GuZgUzSqefZWU8i62jmpkVXDPaYxlQdZNKJ5lsYFtNNvYNjBv63t5sLvZTgRZbdtGKpUSjfmzZ8/EKz05OcF4PF7gwx89eoRkMonRaIRGo4HT01PYto10Oi1VIieTCW5ubnBzc4Pr62vMZjMJnjJrdTgcijqGxcsASHOPfD4v0kdWeOx2uxJwrdfr6PV64qF7vV6RTbI+DYOtlDIC86AyE6C0qoYPlhvmuMDbwB3pIHrxOujKUsimooYxAqcSBiagOtEtToFV/byOsmaZemaZFw+8jRWYoGG+vo7qhrYOoJm6ePPvTSmLXbnb2MXr7oLX/CnaTnjwg8EAiUQCs9kMxWIRp6enePr0KQCgUCggmUyiVCpJg+3PPvsMxWIRjUYDk8kE5+fn0jkJmNevefbsmahySqWSlOlNpVIA5hQMFTSz2QzX19doNBpIpVKijMlms3I8yx2Qd7dtWxQ1AATcqaixLAvxeBzxeBydTgder3eBe2cAk6oaNu2gooaAzHowOsCm6+FQB896NuTgqau3LGtBF08P3YnL1vtNqsUEc1atXOW9ax7fDLICcAR1J8B24tq3bU6e9W2BxwR/N17cLQj6vgKhy+xDUjYPdwHbsZ3w4Hu9HgKBAKbTKU5OTtBoNKTfaTQaxdHRkWSAVioVBAIBHBwcSJmBcrmMSqUCj8eDUCiEYrGIVCqF4XCITqeD6+tr1Go1AeFMJoO9vT1J7+/3+1I5khw4i4uxvg1r1lDq2Ol0BOBJ30ynU1kcwuGwePJsDs75Ekw5Fr14PpOm0bp4bVTVaC/e1Mqb3rvJ45umJZMm/WICt+mxOy0G2ss39zt57ss4eOBdsDe9+2VmBpe3Yavomk3P29bx27AHb3puHyIIv23bCQ+e1Euv18PTp08RCARwfX2NUCiE8XiMJ0+e4PXr11KZsdvt4uDgANVqFbZto1Qq4fz8XHTne3t72N/fx6tXrzAcDnFzc4NSqSQlgQOBAPb39/Hy5UvRn1erVVQqFUSjUZkXW/TNZvOWfuS5SaFQF08Q5cLA5h6TyUQSn5g05fP5JIhKzr3b7YoMkkBPTl4HaIHFPq7A22Jk9Oh1kJXgR05e899OHuV0OpXr0uPn61oWyfM0TaNVNTqpSb/OOZhcPOe9Cci7mbl/00DsKn58U757UzXPXWIBm5rb3YPT9d/nHcWnAKy7Yjvhwdu2LclDwWAQR0dHqFarEkA9Pj5GKpXCbDZvxHF5eSnedS6Xg8fjwfn5udR6j0ajOD4+RjQahdfrRbvdxsXFBdrtNmazmZQczmazSo/2PAAAIABJREFUInNk1iy98uFwKKWHU6kUMpmMZLdOp1Ph7xuNhtR5Z3YrSwswu1UnPhH8gbn3qbl4zcfr7F4GRPnD0yWFCebsOKW7T3Gbqh8uDpqHNz1bTZ1ob97No1/G2Zt0jdPDBHanvzk/J8B3el7X7uptm6C3Lvhsi/bZxphu595F2bPKthEUfl/2sS8oO/FJs4/qaDRCq9XC48ePBdguLy8RiUSk1MB0OsXp6anozLPZLBKJhMgeyYnv7+8jlUrJXQBLE1ChwsWBJQXG4zFubm6EHur3+9JBip58MplcqPrY7XbRbDbRbreFqqH36/P5RDbJ0gXsJqWrPTKJajAYiDevM1s1wGtVCjn6ZXSNpml0MxBe3/yhmbQJvfNlAO5Ex7jJJdcF+XW8didP3fz7fXC4d9Xb62O2Fai8j0XgY7FtfR+fiu0EwEciESk4dnl5iePjY+GLy+UyRqOR1KIJBoM4OztDr9dDOBxGMplEsVhEp9ORTNN+vy9lB+LxOPx+vyhqOp0ORqMREokE8vk8kskkotGoBE354ELBOjOsUcOa9LZtL3RpYoyAwVNmnBLYo9Go6OwJtvpugICuvXkCPGkaE9ydAN78m/s1wGvZpQZ6DYymJ70K5J0CrU4B12W8+yotvD6O81wG9JvYKoC9zT4naeO2qA6Ote7CsG0P/8HWsw/92e0MwE+nU4TDYbx69UpS/lmD/fLyEvl8HtFoFJlMBpVKBdfX17AsC+FwGCcnJ7AsC41GQ3qpxmIx7O3tSeGwdruNcrm8UPMmn88jnU5L3ZlutytjsKE3K0amUilks1nEYjEpIjYcDkVdw25OVOUAEN0+vXhN0xBsbdvGeDwWbl8HWvlgeQB6ycAiyJst/kz5pFmAjB48wV1TRhpoNVg7JTU5JUAto27W8eTXeXCupt23x74OteAWxF52vNtrt5VjrjP+hwaeTeyBk7+97QTAk86IRqNSRTKXy4kX+uLFCymRWygUMJ1OcX5+jtFohOl0isPDQySTSan6eHl5Cdu2Rfeey+Uwm82kaUitVsNsNhNwp6dPioj9WFlCgHXfM5mMNPVg+V7q4gn0/X5flC8ej0fuAFgGgTSNrhczHo/Fi9eKGj5I02hdPJOXCPAEeb/fvwDsTj1bmd0K4B0v0InzNgF/Xf59Gag7AbcOzOpjzPnQnF7T+/TzbW1ZgtEm57+v82hOcZZt26bB6w8Bxusuyp+q3QngLctKWZb1jy3L+tKyrL+yLOtfsSwrY1nWP7Ms69mb5/Sqcag2Ybr95eUl0uk0/H4/MpkMzs/PMR6P4ff7xYu+vr4WIEyn08jn87BtG6PRCKVSCb1eT4KclEQ2m000m03c3Nws7Ce/DkCaf3Oh0UCdTqeRSCQEpIG3OnaWMeh2uws14IPBoMgkmQRFqsnv94smnABOqoYUjUnTEATpKZog78THmzw8M3FXySZX0S1uMkknWmbVgwuXW8AVWA7Wt5VNrmOrqBan4247/vvmkHd1rAfbjt3Vg/91AP+nbdvfAfCzAP4KwK8B+F3btp8C+N0320ut1+stJCC9ePFCkoXYrq/ZbMKyLMRiMRSLRQHgRqOBcDiMw8NDSe4pl8uoVqtCrzBzleDN8fx+P+LxuHj5Pp8Po9FooVaNTv9PJpNC+UQiEXi9XvG6CfCdTkcafVCyyOQnTdMQ5HViEatNEty1B09NvO6lSqAxywSb4G6CvFbS6EqTpq3y4NcJvq4D9hqcnfj2dWWTmpvfhu2C57vtse8SgL3LorjucR/jIrHLc741wFuWlQTwrwL4DQCwbXtk23YDwC8B+K03h/0WgH9n1VidTgeRSAS9Xg+PHj3C+fm58LsHBwewLAulUkm84sePH2M0GglYj0YjqRwZCoXQbDZxcXEhiUdsxzebzZOaqtWqlC8IhUJIJpMoFAqIxWKYzWbScLtaraLb7UqHJnr7rPtOmoZevFbgsIQBi5CRqiHAh8NhoWo0UBLg9YN14t0Sn5xoGieQd6oXr+8ETHOiTdz4dje6xtzvpqBxU9RwHm7g7kbR8Bj9bP69zO7Ls91lMNjEPiap467YB6Go7nDuYwAVAP+LZVl/YlnW/2xZVhRAwbbtqzfHlAAUnE62LOuHlmX9kWVZf9Tv9wEA3W4X+/v70nVpMpkgGo0in8/j+vpaMkiPjo4QDocXOi2xYmQ8Hsd0OpXaNbPZDPF4XNrzTSYTCbgOBgN4vV6Ew2FR1DAZp91uS/nhfr+P6XSKSCQiWa3JZBKRSAQARMuuaRpSK/SwA4GAZLeS4qGihj+W6XS6IJnUVA29eMYdFr7EN5y6m6LGDLRqgHeqTwMstvTT4G7KKFcB+Tq0zTKFzCaA7mTbDLyuE+xbBuZ62+yHuu4i8KElkB/zArUtGerHZHcBeB+AfwnAP7Bt+68B6MKgY+z5r8vxF2bb9o9s2/5527Z/PhAIiHIkEokgHo+jVqsJEBwfH6PRaEj2KIOjs9m8afXV1ZUkJSUSCfj9fpTLZdHWs3m37uhULpfRbrdh2/PaLezkRNqk3++jVqst8OoMtupCYszGHI/H6HQ68mDRMNI0BHgtm2SwVntDpGk0yJsevJn4xEVEg7xJzejXNE1jVqo0vqN3ZJKkktw893WUM6bXro9bJY9cFgS+q93HD3gTsF4F3relPe563DbtYwTJj9nuAvDnAM5t2/7nb7b/MeaAX7Ysax8A3jxfrxooHA6jXq8DmAc5T05OUK1WMZ1OUa/X8ejRIwlClstl+Hw+7O/vw7LmadXn5+eYzWbCkadSKdTrdVSrVWlwvbe3h3Q6jWAwiNlsJrVnCMLJZBK5XA6RSERa9Zkt+phcRYBPJBJSIoCFwsjDE+Sn06l42NTFRyIReZiJT7PZTCSTmq6hF28mPgFYAGqnapNOQG9y8dqj1Lp4M/hJymaZgmaZwsaNfzcB2wnw9XHm37RNPfy7mOmF097HQvE+aJ+7LhTvE8wfFg5nuzXA27ZdAnBmWda337z0iwB+AuCfAPiVN6/9CoDfWTVWNBqVYOn5+TmePHmCbrcLr9eL8/NzFAoF0cqzNMDBwYFIJ6+urtBsNhEOhxGPx1EsFjEcDlGtVtFoNNDpdERpE41G4ff7pV48vXNmyzLxybIskUwyW5WcfTQalcUkHA5LTfXBYCCSSS4K4/FYQJPdqEjTkIvXNA0BnoCuAd4p8YmKEKfEJ1NVY6ppTOmkSdWYQUsn8HaiX5btX+exLKjK52XSSH3MXcDeCURNINl2LXbz702CprcNsH4o27X5fIp212Jj/zGAf2hZVgDACwD/IeaLxj+yLOtXAZwC+OVVgwSDQQyHQxwcHOD169f4zne+I7QDPfZMJgMAGI1GuLq6QqFQQDgchmVZODs7w/X1tSRH7e/vw+fzodlsIhAIoF6vo1AoIJfLod/vIxaLCXfPcgZMekomk5hO5000yPFTYkm6JxwOw7bnNew1D8/qlVycOp0OhsOhSCq13p93JOFwWO4YSPWQirFte6FkAQPABHhKLAEsyB41TQNgAfC5rUHdtu0FmsaJj+czr2cCuvbqnegXp316nFUAvypg6rTf6bhdsLsqVtahbO4qA72tbRu0HxaBu9mdAN627R8D+HmHXb+4yTj0QCORiJT11YXCOp0O9vb20Gw24fP5cHZ2hs8++0w84NlshsvLS8TjcViWhb29PcTjcXS7XYRCIVQqFXz7299GOp1Gs9lEMpnE6empdHEibZNKpZBMJjGZTBCJRFCtViXrlSUQSOEAkCYlDI6yUBgBngFa6t1ZzjgcDgvVwsxWAEKHEPwJ8PTgufDoxty69yo9cCeA14lV2nPnIkFvUXdbevMdy7OmT3ieyasT7HVtGo7pBvBugK4VPKs8eqdnh/9XueMxXyfdd1dzA99NefRtmtNcltEq2/w81rFdAPFdmMN92E6UCx4Oh0gmk1JMrFarSX9Vy7Jwfn6OfD6PbreLRCKBs7MzAUeWA7i6usLJyQl8Pp802K5UKhgMBpIURfXLYDBAMBiUujM+n088+1Qqhel0imQyiUqlIhUuWciMZQZ8Pp8EdRkABSAljT0ejxQgC4fDcivPxCfKHamoIUDyh8USvDrQagI8a9nwB6m17RrgNRdPYNaAD8CRjwecvXZua26dgOwkq+S53E/tvzkmz3VbAJx4ex274GvauMCY3v5t+OXb8N4fmjZ5UNds3z6m2MJOiFk7nY7QJ7lcDi9fvkQul8N0OkUmk8HLly8RjUbh8/mQz+elXgw57b29PdRqNbRaLXQ6HQSDQRweHkpHpJubG1SrVQSDQcRiMeRyOcTjcSkWVq/X0Wq1RCXDUsSBQADD4RC9Xk8CrtTi68Qnatu9Xi9Go5EkPnE+9NZt24bP55PmH7rCpC7ryxwALhyaf9c8vFPik27r56SgceLhTbmkWXyMphUrGsDdePh1dPBO4L5KQWPOyc1z3zVqZl3bFmWyDSDalTG+qXbX/+GdAHi27Gu1Wjg8PMTr16+Fpy4UCiiVSrBtGx6PB+l0GrZtS0Ntj8eDg4MDKfzVbrcxnU5RLBbFc2atd8uypFNTJpPBdDqVAmNU8VCmmc1mEQ6HpesSA64sJubxeKTUASWPbNpB/TrVNJQ5sqk1a7azfAF5eEo0gbcBTWrs9YMAzwJkGuDpwZuySTcljRlk1Y1FTNOeuhPnrj34VWC+im/XoK69cze6hrZsMdi23UfAddNrmq/vApjeVuHzkDz1rn0SHvx0Ou/202q1pNmHx+PBaDRCoVCQwKVlve1zWiqVMJvNJNuVQE7OPpfLIRaLiRdeKpUwGo3g8/mkVLBlWRgOh5LUNB6PpbxBJpNBLBYDAEmOqtVqkqEKQBKfKJ0Mh8MLyUoEeF0wjGqaYDAoXjyzWnVjbgKoLkSmH1oT75T4pBOaTGB3U9Po2jZuzblN2aQTaC/TxpvnmRy+U7CV11/20MeY5haYddp+X7ZN6mdbXva6APuhvPr3KcfchYVyG7YTHLzf70ev15M67ZZlodvtYjqdIpFIIBgMisfu9XpxcHCA6+trHB0dYTab4fHjx0ilUgK+lUoFX3zxhTTWtqx5qYNWq7XQtzUSiUhAk5JJlhbIZrNIp9OSDTscDoUGsixLqJV4PI5EIiExgVarJQsAaRouTsPhELFYTDxqACKXZM0bAi2BRyc+cT/vCGazmcQtuDAA839Ov98vgVoN6hzTjYMH3At2mVSI6YkTxLmP+zmmCf76dSePfpmXr4OBbub2Pu5i6wRPt6VucRt/nXPcFq7b0D8fG9h9bPO9T9sJDz4ajeLm5kakiYVCATc3NwDmYFQsFgXge70eTk5OpJJko9GQMsIE66urK3i9XqTTaSkMRk38aDSCx+ORAmTAXOLI/bybSCaTSKfT0tJvPB6LBr/b7WI4HMLn80nPVT7rGu8sbtbtdhcac9O7ZsCVIE9dvwZa1qfRnrvObqUXr71XnfhE79zUxS/TwpOXp2lO3lS4rJJErlOqwG3/Kq99XcrmtrYNj9Hp2Nt637tEwwCbef0P9mFsJ76dcDiMarWKRCKBcrmM4+Nj1Go1+P1+tNttnJycoNFoAACazSaKxaI0sabHXCwW5R//+voaw+FQ+PFEIoFut4tarYZ+v4/JZCIdmihRZJ14giWbi1CpM5vNpLEHFxfOnbXeWfIYgChgSC9pgOcPg+ULCO4sXWBmtpKm0eBOvTy5fQ2IHF8DttnZyYmDX1abhnPRnLgJ6JqXdwJ5nYFrev9OfLubB8/9+tk0J1pm1TnatgGidwG/u6p07iM+sCsLyzbsU3ovy2wnAJ6dmzKZDF6/fo1Hjx6h1WohFAqhVCrh+PhYGlrf3NyIZz4ajaThRrFYFHCs1WpoNBqiUsnn8+Kls+dpLBZDPp+XUgFs+ccFIBQKidomFArBsiwpJkavfDqdSuISvXgea9tzDXu32xWAJ4WkteoEeF1hUvdMJTDq8gXamyfAa5AF4BhoXcbDsxCZuTiYmmlTjriManEDcifOfZmCZlUQdpkX7+bR38azv61Hvym4vi/VzDcF5HbJ3nf8YicAnsHHaDSKUqkkWnQCfCqVkj6oVLvs7e0tUDLpdFoClt1uF6VSSTJH9/b2RMtONYzf7xcAD4fDGAwGqFQqkpxExQ5LFwQCAak1w3IEw+FQQJoaepZC8Hg8mEwm6Ha7C0XImJFKtQtrxZuKGl2+gACvG4JogGeFSYIl8C5Ns0wuSXDnsdqL1w+ak8rFSSK5TtkCJyWOEydvgjlNL2puQVbuc7Pb0jqb8Nl3oTPuShXdltrZpQVj20Hlb4rtBMCzSTZ/3MPhcCGT1bZtqR7JWu3FYhHAPEB7fn6+UOkRAK6urjCbzeWMbLVHkK3X65jN5i376HkDkAWg2+1iNptJYhR5fKpp2J5vMBgI1UJFDTNdGfxk4hMzXPv9voCSpmnovRPgCbjA28xWJ108gd70lLUnrnu3Onn03KcpGjeaxgRXJ3rGfLhRM+YC4Za5uszDd5qX0yKwjn0o4PgYAetjnPM30XZCRdPr9bC3t4d2u41kMolyuYxUKiU/zHq9LolPHo8H5XIZuVwOFxcXCIfDqFQqGI/HUjYgFApJ/XjbtpFIJJDNZnF1dYV+v496vS7ae2aiUqbZbrfh9/sxHA4RiUSkE1QsFkO5XBaqhQtBPB4XSoWKGlaRBN629GNT78FgIGUGLGve75XB1dlsJjw8YwEABCTp9RPYuU09PCkW7cUDWAB3AqXe5jEMEGuKhosksAicpoxRB125nw/LshyTnThXrapxonGWUTJaMeIE+k7euHnM+5TZ6dedPPplCph1x73t3O773G2O8WDr2U548IPBAPl8HpVKBScnJ3jx4gUKhQL6/T6SySRev36NfD6P6XSKWCyG169fIx6PIxgMIpFISMVHUhzZbFbqyIxGIwSDQfH4h8OhZKVqaiUSiQjHzgzUQCAgXjybgdAj53Hs2qTvIHSwlJmtmqqhx23bttBI9OJ5B6ApFAI8vXjSNJqXJ8gTKHVWqsfjeYeScdrWWa1cLJb1bdWKGuBdHt6Jf3ejbJyCqk60jFvgdZmSRs93XW/etNsGM9dV0bjRN3eRMd41ULtt2wVg/xBz+JDveycAnkqPer2Ok5MTnJ2dIZVKodvt4uDgAK9evZIqj9lsFpeXl+L9UoNeLpcFqLg4NBoNUbsUCgWEQiFMp1O0Wi1cX19LUTPWkB+Px6KUaTabC/Xfqbih/JFcPmvQ+Hw+xGIx4exZBngymYjqRjcCYUVI8vA6s1WDPL1qXW1SK2rMpCed2apr05jqGTPYqoFdA7zJwWue3+nvdTh1XZDMicd34uCdVDROqhqaeYexa+b0o1/G0ZuLwjJQvytfvSsguCvz+JhtJwA+FAqJKiUej0uQs9/vY29vTzJbycVTekgdeTgcRqlUkvH29vZgWZZIGnmHEIvFYFnzjk43NzeYTCYIBoNSmgCASBvr9TrG47GoZOjlU5dOL15LH1mPniDNKpUsN0BFDWWOpJyY2UqQD4VCAvCkfwiM2mvXf+usVp3Zyh88F1E+mzJJgrxZnwaAKx8POJf7XRVIXaaqWVcp4+TdL/PgPwTQvy81zPuw96UEuuu5D7ZoO8HBRyIRlMtlCWSGQiEBzkQiIZQEeW4AopNnu71KpYInT55gMpkgl8shHA6j1WrB5/Oh3W5jf38fyWRSMlPJ0RPUstks/H6/vFav16VNXzgclg5O3W5X1DFs52dZ8xo3XCxisRjG47Fo7Ol193o9qU3j8Xgk41VntnLBCgaDQuEAWNDPE9QZQKUHr6WV5LiBxZ6t/KES1OmB83Pgudr7d1JhaKClOXHrmoMn2DtluC7z4PWxThz7ssDrOmCv+fxt2McMUKtiAHcZ90PZx/x93NV2woMPh8O4uLhAoVDA1dUVjo6OcHNzI95rIpFAtVqVYGEikUCpVJKyswcHB1Kvvd/vi0dOaqT6/7f3rSGOpel5zyeVpNJdpUtd1dPdNdPumXHv7IWxd41t2PgS1ibE/uGATYj3x8L+iCHOBRKbQEL+xRDixBBMNtjYgeCYJA42JuBs1ob82aynbc+sZ3Z6urt6qqvrqvv9Ln35Ib1vv/r6HElVkqpU3XpBlHR0dM7RKek5r573eZ83m4XX60U0GuXO1Gw2i2KxCKBv4UveNWQWRuP+KMMmGofkmo1Gg8f5EQfucrkQCAT45vP5WC5J1I6kaVqtFoO49KeR3jRut3uoyEnbMjN4AnnK5M0wO1tNDl52t0rKxoojtnKbNJU0JuduRdOMA3aZqZtuk7T/SQHcbrndulYh6Q8rmmQWBVCr14yTX06yfBou/6Kvm+c+r2q71y0WAuCV6o/HW19fx/7+PnZ3d3F2doZAIIB8Po9kMomzszO43W40Gg0kk0l2h6zX69je3ka73WYKxOl0YnNzk/Xj6XQavV4P0WiU+fFiscg0DADEYjGEw2EA4OHehUIBvV6P+fVoNMqe6t1ulwutVPB0OBxsPUAgTw6TRNNQkxQZkEnrArOzlagaCahE+chiq6RpiPqR4CUthO2shK06WimDlz41MkwgHuckaec+aUfbmBm8FRVz3gzeDuwvI+y4czswuo42AOOAddEKvy97LMQniGSKfr8fqVQKm5ubKBQKCAaDSKVSQ9YFxWIROzs7KJVK6PV6KJfLiMVicLvdnLGTCyX93KTsnnTqoVAIjUaDh24T9x+JRLioWS6XUSgU+ALg9Xq5kYmojEqlwkM9SJJJAE83Mvgi7t6csSrljy6Xi4GdMnhS41CYhVYCegJ36REvqQdTE2/60MhsXvLwksO3CjtQnQTgzXWtuHU7+eQkN3k85q8Nu/ci4yoKlPMGtkVqXppmf6PqQufd1sscUwG8UuofKaU+Ukp9qJT6PaXUqlLqtlLqO0qpx0qp31f9ea0jo1arIR6Ps/f66uoqWq0WAoEATk9Psb29jWq1CpfLhVwuh83NTQaycrnMQ7AJaIvFItsQULGVrAuoIanb7SKbzTJd4vf7EYvFmMKp1Wp8Aej1ekzTBAIBeDweLgITD0/ad2p6In8aaR5G1gqUwUvrAgJdomkkyFNmTdw20TQykyewtwJ4YLiz1crGQIK6KZeU1gmmfYEJzpNYAo/rapWvpX1MmrWbPP1F4jy0zbQxS8A9L1UySo2zqHHdjveq48IAr5TaAfAPALyrtb4HwAng5wH8GoBf11q/ASAP4GvjtlWtVnHz5k2kUimEw2GUSiX4fD6srKwwMBPA5HI5Bk4CzF6vh42NDc7G0+k0gzHROiSjXF1dRTQaZesC0qa7XC7EYjG2JZA8PNEvNKJvdXWVC7Kkma9Wq0NFWRocQjw8AJZMSvMx4uEJgKUySHrTyIxFTnsypzwRyJveNNLn3U4Lb5fBW3nSWMWkIG6X4Y9bPmnWDrxI48jjM2MaALcqQJ/3teddx9znpEAt17kI/XORi8ssYhG2fV0vLNNSNCsAvEqpFQA+ACcAfgzAfx88/7sAfnbcRrrdLjY2NnB0dITXXnsNT58+RSKRQLPZBPB84lOv10O9Xke328Xa2hqDWqlUwubmJis2CMwJjAHg7OwMWmvOxIPBIFMsRPeQdYHP5wMAnsNar9ehteZOVeLIO50Ob0OqYwikg8HgkDcNFWel+RiphQBww5SkacwsHnhO08gsXt4kwI8CeSuaxtTCS17e1MRbUSEE4LRsFHiPcpS0Am+75+W+6L78a96Xcd7lF43zAMRVFifPs6/rCnrTxHV7zxcGeK31EYB/A+AAfWAvAvgLAAWtNck4DgHsWL1eKfV1pdR9pdT9Xq/H9MuNGzdwcHDA1gWhUAipVArRaJTpjGKxiI2NDX6cyWQQj8cZuLLZ7NCwELfbjVQqxZm43+/H2toaj9ajQmkoFGJQJnklTWRqt9vwer1DI/oAMFiTOgYAe70HAgH4/f6hUXyUcRNFQ786iCs3eXgqtEqPGAnw0p9GKmnMDFrSK3Y8vLnMlEpa8Z4m+AIYS8GYahq7wuooVY0J/PJ4rP5eRcwS1Gd9gZgmrhvIvcoxDUWzBuBnANwGsA3AD+Ark75ea/0NrfW7Wut3I5EIt/3H43GcnJwgHo8jl8thY2MDBwcH2NzcRL1eh8/nw/HxMdbX11lrfnx8jHA4zNlvoVBApVJh24BgMMh0jNYaXq8X8XicfxEQzUJZdyAQ4OarYrHIIE/WCKSQIR6+VqvxxYAGhhDAE2dP6pt2u83FYGkhTEBE1gWyu5UyeAJdAj0Cc0nPmKP8CPwkh27V7GSqaUx3SelRT9sww+TNragZUs5IsLaTUlpl75MUU0eBvLn+ZccsOfSL7muWAL1Ihc7lhefFmIai+QkAn2qt01rrNoA/APDDACIDygYAkgCOxm3I7/djf38f0WiUC4Qej2doCHc0GkW9Xkc8HsezZ88QjUaZNjk9PR3yVidpJLX8R6NRVCoVFAoFNuXa2NjAysoKz2TN5/OsYycQbzabDPCVSgVOp5MvAKFQiOe9EsCXy2W02232fqELhnSIJLmkzPplxi2HckuAJ/AlqocAXBZapSZedrWaIG+libfypJHWwdLbxiyymkXOcQqZcfTMpNm7HVVjBfJWWb7V8V9WTKpzn2a708R1lGiOikWjvS4rpvkvHgD4klLKp/pn5scBfA/AnwH4ucE6XwXwh+M25HK58OzZMySTSZyeniIWi3HHKBVd3W43Wq0WotEoUqkUZ8TBYBDFYpHdH6koeXZ2NtSl2u12kc/nmRKJx+PcOVur1ZDNZgH0O0nJU6bX6zEQl0olaK2HFDI0ZJsAmzzigf4XhMzMqFhKnaNWNA1d2EzrArrJ7Bp4btVrJZmUPLwJ8HSzA3mpk7fSwps8vAwTeEc1M5nPjxsMcp7M3TyeSbh48zUXCatzMk7OZz43K5A5T/F33vt8GYHzusQ0HPx30C91j32qAAAgAElEQVSm/iWAvx5s6xsA/hmAf6yUegwgBuC3JtgWCoUCtre3cXBwgBs3bvCMVikx7PV6Q3YBlCVTpk0ZO1n7Av0P19raGlwuFwqFAmvRqdAK9BubcrkcUz4+nw+RSAQOh2MI4MmbRipkyB+Gsnjye1dKDfHwBNJaawZhE+DJjsDsbCWKh4q1AHh9kkySv42ppJFqGqlpN+WQVnz8qCze7v9oZsx2Gb0J8nZF1HFZu5mZ24E9PT/qM3jemEThMm79aWNWKpDL9JpZxuXFVL/DtNb/Umv9ptb6ntb672mtm1rrJ1rrH9Rav6G1/jta6+a47dTrdVa9PHv2DLu7uzg6OsLa2hqP8isUCgx8DocDhUJhKNM9OztjzXksFkMmk+GLAg3tKJVKbBdAhVYazJHJZFhrT7YG5AsvnSnpeSrIStqFzM3IKoAuNlSUJV0+Zd1kI0wFUvKAl5OXJE3jdrv5lwvwHDwlFy/H+JkWwhSjCq1WU57Mbla7gisdkwT5ccqZSeiZUdz84HM4drn4zF5p4XWaeNlok2XMPxbiE1OpVPDaa69x0TEejyOVSmF9fR3Hx8e4efMmTk9P4fV60Wq1EIvFcHZ2xoZc0WiUR/T1ej0kEgnWplPWLb1pisUiXC4X6+FJb18qlQCAnwsGgwycVLglCiUYDCIcDg/RLlSsJRqILA6owcrn8/GADlK+UAHX9KahLlYplTQthAkwpURSUjTmvFbgeeZlFlDtwF1m86Zc0gzaxyRadzt6ZhJQH7cMGC+VtPq1MS4uK1sepW+fdNk0cd5fJvOMRSriXsdYCIBvNpu4desWTk5OEAwGmVsOh8NsPpZOp5lv39raQjqdxurqKprNJtbX15HJZAD0qZxEIoFerzfUpJRIJBhYC4UCut0uIpEIg6cc5edwODjrB8Bae+LhV1ZW2PJAOkbSAHBStSilhmatkke81nooiyd7BRoEIlUuZgbvcrmGdOiy0Gp2t9JzcjqU1s8nNtl1tsoJUGYGPyp7p+3TXysQHsWz2xVWzfWsti/3PynInydmsY1FjyVYLnZc5P+zEAAPADs7O/j000+xs7ODTCYDr9fL2vhEIoFisQi/349sNovt7W1ks1m4XC5Uq1VsbW2hXC6j0+mg2WwiEonA7XajVCox8K2vrwPoc9fk4x6JRBiAadITSRZp0hNRODTtiQB4dXWVLxCUUVPTEwE2AObsqThLPHyv12MenrxprACesnhJ0VChlcDMnNlqTniSGTyFFcBbSSOlj7zVzQwr2sWOZ7e6Pyp7P69UctRF4DwxL0Cf5hfBeTj+adddxvWNhQB4kgGenp7ijTfewN7eHuveyd+FpJPZbBabm5uo1WpwOBwolUqcnZNkkYqklE2Xy2UkEgl4PB4G60KhwPSJ3+9Hr9dDNpsdamoiHh4Az3IllYz0piGNOFkXEGB3u10u2hJNQxk/Zdeyo5UKpcBzCsVU0xClRD/LCeAlLSP18KY3DYUEadNh0sqyQBZaZcHVDCsOXgK9lYOkHY0zio4ZxcnbhdU6Vo/PA+iXBYjzaHRaNDBftON5GWIhAN7v97PqJZFIYH9/Hzdu3BiiZQKBALTWqFarnDUTWFNjEmXOxMNTwTOfzyMcDnNzEvnMkEqFuPR8Ps9FT+LhSXZJGT6ZijmdTtbLk8adLibE9bfbbaysrPCvBJJMSiUMKWlkFg88H9JB3jZUUDZtC4Dn1gWmP/wo8zEqtEqAt+totVLTyP1LyohikqzajoO3y/btQH0UH2++b/OYzGVmTJO9T8qNz0sTf9HjuYxYlON42WMhAN7r9WJvbw+xWAxK9b3hqdC6ubmJk5MTxGIxtuSl4dsEalprRCIRBnQa9UfZYj6f52lLNCQkk8lAKYXV1VXWtFNhlrxnpD2w1pqdIynLlo6RLpeLJz3RNlqtFpzO5zNX5axWAFyclRk8XaAIRImmkQ1PJJeUahqpmpGZu5V1AYUdTWPy71bgLiWXMkyeXHau2nHq9Jxddj8pmFvRNeaxLRp/PqqguoxlTBsLAfBOpxNPnjzB7u4ustksZ725XA7JZBIHBwfY2tpCpVKB1+tFPp9HPB5nwK/VakgkEvy4UChgfX0dWvcLitlsFlpr1sM7HA6k02l0Oh243W4GanJ6JJ09ZejSdphUMr1ejwutlGFTYZe2Q2MGSe5INA2pVAjgKXsnNQ0BJAG89IY3vWmA0eZj0rZAgqJJt4zTw5s0jQR3CUx2AEuZulXmTuBOr7Pi3ifJ3CfNzCfh4xftQnDemLS56jwqlWkvQMsL2OXHQgB8q9VCPp/H7u4uHj16hGQyyQqYRCKBo6MjJBIJ5HI5xONxHB4eYmtrC7VaDR6PB6lUCltbWzxPlczJSDNPvDgVX1dXV5HL5VCr1VgRE4vF0G632ZagXq+zJQEVR8mbhrJz8qYhlQwA1raTCVmv12N6hQBeTmmShVYCetLRU9OTycNbmY9ZecRL0LezLbDzphkF9ETvyKxz1MQnk5e38n23olZMYLfS0NuB+zhaRv6SGcfLTxqzysJHNZPR85Nu57yvkTGN7n75i2QxYiEAvlarcePQ06dP8frrr+Pk5IR149VqFT6fjzXyR0dHnMEHAgGcnJywN43H42EJJdEhND81GAxyd2mpVEK5XIZSir3ggT5Ak68M0TpkgUAj94hScblc7D5JVA7ZFhDIE19Pg0DoGCiLpyEnEuTb7TYDEL3WzOAlbSLpDSo2W2XxEjSBYZA3te5W4G6uK7/Eki6SNI0dmFtx61acvHmBOE8Wb8W/yzCVRbT9Wcc0QDdJhn1ZHL7Vvi/zdcs4fywMwN+6dYv9XJLJJA4PD7G+vo5KpYLV1VVu6IlEIshkMvD7/Wi32wgGg1yMJU69UCgAAGfepJAh6SWN7KOZqw6HgzN+4sJLpRKcTudQk1Kr1WIZZLVahVJq6HmPx8PDtQnkyXyMXCKlbYEEePKnISUNgZP0ppF/ze5SycHLbN6KgycQk1SLVNJIYJfe8fICIAuto6wLKMyM3KRr7KiYSZ6zAn95DOMye7uQr5lFjOshGPW6afZ5kfXmodpZxuXHQgB8u93GW2+9hadPn3KX6PHxMXZ2dnBycoKNjQ22KvB6vahWq0xNBINBZLNZLg6urq4OUSwEWKlUitv/abg26d57vf5Abp/PxwCdz+ehtebsPBQKAQBn5jRJSg7ZpgsRZfEk0ySqRRZaKQvXWg+ZjxH9Q4AsAV5SNRLkgeFCq5VUkkCeMnngRZrGatgHXYjMTF4WWc0schzg0v7tVDJ20slxgD8uax/Ft182534RUBxFe8z7V8KsYl4Xg+VFxjoWAuCdTieSySQ+/vhj3Lp1iwuZiUQCT58+xe3bt3F8fIxAIIBOpwOPxzPkG0M8tix25nI5hEIhtgtIpVLo9fp2vOQdTzNXm80mQqEQT41qt9vI5/NotVrcqBQOh5kuoo5VaT5Gk6AIsEeZj0lFDADmzEkTL90lAfCFyWx6ko6PBHgE8KZHPIG/BEYCDOLUx3HvprukfD1g/SUbp3ufxDbYjqsH8MJraNmk3LwMK35+VjFrAFo0QFvSNYsZCwHwXq8XvV4Pp6en2N3dxbNnzzj7ps7Vs7MzRKNRFItFNhMjdYvb7UahUGDOngqvXq8XSimEw2HkcjlWtVDGXS6XOXP2er2IRCJQSqHb7aJQKKBWq3FRVvq/E09PPDxl5oFAAEopBlb6JUEKESrokvGYHAIiHSZlJyrwnIeXIC/pFOA50BGgmvYFsqvVynxslHWBaSVs8vCAfZFV3rcCdDswt5JWSmpHFl7p/cu/FOPoGBPwJwV3ukCOi2nomFmB33k6WecV89b3z3P96xwLAfB+vx/Hx8fodrtIJpN48uQJtre3GewikQhyuRwikQiy2Sy2traQSqXg9/vRarUQCoWQTqfh9XqhtebHBKBra2vcTAT0O2fX1tZYykhqmkgkwoBJxmEErqSWIedIKrbSBYUAnhqwaCA4afO1fu5hQzw8uV9KX3cq4MrCKIGspGpkoVaaj9lJJk0ljaRpTICXvLuV2ZikaQAM1QLMsMqYRwG7PDYrzt1cbu5jVNHV7niuKubFc887m77qou6rBNDTxkIAvMfjwYcffojNzU2srq7i4OAAu7u7OD4+5gHYrVYLwWAQZ2dnuHHjBk5PTxEIBFCpVLC5uclukzT2L51OM3URjUbR7XZ5uLbT6UQ8HmdHx3K5DK31kLtktVpFoVBgYCZNfLfbRbPZZN8ZAKyQoSyfQJbWIeMvGgIis/iVlRUuikqAJ1qFeHhqcJJKGgJ9SdHQvk3+XWbx0nyMwtTCm140dJGRFM0oywIKE1xHceqUkY8rvMpt2YG5pHEkdSOPy+qv3fPXIa46S57VL4XrwPlfl1gIgNda4+HDh3j77beRTqfRarV4+EcymWTVjNPpRKFQQCKRQKlUgtfrZXfJTCYDl8uFRqOB9fV1FItFpjlCoRAP/CCQI8dJAutms4m1tTVWuDQaDeTzeQZmmvKklEKn0+EhIJRhE42zuroKoM8907AQAmvgufmYnNQEgEFZNjxNYj5mRdNIkDelkpTJjwJ400LYlE/aedPY0TRW9IoVJSOBeFR2b1dgHZWtj1s27vM5juqxinll0qNosfPs+zLUOcu42lgIgCeaZHd3F59++ilTJel0GslkEs+ePUM8HueCpcvlYr/1UqnEXu9K9Q2/otEoUyndbhderxd+vx+lUol14iSLJCqlWq1yoZTUMFRo7fV67B4ppY005Ul6xPt8Plb4NBoNpoHIpZK8ZUha6Xa7GcCsAF7y8CZNIwutpIenQqtJ05g8vMk1m5JJO6C38qOxk/9ZgeI4Ssau4GqXoY8CeKtjuI6Z+aziopLJi2zjvLGkX+YTCwHwtVoNkUgE0WgUjx8/xs2bN5kz39zcxMHBAba3t1EsFuHxeFhJo7XmJikCtkajwZ4zstGIpkORb3s4HB6aqVosFoeAFwAKhQJ3lno8Hu6EBcCvaTQaAMBNT1ToVUrxrwMC7G63y8ofyuSJYiHgHQXwBPLSk4bklvKLIUf5mRSNVNNIwJPUi0nVjLuN+3KagDwKxGUxVmb94xQ0k9zk8Yy7P6+4SEFwFjLGWYD7VcaiHteix1iAV0r9tlIqpZT6UCyLKqW+qZR6NPi7NliulFK/oZR6rJT6rlLqC5McRK1Ww71791Aul3F2doa7d+9if3+fM+qzszNW0sRiMaTTaUSjUc7QyZ+dpIEul4tVMgRo6+vrqNfr6HQ6KJVK7D+jtUar1UIul4PL5WLdu8fjYXOxTqcDp9PJ/vFUSCXbAq010zjkLulwONBut7kYS8dGWn4CeOkuaQK8nPJEgEuqHknTmPbBVLQ1M3gpl7SiaWR2PollgZn1W4WZNZuAbUW/yGzf5OftMnu7wqsJ4PKXi3mhmEVMQ8tcRvFykgvGosRSejl9TJLB/w6ArxjLfgXAt7TWdwB8a/AYAH4KwJ3B7esAfnOSg9Ba4969e3j48CE8Hg82Njawt7fHnjSdTocbmjY3N9mLplQqwe12o1qtIhqNcvNRu91GNBpFpVKB1n2LYekuWSwWWTVDBUoyJJNFUPKlaTQa6PX6s119Ph+PBiyVSnwBADBkRUAXAdK2UwMT6eHlfohDJ/UNedJQsVVrzUBKIC87W4k+kedTZuwS3O3sg2XB1CpLt+PeTaAHrM3HzP83vd9Rkkmz2xV4EfRHgfqo5aOomnln8vMoIi6CFPK8MY3XzasWF/1fjj3DWuv/CyBnLP4ZAL87uP+7AH5WLP/Puh//D0BEKbU1bh9utxsbGxt48OABtra24HA4cHp6itdeew1nZ2cIBoPo9Xqo1+uIx+M4OTlhGwO/389Tn0i2WKlUsL6+jlqtBqAveYxGowzmlUqFR/YRaFFXK4FvMBgc4tA7nc5Qhq6U4guA5OnJPpi4den1Tl2tBMxSSTM410OWBbLQKgHe5OBJ3UI36UtjR9FIjxhgWAtvlcVb8fJEz1g5TJp+9WYmToAtqRiTppGvPS8dYwXmFp/tob92y+YZ5wW5WdA1s9rXZRzDIof5PhfxfV/0ErqhtT4Z3D8FsDG4vwPgmVjvcLDshVBKfV0pdV8pdR/of9GfPHmCu3fvIpPJoNFoYGdnB/v7+9jZ2UGxWAQA9p6JRqMolUoIh8M4PT3FxsYGqtUqd6iur69zxkwDPwh0ya+dirkul4vpFqJAwuEwut0ud6M2m03uWCUppVTJtNttuN3uoU5VpdTQQA8qEkuAl/bBAF7wpiHufnDOhjJ4qYc3h4AQwNuBvEl30PZH2RaYYC/Xl0VW+UE3qQ+TfpHr2NEudtJKq+em4ePNsFp/HnGRTHbWTVBXLbFcxnxi6t9Iuv/pP/c3QGv9Da31u1rrd9fX1/HJJ59Aa43XX38dDx8+xNraGrxeLw4PD7G7u4vDw0P2kGk2m/B4PGg0GlhbW8PJyQni8TiazSa8Xi9SqRTi8ThLGHO5HDcjAc8LpDQ0m/xryGDM7XZjbW0NAJhiqVarbA9MGXqz2WQrYuLXpfuk0+nkQitth9Q/0iOeOHuSYBIPT5k86daJPjEB3rQP1loPZetWI/xkxmzSNFYAb+rjpU7eip6xUtSYFxQ6Bivt+6jM/rwZPe3fPBaLz+QL604bV9EUNIv9LJKqZd7HcB0y8YvGRQH+jKiXwd/UYPkRgBtiveRg2chwOp14//33cePGDfh8Pjx58gS3b99m8Nza2sLR0RHW19dRKpW4oKp13wwsn8/D5/Oh2+3C5/Mhm82yXHFlZYUbmchxUmuNXC7HKha/349ms8m/Eqig6nK5OPMul8twOp1c+PV6veh0OpzhSxsEsip2uVws16RMvtPpQCnFahq6GNAgEtLmj3OXlNYF0kpAFmxHuUtKkKcMWsod6dyNUtRIzp4uClbAYAWuVjy55N4B+8zeLnOXFwK7fY2jYy6LmrmssCuqLjqILfrxTROX+d4uCvB/BOCrg/tfBfCHYvkvDtQ0XwJQFFSObXQ6Hezt7eHtt99GpVJBJpPhTlYC4Ewmg/X1daTTaSQSCRSLReagCVypgEmZuMfjgcPhYIolFAoxKGWzWZYc+v1+9p8hcAiHw1hdXR3qdu31ekMA3uv1mO6haVKyeEqSTgnwrVZriEuX2f44gAeGaRpzTqsEeALKScBdUjSmFt5OPSMtDUxHSati6ziKxQRnO6WMecwXydjt1hsV4zh9+oVlFReRRo5aPmuAuE5c+zT7fJkvGnYxiUzy9wB8G8BdpdShUuprAP41gJ9USj0C8BODxwDwvwA8AfAYwH8C8PcnOQiiQO7du4e9vT0opZBMJvHw4UNsb2+j2WyiWq0iFovh5OQEyWQSp6enXHx1Op2o1+tMVxAo0hCOTqeDYrGIcDjMkkqayUoXEJfLhVwux2AaDAZ57itRMfQLgbh4AOw3Q2oaaqqiCwE1PFHBttlsAnhuIEaFVsrEtdbsBkkXF2kfTNk/vVdTSUNgS30BBPJmN6uVR7ws1Nrx8FYUjQR8q5Bc+yiJo1UDlJWE0orKsbuZski7DH7UBUH+IrjMmAbQrxOYLdU084uVcStorX/B5qkft1hXA/il8x5EpVLBzZs3EYvF8NFHHyGZTMLlcuHw8BBf/vKXcXx8zICZTqfxuc99Du+99x62t7dRqVQQDoeRzWaZY3c4HDzBiTLeTCbD65F9MGnmiS4pFotM/ZC7JK1HTU2UvZOaRg73aLVaL3SqAmCgpmzf7/czUNP7ol8jSj13lySKSBZaAVhm8LK7d/C/YJCX2TttXw7DliBoBfC0T7oBw5p5OucmxWMXowDfrmgq3xPdl9ux4+Xluua+iK4jMLTSwtuB+qiMfVTMA3hnXXCdJuT/nc7vdY1F+f9OEwtx6Wy323jnnXdQKBRwcnKCO3fusL3vzs4Onj59io2NDc6Eo9Eou0vSgO1UKsV+8X6/nwFfaw2fz8dj/KghicDW6XSyLJI0741GAx6Ph4u6vV7vBfdIkkMSlUJySvKloUyerA3ISEz6w1PTE02Doi5ZWRglJY3UrkvbAgJ5c+qSzM6taBr6pSJvgDXAS1rG1MKbfLy8AdYmXibFYpWF2xVgrS4Ccrt03+oCYRdXxcFfJhgsGvBcp7jOvzAW4sgdDgc+85nP4NGjR+h0OuxJQ4M2yHQsn89z9lqr1RAIBJDL5bCxsYGzszMulkYiEaTTafj9fqZbstksUwrBYBDNZhPlcpl55Egkwk6OzWYTDoeDC61aax7gQfsnOSQVUUkl43Q6hwCeKCIr+2C6uMgBIKa7pAR4AkJpPiYLrgTykocnkLSzLRjlD29y7VaySVlctZNKyrDizuWxyiKrXQZud5PrmvswOfdxnP2rEJOC/vLicH1jIQDe6/Via2sL9+/fx9bWFqLRKE93arfbyOVyPKc1Go1y5k2DPjY2NpDL5eD3+1GtVpFIJJBKpbC6uopOp4O1tTXkcjn+WU0WBaSacTj6M1k7nQ5n2d1uF2trayxhrNfrvP7Kygr7v2utuUZAnbNE05AEkrJomuVK2bNS6gV3SRrjJ90lpX0wWTNI+2BpWWAF8KMKreP84a0KruavBVlsNWkbM+yanqxoFJN7n+R5k3+XFxDA3p/e7jjnHbMqGk6ynetYoJxV4fqi+7nusRAAT5n4p59+iu///u9HtVrF8fEx3nzzTRwfH3M2fXJywp40kUiEQTAUCqFWq8HlcjHA53I5pkdouEer1QIA9pqhuau9Xo87XSkbbzabbEjmdDrRarXYbpgGd9CcVprjSkobmvIk7YNJTUPWxNSdSl7yJsATKEuAJ+5cdplK2wICfQJWycOP8oe38qWxUtPY6eMl0Mvs35TomdnyKEplFC9vBeijeHurLN18zjxGM+YJ9vIcXUc6wEqKueiAuejHN6tYiE+Tx+PBxx9/jF6vh9dffx3Pnj1Dp9NBMpnE/v4+EokEAKBYLGJzc5M18eVymXXYFI1GA+FwmP1jyA+efGEAcHGzWCwyfUF0DNEitVqNB2S7XC70ej32mKdjDoVCfBEh8G6328yRUwEVAMsfpfGYvBhIf3hJq9DxSEAGntsHS0UNUTyy0ErgZ3a2SoWNlX0wnVczSx8F7pKisbIPprCSOdo1NI3i3sfd5PblfRPYrR6PikkvCnYxi2x7UbLxWUlAlzGfWAiAV0rh/v372NnZAXW1xmIxBINB7O/v47XXXmMv91gshqOjI2xvb6NQKMDr9bKNAGX0Ur3S6XTYwpcmMFEjExmZkcWwqXuXBVAArKTRWsPlcvEgkV6v7+VeLpfZHoEAnvYN4AVvGwJ44uBJLulwOF7I4EfZB0se3rQtIGC0KrKO8oc3efhRzU4mDz+Jgobum8VRu+x8lKxyUmAfdd8urJQ1s4h5AvS0evmLXoDOs795N19d9YXkqvdPsRAA32g08ODBA/zAD/wA6vU69vb28Oabb6JcLiObzeL111/HwcEBSxMzmQw2NjaQyWQQjUaRzWbZTZLC5/OhVqtB634xMxgMMofe7XZ5SEi73UatVmPKhYCvWCwOZeFOpxOlUmnIbkBSMO12mztvgT4AE08v3SJJvUPadqm6kQBPACYHcRPA03uSU55kkVb6w5t6+HH+8DKsrAvGecRPCvKjsnJZF7DL4Ef9PS9dcx7An2dcBiich9O+LP77ojFtkXjR6bCZyFZncBxTR6VSwcrKCj7zmc/g8ePHqNVquHv3Lp48eYKVlRWsr6+z6RgN8fD7/SgUCtz8JO2DyaOGMnYqmJZKJWit0Wg0kEgkOOOnodvhcJj570KhAABDA7Jp8hNl0aFQiP3cu90ue86Y81epUCuLuMSJU/etdJeUdgNSKknSSa01g67UxEvJpMl9S1CXfDxl8FKqCGAI2E3u3a671aRorL5Akn+3AmIryeQ4asaKy7eSUY4Dd3OZ1eN50DHj1pv2iz4r8B5Fu11034tysbjqmNd5WAiAbzQauH37NuLxOB48eIBQKITt7W08fPgQGxsbWFlZwdnZGXZ2dpDJZJiCISA/Oztj+wJqWIrH46hUKtzlKjP8Wq3GZmIEulprnrkKgPl5OZyDKBayBw4EAqx1B553tRLgkF6eFC69Xo8BngCbCq1E0Zj2wcTDS8MwUuCQll7SM9KXhoKAU0oj5V+7CU9m0VRSMSb3bj6mbVBIZY8VTWNF1Vjx8hLs7bpYrbZtAvk4wDfvm3HZmf6obHPedMcyFi8m/f8uBMB3Oh188YtfRKVSwaNHj7C7uwuXy4W9vT3s7u6iXC6jWq1ie3sbh4eHXGAF+gqcTCaDWCzGc1XT6TSvQ3NbpT98pVLhAikpYGjoNgEU0S0EumRbQCoY4vYDgcAQwFOhtdfrwePxDNkLA8/nz1IWr7UeAng5AIQA3moQtyyCSiWN7IiVNM2k9sFWIC9/LRDvb9I1JshL4zEzJK89imKhc2BXfJWvH5W1W2X4Vscya0C/ziB7nY99Gc9jIQDe7XbjC1/4Aj744AMe33d8fIxyuYw7d+5gf38fXq8XoVAIR0dHuHnzJk5PT+Hz+aCUQq1Wg9fr5WJpJpNBIpFAo9GA2+1GPp9HPB5nBUqpVILP52OTMVK3hMNhzoCJbiHgpLmvsttVDvgg0zMqtJKckhqiiKaRMkxS+hCPTgBPA0WIPyeZJJmQEUVEIG/SNJKHt3KXHDcAxKRpTIAfJ5UcJ5ekkHSNFe1igvao9e3oGbuM3e45u8dWx32ROC9wzpsnviwgX14wriYWAuCpGHn//n0kEgkkk0mmamKxGPb29rC1tcWeMNvb2zg+PkY8Hkej0YDT6eQvpdfrRT6fRzAYRKfT4WaoYDDIXxaiboLBIANfsVjkbJt4fHKlpIHaDoeDVTD1ep0BnAqclOGTSsbpdLKzJAEvFXXJgVI2LkmppLQckPNZZaGVQFSCO2Xz0j21by4AAAvwSURBVD4AwJBiRkolJ83gzWLrJFp4O2AHrDtKzeYnK1rGBO+LyCZHxXmy+MumaaaJeXL8VxmLfOzzOLZzJwgzP4ILhM/nQyaTwePHj3H37l243W48ePAAt27dglIKJycnuHnzJnejEu8ej8dRLBa5g5Uy2XK5zFkwTWsiEAbAhdBQKMRf/kKhwCoWj8fDqhjiuYPB4JA/fL1eZ18buij0en3PGgJjem8kg5Sa+WazOTQohLJ4kmUSWJLEkUCeMniZZZNUkjh4KZVUSvF7lMBuJ5U0Ad5KLmnnS2OVucvtmB9OK/7dpEzsQFx239p1w5r7mCTjp9ecNy4K/tNKCy+ynfNsY5EBdJ4xzz6EWW5jXCwEwLvdbrz33ntoNBp45513kMlkcHh4iHv37iGdTqNcLuPWrVs4ODhgoC0UCmxJQJJJcpMkwKJMuF6vQykFn8/HvDaN7AP6JzqXyw1l0QC405X8a0hz32q1UKlU0Ov1uBmKrIFlEZV+UUi/GQJ4+hVAShoCadqW1QAQuo3zhyc6SGbxEsTNDF6qaSRwmoVWK+uCUVk8bcMMk26x0sGfRzopt2lHw5jLJsnkJ1lv1vGqAuqixnX/fywEwHc6HXz729/GG2+8gWQyiY8//hgAcOfOHTx8+BA+nw/xeByffvopkskkyuUyut0uz2fd3t7G6ekpIpEIj/OjgdxKKfaBIQ+abrfL/vCkYsnn8wDAWbzT6WRrAq0100jEhVOTFGXo1NBEUsp6vY5OpzPkNUMNUwTwxMUTl24CPBVvZQZP2T+BnAR4s+lJcvD0vkcVWk3wBF70prEb/mHX/GSGBEyrbJ2WS1AfReeM499HgbrdMqtjHtXwtCgXgfPy9dcdvC47ruP5WgiAr9VqODw8xLvvvotut4tPPvkEOzs7CAQC2Nvbw/b2NgAgnU4PDfsgtQvZBYfDYVbSkPlYr9fjJiWiZFwuF/L5PCtgVldXUSqV0G63h+SKpVKJJZFUUCVtOsklaX2fz4eVlZWhgdk0iJtAWzZFEcBTZyxZAUiAJzUNaddlBk+ADDy3LZAgL7XqwHOQogze6mZy3oA9TWOVxZt8vcnhU9hl1XbFUSup5KR8vHnBMh+bx2EenxkmBTSLMIHDSmY6yXPn3c8ixqi6zUW2dVmxqOd2YQB+ZWUFn//853F4eIjj42O89dZbqNVqODk5we3bt1EoFHg+6+HhIRKJBANeJBJBPp9HKBRCuVxGLBZDOp1GOBxGu93G6uoqF1611vB4PMhms2wkRk1MpLqhMX7lcpkLodS5CvQBmrJ0Gp5NDU3ErRNNIztVqSnKdIrs9XoM8LLQSjy6bE4iikiCsPSlMXl4+nUgO1pN+2A7Z0lZKJ1E+07SSDse3iqsOHirxqZRPPyorJ32YffXisaZJhahsHbVYHPV+7eKcZ/DeexrEWIhAL7RaOCzn/0stre38f7770NrjXfeeQePHz9mf/j9/X22Ezg4OMCNGzeQzWaZjqhWq/D5fCgWi1hfX8fZ2Rn7vodCIX5MvDnNZHU4HOwjXy6XGRyDwSADdavVYs944sWlP7ycrUoqGcrO6QJCIO9wOFgNQ9uX/jK0HhWFpS+N3QAQKrSatgUE/PSBI1A0uffzzmmVmbuVFt7k4mV3K4WphZfHZ2bppsLnvEVUOypHhtWvCrtsf1FCUnDnfd3LGC/r+5omFgLglVL40R/9UeRyOXzve9/DjRs3sLW1hQ8++ACJRALRaBSPHz9GMplEu91GoVAY4t1pktLKygqq1Sri8ThyuRx8Ph/q9TrW1taQzWbh8XjQ7XYRCATYioDMvrTu+8MTMJEdMVExWvc7XcnOl9wjicMnXb05ZBvAULMUqW0o05fWB06nky0LiGqh7JsAXtI0BMiU/RPImxSNw+EY4uAlqFsN/5D2wTKDHyeTtOpmldugsAN3E0glkFsBv12T0yT8u9znJNTMLGJeADQLumZWqpFZqYKWMZtYCIBfXV3FW2+9hY8++gj5fB7f933fh06ng729Pdy+fRvdbhfHx8e4desWcrkcer0ez0uNx+PMpxOVEQwGmfZptVo82o+mMwWDQR68QbSI0+lEsVhkMKLGplqtxtpzaS5G7pHE6VPmDQxz7PQ8Zflut5vBVnL1BHRE+UiahgBeFlslwFOhldaXDU8EujJksXVcsxMwrIe308bbUTYm0FuFyY9bgfc0RVW75VZc+qiM/TIz+WnBT15UrxpIZ73/q34/1ykWAuC9Xi/i8Ti++93vAgDeeOMNFAoFpFIp3Lp1C9VqlbP2TCbDhctCoYBoNMr8O6lLPB4PWq0W892BQAC1Wo014V6vl2kSyV9XKhUA/S9yIBAAAOa+STFD6pROp4NqtQqtNWfeJNOkCw3x61LnLv3eCeClrl0qYqSSxnSDlJw58eP0XuRkJ5lNS8rD9KOxkiHKMHl4KyWNpGTM+5KPpzAB2O7+OKAexdFbUS9mWC1fVFpmXMyj83UJqNc31CJ8kJVSZQCfXPVxLEDEAWSu+iCuOJbnoB/L87A8B8D4c3BTa52we3Jl9sdzofhEa/3uVR/EVYdS6v6rfh6W56Afy/OwPAfA9OdgISiaZSxjGctYxuxjCfDLWMYylvGSxqIA/Deu+gAWJJbnYXkOKJbnYXkOgCnPwUIUWZexjGUsYxmzj0XJ4JexjGUsYxkzjiXAL2MZy1jGSxpXDvBKqa8opT5RSj1WSv3KVR/PvEIp9dtKqZRS6kOxLKqU+qZS6tHg79pguVJK/cbgnHxXKfWFqzvy2YVS6oZS6s+UUt9TSn2klPrlwfJX7TysKqX+XCn1weA8/KvB8ttKqe8M3u/vK6Xcg+WewePHg+dvXeXxzzKUUk6l1F8ppf548PhVPAf7Sqm/Vkq9r5S6P1g2k+/ElQK8UsoJ4D8A+CkAbwP4BaXU21d5THOM3wHwFWPZrwD4ltb6DoBvDR4D/fNxZ3D7OoDfvKRjnHd0APwTrfXbAL4E4JcG/+9X7Tw0AfyY1vqzAD4H4CtKqS8B+DUAv661fgNAHsDXBut/DUB+sPzXB+u9LPHLAD4Wj1/FcwAAf0Nr/TmheZ/Nd+KibnyzuAH4IQB/Ih7/KoBfvcpjmvP7vQXgQ/H4EwBbg/tb6Dd8AcB/BPALVuu9TDcAfwjgJ1/l8wDAB+AvAXwR/Y7FlcFy/m4A+BMAPzS4vzJYT131sc/gvScH4PVjAP4YgHrVzsHg/ewDiBvLZvKduGqKZgfAM/H4cLDsVYkNrfXJ4P4pgI3B/Zf+vAx+Yn8ewHfwCp6HATXxPoAUgG8C2ANQ0Fp3BqvI98rnYfB8EUDsco94LvHvAPxTAGQvGsOrdw4AQAP430qpv1BKfX2wbCbfiUWxKnjlQ2utlVKvhGZVKRUA8D8A/EOtdUmaWb0q50Fr3QXwOaVUBMD/BPDmFR/SpYZS6m8BSGmt/0Ip9eWrPp4rjh/RWh8ppdYBfFMp9UA+Oc134qoz+CMAN8Tj5GDZqxJnSqktABj8TQ2Wv7TnRSnlQh/c/4vW+g8Gi1+580ChtS4A+DP06YiIUoqSLvle+TwMng8DyF7yoc46fhjA31ZK7QP4r+jTNP8er9Y5AABorY8Gf1PoX+x/EDP6Tlw1wL8H4M6gcu4G8PMA/uiKj+ky448AfHVw/6voc9K0/BcHFfMvASiKn2vXNlQ/Vf8tAB9rrf+teOpVOw+JQeYOpZQX/TrEx+gD/c8NVjPPA52fnwPwp3pAwF7X0Fr/qtY6qbW+hf73/k+11n8Xr9A5AACllF8pFaT7AP4mgA8xq+/EAhQYfhrAQ/Q5yH9+1cczx/f5ewBOALTR582+hj6H+C0AjwD8HwDRwboKfXXRHoC/BvDuVR//jM7Bj6DPN34XwPuD20+/gufhHQB/NTgPHwL4F4PluwD+HMBjAP8NgGewfHXw+PHg+d2rfg8zPh9fBvDHr+I5GLzfDwa3jwgDZ/WdWFoVLGMZy1jGSxpXTdEsYxnLWMYy5hRLgF/GMpaxjJc0lgC/jGUsYxkvaSwBfhnLWMYyXtJYAvwylrGMZbyksQT4ZSxjGct4SWMJ8MtYxjKW8ZLG/weC041Db1OZIQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x216 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F8EW7-BqBg8y"
      },
      "source": [
        "그림으로 표시된 행렬의 각 행은 단어의 위치별로 다르게 계산된 숫자 512개로 구성되어 있고 이 행 벡터가 단어의 위치 정보를 나타내는 고유한 값이 되는 것입니다.\n",
        "\n",
        "논문에서는 위치 인코딩을 어떤 방법으로 해도 상관없다고 이야기합니다. 토큰 위치에 따라 유일하게 다른 값들을 계산해주는 함수라면 뭐든 가능하단 이야기입니다. sin, cos같은 주기 함수를 사용하면 위치에 따라 서로 다른 값을 계산할 수 있는 것 이외에도 상대위치의 선형성을 확보할 수 있다는 장점도 있습니다. 상대위치의 선형성을 자세히 설명한 글은 이 블로그[[6](https://kazemnejad.com/blog/transformer_architecture_positional_encoding/)]를 참고하시면 좋겠습니다.\n",
        "\n",
        "여기서는 간단한 실험으로 상대위치의 선형성을 직관적으로 알아보겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJFeOstIzeeZ"
      },
      "source": [
        "def pe_test(pos, i, d_model=512):\n",
        "    PE = []\n",
        "    PE.append( np.sin( pos / (10000**(2*i/d_model)) ) )\n",
        "    PE.append( np.cos( pos / (10000**(2*i/d_model)) ) )\n",
        "\n",
        "    return np.array(PE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UPUOViPw0B6y",
        "outputId": "f1ad3eb3-49ad-4dc2-e090-7842e859a857"
      },
      "source": [
        "k = 10\n",
        "d_model = 512\n",
        "i = 1\n",
        "\n",
        "pos1 = 100\n",
        "pos1k = pos1 + k\n",
        "\n",
        "pos2 = 172\n",
        "pos2k = pos2 + k\n",
        "\n",
        "M_k = np.array([[np.cos(k/(10000**(2*i/d_model))), np.sin(k/(10000**(2*i/d_model)))], \n",
        "                [-np.sin(k/(10000**(2*i/d_model))), np.cos(k/(10000**(2*i/d_model)))]])\n",
        "\n",
        "pe_pos1 = pe_test(pos1, i)\n",
        "pe_pos1k = pe_test(pos1k, i)\n",
        "pe_pos2 = pe_test(pos2, i)\n",
        "pe_pos2k = pe_test(pos2k, i)\n",
        "\n",
        "print(f'pe at {pos1}:', pe_pos1)\n",
        "print(f'pe at {pos1k}:', pe_pos1k)\n",
        "print(f'linear transformation of pe at {pos1} by M:', np.dot(M_k, pe_pos1))\n",
        "\n",
        "print(f'pe at {pos2}:', pe_pos2)\n",
        "print(f'pe at {pos2k}:', pe_pos2k)\n",
        "print(f'linear transformation of pe at {pos2} by M:', np.dot(M_k, pe_pos2))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "pe at 100: [ 0.79754236 -0.60326294]\n",
            "pe at 110: [-0.64526647  0.76395758]\n",
            "linear transformation of pe at 100 by M: [-0.64526647  0.76395758]\n",
            "pe at 172: [ 0.55020692 -0.83502835]\n",
            "pe at 182: [-0.3529983   0.93562396]\n",
            "linear transformation of pe at 172 by M: [-0.3529983   0.93562396]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CD2i1Sw69nNl"
      },
      "source": [
        "위 소스를 보면 위치 100에서 위치인코딩을 한 값과 `k=10`만큼 떨어진 위치 110에서 위치인코딩 값을 출력합니다. 그리고 위치 100에서 위치 인코딩 값에서 변환형렬 `M_k`을 곱한 값도 함께 출력합니다. 값을 비교해보면 100에서 10만큼 떨어진 위치 110에서 인코딩 값이 100에서 위치 인코딩 값을 M을 사용해 선형변환한 값과 똑같은 것을 알 수 있습니다. 동일한 실험을 기준 위치 172에서 해도 172에서 위치 인코딩 값에 `M_k`을 곱한 갑과 172+10에서 위치 인코딩 값이 동일한 것을 확인할 수 있습니다. 즉 어떤 위치를 기준으로 잡든 `k`(offset)만큼 떨어진 위치에서 인코딩 값은\n",
        "\n",
        "$$\n",
        "PE_{\\text{pos}+k} = M(k) \\times PE_{\\text{pos}}\n",
        "$$\n",
        "\n",
        "라는 선형관계에 있게 되는 것입니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ppsnR1894K0Z"
      },
      "source": [
        "위 코드셀에서는 논문에 나온 수식을 그대로 구현하였고 아래 코드에서는 수치 계산에서 안정성을 위해 논문에 식을 그대로 사용하지 않고 아래처럼 변환해서 구현합니다.\n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\frac{pos}{10000^{2i/d_{\\text{model}}}} \n",
        "&= pos \\times \\left[ \\exp\\left\\{{\\log\\left(  \\frac{1}{10000^{2i/d_{\\text{model}}}}\\right)} \\right\\} \\right] \\\\\n",
        "&= pos \\times \\left[ \\exp \\left\\{ \\log \\left( \\left( 10000^{2i/d_{\\text{model}}} \\right)^{-1}  \\right) \\right\\} \\right]\\\\\n",
        "&= pos \\times \\left[ \\exp\\left\\{\\log \\left( 10000^{-2i/d_{\\text{model}}} \\right)\\right\\} \\right] \\\\\n",
        "&= pos \\times \\left[ \\exp\\left\\{ -\\frac{2i}{d_{\\text{model}}} \\log(10000) \\right\\} \\right]\n",
        "\\end{aligned}\n",
        "$$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X-jRt5-U0BXe"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    \"Implement the PE function.\"\n",
        "    def __init__(self, d_model, dropout, max_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        \n",
        "        # Compute the positional encodings once in log space.\n",
        "        pe = torch.zeros(max_len, d_model)\n",
        "        position = torch.arange(0, max_len).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2) *\n",
        "                             -(math.log(10000.0) / d_model))\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0)\n",
        "        self.register_buffer('pe', pe)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        pe_val = self.pe[:, :x.size(1)]\n",
        "        pe_val.requires_grad = False\n",
        "        \n",
        "        x = x + pe_val\n",
        "        # x = x + Variable(self.pe[:, :x.size(1)], requires_grad=False)\n",
        "\n",
        "        # page 7\n",
        "        # In addition, we apply dropout to the sums of the embeddings and the\n",
        "        # positional encodings in both the encoder and decoder stacks.\n",
        "        # For the base model, we use a rate of Pdrop=0.1.\n",
        "        return self.dropout(x)\n",
        "        \n",
        "        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ELQTovh2tT9t"
      },
      "source": [
        "그리고 마지막 출력을 `nn.Dropout`을 통과시켜 규제regularization 효과를 주고 있습니다.\n",
        "\n",
        "지금까지 설명한 임베딩과 위치 인코딩이 어떻게 작동하는지 실제 간단한 예로 알아보겠습니다. 다음 코드는 열한개 단어를 사용하여 문장을 만들 때 각 단어를 길이 10인 벡터로 임베딩하는 임베딩 층과 이에 대한 위치 인코딩층을 실험한 코드입니다. 코드에 자세한 주석을 달았고 주석을 보면 알 수 있지만 실제 단어로 구성된 문장이 아닌 숫자가 단어라고 가정한 예제입니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 260
        },
        "id": "oenqh38-ESNf",
        "outputId": "716b1147-6143-4126-c309-c72aa7a24a84"
      },
      "source": [
        "torch.random.manual_seed(15)\n",
        "\n",
        "# 단어장 크기 11, 임베딩 벡터 길이 10인 임베딩 층 선언\n",
        "V = 11\n",
        "d_model = 10\n",
        "emb = Embeddings(d_model, V)\n",
        "# 길이 10인 문장의 위치를 인코딩하는 위치 인코딩 층 선언\n",
        "pe  = PositionalEncoding(d_model, 0.1)\n",
        "# 그림 그리기 위해 positional encoding을 복사해둠\n",
        "pe_ = pe.pe.clone()\n",
        "\n",
        "# 문장 길이는 5이고\n",
        "n_seq = 5\n",
        "# 0~10까지 숫자 5개를 무작위로 뽑아서 문장을 구성\n",
        "# 이 예에서 문장은 실제 단어로 구성된 것은 아니고 0, 1, 2, 3, ..., 10인 \n",
        "# 숫자를 단어로 간주함\n",
        "x = torch.randint(0, V-1, (1, n_seq,), requires_grad=False)\n",
        "print(\"integer tokens:\", x)\n",
        "\n",
        "# Feed forward Embeddings-PositionalEncoding\n",
        "# 숫자(단어) 다섯개로 구성된 입력을 임베딩층을 통해 (n_seq, d_model)로 변환\n",
        "embedded = emb(x)\n",
        "\n",
        "# 임베딩 벡터에 대한 위치 인코딩 정보를 구해서 임베딩 벡터에 더함\n",
        "embedded_pe = pe(embedded)\n",
        "\n",
        "# 임베딩 벡터와 위치 인코딩이 더해진 입력 벡터를 그림 \n",
        "fig, axs = plt.subplots(figsize=(18,3), nrows=1, ncols=3)\n",
        "axs[0].imshow(embedded.detach().numpy()[0], aspect='auto', cmap='gray')\n",
        "axs[0].set_xlabel('d_model')\n",
        "axs[0].set_ylabel('n_seq')\n",
        "axs[0].set_title(f\"embedded vector x\")\n",
        "\n",
        "axs[1].imshow(pe_.numpy()[0][:x.shape[1]], aspect='auto', cmap='gray')\n",
        "axs[1].set_xlabel('d_model')\n",
        "axs[1].set_ylabel('n_seq')\n",
        "axs[1].set_title(f\"positional encoding\")\n",
        "\n",
        "axs[2].imshow(embedded_pe.detach().numpy()[0], aspect='auto', cmap='gray')\n",
        "axs[2].set_xlabel('d_model')\n",
        "axs[2].set_ylabel('n_seq')\n",
        "axs[2].set_title(f\"embedded vector x + positional encoding\")\n",
        "\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "integer tokens: tensor([[8, 4, 6, 8, 1]])\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABBYAAADhCAYAAAB8+jYYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5gcBZnv8d8vM7lOkGAQNSQhHEGWhAWUQDzAuoiKIDd3efYcQC56zm4ej6LEZWWJKJtdODx79vAouCtqFiErkYtcXDk8gLgPsizCBkK45sI9VwIhSBJCzJX3/FE1UjPpmenUTFVNd38/z9MP3V3d9b41mflV8U51jSNCAAAAAAAAeQypugEAAAAAANC4GCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCygX2zfb/vPB2hds2zP7WX5Utufyrnu3O8FgEZg+4e2v93L8m/avqaEPgZsv1CE7L7G9kTbG223Vd0X0Gg4BkRf2C/Vp1n2S+1VNwA0I9tzJK2MiG9V3QuA1hARX+q8b/sYSXMjYnxm+eVV9DWYRcRySaOr7gNA8+AY8F3sl3ZdI++XOGMBGIQacUoJAACA/hnIY0DbX0gHHUDhGCy0INvjbN9m+3XbL9v+WmbZLNu32J5r+y3bT9v+sO2ZttfYXmH7uG6r/JDtR2xvsP0L2+/NrO9jth+yvc72k+m0snPZvrb/Pa3zK0l7duvzbNvLbL9h++Juy4bYvsj2i+nyn3Wr2+N7u61nmu1XsyFu+09sP1VnnaMz27ciDfDpkj4v6cL0VKb/l772wPRUrHW2F9o+JbOeObZ/YPsu229L+kS3Pt9re6Xtk9PHo22/YPucnrYNwOCVnpo70/Yi22/avs72iMzyv0h/xn9r+w7b49Lnbfu7aR5vSDP6oHTZHNuX2e6QdLekcWkGbUxzv8upxrZPSbNoXZpNB3br769sP2V7ve2bO/uzvYftO9N9yJvp/d//BqqP7e4xU21Psh22z7W93PbabH7bbnNy2uyL6X7jMdsT0mVH2n407fVR20dm3tfjviZTsz19fL/tS23/Jn39vbazrz8ns2/5tjnFGg3GHANm18MxYNc67JfYL/VPRHBroZuSYdJjki6RNEzSf5H0kqTPpMtnSdos6TNKPirzE0kvS7pY0lBJfyHp5cz67pe0StJBkjok3abkNCdJ2lvSG5I+m9b9dPr4fenyhyV9R9JwSR+X9FbmvZMlbUyfH56+brukT6XLz5f0n5LGp8t/JOnGet5b42vyoqRPZx7fIumiOursk/Z8Rvq1GSvp0HTZHEmXZdY5VNILkr6Zft2PTd97QOb16yUdlX6tRtTo8zhJr0raS9I/S7q16u8nbty45btJWirpGUkTJL1X0m86MyPNh7WSPprmzj9KeiBd9pk0w8dIsqQDJX0wXTYns45jlJyKm605K5OxH5b0dprLQyVdmGbUsEx/j0gal/a3WNKX0mVjJZ0maZSk3dLM/NdMnfsl/XkP291bpk6SFGm+jZR0iKQtkg5Ml39D0tOSDki3/ZC0l/dKelPS2Ur2W2ekj8em7+ttX9NZsz3T+4vp12dk+vjv02Wd+5ajleT4FZK2qYd9Czdug+0mjgFrfU2a+hhQ0hckzanztUvFfon9Un8ypuqQ41byP7g0TdLybs/NlHRden+WpF9llp2cfsO2pY93S7/Zx6SPf//NnT6eLGmrpDZJfy3p+m61finpXEkTlQR9R2bZDZkfqksk3ZRZ1pGut3OnsljSJzPLP5j+ILX39d4aX5PLJF2b2b63Je1TR52Zkn7ewzrnqOtO5Y+U7BCGZJ67UdKszOt/Use/3z8qCbBVneHEjRu3xrspOUD6UubxZyW9mN7/saR/yCwbnebOJCUHd89J+lg2T9LX/T531PcB3Lcl/SyzbEiaK8dk+jsrs/wfJP2wh205VNKbmcf3q+cDuN4ydZKS/cv4zPJHJJ2e3n9W0qk11nm2pEe6PfewkgPqvvY1nTWzB3Dfyrz2y5LuSe9fovRgM308Sr3sW7hxG2w3cQxY62vS1MeA2vXBAvsl9ku5b3wUovXso+Q0pHWdNyUT1PdnXvNa5v7vJK2NiB2Zx1LXi4qsyNxfpmTKuGda68+61TpayQ/sOCU/8G93e2+ncdn1pq97o9t2/Dyz3sWSdqTb0dd7u7tB0p/aHi7pTyUtiIjOXnqrM0HJBLEe4yStiIh3um3v3pnHK9S32Up+MzAnInrbJgCDX/fsHJfeH6dMHkbERiUZtndE3CfpnyR9X9Ia27NtvydH7e413kn7yWbSq5n7m5Tmvu1Rtn+Unnq5QdIDksa4vs8F95apvdZVz5nbZVtSnfna176mlp7qd9+3bFLv+xZgsOEYcGdNdwxo++pMz1dLOjPz7/BUHzXYL7Ffyo3BQutZoeQ0tjGZ224R8dl+rHNC5v5EJVO+tWmt67vV6oiIv5e0WtIe6Weusu/ttDq7XtujlJxalN2OE7qte0RErKrjvV1ExCIlP9AnSDpTyU6mnjorJH2op9V2e/yKpAm2sz9zE5VMYnt6TxdpOM5Wcmril23v19vrAQx63bPzlfT+K0oOdCRJaU6OVZoXEfG9iDhMyW8HP6zkVMzues2TGjWc9rOqx3e86wIlp31Oi4j3KDmNU0pOA+1Lb5laz3trZW6XbUl15mtf+5pdsVrJqbKSJNsj1cu+BRiEOAbsphmPASPiy539Kvnt9g2Z/g/urY7YL7Ff6gcGC63nEUlv2f5r2yPTi44cZPvwfqzzLNuT0/D+OyWf+9ohaa6kk21/Jq0zwvYxtsen0+D5kv7W9jDbRys55a7TrZJOcnJhnGHperPfrz+U9L9t7yNJtt9n+9Q631vLDUo+Y/VxJZ/LqqfOTyV9yvZ/s91ue6ztQ9Nlryn57GKneUomjBfaHurkAkYnS7qpj76yvqkklP+HpP8r6Sd1TmIBDE5fsT3eyUWiLpZ0c/r8jZK+aPvQ9Ldol0uaFxFLbR/u5IJjQ5WcsrtZ0js11v2apLG2d++h9s8knWj7k+m6LlDyudGH6uh7NyW/uVyX9v439W2upN4ztS/XSLrU9v5OHGx7rKS7JH3Y9plpFv93JQe3d9axr9kVtyrZpx2Z7ltmqb6DVmCw4BiwNo4B38V+if1SbgwWWkwa9icp+ezRy0qmytdI6umHvB7XK/kM1auSRkj6WlprhaRTlYTh60qmet/Qu993Zyr5vN9vlQTATzJ9LpT0FSVhv1rJBU9WZmpeJekOSffafkvJRVem1fneWm6U9MeS7ouItXXWWa7k82cXpNvwhJKLtkjJZ9EmOzmt6l8jYquS0DhBydf8aknnRMSSPvqSJNk+TNJfpu/ZIen/KNnBXFTP+wEMSjdIulfJxdNeVPJZX0XEvyn5rOltSjLsQ5JOT9/zHiUXkXpTyW/Z3lBykNlFmi03SnopzaFx3ZY/K+ksJZ/ZXaskn05Os6ovVyq5gNRaJZl4T91b3Eum1uE7Sg4875W0QUnOjkxPCT5JSRa/oeSCXydlsrzHfc2uSPctX1XyPwOrlXz2fI2SA19g0OMYsEccA76L/RL7pdwc0ddZKQAAYCDZXqrkQlL/VnUvyMf2aEnrJO0fES9X3Q8A9Af7pcZX9X6JMxYAAADqYPtkJxcJ61DyZ72eVnKlcgAASjeY9ksMFgAAAOpzqpKLcr0iaX8lf3KMUz8BAFUZNPslPgoBAAAAAABy44wFAAAAAACQG4MFAAAAAACQW3vVDWSNGDEiOjo6Sq+7++79+Ss7+djV/InRjRs3ll5z+PDhpdeUpM2bN1dSd8SIEaXX3GuvvUqvKUlr1qwpveaKFStKrylJEdESf69+1KhRMWbMmNLrjhs3ru8XDbBXXnml9JoS21qGKraXbS3W0qVLtXbt2pbIYUkaPnx4jBw5svS6VdSs6jjxd7/7Xek1hw4dWnpNSdq6tZ6/2Djwhg0bVnrND3zgA6XXlKTVq1eXXrOqfXtPx8SDarDQ0dGhE088sfS6xx13XOk1q/ifT0l68MEHS6+53377lV5TkhYuXFhJ3SlTppRe87zzziu9piRdddVVpdecMWNG6TVbyZgxYzR9+vTS686aNaslalZVt5W2taq6bGuxpk6dWnrNKo0cOVLHHHNM6XUnT55ces3999+/9JqS9OSTT5Ze8/3vf3/pNSVp+fLlldSdOHFi6TVnzpxZek1JuvTSS0uveckll5Reszd8FAIAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAOTGYAEAAAAAAORW6GDB9vG2n7X9gu2LiqwFANgZOQwA1SOLATS7wgYLttskfV/SCZImSzrD9uSi6gEAuiKHAaB6ZDGAVlDkGQtHSHohIl6KiK2SbpJ0aoH1AABdkcMAUD2yGEDTK3KwsLekFZnHK9PnurA93fZ82/O3bNlSYDsA0HJ2OYc3bdpUWnMA0CJ2OYu3bt1aWnMAMBAqv3hjRMyOiKkRMXX48OFVtwMALSebw6NGjaq6HQBoSdksHjZsWNXtAMAuKXKwsErShMzj8elzAIBykMMAUD2yGEDTK3Kw8Kik/W3va3uYpNMl3VFgPQBAV+QwAFSPLAbQ9NqLWnFEbLd9nqRfSmqTdG1ELCyqHgCgK3IYAKpHFgNoBYUNFiQpIu6SdFeRNQAAPSOHAaB6ZDGAZlf5xRsBAAAAAEDjYrAAAAAAAAByY7AAAAAAAAByY7AAAAAAAAByY7AAAAAAAAByY7AAAAAAAAByY7AAAAAAAAByY7AAAAAAAAByY7AAAAAAAAByY7AAAAAAAABya6+6gawdO3Zo/fr1pde9/vrrS6953nnnlV5Tko444ojSaz7xxBOl15SkM844o5K6q1atKr3mZZddVnpNSRo+fHjpNa+44orSa1555ZWl16zKXnvtVUk+Pfjgg6XX3HPPPUuvKUkRUXrNtra20msC6J8dO3aUXvOhhx4qvebBBx9cek2pmmPiZ555pvSaUnXHxEuWLCm9ZlXHxFu3bi295syZM0uved111/W4jDMWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAbgwWAAAAAABAboUNFmxfa3uN7WeKqgEA6B1ZDADVIocBtIIiz1iYI+n4AtcPAOjbHJHFAFClOSKHATS5wgYLEfGApN8WtX4AQN/IYgCoFjkMoBVwjQUAAAAAAJBb5YMF29Ntz7c9f+vWrVW3AwAtJ5vDb7zxRtXtAEBL4pgYQCOrfLAQEbMjYmpETB02bFjV7QBAy8nm8NixY6tuBwBaEsfEABpZ5YMFAAAAAADQuIr8c5M3SnpY0gG2V9r+n0XVAgDURhYDQLXIYQCtoL2oFUfEGUWtGwBQH7IYAKpFDgNoBXwUAgAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5MZgAQAAAAAA5NZedQNZHR0dOvzww0uvu2zZstJrPvzww6XXlKQVK1aUXjMiSq8pSUuWLKmk7oYNG0qvuWjRotJrStLZZ59des3nn3++9Jq2S69ZlSFDhmjEiBGl17388stLr3nOOeeUXlOStm3bVnrNtra20mtWqZV+ZtGcOjo6NG3atNLrLliwoPSa8+bNK72mJL3++uul13zrrbdKrylJkyZNqqTuypUrS6+5ffv20mtK0iGHHFJ6zddee630mkOHDu1xGWcsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3Oq6xoLtj/a2PCLK/0AWALQYshgAqkUOA0Bt9V688WpJH5X0lCRL+kNJj0naLCkkHVtIdwCALLIYAKpFDgNADfV+FOIVSYdFxNSIOExJoK6KiE9EBAEKAOUgiwGgWuQwANRQ72DhgIh4uvNBRDwj6cBiWgIA9IAsBoBqkcMAUEO9H4V4yvY1kuamjz+v5BQwAEB5yGIAqBY5DAA11DtY+KKk/yXp/PTxA5J+UEhHAICekMUAUC1yGABqqGuwEBGbbf9Q0l0R8WzBPQEAaiCLAaBa5DAA1FbXNRZsnyLpCUn3pI8PtX1HkY0BALoiiwGgWuQwANRW78Ub/0bSEZLWSVJEPCFp36KaAgDURBYDQLXIYQCood7BwraIWN/tuejtDbYn2P617UW2F9o+v7fXAwD6tEtZTA4DwIDjmBgAaqj34o0LbZ8pqc32/pK+JumhPt6zXdIFEbHA9m6SHrP9q4hY1I9+AaCV7WoWk8MAMLA4JgaAGuo9Y+GrkqZI2iLpRkkbJM3o7Q0RsToiFqT335K0WNLe+VsFgJa3S1lMDgPAgOOYGABqqPevQmySdLGki223SeqIiM31FrE9SdJHJM2rsWy6pOmStPvuu9e7SgBoOf3J4npzeMKECQPVLgA0HY6JAaC2ev8qxA2232O7Q9LTkhbZ/kad7x0t6TZJMyJiQ/flETE7IqZGxNSOjo5d6R0AWkreLN6VHN5zzz0HvnEAaBIcEwNAbfV+FGJyGoCfk3S3kqvfnt3Xm2wPVRKgP42I23N3CQCQcmQxOQwAA4pjYgCood7BwtA0ED8n6Y6I2Ka+r4BrST+WtDgivtO/NgEA2sUsJocBYMBxTAwANdQ7WPiRpKWSOiQ9YHsfJRer6c1RSia4x9p+Ir19NnenAIBdzWJyGAAGFsfEAFBDvRdv/J6k73U+tr1c0icyj8+NiH/p9p4HJXmA+gSAlrerWUwOA8DA4pgYAGqr94yFLiKxPfPU+QPUDwCgTmQxAFSLHAaARK7BQg1MYQGgemQxAFSLHAbQkgZqsNDrRWsAAKUgiwGgWuQwgJbEGQsA0DzIYgCoFjkMoCXVdfFG28MlnSZpUvY9EfF36d3fDHhnAIAuyGIAqBY5DAC11TVYkPQLSeslPSZpS/eFEXHeQDYFAKiJLAaAapHDAFBDvYOF8RFxfKGdAAD6QhYDQLXIYQCood5rLDxk+w8L7QQA0BeyGACqRQ4DQA31nrFwtKQv2H5ZyWlfVvKnew8urDMAQHdkMQBUixwGgBrqHSycUGgXAIB6kMUAUC1yGABqqGuwEBHLim5EkjZv3qznnnuujFJdHHfccaXXHDFiROk1JWnTpk2l19xvv/1KrylJjz/+eCV1p0yZUnrNG264ofSaknTVVVeVXnPGjBml1xwsysjitWvX6tprry26zE7uvvvu0mt+/etfL72mlOzrytbeXu/vEQD0pqxj4k2bNmnBggVllOpi2rRppddcv3596TUlaezYsaXXPOigg0qvKVV3TDxx4sTSa86cObP0mpJ06aWXll7zkksuKb1mb+q9xgIAAAAAAMBOGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcGCwAAAAAAIDcChss2B5h+xHbT9peaPtvi6oFAKiNLAaAapHDAFpBe4Hr3iLp2IjYaHuopAdt3x0R/1lgTQBAV2QxAFSLHAbQ9AobLERESNqYPhya3qKoegCAnZHFAFAtchhAKyj0Ggu222w/IWmNpF9FxLwi6wEAdkYWA0C1yGEAza7QwUJE7IiIQyWNl3SE7YO6v8b2dNvzbc/fsmVLke0AQEvqK4uzObxx48baKwEA5MYxMYBmV8pfhYiIdZJ+Len4GstmR8TUiJg6fPjwMtoBgJbUUxZnc3j06NHVNAcALYBjYgDNqsi/CvE+22PS+yMlfVrSkqLqAQB2RhYDQLXIYQCtoMi/CvFBSf9iu03JAONnEXFngfUAADsjiwGgWuQwgKZX5F+FeErSR4paPwCgb2QxAFSLHAbQCkq5xgIAAAAAAGhODBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBuDBYAAAAAAEBu7VU3kLVt2zatXLmy9LoRUXrNww47rPSakrRkyZLSay5btqz0mpJ02mmnVVL30UcfLb3mLbfcUnpNSbr++utLr3nbbbeVXvPCCy8svWZVXn31VV1xxRWl150yZUrpNffYY4/Sa0rSpk2bSq/Z1tZWes0q2a66BaBfNm/erMWLF5de98gjjyy95rnnnlt6TUm6+eabS6/ZasfE9913X+k1qzomnjt3buk1B9sxMWcsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3AofLNhus/247TuLrgUA2Bk5DADVI4sBNLMyzlg4X9LiEuoAAGojhwGgemQxgKZV6GDB9nhJJ0q6psg6AIDayGEAqB5ZDKDZFX3GwpWSLpT0TsF1AAC1kcMAUD2yGEBTK2ywYPskSWsi4rE+Xjfd9nzb87du3VpUOwDQcvLk8DvvcMwLAAMpTxZv3769pO4AYGAUecbCUZJOsb1U0k2SjrU9t/uLImJ2REyNiKnDhg0rsB0AaDm7nMNDhvDHggBggO1yFre3t5fdIwD0S2FHkBExMyLGR8QkSadLui8iziqqHgCgK3IYAKpHFgNoBfxqCgAAAAAA5FbKeVYRcb+k+8uoBQDYGTkMANUjiwE0K85YAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTFYAAAAAAAAuTkiqu7h92y/LmlZjrfuKWntALczWLXStkqttb1s6+C1T0S8r+omytCPHJYa79+1P9jW5sS2Dl4tk8MSx8R1YlubVyttb6Nta49ZPKgGC3nZnh8RU6vuowyttK1Sa20v24pG10r/rmxrc2Jb0eha6d+VbW1erbS9zbStfBQCAAAAAADkxmABAAAAAADk1iyDhdlVN1CiVtpWqbW2l21Fo2ulf1e2tTmxrWh0rfTvyrY2r1ba3qbZ1qa4xgIAAAAAAKhGs5yxAAAAAAAAKtDwgwXbx9t+1vYLti+qup+i2J5g+9e2F9leaPv8qnsqmu0224/bvrPqXopke4ztW20vsb3Y9n+tuqei2P56+v37jO0bbY+ouif0HzncvFolhyWyuOqe0H9kcfNqlSwmhxtbQw8WbLdJ+r6kEyRNlnSG7cnVdlWY7ZIuiIjJkj4m6StNvK2dzpe0uOomSnCVpHsi4g8kHaIm3Wbbe0v6mqSpEXGQpDZJp1fbFfqLHG7abe3UKjkskcVoYGRx025rp1bJYnK4gTX0YEHSEZJeiIiXImKrpJsknVpxT4WIiNURsSC9/5aSH7S9q+2qOLbHSzpR0jVV91Ik27tL+rikH0tSRGyNiHXVdlWodkkjbbdLGiXplYr7Qf+Rw02qVXJYIotFFjcDsrhJtUoWk8ONn8ONPljYW9KKzOOVauJg6WR7kqSPSJpXbSeFulLShZLeqbqRgu0r6XVJ16WnuF1ju6PqpooQEaskXSFpuaTVktZHxL3VdoUBQA43r1bJYYksJosbH1ncvFoli8nhBtfog4WWY3u0pNskzYiIDVX3UwTbJ0laExGPVd1LCdolfVTSDyLiI5LeltSUn4u0vYeS357sK2mcpA7bZ1XbFbDryOGmRBYDDYYsbjrkcINr9MHCKkkTMo/Hp881JdtDlQToTyPi9qr7KdBRkk6xvVTJqXzH2p5bbUuFWSlpZUR0TtpvVRKqzehTkl6OiNcjYpuk2yUdWXFP6D9yuDm1Ug5LZDFZ3PjI4ubUSllMDje4Rh8sPCppf9v72h6m5KIXd1TcUyFsW8lnjhZHxHeq7qdIETEzIsZHxCQl/6b3RUTDT/FqiYhXJa2wfUD61CclLaqwpSItl/Qx26PS7+dPqkkvytNiyOEm1Eo5LJHFIoubAVnchFopi8nhxs/h9qob6I+I2G77PEm/VHI1zWsjYmHFbRXlKElnS3ra9hPpc9+MiLsq7AkD46uSfpoeCLwk6YsV91OIiJhn+1ZJC5Rc0flxSbOr7Qr9RQ6Tw02ELEbDIovJ4iZBDjcwR0TVPQAAAAAAgAbV6B+FAAAAAAAAFWKwAAAAAAAAcmOwAAAAAAAAcmOwAAAAAAAAcmOwAAAAAAAAcmOwAAAAAAAAcmOwgEHJ9izbfzUY6pTVCwAMJuQwAFSPLEajYLAAAAAAAAByY7CAQcP2xbafs/2gpAN6ed39tr9re77txbYPt3277edtX5Z53V/afia9zeirju0P2b7H9mO2/8P2HxS1rQAwGJHDAFA9shiNqL3qBgBJsn2YpNMlHark+3KBpMd6ecvWiJhq+3xJv5B0mKTfSnrR9nclTZL0RUnTJFnSPNv/rmSY1lOd2ZK+FBHP254m6WpJxw7kdgLAYEUOA0D1yGI0KgYLGCz+SNLPI2KTJNm+o4/Xdy5/WtLCiFidvu8lSRMkHZ2u7+30+dvTGkNq1bE9WtKRkm6x3Vlj+MBsGgA0BHIYAKpHFqMhMVhAo9qS/vedzP3Ox3m+r4dIWhcRh/a3MQBoEeQwAFSPLMagwDUWMFg8IOlztkfa3k3Syf1c33+k6xtlu0PSn6TP1awTERskvWz7zyTJiUP62QMANBJyGACqRxajIXHGAgaFiFhg+2ZJT0paI+nRAVjfHEmPpE9dExGPS1IvdT4v6Qe2vyVpqKSb0tcBQNMjhwGgemQxGpUjouoeAAAAAABAg+KjEAAAAAAAIDc+CoFBy/b3JR3V7emrIuK6KvoBgFZDDuzhtq0AAABESURBVANA9chiNAI+CgEAAAAAAHLjoxAAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACA3BgsAAAAAACC3/w94XnqjJRNc6gAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 1296x216 with 3 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "suX7jeqbv4Z2"
      },
      "source": [
        "오른쪽 그림을 보면 위치 정보가 임베딩 벡터에 대해져서 원래 임베딩 벡터와는 색이 조금씩 달라진 것을 확인할 수 있습니다. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7folB27QqXR6"
      },
      "source": [
        "## Encoder\n",
        "\n",
        "이제 인코더로 데이터를 입력하기 위한 준비를 마쳤으니 인코더 구조를 알아봅시다. 글 시작 부분에 제시된 트랜스포머의 그림에서 Multi-Head Attention, Add & Norm 부분을 조금 더 자세히 그리면 아래와 같습니다.\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=10OW0QpQ5jTiW_K__FjmNZphB3fKuSacy)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HCSpqUBUTOWO"
      },
      "source": [
        "임베딩과 포지션 인코딩이 끝난 입력이 Q, K, V라는 세 입력으로 인코더로 들어갑니다. 일반적으로 Q(쿼리), K(키), V(벨류)는 다른 값이 되어야 하지만 여기선 셀프 어텐션을 수행하기 때문에 같은 값으로 입력됩니다. 어텐션은 트랜스포머 이전까지 인코더와 디코더 간에 일어나는 연산으로 개발되었으며 셀프 어텐션에 대응되는 용어로 크로스 어텐션이라고 부르기도 합니다. \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_ZLYYG2ktDCk"
      },
      "source": [
        "### Attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zOedrrabjOr1"
      },
      "source": [
        "\n",
        "어텐션의 기본적인 동작 방식은 쿼리를 가지고 키로 질의를 해서 얻은 어텐션 가중치를 벨류에 적용하는 것입니다. 보통 어텐션에서 키란 인코더에서 생성한 정보로 이 정보를 디코더에게 효율적으로 전달하는 것이 어텐션의 주 목적입니다. 왜냐하면 디코더는 좋은 결과를 생성하기 위해 인코더의 정보를 가능한 많이 활용하는 편이 도움이 되기 때문입니다. 이전 seq2seq는 인코더가 인코딩한 마지막 결과만 디코더로 전달하기 때문에 디코더 입장에서는 좋은 결과를 생성하기 힘든 것이죠. 전체적인 과정은 대략 다음처럼 진행됩니다.\n",
        "\n",
        "1. 인코더는 인코딩 중에 생성된 정보 다시말해 키를 모두 가지고 있습니다.\n",
        "2. 디코더가 정보를 생성할 때 디코더가 만든 중간 결과 즉 쿼리를 가지고 이 쿼리와 가장 관계가 높은 정보가 어떤 것인지 인코더 쪽에 물어 봅니다. \n",
        "3. 인코더는 자기가 인코딩하면서 생성한 정보(키)들 중에 요청받은 쿼리와 어떤 키가 얼마나 적합한지 가중치 계산을 합니다.\n",
        "4. 인코더는 이렇게 계산된 가중치를 디코더로 전달할 벨류라는 값에 가중합하여 디코더로 전달합니다.\n",
        "5. 디코더는 4에서 전달받은 정보와 현재 디코더가 디코딩한 정보를 합하여 최종 결과를 만들어 냅니다.\n",
        "\n",
        "이렇게 크로스 어텐션인 경우 대충 아이디어만 들어도 충분히 성능 향상에 영향을 미치겠다는 생각이 듭니다. 하지만 트랜스포머는 이 어텐션 아이디어를 인코더 또는 디코더에만 적용하는 식으로 적용범위를 넓혔습니다. 인코더는 입력 정보끼리 어텐션을 계산해서 디코더가 디코딩하기 좋은 인코딩 정보를 만들어 낼 수 있고 디코더도 마찬가지로 디코딩되는 정보 끼리 어텐션을 해서 더 좋은 결과를 만들어 낼수 있게 되는 것입니다. 이런 어텐션을 셀프 어텐션이라 합니다.\n",
        "\n",
        "셀프 어텐션은 논문에서 제시하는 다음 수식으로 수행됩니다.\n",
        "\n",
        "$$\n",
        "\\text{Attention}(Q, K, V) = \\text{softmax} \\left( \\frac{QK^T}{\\sqrt{d_k}} \\right)V\n",
        "$$\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8IRPp4bJbVGi"
      },
      "source": [
        "이렇게 셀프 어텐션이 되면 어떤 점이 좋아지는지 개념을 이해하기 위해 간단한 그림을 보도록 하겠습니다.\n",
        "\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1c_TAUg5FY3YNknOMKZMW8XGGbwTNGlzF)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HDyKVcGSb8wI"
      },
      "source": [
        "위 그림은 i love you so much라는 토큰들이 임베딩되는 상황을 나타냅니다. 우선 $W^Q_i$, $W^K_i$, $W^V_i$를 각각 곱하여 $Q$, $K$, $V$를 만들고 그렇게 만들어진 텐서는 색깔을 다르게 표현했습니다. 이후 $\\text{softmax} \\left( \\frac{QK^T}{\\sqrt{d_k}} \\right)$가 실행되면 $(n_{\\text{seq}}, n_{\\text{seq}})$인 `p_attn` 행렬이 만들어지는데 이 행렬은 각 토큰들간의 관계가 계산되도록  $W^Q_i$, $W^K_i$가 학습되길 바라는 것입니다.  $W^Q_i$, $W^K_i$가 잘 학습되어 i love you so much라는 문장에서 각 토큰간의 관계가 잘 만들어졌다고 가정하겠습니다. 그러면 love라는 토큰은 한 문장안에서 love, you 라는 토큰하고 관계가 높게 표현될 수 있습니다. 그림에서  `p_attn`을 보면 love에 해당하는 행에서 love와 you에 해당하는 열이 붉게 빛나는 것을 볼 수 있습니다. 이렇게 인코딩된 어텐션 행렬과 $V$를 곱하게 됩니다. 행렬곱은 앞에서 곱하는 행렬의 행으로 뒤에서 곱하는 행렬의 행을 선형조합하는 것과 같습니다. 최종적으로 계산된 `head_i`에서 love에 해당하는 행은 $V$ 행렬의 행 다섯 개가 선형조합된 것인데 이때 $V$에서 love와 you에 해당하는 행에 높은 가중치가 부여되어 조합된 벡터가 됩니다. \n",
        "\n",
        "이렇게 셀프 어텐션되어 출력되는 결과는 각 토큰이 `d_model`사이즈로 변환된 벡터를 가지는데 그치지 않고 각 벡터들이 서로 관계가 높은 토큰들끼리 잘 조합되어 만들어진 벡터가 되게 됩니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ayUe45lhbTjk"
      },
      "source": [
        "아래 코드는 어텐션을 구현한 것입니다. 쿼리, 키, 벨류로 무엇을 입력는가에 따라 셀프 어텐션과 크로스 어텐션으로 나눌 수 있으므로 구현 코드는 두 경우 모두 동일합니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0a6MWdYRz6Hb"
      },
      "source": [
        "def attention(query, key, value, mask=None, dropout=None):\n",
        "    \"Compute 'Scaled Dot Product Attention'\"\n",
        "    # 아래서 nbatches는 미니배치 크기, 코드에서 nbatches라는 변수명을 쓰므로\n",
        "    # 표기를 통일하기 위해 nbatches로 표기\n",
        "    # query: (nbatches, h, n_seq, d_k)\n",
        "    # key:   (nbatches, h, n_seq, d_k)\n",
        "    # value: (nbatches, h, n_seq, d_v) 인데 d_k=d_v로 두었음\n",
        "    # 이 함수는 아래쪽 MultiHeadedAttention 클래스의 foward 함수에서 호출됨\n",
        "\n",
        "    d_k = query.size(-1)\n",
        "    \n",
        "    # Scaled에 대한 여러 참고 링크들\n",
        "    # https://stats.stackexchange.com/questions/318243/variance-and-expectation-of-dot-product\n",
        "    # https://www.reddit.com/r/learnmath/comments/9gbk4q/mean_and_variance_of_dot_product_of_two_random/\n",
        "    # https://stats.stackexchange.com/questions/52646/variance-of-product-of-multiple-random-variables\n",
        "    scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(d_k)\n",
        "    # scores: (nbatches, h, n_seq, n_seq)\n",
        "    \n",
        "    if mask is not None:\n",
        "        scores = scores.masked_fill(mask == 0, -1e9)\n",
        "\n",
        "    p_attn = F.softmax(scores, dim = -1)\n",
        "\n",
        "    if dropout is not None:\n",
        "        p_attn = dropout(p_attn)\n",
        "\n",
        "    # torch.matmul(p_attn, value): (nbatches, h, n_seq, n_seq)*(nbatches, h, n_seq, d_v)\n",
        "    # = (nbatches, h, n_seq, d_v),      p_attn: (nbatches, h, n_seq, nseq)\n",
        "    return torch.matmul(p_attn, value), p_attn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "daF4FY2Anpzi"
      },
      "source": [
        "위 과정을 Scaled-dot product attention이라고 하는데 $\\sqrt{d_k}$로 나누는 부분 때문에 scaled라는 이름이 붙었습니다. $\\sqrt{d_k}$로 나누는 이유는 softmax를 취한 후 값들이 극단적으로 0과 1이 몰리게 되는 것을 막기 위함입니다. 만약 이런 현상이 일어난다면 softmax를 미분했을 때 미분 계수가 거의 0이 되어 그래디언트를 구하기 힘들어지는 문제가 생깁니다. 좀 더 자세한 설명은 아래 기술하였습니다. \n",
        "\n",
        "바쁘신 분들은 스킵해도 좋겠습니다. 🤨\n",
        "\n",
        "인코더는 N개가 누적되면서 인코더의 출력이 다시 인코더의 입력으로 들어가는 구조로 되어 있습니다. 그림을 보면 인코더가 인코딩을 출력하기 전에 레이어 정규화층을 거치게 됩니다. 즉 셀프 어텐션을 위해 입력되는 $Q$, $K$는 레이어 정규화를 거치게 되므로 $Q$, $K$의 행벡터 $\\mathbf{q}$와 $\\mathbf{k}$의 요소들은 대략적으로 평균 0, 분산 1을 따른다고 가정할 수 있습니다.\n",
        "\n",
        "$$\n",
        "q_i , k_i \\sim p(0, 1)\n",
        "$$\n",
        "\n",
        "행렬곱 $Q K^T$에서 $Q$의 길이 $d_k$인 행벡터 $\\mathbf{q}$와 $K$의 열벡터 $\\mathbf{k}$가 내적되는데 이 때 벡터의 요소가 서로 독립이고 위처럼 평균 0, 분산 1을 따른다고 가정하면 $\\mathbf{q} \\cdot \\mathbf{k}$는 평균 0, 분산 $d_k$를 가지게 됩니다. 이는 다음처럼 보일 수 있습니다.\n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\mathbb{E}\\left[\\mathbf{q} \\cdot \\mathbf{k}\\right] \n",
        "&= \\mathbb{E}\\left[ \\sum_{i=1}^{d_k} q_i k_i \\right] \\\\\n",
        "&= \\sum_{i=1}^{d_k} \\mathbb{E} \\left[ q_i k_i \\right]\n",
        "\\end{aligned} \\tag{1}\n",
        "$$\n",
        "\n",
        "위 식(1)에서 기댓값의 성질 $\\mathbb{E}[X+Y] = \\mathbb{E}[X]+\\mathbb{E}[Y]$가 사용되었습니다. 이제 시그마 안쪽 $\\mathbb{E} \\left[ q_i k_i \\right]$를 전개해보면\n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\mathbb{E}[q_i k_i] \n",
        "&= \\sum_{q_i} \\sum_{k_i} q_i  k_i \\ p(q_i, k_i) \\\\\n",
        "&= \\sum_{q_i} \\sum_{k_i} q_i  k_i \\ p(q_i) p(k_i)  \\quad \\because q_i \\text{ and } k_i \\text{ are independent}  \\\\\n",
        "&= \\sum_{q_i} q_i \\, p(q_i) \\sum_{k_i} k_i \\, p(k_i) \\\\\n",
        "&= \\mathbb{E}[q_i] \\, \\mathbb{E}[k_i] = 0 \\quad \\because \\mathbb{E}[q_i] = \\mathbb{E}[k_i] = 0\n",
        "\\end{aligned} \\tag{2}\n",
        "$$\n",
        "\n",
        "식(2)의 결과를 식(1)에 대입하면 \n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\mathbb{E}\\left[\\mathbf{q} \\cdot \\mathbf{k}\\right] \n",
        "&= \\mathbb{E}\\left[ \\sum_{i=1}^{d_k} q_i k_i \\right] \\\\\n",
        "&= \\sum_{i=1}^{d_k} \\mathbb{E} \\left[ q_i k_i \\right] \\\\\n",
        "&= \\sum_{i=1}^{d_k} 0 = 0\n",
        "\\end{aligned}\n",
        "$$\n",
        "\n",
        "이 되게 됩니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uUr0Ihbhj091"
      },
      "source": [
        "분산도 위와 비슷한 과정으로 $d_k$가 됨을 보일 수 있습니다.\n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\text{Var}\\left[\\mathbf{q} \\cdot \\mathbf{k}\\right] \n",
        "&= \\text{Var}\\left[\\sum_{i=1}^{d_k} q_i k_i \\right]  \\\\\n",
        "&= \\sum_{i=1}^{d_k} \\text{Var} \\left[ q_i k_i \\right]\n",
        "\\end{aligned} \\tag{3}\n",
        "$$\n",
        "\n",
        "식(3)에서 독립인 확률변수에 대한 분산의 성질 $\\text{Var}[X+Y] = \\text{Var}[X] + \\text{Var}[Y]$가 사용되었습니다. \n",
        "\n",
        "이제 기댓값처럼 시그마 안쪽을 전개하면\n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\text{Var}\\left[ q_i k_i\\right] \n",
        "&= \\mathbb{E}[q_i^2 k_i^2]-\\left(\\mathbb{E}[q_i k_i ]\\right)^2 \\\\[5pt]\n",
        "&= \\mathbb{E}\\left[q_i^2\\right] \\mathbb{E}\\left[ k_i^2 \\right] - \\mathbb{E}\\left[q_i\\right]^2 \\mathbb{E} [k_i]^2 \\\\[5pt]\n",
        "&=\\left( \\text{Var}[q_i]+\\mathbb{E}[q_i]^2 \\right) \\left( \\text{Var}[k_i]+\\mathbb{E}[k_i]^2 \\right) - \\mathbb{E}\\left[q_i\\right]^2 \\mathbb{E} [k_i]^2 \\\\[5pt]\n",
        "&= \\text{Var}[q_i] \\text{Var}[k_i] + \\text{Var}[q_i] \\mathbb{E} [k_i]^2 + \\mathbb{E} [q_i]^2 \\text{Var}[k_i] + \\mathbb{E}\\left[q_i\\right]^2 \\mathbb{E} [k_i]^2 - \\mathbb{E}\\left[q_i\\right]^2 \\mathbb{E} [k_i]^2 \\\\[5pt]\n",
        "&= \\text{Var}[q_i] \\text{Var}[k_i] + \\text{Var}[q_i] \\mathbb{E} [k_i]^2 + \\mathbb{E} [q_i]^2 \\text{Var}[k_i]\n",
        "\\end{aligned}\n",
        "$$\n",
        "\n",
        "여기서 각 요소들은 $ \\mathbb{E} [q_i] = \\mathbb{E} [k_i] = 0 $ , $\\text{Var}[q_i] = \\text{Var}[k_i] = 1$이므로\n",
        "\n",
        "$$\n",
        "\\text{Var}\\left[ q_i k_i\\right] = \\text{Var}[q_i] \\text{Var}[k_i] = 1 \\tag{4}\n",
        "$$\n",
        "\n",
        "이 되고 식(4)를 윗 식에 대입하면\n",
        "\n",
        "$$\n",
        "\\begin{aligned}\n",
        "\\text{Var}\\left[\\mathbf{q} \\cdot \\mathbf{k}\\right] \n",
        "&= \\text{Var}\\left[\\sum_{i=1}^{d_k} q_i k_i \\right]  \\\\\n",
        "&= \\sum_{i=1}^{d_k} \\text{Var} \\left[ q_i k_i \\right] \\\\\n",
        "&= \\sum_{i=1}^{d_k}  \\text{Var}[q_i] \\text{Var}[k_i] \\\\\n",
        "&= \\sum_{i=1}^{d_k} 1 = d_k\n",
        "\\end{aligned}\n",
        "$$\n",
        "\n",
        "이 됨을 알 수 있습니다.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yYtrEdRMs1AG"
      },
      "source": [
        "두 행렬을 곱한 행렬의 요소 $\\left(Q K^T \\right)_{ij}$는 $\\mathbf{q}_i \\cdot \\mathbf{k}_j$로 계산되는데 앞선 계산에서 본것 처럼 분산이 $d_k$가 되므로 $d_k$가 커질 수록 $\\left(Q K^T \\right)_{ij}$의 값들의 차이도 커지게 됩니다. $\\left(Q K^T \\right)_{ij}$들의 분산이 커지면 $ Q K^T  $가 소프트맥스 함수를 거친 후 요소의 값은 1에 아주 가깝거나 0에 아주 가까운 값들로 구성되게 될 것입니다. 한편 소프트맥스 $\\mathbf{s}(\\mathbf{z})$의 미분은 다음과 같습니다.\n",
        "\n",
        "$$\n",
        "\\frac{\\partial}{\\partial z_j} s_i(\\mathbf{z})=\\begin{cases}\n",
        "s_j(\\mathbf{z}) (1-s_j(\\mathbf{z})), & \\mbox{if } i = j \\\\\n",
        "-s_i(\\mathbf{z})s_j(\\mathbf{z}), & \\mbox{if } i \\neq j\n",
        "\\end{cases}\n",
        "$$\n",
        "\n",
        "위 식을 보면 소프트맥스 함수의 미분계수는 결국 소프트맥스 함수의 요소들 끼리 곱으로 결정되는데 이 요소들이 0에 아주 가깝거나 1에 아주 가깝게 되면 미분계수가 매우 작아지는 문제가 생기게 됩니다. 결과적으로 소프트맥스 함수를 백워드 패스할 때 미분계수가 매우 작아져서 학습에 문제가 생기게 되겠죠. 그래서 앞서 구한 $\\left(Q K^T \\right)_{ij}$의 표준편차인 $\\sqrt{d_{\\text{k}}}$로 나눠서 평균과 분산을 0, 1에 가깝게 만들려하는 것입니다. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-zv0_nz0t3vk"
      },
      "source": [
        "#### Multi-Head Attention\n",
        "\n",
        "앞 절에서 이야기한 어텐션을 한번만 하는 것이 아니라 $h$번하고 그 결과로 나온 $(n_{seq}, d_{v})$인 결과 $h$개를 $d_v$ 방향으로 `concat`시켜 최종적으로 $(n_{seq}, hd_{v})$ 만들게 됩니다. 위 인코더 그림에서 이를 표현하고 있습니다. \n",
        "\n",
        "그리고 이 $(n_{seq}, hd_{v})$를 $(hd_{v}, d_{model})$인 $W^o$와 곱하여 결과를 $(n_{seq}, d_{model})$로 만들게 됩니다. 아래 그 코드가 있는데 매우 교묘하게 코딩되어 있어 주의깊게 볼 필요가 있습니다. 우선 코드에 적혀있는 주석을 읽고 코드를 이해해봅시다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AQTK7gaSzmVa"
      },
      "source": [
        "def clones(module, N):\n",
        "    \"Produce N identical layers.\"\n",
        "    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zUsjvdDQz8S_"
      },
      "source": [
        "class MultiHeadedAttention(nn.Module):\n",
        "    def __init__(self, h, d_model, dropout=0.1):\n",
        "        \"Take in model size and number of heads.\"\n",
        "        super(MultiHeadedAttention, self).__init__()\n",
        "        assert d_model % h == 0\n",
        "        # We assume d_v always equals d_k\n",
        "        self.d_k = d_model // h  # Q, K의 차원\n",
        "        self.h = h\n",
        "        \n",
        "        # Wq, Wk, Wv and Wo\n",
        "        # Wq, Wk, Wv를 각각 h개 만들지 않고 \n",
        "        # Wq, Wk, Wv를 d_model의 1/h 크기로 만듬\n",
        "        self.linears = clones(nn.Linear(d_model, d_model), 4) \n",
        "        \n",
        "        self.attn = None\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        \n",
        "    def forward(self, query, key, value, mask=None):\n",
        "        # query, key, value: (n_seq, d_model)\n",
        "        \"Implements Figure 2\"\n",
        "        if mask is not None:\n",
        "            # EncoderLayer에서 호출될 때\n",
        "            # mask는 src_mask: (nbatches, 1, n_seq_src)\n",
        "            # DecoderLayer에서 호출될 때\n",
        "            # self_attn으로 호출되면 mask는 tgt_mask: (nbatches, n_seq_trg, n_seq_trg)\n",
        "            # src_attn(cross_attn)으로 호출되며 mask는 src_mask: (nbatches, 1, n_seq_src)\n",
        "            \n",
        "            # Same mask applied to all h heads.\n",
        "            mask = mask.unsqueeze(1)\n",
        "        nbatches = query.size(0)\n",
        "        \n",
        "        # 1) Do all the linear projections in batch from d_model => h x d_k \n",
        "        # self.linears는 요소 네갠데 (query, key, value)와 짝을 맞춰서\n",
        "        # 루프는 총 3번 돌아감\n",
        "        query, key, value = \\\n",
        "            [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "             for l, x in zip(self.linears, (query, key, value))]\n",
        "        # 이 라인이 실행되면 query, key, value는 각각\n",
        "        # (nbatches, h, n_seq, d_k) 가 됨\n",
        "        \n",
        "        # 2) Apply attention on all the projected vectors in batch. \n",
        "        x, self.attn = attention(query, key, value, mask=mask, \n",
        "                                 dropout=self.dropout)\n",
        "        # x: (nbatches, h, n_seq, d_v),  self.attn: (nbatches, h, n_seq, n_seq)\n",
        "\n",
        "        # 3) \"Concat\" using a view and apply a final linear. \n",
        "        x = x.transpose(1, 2).contiguous() \\\n",
        "             .view(nbatches, -1, self.h * self.d_k)\n",
        "        # x: (nbatches, n_seq, h*d_k) 여기서 h*d_k=d_model\n",
        "\n",
        "        # 4) matmul x and Wo -> (nbatches, n_seq, d_model)\n",
        "        return self.linears[-1](x)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_f04zeIo4GD"
      },
      "source": [
        "위 코드에서 가장 난해한 코드는 주석 1) 부분입니다. \n",
        "\n",
        "```python\n",
        "# 1) Do all the linear projections in batch from d_model => h x d_k \n",
        "# self.linears는 요소 네갠데 (query, key, value)와 짝을 맞춰서\n",
        "# 루프는 총 3번 돌아감\n",
        "query, key, value = \\\n",
        "    [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)\n",
        "        for l, x in zip(self.linears, (query, key, value))]\n",
        "# 이 라인이 실행되면 query, key, value는 각각\n",
        "# (nbatches, h, n_seq, d_k) 가 됨\n",
        "```\n",
        "\n",
        "1)에서 $(n_{seq}, d_{k})$ 모양의 쿼리, 키, 벨류 $h$개를 동시에 만들고 있습니다. 쿼리를 만드는 과정을 아래 그림으로 자세히 나타냈습니다. 나머지 키와 벨류에 대해서도 동일한 연산이 수행됩니다.\n",
        "  \n",
        "![picture](https://drive.google.com/uc?id=10JMUOLLUgyPJwqQvVyWG2J9MmUIB_gEo)\n",
        "\n",
        "그림 상단에 코드가 주석 1)에 해당하는 코드이며 코드 부분이 진행되는 순서대로 텐서를 표현하였습니다. 1)에서 결과를 앞서 알아본 `attention` 함수로 입력하여 어텐션 결과와 어텐션 맵을 돌려 받는 부분이 2)입니다. 그런 다음 3)에서 $h$개 어텐션 결과를 `concat`하고 이를 4)에서 $W^O$와 곱하게 됩니다.\n",
        "\n",
        "마지막 완전연결층의 가중치 $W^{O}$는 $(hd_v , d_{\\text{model}})$ 크기를 가지므로 이 층의 입력차원은 $hd_v$, 출력차원은 $d_{\\text{model}}$이 됩니다. 입력으로 들어오는 멀티 헤드 어텐션의 출력은 $(n_{\\text{seq}} , hd_k)$입니다. 셀프 어텐션에서 $d_k=d_v$이므로 마지막 완전연결층을 통과하면 출력은 $(n_{\\text{seq}},d_{\\text{model}})$이 됩니다. \n",
        "\n",
        "위 멀티헤드어텐션 `forward()`에서 마지막 줄입니다.\n",
        "\n",
        "```python\n",
        "# matmul x and Wo -> (nbatches, n_seq, d_model)\n",
        "return self.linears[-1](x)\n",
        "```\n",
        "\n",
        "3)과 4) 과정을 아래 그림으로 나타내었습니다. 인코더의 전체 그림과 함께 비교하면 이해하기가 훨씬 쉬워집니다.\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=10KXLvpL_UU1NbWQT5sTlaiF5EHtQdhek)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Nqd8GdVgtUTO"
      },
      "source": [
        "### Normalization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dd4oK0hZCjKv"
      },
      "source": [
        "앞선 단계를 거쳐 얻은 결과는 모양이 $(\\text{nbatches}, n_{\\text{seq}}, d_{\\text{model}})$이고 인코더의 입력도 모양이 $(\\text{nbatches}, n_{\\text{seq}}, d_{\\text{model}})$이므로 서로 요소끼리 더할 수 있습니다. 이렇게 더하는 과정을 스킵커넥션skip-connection이라고 합니다. 스킵 커넥션은 ResNet[[6](https://arxiv.org/abs/1512.03385)]에서 소개된 기법으로 이 후 모델에서는 거의 필수로 사용되는 기법입니다. 이렇게 스킵커넥션까지 거친 텐서를 정규화하게 되는데 여기서는 보편적으로 사용하는 배치정규화를 쓰지 않고 레이어 정규화를 사용합니다. \n",
        "\n",
        "배치 정규화와 레이어 정규화의 차이를 알아보기 위해 구체적인 예를 들어 보겠습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gkZRxTeu7RwM"
      },
      "source": [
        "#### Batch Normalization\n",
        "\n",
        "배치 정규화에 대한 수식은 아래와 같습니다.\n",
        "\n",
        "$$\n",
        "y = \\frac{x - \\mathbb{E}[x]}{\\sqrt{\\text{Var}[x]+\\epsilon}} * \\gamma + \\beta\n",
        "$$\n",
        "\n",
        "식만보면 입력 배치 $x$에 대한 평균과 분산을 구해서 입력들에 대해서 표준화 작업을 하는 것입니다. 그런데 PyTorch 구현체를 보면 상황이 조금 더 복잡합니다.\n",
        "\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1BdMW-mYOwNea6yH6wyk63ZPCBSsHLGij)\n",
        "\n",
        "위 그림은 데이터 하나가 요소 4개짜리 벡터이고 미니배치 크기가 3인 상황을 나타내었습니다. 이 경우 입력 텐서는 (3,4) 또는 (3,1,4)입니다. 트랜스포머로 입력되는 상황을 고려하면 (3,1,4)가 적합한데 각 차원은 (nbatches, n_seq, embedding)으로 샘플은 세 개, 각 샘플당 토큰은 한 개, 각 토큰은 숫자 네 개로 임베딩되어 있는 상황입니다. 노란색, 주황색, 보라색 데이터가 각각 네트워크에 입력되고 길이 여섯개짜리 벡터로 변환되면 출력은 (3,1,6)이 됩니다.   \n",
        "\n",
        "이 출력에 배치 정규화를 적용하면 `(N, C, L)`에서 `C`에 대해 데이터를 구분해서 `C`별로 평균과 분산을 구하게 됩니다. 이것은 이미지에 대해서 배치 정규화를 적용하면 이미지의 채널별로 평균과 분산을 구하는 것과 동일한 방식입니다.\n",
        "\n",
        "실제로 그렇게 작동하는지 아래 pytorch 코드와 직접 만든 배치 정규화 코드의 결과를 비교하였습니다. \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FWbJcixn8iqt",
        "outputId": "dcb8eec7-071d-4d25-cade-b6f3081a52ae"
      },
      "source": [
        "torch.manual_seed(0)\n",
        "\n",
        "# 배치사이즈 3, 단어 한개로 구성된 문장, feature 4\n",
        "x = torch.randn(3, 1, 4)\n",
        "print('\\nINPUT', x.shape)\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION', x.shape)\n",
        "print(x)\n",
        "\n",
        "#===================\n",
        "# batch norm.\n",
        "# pytorch \n",
        "bnorm = nn.BatchNorm1d(x.shape[1], eps=0., momentum=0.)\n",
        "print('\\nPYTORCH BATCH NORM. 1D')\n",
        "print(bnorm(x))\n",
        "\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=(0,2)).reshape(-1,1)\n",
        "var = x.var(axis=(0,2), unbiased=False).reshape(-1,1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY BATCH NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)\n",
        "#===================\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "INPUT torch.Size([3, 1, 4])\n",
            "tensor([[[ 1.5410, -0.2934, -2.1788,  0.5684]],\n",
            "\n",
            "        [[-1.0845, -1.3986,  0.4033,  0.8380]],\n",
            "\n",
            "        [[-0.7193, -0.4033, -0.5966,  0.1820]]])\n",
            "\n",
            "ACTIVATION torch.Size([3, 1, 6])\n",
            "tensor([[[-1.6115,  0.2043,  0.3828, -0.4272, -0.1355, -0.9343]],\n",
            "\n",
            "        [[-0.4144,  0.7838,  0.4598,  0.5056,  0.1713,  1.1928]],\n",
            "\n",
            "        [[-0.5397,  0.4543,  0.7689,  0.4446,  0.1475,  0.2965]]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "\n",
            "PYTORCH BATCH NORM. 1D\n",
            "tensor([[[-2.5974,  0.1627,  0.4341, -0.7971, -0.3537, -1.5679]],\n",
            "\n",
            "        [[-0.7777,  1.0436,  0.5512,  0.6207,  0.1127,  1.6653]],\n",
            "\n",
            "        [[-0.9681,  0.5429,  1.0210,  0.5281,  0.0764,  0.3030]]],\n",
            "       grad_fn=<NativeBatchNormBackward0>)\n",
            "\n",
            "MY BATCH NORM.\n",
            "mean shape: torch.Size([1, 1])\n",
            "var shape: torch.Size([1, 1])\n",
            "tensor([[[-2.5974,  0.1627,  0.4341, -0.7971, -0.3537, -1.5679]],\n",
            "\n",
            "        [[-0.7777,  1.0436,  0.5512,  0.6207,  0.1127,  1.6653]],\n",
            "\n",
            "        [[-0.9681,  0.5429,  1.0210,  0.5281,  0.0764,  0.3030]]],\n",
            "       grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H2ekyQ_lieG8"
      },
      "source": [
        "두 경우 모두 출력이 동일하고 평균과 분산은 숫자 하나라는 사실을 알 수 있습니다. (이 예에서 C=1이기 때문입니다.)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oC8knQQGtzUQ"
      },
      "source": [
        "![picture](https://drive.google.com/uc?id=1RBZJ6wWqq5PABCW8Yw5-Ny9yupcxNRqA)\n",
        "\n",
        "이번에는 데이터가 위처럼 입력된다고 생각해봅시다. 이 상황은 샘플 세 개가 입력되는데 전과는 다르게 각 데이터는 토큰이 두 개로 구성된 문장이고 토큰 하나는 숫자 네 개짜리 벡터로 표현된 상황입니다. 이번 예에서는 C=2이기 때문에 각 샘플의 첫번째 단어끼리 평균과 분산을 구하고, 두번째 단어끼리 평균과 분산을 구해서 각 단어에 대해 표준화를 수행할 것입니다.\n",
        "\n",
        "아래 코드를 보면 그 사실을 확인할 수 있습니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PlNATxHvJcbc",
        "outputId": "70aa81d7-8e73-4c1d-de0e-e11de4eda5f3"
      },
      "source": [
        "# 배치사이즈 3, 단어 두 개로 구성된 문장, feature 4\n",
        "x = torch.randn(3, 2, 4)\n",
        "print('\\nINPUT', x.shape)\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION', x.shape)\n",
        "print(x)\n",
        "\n",
        "#===================\n",
        "# batch norm.\n",
        "# pytorch \n",
        "bnorm = nn.BatchNorm1d(x.shape[1], eps=0., momentum=0.)\n",
        "print('\\nPYTORCH BATCH NORM. 1D')\n",
        "print(bnorm(x))\n",
        "\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=(0,2)).reshape(-1,1)\n",
        "var = x.var(axis=(0,2), unbiased=False).reshape(-1,1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY BATCH NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)\n",
        "#==================="
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "INPUT torch.Size([3, 2, 4])\n",
            "tensor([[[ 5.4329e-01, -3.9516e-01,  2.0553e-01, -4.5033e-01],\n",
            "         [-5.7308e-01, -5.5536e-01,  5.9432e-01,  1.5419e+00]],\n",
            "\n",
            "        [[-1.0925e+00, -8.5194e-02, -9.3348e-02,  6.8705e-01],\n",
            "         [-8.3832e-01,  8.9182e-04,  8.4189e-01, -4.0003e-01]],\n",
            "\n",
            "        [[ 6.2114e-01,  6.3818e-01, -2.4600e-01,  2.3025e+00],\n",
            "         [-1.8817e+00, -4.9727e-02, -1.0450e+00, -9.5650e-01]]])\n",
            "\n",
            "ACTIVATION torch.Size([3, 2, 6])\n",
            "tensor([[[-0.0636, -0.0346,  0.0337, -0.0617, -0.7393,  0.5536],\n",
            "         [ 0.8464,  1.3011,  0.0941,  0.5593,  0.2273,  0.2352]],\n",
            "\n",
            "        [[ 0.6898,  0.7568,  0.4834,  0.5283,  0.2946, -0.0747],\n",
            "         [ 0.6558,  0.7065, -0.0185,  0.2920, -0.5097, -0.1648]],\n",
            "\n",
            "        [[-0.2619,  0.3526,  0.0770, -0.2704,  0.9377,  0.2629],\n",
            "         [ 0.6593,  0.0531,  1.1190,  0.7146,  0.0106, -0.2787]]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "\n",
            "PYTORCH BATCH NORM. 1D\n",
            "tensor([[[-0.6198, -0.5497, -0.3845, -0.6153, -2.2559,  0.8745],\n",
            "         [ 1.0204,  1.9767, -0.5619,  0.4166, -0.2817, -0.2651]],\n",
            "\n",
            "        [[ 1.2042,  1.3664,  0.7044,  0.8132,  0.2472, -0.6468],\n",
            "         [ 0.6196,  0.7262, -0.7988, -0.1456, -1.8319, -1.1065]],\n",
            "\n",
            "        [[-1.1000,  0.3878, -0.2796, -1.1207,  1.8043,  0.1706],\n",
            "         [ 0.6268, -0.6482,  1.5938,  0.7432, -0.7376, -1.3461]]],\n",
            "       grad_fn=<NativeBatchNormBackward0>)\n",
            "\n",
            "MY BATCH NORM.\n",
            "mean shape: torch.Size([2, 1])\n",
            "var shape: torch.Size([2, 1])\n",
            "tensor([[[-0.6198, -0.5497, -0.3845, -0.6153, -2.2559,  0.8745],\n",
            "         [ 1.0204,  1.9767, -0.5619,  0.4166, -0.2817, -0.2651]],\n",
            "\n",
            "        [[ 1.2042,  1.3664,  0.7044,  0.8132,  0.2472, -0.6468],\n",
            "         [ 0.6196,  0.7262, -0.7988, -0.1456, -1.8319, -1.1065]],\n",
            "\n",
            "        [[-1.1000,  0.3878, -0.2796, -1.1207,  1.8043,  0.1706],\n",
            "         [ 0.6268, -0.6482,  1.5938,  0.7432, -0.7376, -1.3461]]],\n",
            "       grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zmp1erKejHfB"
      },
      "source": [
        "직접 구현한 코드를 보면 평균과 분산이 숫자 두개로 구성된 것을 알 수 있습니다. 이 두 경우 모두 구해진 평균과 분산에서 미니배치의 차원인 3이 사라진 것을 확인할 수 있습니다. 미니배치 내 샘플들끼리 평균과 분산을 구했기 때문입니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y0Bjbgh9IkDe"
      },
      "source": [
        "만약 첫번째 예와 같은 상황에서 입력을 (3,4)로 했다면 평균과 분산을 구하는 방식이 조금 달라지게 됩니다.\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1hRLd1iV5wlLDF__VD4HtozaKnNgdgQYv)\n",
        "\n",
        "길이 6짜리 벡터를 더해서 평균벡터와 분산벡터를 구하는 상황입니다. 이런 식으로 동작하기 위해서 `(N,L)`에서 `L`을 `BatchNorm1d()`에 넘기면 됩니다. 아래 실험 코드가 있습니다. 이런 경우를 pytorch 문서에서는 'Temporal Batch Normalization'이라고 부릅니다. 미니배치에 있는 샘플들을 대상으로 `C`차원에 대해서 각각 구분해서 평균과 분산을 구해야 하는데 샘플에 `C`차원이 없어서 이렇게 이름 붙인것 같습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_3ikbGuO_IcG",
        "outputId": "61e947ce-8be5-4647-a48c-01327ca506d8"
      },
      "source": [
        "# 배치사이즈 3, 단어 한개로 구성된 문장, feature 4, 여기선 C가 없음\n",
        "x = torch.randn(3, 4)\n",
        "print('\\nINPUT', x.shape)\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION', x.shape)\n",
        "print(x)\n",
        "\n",
        "\n",
        "# pytorch \n",
        "bnorm = nn.BatchNorm1d(x.shape[1], eps=0., momentum=0.)\n",
        "print('\\nPYTORCH BATCH NORM. 1D')\n",
        "print(bnorm(x))\n",
        "\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=0)\n",
        "var = x.var(axis=0, unbiased=False)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY BATCH NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "INPUT torch.Size([3, 4])\n",
            "tensor([[ 1.8929,  3.1110, -0.4584, -0.3360],\n",
            "        [-1.5700,  1.2315,  1.3946,  1.1711],\n",
            "        [ 0.4335, -1.7343, -1.3360,  0.8871]])\n",
            "\n",
            "ACTIVATION torch.Size([3, 6])\n",
            "tensor([[-0.8547,  0.8803, -0.9615,  0.1794, -0.1448, -1.3131],\n",
            "        [-0.7770,  0.4903, -1.8076, -0.7847, -0.0231,  0.8162],\n",
            "        [ 1.2591, -1.1274,  0.3239,  0.6613,  0.3677, -1.1282]],\n",
            "       grad_fn=<AddmmBackward0>)\n",
            "\n",
            "PYTORCH BATCH NORM. 1D\n",
            "tensor([[-0.7464,  0.9195, -0.1671,  0.2673, -0.9670, -0.8009],\n",
            "        [-0.6670,  0.4708, -1.1326, -1.3363, -0.4102,  1.4099],\n",
            "        [ 1.4135, -1.3903,  1.2997,  1.0690,  1.3772, -0.6089]],\n",
            "       grad_fn=<NativeBatchNormBackward0>)\n",
            "\n",
            "MY BATCH NORM.\n",
            "mean shape: torch.Size([6])\n",
            "var shape: torch.Size([6])\n",
            "tensor([[-0.7464,  0.9195, -0.1671,  0.2673, -0.9670, -0.8009],\n",
            "        [-0.6670,  0.4708, -1.1326, -1.3363, -0.4102,  1.4099],\n",
            "        [ 1.4135, -1.3903,  1.2997,  1.0690,  1.3772, -0.6089]],\n",
            "       grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TqfuardnJroG"
      },
      "source": [
        "출력 샘플 하나가 길이 6짜리 벡터이므로 이런식으로 동작하는것이 좀 더 일반적인 경우라 할 수 있습니다. 버클리 대학교 인공지능 수업 [[cs182](https://cs182sp21.github.io/)]에서도 배치 정규화를 정확히 이렇게 설명하고 있습니다. 아래는 설명을 발췌한 슬라이드인데 수식을 보면 그냥 미니배치 안의 샘플들끼리 평균을 구하고 표준편차를 구하는 것을 확인할 수 있습니다.\n",
        "\n",
        "\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1YYnMN9hD6xJY5zNFvEG1nBslg4caMsVm)\n",
        "\n",
        "\n",
        "이제 배치놈에 대해서 자세히 알아봤으니 레이어 정규화에 대해서 알아볼 차례입니다.\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2UaGFngvorJh"
      },
      "source": [
        "#### Layer Normalization\n",
        "\n",
        "레이어 정규화는 배치 정규화보다 개념이 간단합니다. 그냥 출력층 모양에 맞춰서 평균과 분산을 구하면 됩니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SBNh-Ryctc4N"
      },
      "source": [
        "![picture](https://drive.google.com/uc?id=1jBGGDQPHCWppGLMuY0Nw0J48AYQ6SEHe)\n",
        "\n",
        "위 그림이라면 출력층이 숫자 여섯개인 벡터이므로 출력도 여섯개씩 묶어서 평균과 분산을 구하면 됩니다. 아래 코드에 실험결과가 있습니다.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PSYqF1MflWc6",
        "outputId": "e281ec2b-ceaf-4d79-bfab-3b39d90a7209"
      },
      "source": [
        "torch.manual_seed(0)\n",
        "\n",
        "# 배치사이즈 3, 단어 한개로 구성된 문장, feature 4\n",
        "x = torch.randn(3, 1, 4)\n",
        "print('\\nINPUT', x.shape)\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION', x.shape)\n",
        "print(x)\n",
        "\n",
        "#===================\n",
        "# layer norm.\n",
        "# pytorch \n",
        "# 정규화할 shape을 출력레이어 모양에 맞게 설정한다.\n",
        "lnorm = nn.LayerNorm(x.shape[2], eps=0.)\n",
        "print('\\nPYTORCH LAYER NORM.')\n",
        "print(lnorm(x))\n",
        "\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=-1).reshape(-1,1,1)\n",
        "var = x.var(axis=-1, unbiased=False).reshape(-1,1,1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY LAYER NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)\n",
        "#===================\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "INPUT torch.Size([3, 1, 4])\n",
            "tensor([[[ 1.5410, -0.2934, -2.1788,  0.5684]],\n",
            "\n",
            "        [[-1.0845, -1.3986,  0.4033,  0.8380]],\n",
            "\n",
            "        [[-0.7193, -0.4033, -0.5966,  0.1820]]])\n",
            "\n",
            "ACTIVATION torch.Size([3, 1, 6])\n",
            "tensor([[[-1.6115,  0.2043,  0.3828, -0.4272, -0.1355, -0.9343]],\n",
            "\n",
            "        [[-0.4144,  0.7838,  0.4598,  0.5056,  0.1713,  1.1928]],\n",
            "\n",
            "        [[-0.5397,  0.4543,  0.7689,  0.4446,  0.1475,  0.2965]]],\n",
            "       grad_fn=<AddBackward0>)\n",
            "\n",
            "PYTORCH LAYER NORM.\n",
            "tensor([[[-1.7441,  0.9143,  1.1757, -0.0102,  0.4169, -0.7525]],\n",
            "\n",
            "        [[-1.7336,  0.6699,  0.0201,  0.1118, -0.5586,  1.4903]],\n",
            "\n",
            "        [[-1.9794,  0.4748,  1.2514,  0.4509, -0.2829,  0.0851]]],\n",
            "       grad_fn=<NativeLayerNormBackward0>)\n",
            "\n",
            "MY LAYER NORM.\n",
            "mean shape: torch.Size([3, 1, 1])\n",
            "var shape: torch.Size([3, 1, 1])\n",
            "tensor([[[-1.7441,  0.9143,  1.1757, -0.0102,  0.4169, -0.7525]],\n",
            "\n",
            "        [[-1.7336,  0.6699,  0.0201,  0.1118, -0.5586,  1.4903]],\n",
            "\n",
            "        [[-1.9794,  0.4748,  1.2514,  0.4509, -0.2829,  0.0851]]],\n",
            "       grad_fn=<AddBackward0>)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RkBUeGbRpLCx"
      },
      "source": [
        "그림처럼 출력을 층 모양에 맞게 묶어서 평균과 분산을 구하면 숫자가 세 개 구해지게 됩니다. 위 코드의 출력에서 마지막에 구한 `mean`, `var`의 `shape`을 확인해보면 숫자 세 개로 구성된 것을 알 수 있습니다. 적절한 자리에 빼고, 나누고 하기 위해 `reshape`을 적당히 해주면 pytorch 결과와 똑같은 결과를 만들 수 있습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ElOAxxKZSgpm"
      },
      "source": [
        "레이어 정규화를 두번째 예에 적용해보면 다음처럼 각 토큰별로 계산하게 됩니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Xl3ZJIt1tcxx"
      },
      "source": [
        "![picture](https://drive.google.com/uc?id=1TXKgQMek9cq4a0-fS_krZ1U3lTIZO7mO)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6tZgt1i4nBGF"
      },
      "source": [
        "# 배치사이즈 3, 단어 두 개로 구성된 문장, feature 4\n",
        "x = torch.randn(3, 2, 4)\n",
        "print('\\nINPUT', x.shape)\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION', x.shape)\n",
        "print(x)\n",
        "\n",
        "# pytorch \n",
        "lnorm = nn.LayerNorm(x.shape[2], eps=0.)\n",
        "print('\\npytorch layer norm.')\n",
        "print(lnorm(x))\n",
        "\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=-1).unsqueeze(-1)\n",
        "var = x.var(axis=-1, unbiased=False).unsqueeze(-1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nmy layer norm.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5L35Mg9r98c"
      },
      "source": [
        "NLP에서는 시퀀스의 길이가 샘플마다 모두 달라서 패딩을 하게 되고 또 시퀀스 길이가 대체로 길어서 미니배치 사이즈를 크게 가져가지 못하기 때문에 배치 정규화를 사용하기 부적합하다고 판단하고 레이어 정규화를 사용했습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f6IND4rKzp_O"
      },
      "source": [
        "class LayerNorm(nn.Module):\n",
        "    \"Construct a layernorm module (See citation for details).\"\n",
        "    def __init__(self, features, eps=1e-6):\n",
        "        super(LayerNorm, self).__init__()\n",
        "        self.a_2 = nn.Parameter(torch.ones(features))\n",
        "        self.b_2 = nn.Parameter(torch.zeros(features))\n",
        "        self.eps = eps\n",
        "\n",
        "    def forward(self, x):\n",
        "        mean = x.mean(-1, keepdim=True)\n",
        "\n",
        "        # torch.nn.LayerNorm()과 맞추기 위해 unbiased=False로 수정\n",
        "        std = x.std(-1, unbiased=False, keepdim=True)\n",
        "        # std = x.std(-1, keepdim=True)\n",
        "        \n",
        "        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A4uqOeU7spJ8"
      },
      "source": [
        "위에서 만든 레이어 정규화 층을 테스트 해봅시다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rWZBASxl6VTt"
      },
      "source": [
        "torch.manual_seed(0)\n",
        "\n",
        "# 배치사이즈 3, 단어 한개로 구성된 문장, feature 4\n",
        "x = torch.randn(3, 1, 4)\n",
        "print('\\nINPUT')\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION')\n",
        "print(x.shape)\n",
        "\n",
        "#===================\n",
        "# layer norm.\n",
        "# pytorch \n",
        "lnorm2 = nn.LayerNorm(x.shape[-1], eps=0.)\n",
        "print('\\nPYTORCH LAYER NORM.')\n",
        "print(lnorm2(x))\n",
        "\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=-1).reshape(-1,1,1)\n",
        "var = x.var(axis=-1, unbiased=False).reshape(-1,1,1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY LAYER NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)\n",
        "\n",
        "# 레이어 층\n",
        "lnorm = LayerNorm(x.shape[-1], eps=0.)\n",
        "print('\\nclass LayerNorm')\n",
        "print(lnorm(x))\n",
        "\n",
        "\n",
        "#===================\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-DfhA-Hatc6E"
      },
      "source": [
        "pytorch의 `nn.LayerNorm()`과 앞서 직접 만들어본 레이어 정규화 결과와 클래스로 제공된 코드의 결과가 동일한 것을 알 수 있습니다.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bGrIJKhoVth5"
      },
      "source": [
        "#### Instance Normalization\n",
        "\n",
        "참고 삼아 인스턴스 정규화에 대한 실험 코드도 아래 제시 했습니다. 인스턴스 정규화는 배치 차원과 채널 차원을 제외한 차원에 대해서 평균을 내는 방식입니다. 지금까지 사용한 예를 들어 알아보면 데이터가 (3,2,4)일 때 인스턴스 정규화를 적용하면 (3,2,$\\cdot$)차원에 대해서는 계산하지 않고 마지막 ($\\cdot$, $\\cdot$,4)차원인 네 개 숫자를 모두 더해서 4로 나눈 것이 평균이 됩니다. 그래서 평균과 분산은 (3,2,1)이 되고 이를 각 샘플과 채널에 대해서 정규화 하는 방식입니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gNEkIwdtGddZ"
      },
      "source": [
        "# 배치사이즈 3, 단어 두 개로 구성된 문장, feature 4\n",
        "x = torch.randn(3, 2, 4)\n",
        "print('\\nINPUT', x.shape)\n",
        "print(x)\n",
        "\n",
        "# 4차원 벡터를 6차원 벡터로 변환하는 완전연결 층\n",
        "linear = nn.Linear(x.shape[-1], 6)\n",
        "x = linear(x)\n",
        "print('\\nACTIVATION', x.shape)\n",
        "print(x)\n",
        "\n",
        "#===================\n",
        "# batch norm.\n",
        "# pytorch \n",
        "bnorm = nn.BatchNorm1d(x.shape[1], eps=0., momentum=0.)\n",
        "print('\\nPYTORCH BATCH NORM. 1D')\n",
        "print(bnorm(x))\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=(0,2)).reshape(-1,1)\n",
        "var = x.var(axis=(0,2), unbiased=False).reshape(-1,1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY BATCH NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)\n",
        "#===================\n",
        "\n",
        "#===================\n",
        "# instance norm.\n",
        "# pytorch \n",
        "inorm = nn.InstanceNorm1d(x.shape[1], eps=0., momentum=0.)\n",
        "print('\\nPYTORCH INSTANCE NORM. 1D')\n",
        "print(inorm(x))\n",
        "# 직접 만들기\n",
        "mean = x.mean(axis=2).unsqueeze(2)#.reshape(-1,1)\n",
        "var = x.var(axis=2, unbiased=False).unsqueeze(2)#.reshape(-1,1)\n",
        "gamma = torch.ones(x.shape)\n",
        "beta = torch.zeros(x.shape)\n",
        "print('\\nMY INSTANCE NORM.')\n",
        "print('mean shape:', mean.shape)\n",
        "print('var shape:', var.shape)\n",
        "print(((x - mean) / torch.sqrt(var))*gamma + beta)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_YpOFnAEVsDz"
      },
      "source": [
        "\n",
        "이렇게 인코더를 구상하기 위한 기본 요소를 모두 알아봤습니다. 이제 이것들을 모아서 인코더를 만들어볼 차례입니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gs3nqJcxtkeN"
      },
      "source": [
        "### EncoderLayer와 Encoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dgBBKFT1W3DC"
      },
      "source": [
        "인코더는 다음 그림처럼 `SublayeConnection`, `EncoderLayer`, `Encoder`로 구성됩니다. 트랜스포머는 인코더와 디코더가 각각 N번 반복됩니다. 여기서 N번 반복되는 단위를 `EncoderLayer`로 구현하였고 이를 N번 복사하여 가지고 있는 클래스가 `Encoder`가 됩니다. `EncoderLayer`내부에는 `SublayerConnection`이 있게 됩니다. 각 클래스가 좀 복잡할 정도로 조각조각 나있다는 느낌이 드는데 전체적인 구조는 다음 그림과 같습니다.\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1a_MqSdbp9Ngjhkl6cNAUICb5tuDfBVjk)\n",
        "\n",
        "\n",
        "그림은 보면 인코더는 `SublayerConnection`을 두 개, 디코더는 세 개 가지는 것을 알 수 있습니다.\n",
        "\n",
        " 먼저 가장 기본 단위인 `SublayerConnection`부터 알아봅니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wGmkKSRLzreJ"
      },
      "source": [
        "class SublayerConnection(nn.Module):\n",
        "    \"\"\"\n",
        "    A residual connection followed by a layer norm.\n",
        "    Note for code simplicity the norm is first as opposed to last.\n",
        "    \"\"\"\n",
        "    def __init__(self, size, dropout):\n",
        "        super(SublayerConnection, self).__init__()\n",
        "        self.norm = LayerNorm(size)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x, sublayer):\n",
        "        \"Apply residual connection to any sublayer with the same size.\"\n",
        "\n",
        "        # (어텐션 or ff)-[드랍아웃]-Add and Norm으로 바꿨음\n",
        "        # 드랍아웃은 논문에 sublayer에 썼다고 나와 있음\n",
        "        # page 7        \n",
        "        # We apply dropout [33] to the output of each sub-layer, \n",
        "        # before it is added to the sub-layer input and normalized. \n",
        "        # 층구성이 바뀌어서 아래쪽 hyperparam. warmup을 좀 키워야 됨\n",
        "        # return x + self.dropout(sublayer(self.norm(x)))\n",
        "        return self.norm(x + self.dropout(sublayer(x)))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UXnnsxlauzzB"
      },
      "source": [
        "`SublayerConnection`은 내부적으로 `LayerNorm`과 `Dropout`만 가지고 있습니다. 포워들 할 때 외부에서 `sublayer`를 넘겨받아 이 `sublayer`를 포워드 시키고 내부에 있는 드롭아웃과 정규화를 적용하는 식으로 동작합니다. \n",
        "\n",
        "그래서 `sublayer`가 어텐션 레이어이면 위 그림 아래쪽 회식 박스에 해당하고 피드 포워드 레이어이면 윗쪽 회색 박스에 해당하게 되겠습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9LQf0QJhXMV0"
      },
      "source": [
        "`EncoderLayer`는 `SublayerConnection`을 두 개 가지고 있습니다. 그리고 이 위 그림에서 확인할 수 있듯이 `SublayerConnection` 두 개는 각각 (어텐션 레이어, Add & Norm)과 (피드포워드 레이어, Add & Norm)으로 구성됩니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a3l2Ya1Bzs7U"
      },
      "source": [
        "class EncoderLayer(nn.Module):\n",
        "    \"Encoder is made up of self-attn and feed forward (defined below)\"\n",
        "    def __init__(self, size, self_attn, feed_forward, dropout):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        \n",
        "        # self_attn: MultiHeadedAttention\n",
        "        self.self_attn = self_attn\n",
        "\n",
        "        # feed_forward: PositionwiseFeedForward\n",
        "        self.feed_forward = feed_forward \n",
        "        \n",
        "        # SublayerConnection이 2개\n",
        "        self.sublayer = clones(SublayerConnection(size, dropout), 2) \n",
        "        self.size = size # d_model\n",
        "\n",
        "    def forward(self, x, mask):\n",
        "        \"Follow Figure 1 (left) for connections.\"\n",
        "        # 1. 어텐션으로 람다 함수를 만들어서 sublayer[0]에 x와 함께 전달\n",
        "        #    이때 어텐션 \n",
        "        # 2. sublayer[0]가 포워드 되면서\n",
        "        # 3. 레이어노멀 하고\n",
        "        # 4. 인자로 넘긴 람다 함수가 sublayer로 돌면서 어텐션하고\n",
        "        # 5. 드랍아웃하고 \n",
        "        # 6. x와 더해져서 리턴 \n",
        "        # 그냥 self_atten을 바로 넘기지 않는 이유는 self_attn에 x, mask를 같이 받기 때문에\n",
        "        # x만 인자로 만들려고\n",
        "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))\n",
        "        # 원래 코드는 이 시점에서 \n",
        "        # x는 레이어놈-어텐션-드랍아웃-스킵커넥션 까지 됨\n",
        "        # 하지만 위 SublayerConnection.forward에서 순서를 원래 논문 순서로 맞춰서\n",
        "        # 어테션-드랍아웃-스킵커넥션-레이어놈 이 되었음\n",
        "        \n",
        "        # 7. 위와 마찬가지로 \n",
        "        # 원 코드 순서는 레이어놈-ff-드랍아웃-스킵커넥션\n",
        "        # 코드를 고쳐서 여기서도 \n",
        "        # ff-드랍아웃-스킵커넥션-레이어놈\n",
        "        return self.sublayer[1](x, self.feed_forward)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N5bU41I3XYAZ"
      },
      "source": [
        "`EncoderLayer`는 내부적으로 `MultiHeadedAttention`과 `PositionwiseFeedForward`을 가지고 있습니다. 그리고 `SublayerConnection`을 두개 가지고 이 `SublayerConnection` 포워드 시킬 때 `layer`인자로 `MultiHeadedAttention`과 `PositionwiseFeedForward`을 각각 전달하게 됩니다. 그림을 보면 그림과 동일하게 구현되어 있음을 알 수 있습니다.\n",
        "\n",
        "이제 마지막으로 `Encoder`입니다. `Encoder`는 생성자로 넘어오는 `EncoderLayer`를 N개 복사하고 포워드시 각각을 포워드 시키는 간단한 구조로 작성되었습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xZR1YxShzoJc"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    \"Core encoder is a stack of N layers\"\n",
        "    def __init__(self, layer, N):\n",
        "        super(Encoder, self).__init__()\n",
        "        \n",
        "        # make_model()함수에서 아래처럼 생성될 예정\n",
        "        # Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N)\n",
        "        # 여기 layer는 EncoderLayer\n",
        "        self.layers = clones(layer, N) \n",
        "        self.norm = LayerNorm(layer.size)\n",
        "        \n",
        "    def forward(self, x, mask):\n",
        "        \"Pass the input (and mask) through each layer in turn.\"\n",
        "        # self.layers에는 EncoderLayer 여섯 개가 순차적으로 있음\n",
        "        # 이 함수는 다음처럼 호출됨\n",
        "        # EncoderDecoder.encode(\n",
        "        #     self.encoder(self.src_embed(src), src_mask)\n",
        "        # )\n",
        "        # src_mask: (nbatches, 1, n_seq_src)\n",
        "        \n",
        "        for layer in self.layers: \n",
        "            # 여기 layer는 EncoderLayer고 EncoderLayer를 포워드 시킨다.\n",
        "            # EncoderLayer포워드는 위 EncoderLayer 주석참고\n",
        "            x = layer(x, mask) \n",
        "        \n",
        "        # 논문 그림 구조와 맞추기 위해 SublayerConnection에서 norm하고\n",
        "        # 여기선 안하는 것으로 바꿈\n",
        "        # return self.norm(x)\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QNN5Hxlotn0Y"
      },
      "source": [
        "### Positionwise Feed Forward"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wg2ZuCqpz-bD"
      },
      "source": [
        "class PositionwiseFeedForward(nn.Module):\n",
        "    \"Implements FFN equation.\"\n",
        "    def __init__(self, d_model, d_ff, dropout=0.1):\n",
        "        super(PositionwiseFeedForward, self).__init__()\n",
        "        self.w_1 = nn.Linear(d_model, d_ff)\n",
        "        self.w_2 = nn.Linear(d_ff, d_model)\n",
        "        self.dropout = nn.Dropout(dropout)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.w_2(self.dropout(F.relu(self.w_1(x))))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hwaq4h6cj7S7"
      },
      "source": [
        "인코더에서 $(n_{\\text{seq}}, d_{\\text{model}})$ 크기를 가지는 텐서를 두번 Linear 층에 통과 시키게 되는데 이를 positionwise feed forward라고 이야기했습니다. 위 구조 그림에서 하늘색으로 표시된 부분입니다. 코드를 보면 가중치 행렬을 두번 곱하게 되는데 각 곱에 대해서 크기는 다음처럼 변하게 됩니다.\n",
        "\n",
        "$(n_{\\text{seq}}, d_{\\text{model}}) \\to (n_{\\text{seq}}, d_{\\text{ff}}) \\to (n_{\\text{seq}}, d_{\\text{model}})$\n",
        "\n",
        "이 변환은 $n_{\\text{seq}}$의 각 자리에 해당하는 토큰 표현 벡터를 $d_{\\text{model}}$에서 $d_{\\text{ff}}$로 변환했다 다시 $d_{\\text{model}}$차원으로 되돌리는 것입니다. positionwise라고 이름지은 이유는 각 토큰 표현에 해당하는 n_seq개 벡터가 각각 $d_{\\text{ff}}$로 변환되었다 다시 $d_{\\text{model}}$ 크기로 돌아오기 때문입니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vkzmc8hxYvNQ"
      },
      "source": [
        "이렇게 인코더에 대해서 모두 알아봤습니다. 인코더를 잘 이해하면 디코더를 80% 이상 이해한것입니다. 디코더에서는 추가로 고려해야할 사항을 중심으로 이야기 하도록 하겠습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H9pXBAPMzzfB"
      },
      "source": [
        "## Decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YH77gBTLqj_N"
      },
      "source": [
        "### 디코더에 적용되는 마스크\n",
        "\n",
        "디코더에도 셀프 어텐션이 적용되는데 이 때는 이미 생성된 토큰 끼리만 어텐션 해야 합니다. 아직 생기지도 않은<sup>&#8224;</sup> 토큰과 어텐션할 수 없기 때문입니다.  seq2seq 모델처럼 시간순으로 타겟 토큰을 입력한다면 이는 자연스럽게 해결되지만 트랜스포머는 학습시 디코더에 모든 정답 토큰이 입력되므로 아직 어텐션 하지 않아아 될 부분을 지워내야 합니다. 그렇게 하기 위해 디코더에 입력되는 정답 시퀀스 길이를 행과 열로 가지는 마스크 행렬을 만드는데 이 행렬은 하삼각 요소와 주대각선만 1이고 상삼각행렬은 0을 가지게 됩니다. \n",
        "\n",
        "\n",
        "---\n",
        "<sup>&#8224;</sup> \"아직 생기지도 않은\" 이란 표현에서 뭔가 혼란스러움을 느꼈다면 충분히 그럴 수 있습니다. 트랜스포머에는 모든 정답 토큰이 디코더로 한꺼번에 입력된다고 했는데 마치 시간 순으로 토큰이 디코딩 되는 듯한 \"아직 생기지도 않은\"이란 표현은 뭔가 어색하기 짝이 없습니다. 이런 이유 때문에 디코더쪽 마스킹을 이해하기 힘들어지는데 어떻게 표현하면 더 자연스러울지 잘 떠오르지 않아서 계속 고민중에 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "tegWgGgDz3EU"
      },
      "source": [
        "def subsequent_mask(size):\n",
        "    \"Mask out subsequent positions.\"\n",
        "    attn_shape = (1, size, size)\n",
        "    subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype('uint8')\n",
        "    return torch.from_numpy(subsequent_mask) == 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BRy3OXfI0Gzo"
      },
      "source": [
        "위 코드는 행과 열 수가 입력 토큰 수이고 상삼각 행렬이 모두 0인 행렬을 만드는 코드입니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 355
        },
        "id": "FwfScgrCz4dP",
        "outputId": "adeb160c-2dac-4ca8-d9cd-1b0348e806b9"
      },
      "source": [
        "plt.figure(figsize=(5,5))\n",
        "\n",
        "sub_mask = subsequent_mask(20)\n",
        "print(sub_mask.shape)\n",
        "plt.imshow(sub_mask[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([1, 20, 20])\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7fb644fc7fd0>"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAATsAAAEvCAYAAAA6m2ZKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASrklEQVR4nO3df6xcZZ3H8fdnC7hZJAKlIpQi6hISNAuSm6q7aHBxoTSEqmHdNmZFZVNxJZFkNwbXBI37z7pGTVyMpEoDGhbJqmizFqGLJmgiPwopUORXJRhasEXqgoguW/zuH/fUnd7OtLczc29v+7xfyWTOPM8z53x7Zu6Hc2bmPKSqkKSD3R/t7wIkaTYYdpKaYNhJaoJhJ6kJhp2kJhh2kppwyP4uoJ9jjp5XJy06dJ+f98h9fzID1Ug6UPyO3/Bi/U/69c3JsDtp0aHcefOifX7eucefPgPVSDpQ3FG3DuzzNFZSE0YKuyRLkjycZFOSy/v0vyzJDV3/HUlOGmV7kjSsocMuyTzgS8B5wKnAiiSnThl2MfCrqvpT4AvAZ4bdniSNYpQju8XApqp6rKpeBL4BLJsyZhlwbbf8TeDsJH0/PJSkmTRK2C0Enuh5vLlr6zumqnYAzwLzR9imJA1lznxBkWRlkvVJ1j/9zEv7uxxJB5lRwm4L0Pv7kBO6tr5jkhwCvAJ4pt/KqmpVVU1U1cSC+fNGKEuSdjdK2N0FnJzkNUkOA5YDa6aMWQNc1C1fCPygnEBP0n4w9I+Kq2pHkkuBm4F5wOqqeiDJp4H1VbUGuBr4epJNwHYmA1GSZt1IV1BU1Vpg7ZS2K3qWfwf89SjbkKRxmDNfUEjSTDLsJDVhTk4EMKybn9ywz89x8gCpDR7ZSWqCYSepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kppg2ElqgmEnqQmGnaQmGHaSmnBQTQQwjGEmDwAnEJAONB7ZSWqCYSepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kppg2ElqwtBhl2RRkh8m+WmSB5J8tM+Ys5I8m2RDd7titHIlaTijXC62A/iHqronyRHA3UnWVdVPp4z7UVWdP8J2JGlkQx/ZVdVTVXVPt/xr4EFg4bgKk6RxGstndklOAt4I3NGn+y1J7k1yU5LXj2N7krSvRp71JMnLgW8Bl1XVc1O67wFeXVXPJ1kKfAc4ecB6VgIrAU5cOPcnYxlmthRnSpH2n5GO7JIcymTQXVdV357aX1XPVdXz3fJa4NAkx/RbV1WtqqqJqppYMH/eKGVJ0m5G+TY2wNXAg1X1+QFjXtWNI8nibnvPDLtNSRrWKOeLfwH8LXB/kp3ndP8EnAhQVVcBFwIfTrID+C2wvKpqhG1K0lCGDruq+jGQvYy5Erhy2G1I0rh4BYWkJhh2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCYSepCXP/ivuDyDCTB4ATCEjj4JGdpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCYSepCYadpCY468kBwNlSpNF5ZCepCYadpCaMHHZJHk9yf5INSdb36U+SLybZlOS+JGeMuk1J2lfj+szu7VX1ywF95wEnd7c3AV/u7iVp1szGaewy4Gs16XbgyCTHzcJ2JekPxhF2BdyS5O4kK/v0LwSe6Hm8uWuTpFkzjtPYM6tqS5JXAuuSPFRVt+3rSrqgXAlw4kJ/ESNpvEY+squqLd39NuBGYPGUIVuART2PT+japq5nVVVNVNXEgvnzRi1LknYxUtglOTzJETuXgXOAjVOGrQHe130r+2bg2ap6apTtStK+GvV88VjgxiQ71/XvVfX9JJcAVNVVwFpgKbAJeAH4wIjblKR9NlLYVdVjwGl92q/qWS7gI6NsR5JG5RUUkppg2Elqgr/xOIgNM1uKM6XoYOWRnaQmGHaSmmDYSWqCYSepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kppg2ElqgmEnqQlOBKBdDDN5ADiBgOY+j+wkNcGwk9QEw05SEww7SU0w7CQ1wbCT1ATDTlITDDtJTTDsJDVh6LBLckqSDT2355JcNmXMWUme7RlzxeglS9K+G/pysap6GDgdIMk8YAtwY5+hP6qq84fdjiSNw7hOY88GflZVPx/T+iRprMYVdsuB6wf0vSXJvUluSvL6MW1PkvbJyLOeJDkMuAD4eJ/ue4BXV9XzSZYC3wFOHrCelcBKgBMXOhnLgWaY2VKcKUWzaRxHducB91TV1qkdVfVcVT3fLa8FDk1yTL+VVNWqqpqoqokF8+eNoSxJ+n/jCLsVDDiFTfKqJOmWF3fbe2YM25SkfTLS+WKSw4G/Aj7U03YJQFVdBVwIfDjJDuC3wPKqqlG2KUnDGCnsquo3wPwpbVf1LF8JXDnKNiRpHLyCQlITDDtJTTDsJDXBsJPUBMNOUhMMO0lNMOwkNcGwk9QEr7jXfjPM5AHgBAIajkd2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCYSepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kprgrCc64DhbiobhkZ2kJhh2kpowrbBLsjrJtiQbe9qOTrIuyaPd/VEDnntRN+bRJBeNq3BJ2hfTPbK7Blgype1y4NaqOhm4tXu8iyRHA58E3gQsBj45KBQlaSZNK+yq6jZg+5TmZcC13fK1wDv7PPVcYF1Vba+qXwHr2D00JWnGjfKZ3bFV9VS3/Avg2D5jFgJP9Dze3LVJ0qwayxcUVVVAjbKOJCuTrE+y/ulnXhpHWZL0B6OE3dYkxwF099v6jNkCLOp5fELXtpuqWlVVE1U1sWD+vBHKkqTdjRJ2a4Cd365eBHy3z5ibgXOSHNV9MXFO1yZJs2q6Pz25HvgJcEqSzUkuBv4F+KskjwLv6B6TZCLJVwGqajvwz8Bd3e3TXZskzappXS5WVSsGdJ3dZ+x64O96Hq8GVg9VnSSNiVdQSGqCYSepCc56omYMM1uKM6UcPDyyk9QEw05SEww7SU0w7CQ1wbCT1ATDTlITDDtJTTDsJDXBsJPUBMNOUhMMO0lNMOwkNcGJAKQ9GGbyAHACgbnIIztJTTDsJDXBsJPUBMNOUhMMO0lNMOwkNcGwk9QEw05SEww7SU3Ya9glWZ1kW5KNPW2fTfJQkvuS3JjkyAHPfTzJ/Uk2JFk/zsIlaV9M58juGmDJlLZ1wBuq6s+AR4CP7+H5b6+q06tqYrgSJWl0ew27qroN2D6l7Zaq2tE9vB04YQZqk6SxGcdndh8EbhrQV8AtSe5OsnIM25KkoYw060mSTwA7gOsGDDmzqrYkeSWwLslD3ZFiv3WtBFYCnLjQyVh0YBtmthRnSplZQx/ZJXk/cD7w3qqqfmOqakt3vw24EVg8aH1VtaqqJqpqYsH8ecOWJUl9DRV2SZYAHwMuqKoXBow5PMkRO5eBc4CN/cZK0kybzk9Prgd+ApySZHOSi4ErgSOYPDXdkOSqbuzxSdZ2Tz0W+HGSe4E7ge9V1fdn5F8hSXux1w/HqmpFn+arB4x9EljaLT8GnDZSdZI0Jl5BIakJhp2kJhh2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCV9xLc8QwkweAEwhMl0d2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCYSepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kprgrCfSAc7ZUqbHIztJTTDsJDVhr2GXZHWSbUk29rR9KsmWJBu629IBz12S5OEkm5JcPs7CJWlfTOfI7hpgSZ/2L1TV6d1t7dTOJPOALwHnAacCK5KcOkqxkjSsvYZdVd0GbB9i3YuBTVX1WFW9CHwDWDbEeiRpZKN8Zndpkvu609yj+vQvBJ7oeby5a5OkWTds2H0ZeB1wOvAU8LlRC0myMsn6JOuffualUVcnSbsYKuyqamtVvVRVvwe+wuQp61RbgEU9j0/o2gatc1VVTVTVxIL584YpS5IGGirskhzX8/BdwMY+w+4CTk7ymiSHAcuBNcNsT5JGtdcrKJJcD5wFHJNkM/BJ4KwkpwMFPA58qBt7PPDVqlpaVTuSXArcDMwDVlfVAzPyr5Ckvdhr2FXVij7NVw8Y+ySwtOfxWmC3n6VI0mzzCgpJTTDsJDXBWU+kRg0zW8qBPFOKR3aSmmDYSWqCYSepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kppg2ElqgmEnqQmGnaQmOBGApGkbZvIAmBsTCHhkJ6kJhp2kJhh2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCYSepCXu9giLJauB8YFtVvaFruwE4pRtyJPDfVbXbT6STPA78GngJ2FFVE2OqW5L2yXQuF7sGuBL42s6GqvqbnctJPgc8u4fnv72qfjlsgZI0DnsNu6q6LclJ/fqSBHgP8JfjLUuSxmvUz+zeCmytqkcH9BdwS5K7k6wccVuSNLRRZz1ZAVy/h/4zq2pLklcC65I8VFW39RvYheFKgBMXOhmLdDAZZraUcc+UMvSRXZJDgHcDNwwaU1VbuvttwI3A4j2MXVVVE1U1sWD+vGHLkqS+RjmNfQfwUFVt7teZ5PAkR+xcBs4BNo6wPUka2l7DLsn1wE+AU5JsTnJx17WcKaewSY5PsrZ7eCzw4yT3AncC36uq74+vdEmavul8G7tiQPv7+7Q9CSztlh8DThuxPkkaC6+gkNQEw05SEww7SU0w7CQ1wbCT1ATDTlITDDtJTTDsJDXBK+4lzUnDTB6w+NwXBvZ5ZCepCYadpCYYdpKaYNhJaoJhJ6kJhp2kJhh2kppg2ElqgmEnqQmGnaQmGHaSmmDYSWqCYSepCamq/V3DbpI8Dfy8T9cxwC9nuZx+rGNX1rEr69jVbNbx6qpa0K9jTobdIEnWV9WEdViHdVjHvvI0VlITDDtJTTjQwm7V/i6gYx27so5dWceu5kQdB9RndpI0rAPtyE6ShjInwy7JkiQPJ9mU5PI+/S9LckPXf0eSk2aghkVJfpjkp0keSPLRPmPOSvJskg3d7Ypx19Ft5/Ek93fbWN+nP0m+2O2P+5KcMQM1nNLz79yQ5Lkkl00ZMyP7I8nqJNuSbOxpOzrJuiSPdvdHDXjuRd2YR5NcNAN1fDbJQ91+vzHJkQOeu8fXcAx1fCrJlp59v3TAc/f4tzWGOm7oqeHxJH3/rznj3B/TVlVz6gbMA34GvBY4DLgXOHXKmL8HruqWlwM3zEAdxwFndMtHAI/0qeMs4D9nYZ88Dhyzh/6lwE1AgDcDd8zCa/QLJn/TNOP7A3gbcAawsaftX4HLu+XLgc/0ed7RwGPd/VHd8lFjruMc4JBu+TP96pjOaziGOj4F/OM0Xrc9/m2NWseU/s8BV8z0/pjubS4e2S0GNlXVY1X1IvANYNmUMcuAa7vlbwJnJ8k4i6iqp6rqnm7518CDwMJxbmOMlgFfq0m3A0cmOW4Gt3c28LOq6vfD77GrqtuA7VOae98D1wLv7PPUc4F1VbW9qn4FrAOWjLOOqrqlqnZ0D28HThh2/aPUMU3T+dsaSx3d3+N7gOuHXf+4zcWwWwg80fN4M7uHzB/GdG+0Z4H5M1VQd5r8RuCOPt1vSXJvkpuSvH6GSijgliR3J1nZp386+2ycljP4TTwb+wPg2Kp6qlv+BXBsnzGzvV8+yOQRdj97ew3H4dLudHr1gNP62dwfbwW2VtWjA/pnY3/sYi6G3ZyS5OXAt4DLquq5Kd33MHkqdxrwb8B3ZqiMM6vqDOA84CNJ3jZD29mrJIcBFwD/0ad7tvbHLmryvGi//qwgySeAHcB1A4bM9Gv4ZeB1wOnAU0yeQu5PK9jzUd2sv6fnYthtARb1PD6ha+s7JskhwCuAZ8ZdSJJDmQy666rq21P7q+q5qnq+W14LHJrkmHHXUVVbuvttwI1Mno70ms4+G5fzgHuqamufOmdlf3S27jxV7+639RkzK/slyfuB84H3dsG7m2m8hiOpqq1V9VJV/R74yoD1z9b+OAR4N3DDoDEzvT/6mYthdxdwcpLXdEcRy4E1U8asAXZ+s3Yh8INBb7JhdZ85XA08WFWfHzDmVTs/K0yymMn9OdbQTXJ4kiN2LjP5gfjGKcPWAO/rvpV9M/BszyneuA38L/Zs7I8eve+Bi4Dv9hlzM3BOkqO607pzuraxSbIE+BhwQVW9MGDMdF7DUevo/Yz2XQPWP52/rXF4B/BQVW3u1zkb+6Ov2fw2ZLo3Jr9dfITJb44+0bV9msk3FMAfM3katQm4E3jtDNRwJpOnRvcBG7rbUuAS4JJuzKXAA0x+q3U78OczUMdru/Xf221r5/7orSPAl7r9dT8wMUOvy+FMhtcretpmfH8wGa5PAf/L5OdMFzP5Ge2twKPAfwFHd2MngK/2PPeD3ftkE/CBGahjE5Ofg+18j+z8lcDxwNo9vYZjruPr3Wt/H5MBdtzUOgb9bY2zjq79mp3viZ6xM7Y/pnvzCgpJTZiLp7GSNHaGnaQmGHaSmmDYSWqCYSepCYadpCYYdpKaYNhJasL/AXkopxz7l3OuAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 360x360 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ut7ZGkxI0ilu"
      },
      "source": [
        "이 마스크가 어떻게 적용되는지는 뒤에 상세히 예를 들어 설명하도록 하겠습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "loOPaKHueQsH"
      },
      "source": [
        "### DecoderLayer와 Decoder\n",
        "\n",
        "`DecoderLayer`와 `Decoder`의 구성은 인코더와 동일합니다. 한가지 다른 점은 `DecoderLayer`에는 어텐션이 두 개 있다는 점입니다. `forward()`함수를 보면 알 수 있듯이 셀프 어텐션과 인코더, 디코더 간의 크로스 어텐션입니다. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "XqWPxChez1h2"
      },
      "source": [
        "class DecoderLayer(nn.Module):\n",
        "    \"Decoder is made of self-attn, src-attn, and feed forward (defined below)\"\n",
        "    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):\n",
        "        # self_attn: MultiHeadedAttention\n",
        "        # src_attn: MultiHeadedAttention\n",
        "\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.size = size\n",
        "        self.self_attn = self_attn\n",
        "        self.src_attn = src_attn # cross attention\n",
        "        self.feed_forward = feed_forward # positional ff\n",
        "        self.sublayer = clones(SublayerConnection(size, dropout), 3)\n",
        " \n",
        "    def forward(self, x, memory, src_mask, tgt_mask):\n",
        "        \"Follow Figure 1 (right) for connections.\"\n",
        "        # 여기서 momory는 인코더로 부터 넘어온 인코딩 (nbatche, n_seq, d_model)\n",
        "        m = memory\n",
        "        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))\n",
        "        \n",
        "        # 크로스 어텐션 할 때 마스크는 src_mask(key_pad_mask)를 전달함!\n",
        "        # 이렇게 하면 입력에 존재하는 패딩 토큰이 쿼리로써 인코딩되어 온 것을 \n",
        "        # key 패딩으로 마스킹 할 수 있게 된다.\n",
        "        # (인코더 출력이 이제는 key로 작용하기 때문에!!!)\n",
        "        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n",
        "        \n",
        "        return self.sublayer[2](x, self.feed_forward)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SsqlIIgdg27H"
      },
      "source": [
        "크로스 어텐션를 실행하는 코드를 보면 \n",
        "\n",
        "```python\n",
        "# x: 디코더에서 만든 (n_seq, d_mode) 텐서 (여기서 n_seq는 디코더로 입력되는 시퀀스의 길이)\n",
        "# m: 인코더에서 전달받은 (n_seq, d_model) 텐서 (여기서 n_seq는 인코더로 입력된 시퀀스의 길이)\n",
        "x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))\n",
        "```\n",
        "\n",
        "셀프 어텐션과 차이는 쿼리로 디코더의 입력이 인코딩된 것을 넘기고 키와 벨류는 인코더에서 입력받는 인코딩 정보를 넘긴다는 것입니다. 이렇게 해야 크로스 어텐션이 계산되겠죠. 또 마스크는 `src_mask`를 넘기고 있는 것을 확인할 수 있습니다. 이에 대한 자세한 설명은 아래 [실험용 데이터와 보조 코드 준비](#cell-id)절에서 모두 설명하도록 하겠습니다.\n",
        "\n",
        "아래는 `Decoder` 코드이고 특별히 언급할 점은 없습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "EutLhZ9Ezu08"
      },
      "source": [
        "class Decoder(nn.Module):\n",
        "    \"Generic N layer decoder with masking.\"\n",
        "    def __init__(self, layer, N):\n",
        "        # layer: DecoderLayer\n",
        "\n",
        "        super(Decoder, self).__init__()\n",
        "        self.layers = clones(layer, N)\n",
        "        self.norm = LayerNorm(layer.size)\n",
        "        \n",
        "    def forward(self, x, memory, src_mask, tgt_mask):\n",
        "        for layer in self.layers:\n",
        "            x = layer(x, memory, src_mask, tgt_mask)\n",
        "        \n",
        "        # Encoder와 마찬가지로 norm 안함\n",
        "        # return self.norm(x)\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_fkVa3LgqArF"
      },
      "source": [
        "### 디코더 끝에 붙는 마지막 Linear 레이어\n",
        "\n",
        "디코더의 출력을 받아서 최종적으로 $(n_{\\text{seq}}, d_{\\text{model}})$을 $(n_{\\text{seq}}, \\text{vocab})$로 변환하는 피드 포워드 레이어를 추가합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "4UeVvfIOzkoF"
      },
      "source": [
        "class Generator(nn.Module):\n",
        "    \"Define standard linear + softmax generation step.\"\n",
        "    def __init__(self, d_model, vocab):\n",
        "        super(Generator, self).__init__()\n",
        "        self.proj = nn.Linear(d_model, vocab)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return F.log_softmax(self.proj(x), dim=-1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9k4OcZItp8aa"
      },
      "source": [
        "\n",
        "\n",
        "## Transformer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4IH3c-I1h9Ry"
      },
      "source": [
        "이상에서 만든 부품들로 트랜스포머를 조립합니다. `Encoder`, `Decoder`를 받아서 전체 모델을 생성하는 `EncoderDecoder` 클래스를 정의하고 이것을 객체로 생성하는 `make_model()` 함수를 정의합니다.\n",
        "\n",
        "아래 그림은 `make_model()` 함수가 조립하는 전체 구조를 주석과 함께 나타낸 것입니다.\n",
        "\n",
        "![picture](https://drive.google.com/uc?id=1aQKj79KnivzAaQ2pVz6103kQ1hf4klBQ)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "N26U07Vmze0L"
      },
      "source": [
        "class EncoderDecoder(nn.Module):\n",
        "    \"\"\"\n",
        "    A standard Encoder-Decoder architecture. Base for this and many \n",
        "    other models.\n",
        "    \"\"\"\n",
        "    def __init__(self, encoder, decoder, src_embed, tgt_embed, generator):\n",
        "        super(EncoderDecoder, self).__init__()\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.src_embed = src_embed # 인코더의 임베딩-위치인코딩\n",
        "        self.tgt_embed = tgt_embed # 디코더의 임베딩-위치인코딩\n",
        "        self.generator = generator\n",
        "\n",
        "    # 이 함수가 모델이 포워드 되는 엔트리 포인트!!!    \n",
        "    def forward(self, src, tgt, src_mask, tgt_mask):\n",
        "        \"Take in and process masked src and target sequences.\"\n",
        "        return self.decode(\n",
        "            self.encode(src, src_mask), src_mask,\n",
        "            tgt, tgt_mask\n",
        "        )\n",
        "    \n",
        "    def encode(self, src, src_mask):\n",
        "        return self.encoder(self.src_embed(src), src_mask)\n",
        "    \n",
        "    def decode(self, memory, src_mask, tgt, tgt_mask):\n",
        "        return self.decoder(self.tgt_embed(tgt), memory, src_mask, tgt_mask)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XrFUFtJ82r89"
      },
      "source": [
        "마지막으로 `Encoder`와 `Decoder`를 한번 더 감싸서 `EncoderDecoder` 클래스를 만듭니다. 이 클래스의 `forward`함수가 모델을 포워드시키는 엔트리 포인트가 됩니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "lKThOQWr0DGO"
      },
      "source": [
        "def make_model(src_vocab, tgt_vocab, N=6, \n",
        "               d_model=512, d_ff=2048, h=8, dropout=0.1):\n",
        "    \"\"\"\n",
        "        Helper: Construct a model from hyperparameters.\n",
        "        src_vocab: 입력을 임베딩할 때 사용하는 단어장 사이즈\n",
        "        tgt_vocab: 출력을 위한 출력쪽 단어장 사이즈\n",
        "        d_mode: 트랜스포머 인코더 디코더에서 사용되는 벡터의 크기\n",
        "        d_ff: feed foward층이 출력하는 벡터의 크기\n",
        "    \"\"\"\n",
        "    c = copy.deepcopy\n",
        "    attn = MultiHeadedAttention(h, d_model)\n",
        "    ff = PositionwiseFeedForward(d_model, d_ff, dropout)\n",
        "    position = PositionalEncoding(d_model, dropout)\n",
        "    model = EncoderDecoder(\n",
        "        # 인코더를 만들기 위해 어텐선 레이어 하나와 피드포워드 레이어 하나가 필요합니다.\n",
        "        Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N),\n",
        "        \n",
        "        # 디코더를 만들기 위해 어텐션 레이어 두개와 피드포워드 레이어 하나가 필요합니다.\n",
        "        # 두 어텐션 레이어 중 하나는 셀프 어텐션을 담당하고 하나는 크로스 어텐션을 담당합니다.\n",
        "        Decoder(DecoderLayer(d_model, c(attn), c(attn), \n",
        "                             c(ff), dropout), N),\n",
        "\n",
        "        # 인코더와 디코더 쪽 임베딩으로 직접 만든 토큰 임베딩과\n",
        "        # 포지션 인코딩을 순차적으로 수행하는 nn.Sequential                   \n",
        "        nn.Sequential(Embeddings(d_model, src_vocab), c(position)),\n",
        "        nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)),\n",
        "\n",
        "        Generator(d_model, tgt_vocab))\n",
        "    \n",
        "    # This was important from their code. \n",
        "    # Initialize parameters with Glorot / fan_avg.\n",
        "    for p in model.parameters():\n",
        "        if p.dim() > 1:\n",
        "            # nn.init.xavier_uniform(p)\n",
        "            nn.init.xavier_uniform_(p)\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W_htqhWmeyKG"
      },
      "source": [
        "마지막으로 트랜스포머를 조립하는 `make_model` 함수입니다. 지금까지 내용을 충분히 이해했다면 여기서 모델을 만드는 코드가 한눈에 들어올 것입니다.\n",
        "\n",
        "\n",
        "이상없이 모델이 만들어지는지 테스트 합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "Aa2is5Bd0FGO"
      },
      "source": [
        "# 무소식이 희소식! 아무 에러없이 실행되면 OK\n",
        "tmp_model = make_model(10, 10, 2)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WFTdWsxYDoZU"
      },
      "source": [
        "<a name=\"cell-id\"></a>\n",
        "## 실험용 데이터와 보조 코드 준비"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J_2JoQDfe4O1"
      },
      "source": [
        "이렇게 모델에 대한 코딩을 모두 마쳤고 이제 간단한 실험을 위한 코드를 알아보도록 하겠습니다. 먼저 `Batch`클래스입니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "sUD6rROm0Hf2"
      },
      "source": [
        "class Batch:\n",
        "    \"Object for holding a batch of data with mask during training.\"\n",
        "    def __init__(self, src, trg=None, pad=0):\n",
        "        # src: (nbatches, n_seq_src)\n",
        "        self.src = src\n",
        "        self.src_mask = (src != pad).unsqueeze(-2) # (nbatches, 1, n_seq_src)\n",
        "        if trg is not None:\n",
        "            self.trg = trg[:, :-1]\n",
        "            self.trg_y = trg[:, 1:]\n",
        "            self.trg_mask = self.make_std_mask(self.trg, pad) # (nbatches, n_seq_trg, n_seq_trg)\n",
        "            self.ntokens = (self.trg_y != pad).data.sum() # 패딩 토큰이 아닌 토큰 수\n",
        "    \n",
        "    @staticmethod\n",
        "    def make_std_mask(tgt, pad):\n",
        "        \"Create a mask to hide padding and future words.\"\n",
        "        tgt_mask = (tgt != pad).unsqueeze(-2) # (nbatches, 1, n_seq_trg)\n",
        "        #tgt_mask = tgt_mask & Variable( subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data) )\n",
        "        tgt_mask = tgt_mask & subsequent_mask(tgt.size(-1)).type_as(tgt_mask.data)\n",
        "        return tgt_mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ui_CyOaWfAzH"
      },
      "source": [
        "`Batch`클래스는 입력 시퀀스와 타겟 시퀀스를 받아서 입력과 타겟의 마스크를 만들고 타겟을 디코더에 입력되는 타겟(`self.trg`)과 출력 타겟(`self.trg_y`)으로 분리합니다. 예를 들면 다음과 같습니다.\n",
        "\n",
        "- `src`: `I` `am` `a` `student.`\n",
        "\n",
        "- `trg`: `나는` `학생` `입니다.`\n",
        "\n",
        "    - `trg`: `나는` `학생`\n",
        "    - `trg_y`: `학생` `입니다.`\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YN-7kibnl2zp"
      },
      "source": [
        "다음으로 마스크를 만들게 되는데 마스크를 만드는 과정(`make_std_mask` 함수)은 코드만 봐서는 이해가 어려우므로 다음 예를 보면서 알아보겠습니다. \n",
        "\n",
        "이 글에서 어쩌면 가장 복잡하고 설명하기 지저분한 부분이니 잠시 쉬고 와서 읽도록 합시다. 😎"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eR2XCJltHa1F"
      },
      "source": [
        "### Source mask (key pad mask)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EJ2_DBN-l4xZ"
      },
      "source": [
        "먼저 source mask에 대해서 알아봅시다. 아래 예에서는 시퀀스 길이 5인 샘플 두 개가 (2,5)인 행렬에 들어있는 상황입니다.\n",
        "\n",
        "예를 들어 각 샘플이 다음과 같다고 하겠습니다.\n",
        "\n",
        "*   `i,     love,  you,   [PAD],  [PAD]`\n",
        "*   `good,  job,  [PAD],  [PAD],  [PAD]`\n",
        "\n",
        "이렇게 구성된 샘플에 셀프어텐션을 하는 경우 [PAD] 토큰에는 어텐션이 되면 안될 것입니다. 예를 들어 'i'와 관계가 있는 토큰을 알아내는 것이 셀프어텐션인데 그 후보 키 토큰에 [PAD]가 들어가는 것은 별로 바람직하지 않습니다. 물론 [PAD]가 쿼리로 작동하는 것도 피해야 합니다.따라서 [PAD] 가 있는 위치를 마스킹할 수 있는 마스크를 만들어야 합니다. \n",
        "\n",
        "마스크를 만들고 이를 직접 그림으로 보이는 것은 것은 트랜스포머 모델을 실제로 포워드 시켜야 하는 과정을 거쳐야 해서 매우 귀찮은 작업이지만 하나 하나 따라 가보도록 하겠습니다.😵\n",
        "\n",
        " 단 문제를 단순화 하고 그림을 그리기 위해 헤드 하나를 가정합니다. \n",
        "우선 모델과 관련된 변수를 적당히 세팅합니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "eT9KxTQaxhlI"
      },
      "source": [
        "model_const = {'dv':3, 'dk':3, 'h':1, 'd_model':7, 'src_len':5, 'target_len':6}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "BdeZzzmpBfMo",
        "outputId": "6919399f-9991-49aa-a811-56dd05ce7b8a"
      },
      "source": [
        "pad = 0\n",
        "# 길이 7인 샘플 두 개, src:(2,5)\n",
        "# 1번 샘플은 길이 5, 패딩 2, 예를 들어 i     love  you   [PAD]  [PAD]  \n",
        "# 2번 샘플은 길이 2, 패딩 3, 예를 들어 good  job  [PAD]  [PAD]  [PAD]\n",
        "# 임의 숫자를 토큰 번호로 가정하고 randint를 사용\n",
        "src = torch.from_numpy(np.random.randint(1, 100, size=(2, model_const['src_len'])))\n",
        "paddings = torch.LongTensor([2, 3]).reshape(-1,1) # 각 샘플당 패딩 개수\n",
        "pad_idx = src.shape[1] - paddings\n",
        "col_idx = torch.arange(src.shape[1]).reshape(1,-1)\n",
        "# src = src * ~(paddings > col_idx)\n",
        "src = src * (pad_idx > col_idx)\n",
        "\n",
        "# 마지막 토큰들은 PAD토큰이 되었음\n",
        "print('src', src, '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "src tensor([[97, 53, 93,  0,  0],\n",
            "        [95, 83,  0,  0,  0]]) \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rQPR8n5CBgnx"
      },
      "source": [
        "이렇게 가상으로 만들어진 src에서 0인 자리를 `False`, 0이 아닌 자리를 `True`로 가지는 마스크를 생성합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "-VHv2zCgBrlS",
        "outputId": "42de507a-5a57-4cbd-b222-a8a0b6c507be"
      },
      "source": [
        "# (src != pad): (2,5)\n",
        "# 패딩 토큰자리만 False인 src와 모양이 같은 마스크\n",
        "print('\\n(src != pad).shape', (src != pad).shape)\n",
        "print(src != pad, '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "(src != pad).shape torch.Size([2, 5])\n",
            "tensor([[ True,  True,  True, False, False],\n",
            "        [ True,  True, False, False, False]]) \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KxsUaK5kB2U0"
      },
      "source": [
        "이제 멀티헤드 어텐션에 마스크를 적용하기 위해 중간 차원을 하나 더 늘립니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "pGoHTIc6B688",
        "outputId": "45f0dd61-2484-44c9-d1cc-1fb55127c858"
      },
      "source": [
        "src_mask_enc = (src != pad).unsqueeze(-2)\n",
        "print('\\n(src != pad).unsqueeze(-2).shape:', src_mask_enc.shape)\n",
        "print(src_mask_enc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "(src != pad).unsqueeze(-2).shape: torch.Size([2, 1, 5])\n",
            "tensor([[[ True,  True,  True, False, False]],\n",
            "\n",
            "        [[ True,  True, False, False, False]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8bXmUJuNB8Vx"
      },
      "source": [
        "결과는 (2,5)인 마스크가 (2,1,5)가 되는데 마스크가 적용될 시점의 코드를 다시보면 \n",
        "\n",
        "```python\n",
        "# MultiHeadedAttention.forward()에서\n",
        "    # Same mask applied to all h heads.\n",
        "    mask = mask.unsqueeze(1)\n",
        "    ...\n",
        "    x, self.attn = attention(query, key, value, mask=mask, \n",
        "                                 dropout=self.dropout)\n",
        "\n",
        "# attention()에서\n",
        "    # scores: (nbatches, h, n_seq, n_seq)\n",
        "    if mask is not None:\n",
        "        scores = scores.masked_fill(mask == 0, -1e9)\n",
        "```\n",
        "\n",
        "위 코드에서 `MultiHeadedAttention.forward()`를 거치면서 전달된 `src_mask`는 (2,1,5)가 (2,1,1,5)로 되며 이를 `(nbatches, h, n_seq, n_seq)`인 `score`에 적용하면 결국 h개 헤드에 행 방향으로만 마스크가 주어지게 됩니다. \n",
        "\n",
        "마스킹을 해서 그림을 그려보면\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 578
        },
        "id": "R7aixZgtDFpi",
        "outputId": "396c610b-4bf5-4325-8969-315787c51e1c"
      },
      "source": [
        "# 샘플 두개 헤드 하나짜리 (5,5)인 어텐션 스코어\n",
        "encoder_self_attn_scores = torch.rand((2, 1, model_const['src_len'], \n",
        "                                       model_const['src_len']))\n",
        "src_mask_enc_reshape = src_mask_enc.unsqueeze(1)\n",
        "\n",
        "# -1e9로 마스킹을 하면 마스킹 되지 않는 부분이 모두 같은 색으로 \n",
        "# 나타나므로 여기서는 그냥 0으로 마스킹\n",
        "masked_encoder_self_attn_scores = encoder_self_attn_scores.masked_fill(\n",
        "    src_mask_enc_reshape == 0, 0)\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(12,5), nrows=1, ncols=2)\n",
        "\n",
        "ax[0].imshow(masked_encoder_self_attn_scores[0][0], cmap='BrBG')\n",
        "ax[0].set_xticks([0, 1, 2, 3, 4])\n",
        "ax[0].set_xticklabels(['i', 'love', 'you', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[0].set_xlabel('Key', fontsize=15)\n",
        "ax[0].set_yticks([0, 1, 2, 3, 4])\n",
        "ax[0].set_yticklabels(['i', 'love', 'you', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[0].set_ylabel('Query', fontsize=15)\n",
        "ax[0].set_title('Sample 1', fontsize=15)\n",
        "\n",
        "ax[1].imshow(masked_encoder_self_attn_scores[1][0], cmap='BrBG')\n",
        "ax[1].set_xticks([0, 1, 2, 3, 4])\n",
        "ax[1].set_xticklabels(['good', 'job', '[PAD]', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[1].set_xlabel('Key', fontsize=15)\n",
        "ax[1].set_yticks([0, 1, 2, 3, 4])\n",
        "ax[1].set_yticklabels(['good', 'job', '[PAD]', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[1].set_ylabel('Query', fontsize=15)\n",
        "ax[1].set_title('Sample 2', fontsize=15)\n",
        "\n",
        "plt.show()\n",
        "\n",
        "print(masked_encoder_self_attn_scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "findfont: Font family ['NanumGothic'] not found. Falling back to DejaVu Sans.\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs8AAAFZCAYAAABnpcJ0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5hkZXn3++9PJqAICDiIKJFBtxoP7K0CnqIRjWJAUDlLklfBE2F7wFMM+saIh7xRokIQo4LIKFFAXhHRSDgpJr4oMIYdEFEBRUBOcj4Mg8jc+4+1OtQU3dOrZ6q6uqu/n+uqq2o961mr7mem6+6713rWqlQVkiRJkqb3kFEHIEmSJM0XFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5r3kuyX5MdJ7kxya5ILk3xq1HH1S7I0ybIB7GfdJP+Y5D+S3JPE+01KmlcWYN7ePsmxSS5PsjzJz5N8MMlDBxGnZpfFs+a1JO8DvgCcDuwOvBb4JvDKUcY1ZOsDbwSWA+eOOBZJmpEFmrf3AZ4AfBzYGfgM8C7gK6MMSmsmfkmK5rMkvwFOqaq39LWn5tgPd5KlwNOrarsB7CtVVUneCny6qrLWAUrSLFiIeTvJ4qq6qa/tzcDngSVV9eu12b9ml0eeNd9tDFzf39ifgJN8LMnFSe5Kck2SryR5dF+fK5N8IsnBSa5LcnuST6axc5JL2lOMpyTZpGe7HZJUkh2TfDvJ3UmuSvJX0wWf5HFJTkhyS3sq7/QkT55uu7n2C0aSZmDB5e3+wrl1Yfv8mOneU3PLolEHIK2l/wTeluQq4NtVdfMU/R4F/C/gWmAz4N3Ad5M8vapW9vR7DXA+sD+wLfBRmj8y/wT4APAw4EjgH4D+JHsMcBzwaWA34LNJrqmqb08WUJJNgR8AN7f7Wg4cDJyV5ElVdU/nfwVJmj/M243nASuBK2awjeaCqvLhY94+gP8b+CVQNEnoEuDDwEar2WYd4LHtNn/S034lcDmwTk/b+cDvga172g4FbuhZ3qHd11F973Mm8KOe5aXAsp7lj9Ak4E172jYBbgfe0nH8b6U9YOPDhw8f8+Gx0PN2u82jgRuBpaP+//Ax84fTNjSvVdVFwFNoLjT5ZyA0RxqWJdlgol+SnZKcm+R2mqR6TbvqSX27PKeq7u9Zvhy4sqp+1de2WZJ1+7b9Rt/yycC2SdaZIvyX0iTqO5IsSrIIuBP4MbDW86IlaS5a6Hm7jeFrwF3AO7tso7nF4lnzXlXdW1Xfqqq3VtVTae5E8UTgDdDcIgg4lSbx/g+aU2XPbTfvv03QbX3Lv5uiLUB/Er5xkuVFwOIpQl9McwX2fX2PFwN/OMU2kjTvLdS8nSTAl4GnATtX1a3TbaO5xznPGjtVdUySQ4E/apt2A34L7FPt+bIkWw3hrR81yfLvgckuFAG4heaXw0cmWXfnAOOSpDltAeXtw4FXAS+rqp91DVJzi8Wz5rUkj6qqG/vaNgMeAdzQNj0MuG8iAbf+Ygjh7Aac1rf8477Tib3OBvYGLikvDpS0QCzUvN3e3/qtwN5V9YM1CVZzg8Wz5ruLk3wTOIPmdNtWwHtoroD+UtvnTOAdSQ4HvgU8H/jLIcSyU5K/B75Pc+P/l9EcYZjKp9o4vpvk08BvgM2BFwE/qKrjp9owyU7Aw4FntMt7tqsuKO8XKmluW3B5O8mf09w5ZCnwmyTP7Vl9RVX9dm0Hotlj8az57sM0ie4IYFOae4eeS3Oq71cAVfWdJH8DvA14E/BDYBfgFwOO5Y3AO2guALmF5srrU6fqXFU3tQn074HDaO59eh3NbZAumua9PkvzC2fCSe3z/jTJWZLmqoWYt3dsn/drH73M2/OM3zAoraUkOwDfA7apqp+MOBxJ0jTM21ob3m1DkiRJ6sjiWZIkSerIaRuSJElSRx55liRJkjqyeJYkSZI68lZ1M7D4kRvX47Z89KjDGLh7s96oQxia26+8eNQhDMXy3406As3UrffUTVW12ajjWEjyB4uKh/Z/G/MY+IMxHFNrkxV3jDoECVh9zrZ4noHHbflo/s8ZXxh1GAP3i0VPHHUIQ/Od/bcedQhD8Z9X/37UIWiG/vf/d59fXjPbHrouPOPJo45i8LZ83KgjGJo//dlp03eSZsHqcrbTNiRJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ6BJEuTLBt1HJKk1UuyQZJKst+oY5G0MC0adQBzxEeAh406CEmSJM1tFs9AVV0x6hgkSZI09zltA6dtSNJMJHlrkquT3J3klCR/2k6l2KFdv36SI5Jcn2RFkguS7DjFfi5Lcm+Sy5O8c5I+eyT5RZJ7kvw78EfDH6EkTc3iWZLUWZLdgE8DpwK7ARcBx/R1OxrYH/j7ts/VwL8meUHPft7Us59dgZOATyY5uKfPs4ATgf8Cdge+BXxtKAOTpI6ctiFJmon3A9+pqre0y2ckWQwcCJDkKcC+wP5V9aW27XSaIvsDwMuTPAQ4BFhaVe/u2c8jgPclObyqVgAHA78A9q6qAk5Lsi7w0dkYqCRNxiPP00jy5iTLkiy76ZbbRh2OJI1MkkXAM2mOFvfqXd4eCM2RZACqamW7PHHkeUvgMb19WicCGwHbtMvPBk5tC+cJJ08T43/nbO77/bRjkqSZsnieRlUdVVXbVdV2izfdeNThSNIoLQbWAX7b1967vAVwV1Ut7+tzA7B+kvXaPhNt/X0ANm2fHw3c2Nenf3kVvTmbP/DkqqTBs3iWJHV1E3A/sFlfe+/ydcAGSdbv67M5sLyq7m37ADxqkj4At7TP10/Sp39ZkmaVxbMkqZOq+j1wIfCqvlWv7Hl9AVDAnhMNSdIu/6Btuga4Ftirbz97A3cAF/fs65Xt9hN2X4shSNJa85yWJGkm/gH4epIjaeY6/zHwinbdyqq6NMnxwJFJNgSuAN5Ec4u5A6GZA53kEODzSW4GzgRe1K5/f3uxIMDHgfOAryU5Bng68IZZGKMkTckjz5KkzqrqZODtwKuBU2guEHxPu/qO9vlNwJeAvwO+CWwF7FJVP+jZz9HAQTS3svs2zR063l1VH+vpswx4Dc1Fiqe077nPsMYmSV145Bmoqv1GHYMkzRdV9WmaezQDkORvgRXAz9v1y4G3tY/O+5miz0k8+K4cmayvJM0Gi2dJUmdJNgPeB3wPWA68EPgb4JiqumeUsUnSbLB4liTNxO9o5i+/FngEzZ0z/onmC1AkaexZPEuSOquq24GdRx2HJI2KFwxKkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR0tGnUA88o667Nyg2eOOoqB2/D8j446hKE58plvG3UIQ/H8qw8bdQjSnPfEx/9f/POJ3xh1GAP3h//5iVGHMDR/+4FRRyBNzyPPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSR3O2eE6yNMmyUcchSRqMmeb1JEuSVJJdhhmXJM3EolEHIElaMD4CPGzUQUjS2rB4liTNiqq6YtQxSNLamrPTNvoleUaSs5MsT3Jrkq8k2bxn/a+S/OMk252U5Ac9y5smOSrJDUlWJDk3yXNmaxyStFD1T9uYLq/32CjJcUnuTHJjkg/OYtiStIp5UTwn2Qw4B1gf+HPgbcCLgDOTrNt2+xqwV992GwCvAE5ol9cDzgJeCvw18Grgt8BZSR499IFIkoDOeX3CPwLLgT2Bo4EPJnnL7EUrSQ+YL9M23t0+v7yq7gBIchnwI2AP4HiaAvm9SZ5bVT9q++8KrAuc1C7/JfB04GlVdVm7n7OAn7fv8dezMBZJUre8PuGSqjqgfX16kkcB70/y2apaOWsRSxLz5Mgz8GzgjIkEC1BV5wFXAi9oly8EfgHs07PdPsD3q+qGdvmlwI+BXyVZlGTij4fvA9tN9sZJ3pxkWZJlN9100wCHJEkL2rR5vcc3+pZPBh4DbNm/096cffvNNw82Ykli/hTPWwA3TNJ+A7Bpz/KJwF5pbAT8Ge2UjdZi4LnAfX2P/YE/nOyNq+qoqtquqrZbvHjxWg9EkgR0z+sAN06xvEX/xr05+xGPfOTaRylJfebLtI3rgEdN0r45zZHkCScCH6A5arE1zR8HJ/esvwVYBhw4yb7uHUikkqQuuuZ1Juk3sXzdoIOSpOnMl+L5PODAJBtW1Z0ASbYHlgD/fSeNqrokyU9opmtsDZxVVb3n7c4GdgSuqqr+IxmSpNnTKa+3dgM+27O8O03hfM0sxClJq5gv0zY+1T6fnuRVSf6C5ojyxcDX+/qeSFM8v6x93evLwK+Ac5K8PskOSfZI8vEk7xxi/JKkVc0krz8tyeeT7Jjko8AbgH/wYkFJozAviueq+i3wYmAFzRXYnwH+A3hZVf2ur/sJNHObVwKn9O1nRbufM4EPAWcA/wQ8ETh/iEOQJDUKZpzX3wtsRFNUH0DzTYVHzlbAktRrzk7bqKr9+pYvBF7SYbvLgaxm/e3AQe1DkjR7NqS59gSYPq9X1ZU8kM+/MtTIJKmjeXHkWZI0fyXZJMmrgB1oLtqWpHnL4lmSNGwvAv6F5kLAT444FklaK3N22oYkaTxU1Sk0UzYkad7zyLMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1NGiUQcwn1x45S/Z4I1/PuowBu4H2/521CEMzbXfvWfUIUgakd/d/3uuuvO2UYcxcM/eZudRhzBER406AGlaHnmWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjrqXDwn2WaYgUiSBsecLUnDMZMjz/+V5IIkBybZeGgRSZIGwZwtSUMwk+L5JcBPgUOBa5Mcn+RlSTKc0CRJa8GcLUlD0Ll4rqpzqup1wKOBtwKPBU4Hfp3kI0meMKQYJUkzZM6WpOGY8QWDVXV3VX2xqv4EeDJwJfB+4BdJvp9ktwHHKElaQ+ZsSRqsNbrbRpIlSQ6hOYrxPOA7wJuBG4ATkxw2sAglSWvFnC1JgzOTu22sn+S1Sb4HXA78BXA08Liq2rWqjqmqvYEDgDcMJ1xJUhfmbEkajkUz6HsDsA7wdeClVXXOFP0uAG5ey7gkSWvHnC1JQzCT4vm9wFer6vbVdaqqnwBbr1VUkqS1Zc6WpCHoNG0jyUOBd9LMlZMkzWHmbEkank7Fc1WtADYGVg43HEnS2jJnS9LwzORuG18B9h9WIJKkgTJnS9IQzGTO81XA3kkuAE6juRiletZXVX12kMFJktaYOVuShmAmxfMn2+ctgG0nWV+AiViS5gZztiQNQefiuarW6AtVJEmzz5wtScNhcpUkTSnJOUmqfbyjbVva07Yyya+THJtk875tH57k7iTLk2w4yb7369vP7UkuSnJ4kidM0v/Knv67DG/UkjS1GRXPSR6V5ONJzk7yiyRPa9sPSuItkSRpDhlgzv4ezW3vTuhp+1nb9gLgo8ArgFOT9P5eeSWwPvAw4NWr2f9LgOcDewDHADsCFyXZqa/fbsDuM4hbkgZuJl/P/WzgMprkdiXwBGC9dvUWwLsHHZwkac0MOGffUlU/qqrre9rubtvOraqjgXcBz2bV+dX7Ar8EftW+nsoF7b7Oqqp/Ap4JnAt8NckjJjpV1YXAhTOIW5IGbiZHng+jOfrwJOAAID3rzqdJmkORZOf2lN7Wfe1bt+2vapffmuSyJPcmuTzJO/v6L02yrK9tiacAJY2h2c7ZP26flwAk2QR4OXAizRHrlyVZ3GVHVXUv8Haae1WvruiWpFk3k+L5WcA/V9VKVr3dEcDNwKMGFtWDnQ5cC7yur30/4EbgX5O8Cfg0cCqwK3AS8MkkBw8xLkmaq2Y7Zy9pnyeOTu8BrEtTOB9Pc4H6nl13VlWXAtcAzx1ciJK09mZSPN8ObDbFusfT3EN0KKrqfmAp8LokAWifXwf8C823aB0CLK2qd1fVGVX1PuBzwPvar6qVpIVk6Dk7yaIk6yZ5BnAocDUPHIHeF7i0qi6qqouBS5j5UeRrgM2n7SVJs2gmxfOpwIeSPL6nrdrTcO8BTh5oZA/2RWArYId2+cXt8rHAlsBjaI429zoR2AjYZk3fNMmbkyxLsox7f7emu5Gk2TbsnL0tcB9wLw/MQ96zqpYn2YImV/deYHgC8MIkW87gPTJ9l74NenL2XbfeNtPNJWlaMyme/wa4A/gp8O9t2+eAnwP3AH832NBWVVW/BM7hga+b3R84v6ouobn4BR58JGViedO1eN+jqmq7qtqO9dZd091I0mwbds6+FNieZnrI5lW1TVWd367bm+b3y78l2TjJxjTfchhgnxm8x2OZ4RHy3py9wSYbz2RTSeqkc/FcVbfSzD17C/Br4CyaK6gPBv64qu4cSoSr+gKwR5LH0tyu6Ni2/br2uX8O38Tpvlva5xU0c/B6bTLoICVp1GYhZy+vqmVVdWFV3di3bmJ6xnnAre1jWd+61UryFJqzij9cyzglaaBm8vXcVNXvaO7BecxwwpnWycBnaE7/PYQHTgleQ3NB4V40Rzcm7E1z5OXinn5Lkjy0qla0bTsOO2hJGoVR5Ox2mshzaO72cWrf6p2A9yZ5YlVdtpp9rAccAdzGqlM/JGnkOhfPSdafrk9VLV+7cKbd/4okX6E5knJ8Vd3Wtq9Mcgjw+SQ3A2cCLwIOBN7fUyifAnwY+EKSpTT3En39MGOWpFEYYc5+Dc1F3J+oqmv7Yvopzf2g96XJxRO2T3IPzReqPJ3m1npbAXtV1e1DiFGS1thMjjzfxYNvd9RvnbWIpatTaIrnL/Y2VtXR7V01Dmof1wDvrqrDevr8JMnrgQ/QTPv4Ls3c6f8zC3FL0mwaVc7eFzi7v3AGqKobk5zJg4vn77bPd9F8octZwBFVdcUQ4pOktTKT4vn1PDgRT9wE/6nARwYV1DR2pJm/993+FVX1aZp7PU+pqpbS3Pau14yv6JakOW6QOTtJFgH3V2O/qTpW1WrvblRVO/e8XsqD8/HqgliH2TlII0lT6lw8t0luMocn+SzwtIFENIUkT6ZJ+AcCH2pv/C9JmsSAc/buNLeleydw+FqGtjauoJnOIUkjM6MLBlfj6zT3VH7ndB3XwudpLkI5leZCEknSmplJzj4A2LB9fdXQIupmV2C99vWUFxxK0jANqnjenuZG+UNTVTsMc/+StIB0ztlV9fMhx9JZ+02FkjRSM7nbxqGTNK8LPAX4U0Z7Kk+S1MOcLUnDMZMjz3vz4ItPVtDc1eLtwFGDCkqStNbM2ZI0BDO5YHDJEOOQJA2QOVuShqPT13MneXqSzyW5NMmdSe5I8vMkX0zywmEHKUnqzpwtScMzbfGc5CDgQmAf4FLgCzRf9XoJ8GrgnCSHt30fkmS191mWJA2POVuShmu10zaS7AwcBhwK/K+quqNv/YbA+4C/SfIbmq/EfgnwtuGEK0maijlbkoZvujnP7wG+VFUHT7ayqu4E3p9kC+BjwPXAiwcboiSpI3O2JA3ZdNM2tgVO6LCfiT7bVdV5axeSJGkNmbMlacimK54fAvy+w35+DyyvquvWPiRJ0hoyZ0vSkE1XPF8C7NJhP7sAP1n7cCRJa8GcLUlDNl3x/DngLUnelCSTdUjyRuD/BT476OAkSTNizpakIVvtBYNVtTTJc4DPA+9J8i3g1+3qrYBXAE8CPl9VXx5qpJKk1TJnS9LwTfsNg1V1YJLTgYOAtwDrtavuBc4FDq6qbw4vRElSV+ZsSRquTl/PXVWnAKckWQdY3DbfVFX3Dy0ySdIaMWdL0vB0Kp4ntIn3hiHFIkkaIHO2JA3etF/PLUmSJKlh8SxJkiR1NKNpGwvdUx+3FV898qhRhzFwOx/xqVGHMDSXvOvWUYcwFB/88CWjDkGa8za6/052vOPsUYcxcL9a/JejDkFa0DzyLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdTSrxXOSc5JU+3hH27a0p21lkl8nOTbJ5n3bPjzJ3UmWJ9lwkn3v17ef25NclOTwJE+YpP+VPf13Gd6oJWn+Mm9L0qpGceT5e8DzgBN62n7Wtr0A+CjwCuDUJL3xvRJYH3gY8OrV7P8lwPOBPYBjgB2Bi5Ls1NdvN2D3NR+GJC0Y5m1Jai0awXveUlU/6mu7u6ft3CT3AMcB2wIXtO37Ar8E0r4+bor9X1BVd7Wvz0ryOeDbwFeTLKmq2wGq6sIktw5mSJI01szbktSaq3Oef9w+LwFIsgnwcuBEmiMfL0uyuMuOqupe4O3AxjTJW5I0eOZtSQvCXC2el7TP17fPewDr0iTg42mOmO/ZdWdVdSlwDfDcwYUoSeqxpH02b0saa3OmeE6yKMm6SZ4BHApczQNHMvYFLq2qi6rqYuASZn404hpg82l7PTiuNydZlmTZrTffMtPNJWlszcW83Zuzb7nt7hm+nSRNb64Uz9sC9wH3Ahe2bXtW1fIkWwA7sOqFKicAL0yy5QzeI2sSWFUdVVXbVdV2mzxy0zXZhSSNozmZt3tz9qYbP3ymm0vStOZK8XwpsD3wLGDzqtqmqs5v1+1NE+e/Jdk4ycbAaTRJdZ8ZvMdjgRsGGLMkLWTmbUkL0ijutjGZ5VW1bIp1E6f5zpti3Sen23mSpwBbAj9cs/AkSX3M25IWpLlSPE8qyeOB5wCHAaf2rd4JeG+SJ1bVZavZx3rAEcBtrHoKUZI0YOZtSeNuThfPwGuAlcAnqura3hVJfgq8i+Yoxod7Vm3f3m90feDpwAHAVsBeE/cKlSQNjXlb0lib68XzvsDZ/QkYoKpuTHImD07C322f7wKuBM4CjqiqK4YcqyTJvC1pzI2ieE6SRcD91dhvqo5Vtc3qdlRVO/e8XgosnUEQ6wDrdO0vSQuYeVuSWqO428buNLc3OmgE793rCuDyEccgSfOBeVuSWrN95PkAYMP29VWz/N79dgXWa19PeeGKJC1w5m1J6jGrxXNV/Xw232912m+8kiSthnlbklY1V74kRZIkSZrzLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOFo06gPlk+W9+yn8d/IxRhzFwT/rjvxt1CEPzo6fuO+oQhmTpqAOQ5rxF623Epo//s1GHMXC7n3T8qEMYmq1GHYDUgUeeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjma1eE5yTpJqH+9o25b2tK1M8uskxybZvG/bhye5O8nyJBtOsu/9+vZze5KLkhye5AmT9L+yp/8uwxu1JM1f5m1JWtUojjx/D3gecEJP28/athcAHwVeAZyapDe+VwLrAw8DXr2a/b8EeD6wB3AMsCNwUZKd+vrtBuy+5sOQpAXDvC1JrUUjeM9bqupHfW1397Sdm+Qe4DhgW+CCtn1f4JdA2tfHTbH/C6rqrvb1WUk+B3wb+GqSJVV1O0BVXZjk1sEMSZLGmnlbklpzdc7zj9vnJQBJNgFeDpxIc+TjZUkWd9lRVd0LvB3YmCZ5S5IGz7wtaUGYq8Xzkvb5+vZ5D2BdmgR8PM0R8z277qyqLgWuAZ47uBAlST2WtM/mbUljbc4Uz0kWJVk3yTOAQ4GreeBIxr7ApVV1UVVdDFzCzI9GXANsPm0vSVIn5m1JC9FcKZ63Be4D7gUubNv2rKrlSbYAdmDVC1VOAF6YZMsZvEfWJLAkb06yLMmyO1esXJNdSNI4mpN5uzdn33Sz06MlDd5cKZ4vBbYHngVsXlXbVNX57bq9aeL8tyQbJ9kYOI0mqe4zg/d4LHDDTAOrqqOqaruq2m7Dh86Vfy5JGrk5mbd7c/biR24yk00lqZNR3G1jMsuratkU6yZO8503xbpPTrfzJE8BtgR+uGbhSZL6mLclLUhzpXieVJLHA88BDgNO7Vu9E/DeJE+sqstWs4/1gCOA21j1FKIkacDM25LG3ZwunoHXACuBT1TVtb0rkvwUeBfNUYwP96zavr3f6PrA04EDgK2AvSbuFSpJGhrztqSxNteL532Bs/sTMEBV3ZjkTB6chL/bPt8FXAmcBRxRVVcMOVZJknlb0pgbRfGcJIuA+6ux31Qdq2qb1e2oqnbueb0UWDqDINYB1unaX5IWMPO2JLVGcfuI3Wlub3TQCN671xXA5SOOQZLmA/O2JLVm+8jzAcCG7eurZvm9++0KrNe+nvLCFUla4MzbktRjVovnqvr5bL7f6pCmS/4AAA0XSURBVLTfeCVJWg3ztiStym/9kCRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6SlWNOoZ5I8lvgV/P0tstBm6apfeaTeM6LhjfsTmuwdiqqjabxfdb8MzZAzOuY3Nc889sjm3KnG3xPEclWVZV2406jkEb13HB+I7NcUnTG+efp3Edm+Oaf+bK2Jy2IUmSJHVk8SxJkiR1ZPE8dx016gCGZFzHBeM7NsclTW+cf57GdWyOa/6ZE2NzzrMkSZLUkUeeJUmSpI4snuegJEuTLBt1HKszH2JcyJKck6TaxzvatqU9bSuT/DrJsUk279v24UnuTrI8yYaT7Hu/vv3cnuSiJIcnecIk/a/s6b+L45JWlWSD9udov1l+3xnl8SRLRvnzPs6f/3Ed27iOy+J5bvoIsN+og9C89z3gecAJPW0/a9teAHwUeAVwapLeXPBKYH3gYcCrV7P/lwDPB/YAjgF2BC5KslNfv92A3dd8GA8yruOSZtt8/F0zzp//cR3b2I1r0SB2osGqqitGHYPGwi1V9aO+trt72s5Ncg9wHLAtcEHbvi/wSyDt6+Om2P8FVXVX+/qsJJ8Dvg18NcmSqrodoKouTHLrYIYEjO+4pFk1T3/XjPPnf1zHNnbj8sjzHDTTU2lzQZJnJDm7Pb1ya5Kv9J6CSfKrJP84yXYnJflBz/KmSY5KckOSFUnOTfKc2RpHG8PO7Smgrfvat27bX9UuvzXJZUnuTXJ5knf29X/Q/2NGfNpzEj9un5cAJNkEeDlwIs1RgpclWdxlR1V1L/B2YGOaRDdK4zouDUn7eb66PU18SpI/bT+rO7Tr109yRJLr29x0QZIdp9jPlHmh7bNHkl8kuSfJvwN/NPwRPlh/jpouj/fYKMlxSe5McmOSD85i2F2M8+d/XMc2r8Zl8ay1lmQz4Bya0yt/DrwNeBFwZpJ1225fA/bq224DmlM1J7TL6wFnAS8F/prmNM1vaf6SfPTQB/KA04Frgdf1te8H3Aj8a5I3AZ8GTgV2BU4CPpnk4FmMcxCWtM/Xt897AOvS/J8cT3N2as+uO6uqS4FrgOcOLsQ1sqR9HrdxaQiS7MYDn+fdgItoTv/2OhrYH/j7ts/VNLngBT37mTYvJHkWTUHwXzSnkL9Fkx9HqmMen/CPwHKaz9DRwAeTvGX2op3WkvZ5HD//S9rncRvbkvZ5XozLaRsahHe3zy+vqjsAklwG/IjmA3A8zQfgvUme23OqZleaD8dJ7fJfAk8HnlZVl7X7OQv4efsefz0LY6Gq7k+yFHhdkg9VVSUJTTH9L8BK4BBgaVVNjP2MJI8A3pfk8KpaMRuxrokki2j+cH4qcChNETDxV/++wKVVdVHb95K27XMzeItrgMmOVg3VuI5Ls+L9wHeqaqIAPKM9ynUgQJKn0Py87F9VX2rbTqcpsj8AvDzNXM1DmD4vHAz8Ati7mnvFntYWpx+djYGuRpc8PuGSqjqgfX16kkcB70/y2apaOWsR9xjnz/+4jm0+j8sjzxqEZwNnTCRcgKo6D7iS5mIAqupCml8Y+/Rstw/w/aq6oV1+Kc0H51dJFrUfLIDvA7P9XfZfBLYCdmiXX9wuHwtsCTyGB4r+CScCGwHbzE6Ia2Rb4D7gXuDCtm3PqlqeZAua8fZe1HEC8MIkW87gPTKIQGdoXMelIWvzzDNpjhb36l3enub//78/822ReBJtjqN7Xng2cGqt+iULJ6/FEAZl2jze4xt9yyfTjH0mn6dBGufP/7iObV6Py+JZg7AFcMMk7TcAm/YsnwjslcZGwJ+x6odjMc0plvv6HvsDfziEuKdUVb+kOYW5f9u0P3B+VV1CM1548Jgnljdl7rqUphB4FrB5VW1TVee36/amyQn/lmTjJBsDp9EkoH0m3dvkHsvkPw/DNK7j0vAtBtahmSLWq3d5C+Cuqlre1+cGYP12ylnXvPBomulfvfqXR6FrHoep49+C0Rjnz/+4jm1ej8tpGxqE64BHTdK+OQ+cgoGmeP4AzVGMrWk+HL1HXG4BltGeKu1z70AinZkvAEcneR/N3MSJ05rXtc/9Y544PXRL+7yCZlpKr00GHeQMLa+qqS5Gnbiw4rwp1n1yup23p7e3BH64ZuGtsXEdl4bvJuB+YLO+9t7l64ANkqzfV0BvTvOzd2+Srnnh+kn6TJY/Z1vXPM4k/SaWr2M0xvnzP65jm9fjsnjWIJwHHJhkw6q6EyDJ9jQXAPz3nTSq6pIkP6H5y3Fr4KyqurlnP2fT3J/xqqqaC0diTgY+Q3N0/CE8cJT8GpoLCvei+Wt4wt7AHcDFPf2WJHlozxzoB12dPxckeTzwHOAwHnz6eiea+epPnJiLPsU+1gOOAG5j1TMKIzOu49LgVNXvk1wIvAr4fM+qV/a8vgAomguWvgzQXgexJw/kuK554QLglUne1zN1Yy7cL7xTHm/tBny2Z3l3msL5mlmIs7Nx/vyP69jmy7gsnjUIn6I5Wnx6ko8DGwAfo/ll8fW+vicCBwGPAN7Ut+7LwF8B5yT5BM39HR9JMxfv+qo6bGgjmERVrUjyFeAtwPFVdVvbvjLJIcDnk9wMnElzVfqBwPt7CuVTgA8DX2gvQHwm8PrZHMMMvIbmQshPVNW1vSuS/BR4F81f/B/uWbV9mntzrk9zoecBNPPC95q4r+YcMK7j0mD9A/D1JEfS/ML+Y5o7AQGsrKpLkxwPHJnmm86uoMlff0R7pmwGeeHjNIXq15IcQ/Mz9oZZGON0ZpLHn5bk8237n9DEf9CoLhZcjXH+/I/r2ObFuJzzrLVWVb+luaBuBc0V2Z8B/gN4WVX9rq/7CTRzDFfSFJe9+1nR7udM4EPAGcA/AU8Ezmc0JmL8Ym9jVR1N80fAbjQ3Y98XeHdVfaynz09oiuXn0fxCfhEPzKGea/YFzu5PVgDtWYAzefD9Mr9Lc0rsmzSFxFnA/1NVpzF3jOu4NEBVdTLNfWFfTfOZ3x54T7t64gK6NwFfAv6O5mdjK2CXquo9u9YlLyyjKRCe2b7Xq5nZPM5BqzaumeTx99JcBPl1mkLlI8CRsxXwDIzz539cxzYvxpVVL/iV1CvJoTSnXR8/B4+qTCnJOcDNNL+U768RftCTrENz6vdyYNeq+vZa7OscxnBcmnuS/C3wP4FNq+qeUcczDEm+DmxQVS8fdSxdjPPnf1zHNq7j8sizNIkkT07zxQkHAkfOp8K5x+40dys5aMRxXEGTrAZlXMelEUmyWZJPJdk1zTcLHkJTOB8zjoVzkk3SfFPqDjQXac8n4/z5H9exjd24PPIsTaL9a/k5NNMt/sckpy3ntCRPBjZsF0d6AWaSbYD12sXL1mYO2riOS6PVfpHJ8TTXVzyC5uK3rwIfqKr7RhnbMCR5NXAczenu/avqlmk2mRPG+fM/rmMb23FZPEuSJEndOG1DkiRJ6sjiWZIkSerI4lmSJEnqyOJZmkKSQ5Lc1Nf2kCRfSbIiyby4vZMkLQTmbM0Wv2FQ6qj9Ot6jab5+d/eqOn3EIUmSpmDO1rBYPEvdHQm8FniNX4ghSXOeOVtD4bQNqYMknwL+CnhtVX29p/2FSb6fZHmSm5McnWTDdt2m7anC/fr2lSS/THLYrA5CkhYIc7aGyeJZmkaSvwfeAbyxqo7vaf9j4CzgemDPts/OwLEA7RcPfAPYr2+XOwBbA18ccuiStOCYszVsTtuQVu+RwPuBw6rq2L51HwPOrap9JhqS/AY4O8nTq+onwDHAGUkeX1W/bLvtD/y4qi6ehfglaSExZ2voPPIsrd4dwHnAG5I8Y6IxyfrA84CvJVk08QB+ANwHbNt2PRv4NfC6drsNgT1oj3RIkgbKnK2hs3iWVu8+4BXAtcBpSR7ftm8CrAP8c9tn4nEv8AfAHwJUVdEk3de1V37v3W731VkcgyQtFOZsDZ3TNqRpVNXN7f1BzwVOb+fN3QYUcAjwnUk2u7bn9bHAB4EX08ylO6Wqbh1mzJK0UJmzNWwWz1IHVXVVm4z/AziN5gKSHwFPrqoPT7Pt1UnOAD4EvAD4syGHK0kLmjlbw2TxLHVUVZck2YXmau1vAAcDZyZZCfxv4E7gcTSnDP9nVf2iZ/NjgJOAa4AzZzVwSVqAzNkaFuc8SzNQVefSzIF7EfAW4E+AzYDjgG8B7wWuBm7o2/TbwO+BL1XVylkLWJIWMHO2hiHN3HhJw5RkZ5pk/KSqunzU8UiSpmbO1upYPEtDlOQxwBOBTwNXVdUuIw5JkjQFc7a6cNqGNFxvprlv6ArgbSOORZK0euZsTcsjz5IkSVJHHnmWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6+v8B/HE5UNSbzwkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x360 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([[[[0.3935, 0.5685, 0.0896, 0.0000, 0.0000],\n",
            "          [0.4945, 0.2905, 0.8362, 0.0000, 0.0000],\n",
            "          [0.9139, 0.2138, 0.9821, 0.0000, 0.0000],\n",
            "          [0.5684, 0.8328, 0.2671, 0.0000, 0.0000],\n",
            "          [0.1406, 0.7698, 0.6470, 0.0000, 0.0000]]],\n",
            "\n",
            "\n",
            "        [[[0.8680, 0.8078, 0.0000, 0.0000, 0.0000],\n",
            "          [0.5331, 0.2438, 0.0000, 0.0000, 0.0000],\n",
            "          [0.5725, 0.3306, 0.0000, 0.0000, 0.0000],\n",
            "          [0.2783, 0.5039, 0.0000, 0.0000, 0.0000],\n",
            "          [0.3772, 0.6935, 0.0000, 0.0000, 0.0000]]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xLkGnGLkIMUb"
      },
      "source": [
        "[PAD]에 해당하는 열만 마스킹되었습니다. 어텐션 스코어 행렬에서 행은 쿼리에, 열은 키에 해당하는데 이렇게 마스킹되는 것은 결국 키만 마스킹 했다는 의미입니다. 다시말해  i, love, you, [PAD], [PAD]라는 단어 다섯개를 쿼리로 요청하였고 그에 대한 응답으로 [PAD] 두개는 마스킹하고 i, love, you만 키가 되어 요청된 쿼리와 가중치가 계산된 것입니다.\n",
        "\n",
        "어차피 필요없는 [PAD]를 왜 쿼리로 요청했는지 이상할 수 있습니다. 그 이유는 이렇게 키만 마스킹된 인코딩 정보가 디코더로 넘어가 크로스 어텐션될 때는 다시 키로 작용하게 되는데 그때 키가 $K^T$로 전치 되면서 다시 한번 마스킹을 적용하면 마스킹 되지 않았던 쿼리에 대한 [PAD]도 마스킹되어 사라지기 때문입니다. \n",
        "\n",
        "이런 이유 때문에 이 소스코드에서 `src_mask`가 PyTorch에서는 `key_padding_mask`라는 이름으로 제공됩니다.\n",
        "\n",
        "> key_padding_mask: If specified, a mask of shape (N,S) indicating which elements within key to ignore for the purpose of attention (i.e. treat as  padding”). \n",
        "\n",
        "해당 과정은 설명의 마지막 부분에서 구체적인 예를 들어 직접 코드로 확인하겠습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BqBQFZqwJl2X"
      },
      "source": [
        "### Target mask"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qwx0zM31Kvw5"
      },
      "source": [
        "다음은 타겟쪽 마스크를 알아보겠습니다. 타겟 마스크도 먼저 [PAD] 위치를 마스킹 합니다. 이 과정은 앞서 알아본 `src_mask`와 동일합니다. 타겟 데이터는 소스가 번역된 다음 문장으로 가정합니다.\n",
        "\n",
        "\n",
        "*   `나는,  당신을,  사랑,   합니다, [PAD], [PAD], [PAD]`\n",
        "*   `잘,    했어,    [PAD],  [PAD],  [PAD], [PAD], [PAD]`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bp5FhIEkMMAS",
        "outputId": "612b9525-b3ad-4481-913b-ef39479de571"
      },
      "source": [
        "# 디코더에 입력될 타겟을 준비한다. \n",
        "# shape: (2,6)\n",
        "# 임의 숫자를 토큰 번호로 가정하고 randint를 사용\n",
        "trg = torch.from_numpy(np.random.randint(1, 100, size=(2, model_const['target_len'])))\n",
        "paddings = torch.LongTensor([2, 4]).reshape(-1,1) # 각 샘플당 패딩 개수\n",
        "pad_idx = trg.shape[1] - paddings\n",
        "col_idx = torch.arange(trg.shape[1]).reshape(1,-1)\n",
        "# src = src * ~(paddings > col_idx)\n",
        "trg = trg * (pad_idx > col_idx)\n",
        "\n",
        "# 마지막 토큰들은 PAD토큰이 되었음\n",
        "print('trg', trg, '\\n')\n",
        "\n",
        "# src때와 마찬가지로 패딩 토큰자리만 False인 trg와 모양이 같은 마스크\n",
        "# shape: (2,6)\n",
        "print('(trg != pad).shape', (trg != pad).shape)\n",
        "print(trg != pad, '\\n')\n",
        "\n",
        "# src_mask때와 마찬가지로 차원을 늘린다.\n",
        "# subsequent_mask()에서 출력된 마스크와 브로드캐스팅된다.\n",
        "# (2,1,6)\n",
        "print('(trg != pad).unsqueeze(-2).shape', (trg != pad).unsqueeze(-2).shape)\n",
        "trg_mask = (trg != pad).unsqueeze(-2)\n",
        "print(trg_mask, '\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "trg tensor([[29, 66, 61, 43,  0,  0],\n",
            "        [59, 86,  0,  0,  0,  0]]) \n",
            "\n",
            "(trg != pad).shape torch.Size([2, 6])\n",
            "tensor([[ True,  True,  True,  True, False, False],\n",
            "        [ True,  True, False, False, False, False]]) \n",
            "\n",
            "(trg != pad).unsqueeze(-2).shape torch.Size([2, 1, 6])\n",
            "tensor([[[ True,  True,  True,  True, False, False]],\n",
            "\n",
            "        [[ True,  True, False, False, False, False]]]) \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jLungGfHMtVr"
      },
      "source": [
        "디코더 쪽에서 사용하는 타겟 마스크는 앞서 설명한 셀프 어텐션에 대한 마스킹도 함께 해야 하므로 `subsequnent_mask()`에서 생성된 마스크 행렬과 & 연산을 해줍니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VpgjuuMx7mWv",
        "outputId": "8fa5575b-242e-4e2c-f7ea-f2a9e9b3428e"
      },
      "source": [
        "# 입력 타켓으로 subsequent_mask 마스크 행렬을 만든다.\n",
        "# shape: (1, 6, 6)\n",
        "self_attn_mask = subsequent_mask(trg.size(-1)).type_as(trg_mask.data)\n",
        "print('self_attn_mask.shape => ', self_attn_mask.shape)\n",
        "print( self_attn_mask, '\\n')\n",
        "\n",
        "# 패딩 토큰 자리가 False인 trg_mask와 subsequent_mask()의 결과를 & 한다.\n",
        "# 다음 브로드캐스팅되는 과정이 있고 늘어나는 차원은 []로 표시했다.\n",
        "# (2,1,6) & (1, 6, 6) => (2, [6], 6) & ([2], 6, 6)\n",
        "trg_mask_dec = trg_mask & self_attn_mask\n",
        "print('trg_mask_dec')    \n",
        "print( trg_mask_dec )"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "self_attn_mask.shape =>  torch.Size([1, 6, 6])\n",
            "tensor([[[ True, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True]]]) \n",
            "\n",
            "trg_mask_dec\n",
            "tensor([[[ True, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True, False, False]],\n",
            "\n",
            "        [[ True, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dSoA3EJdnYx_"
      },
      "source": [
        "위 코드의 주석을 읽고 마지막 출력 결과를 보면 패딩 토큰이 있는 행과 열은 모두 False이고 데이터가 있는 자리만 작게 하삼각 행렬이 True인 부분 행렬이 마스크로 만들어짐을 확인할 수 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 362
        },
        "id": "H158dtFsNhWq",
        "outputId": "20116805-79c9-4fac-ad6f-1a0380763ff0"
      },
      "source": [
        "# 샘플 두개 헤드 하나짜리 (6,6)인 어텐션 스코어\n",
        "decoder_self_attn_scores = torch.rand((2, 1, 6, 6))\n",
        "trg_mask_dec_reshape = trg_mask_dec.unsqueeze(1)\n",
        "\n",
        "# -1e9로 마스킹을 하면 마스킹 되지 않는 부분이 모두 같은 색으로 \n",
        "# 나타나므로 여기서는 그냥 0으로 마스킹\n",
        "masked_decoder_self_attn_scores = decoder_self_attn_scores.masked_fill(trg_mask_dec_reshape == 0, 0)\n",
        "\n",
        "fig, ax = plt.subplots(figsize=(12,5), nrows=1, ncols=2)\n",
        "\n",
        "ax[0].imshow(masked_decoder_self_attn_scores[0][0], cmap='BrBG')\n",
        "ax[0].set_xticks([0, 1, 2, 3, 4, 5])\n",
        "ax[0].set_xticklabels(['나는', '당신을', '사랑', '합니다', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[0].set_xlabel('Key', fontsize=15)\n",
        "ax[0].set_yticks([0, 1, 2, 3, 4, 5])\n",
        "ax[0].set_yticklabels(['나는', '당신을', '사랑', '합니다', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[0].set_ylabel('Query', fontsize=15)\n",
        "ax[0].set_title('Sample 1', fontsize=15)\n",
        "\n",
        "ax[1].imshow(masked_decoder_self_attn_scores[1][0], cmap='BrBG')\n",
        "ax[1].set_xticks([0, 1, 2, 3, 4, 5])\n",
        "ax[1].set_xticklabels(['잘', '했어', '[PAD]', '[PAD]', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[1].set_xlabel('Key', fontsize=15)\n",
        "ax[1].set_yticks([0, 1, 2, 3, 4, 5])\n",
        "ax[1].set_yticklabels(['잘', '했어', '[PAD]', '[PAD]', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[1].set_ylabel('Query', fontsize=15)\n",
        "ax[1].set_title('Sample 2', fontsize=15)\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs8AAAFZCAYAAABnpcJ0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhsZXnv/e+PjXsDIqAMIiAyCg7gBNGAEw4BDBHFEVAB9eWI0wGHHOM8vCHRDHAQI0L0NUHFgRyD4oAzir4SCWpQFBUEgYjILIibPdznj7Uai6a6e1Xv6q6u7u/nutYltYZn3bXtuvvuZz3PU6kqJEmSJM1svVEHIEmSJI0Li2dJkiSpI4tnSZIkqSOLZ0mSJKkji2dJkiSpI4tnSZIkqSOLZ2meJDk0yVmjjkOS1I15W/1YPGtRSeNVSX6c5MYkv03yjlHH1VrRbuskyQZJzkxy0RBikqSRWsx5O8kOSd6f5PL2vV2Y5KAhxqcRWH/UAUhD9jbg8cCfV9XlSTYG7jfimIYmyX2AzwC3A/cYcTiSNAyLOW9vB/wAeDNwA/A04ONJ9qkqO0DGlMWzFptXAU+uqssBqupW4OcjjWi4DgU+CvwEOGXEsUjSMCzavF1V5wHn9ez6XJL/AxwMWDyPKYdtaLEpYOupDibZPckX2seCNyQ5P8ljeo4/OclXkrwxya/bc/53kvWSHNs+ersmyXuSLOu57tQkL0nyL0muba97f5INpoll/bad3ya5KclHkmwy7Zurel9VvX/AfxNJWsgWdd7u42Zg0wGv0QJi8azF5u3AR5I8ZYrjGwDvBrYFNm//+xNJJp7CrAH2AR4MPBB4ALAH8DHgKe1/PxDYGziyp93lwDuB/6T5JfDg9pzjp4n1r4HHAo+gebR3B3By1zcqSYvE21kiebst3g8Czul6jRagqnJzW1Qb8FKav+xPB7bqcP41wC7tfz8RWAls0nP8Oe2++07a9289rz8MfGVSuw8HbgE2aF8fCXyx/e/7AL8Dduo5/17ATcCWHWJ+IvDTUf9bu7m5uQ1jWwp5uz3/5cB3Rv3v7bZumz3PWnSq6p+Bh9D0VvwoyZ9MHEtyjyTHJTmvfbx3M7AVTW/GhCur6pae178FLqmq30za13sNwJcmxfEDYBWwQ58w9wQur6rLes7/HXAZ8KBu71SSFoelkLeT7ErTy/4/ZjpXC5sTBrUoVdVVwHOSHAecmWTnqloFvJfmkdubaCZx3AD8CkjP5av7NHlDh9ve1GffLfQf27YNsHuSydcsBzbrcC9JWlQWc95Oci/g34H/Va6yMfYsnrWoVdUJSV4PPDTJT4AXAw+qqkvhzvFnU05UGdB9++zbCri6z/5bgR9U1d5DurckLQqLLW+38X4c+GpV/X+Dh6iFxmEbWtSSbEQzTu0mmp6E9YErek55MsP7I/Iuk12SPAG4DvjvPudeCDwoyZZDurckLQqLMG+fSFNvHTerCLXgWDxrUUly+ERiS3Jf4F+BT1bVL4FraRLwMe03Wu0B/B1w6ZBu/9Akr0myIsm2NDOw/7Gq1k4+sX08+QWaxfJ3bOO9d5InDSkWSRoLizlvJ3kl8CTgeVW1Zkgxa8QsnrXY/BnNZJObgW8D/wUcDVBVBTyd5otGbgA+ArwWuByYWPvzDzQztHut7LjvHcCjaHosfkDzTYAnTXPNETRfdvL/J7kF+CHwJ3TT7/6SNI4Wc95+NbAT8Kt2XeiJ7TvTXKMFLs3PpaR1keTDwDeq6sMjDkWS1IF5W7Nlz7M0HKtpFsuXJI0H87ZmxZ5nSZIkqSN7niVJkqSOLJ4lSZKkjvySlAFlww1rvU3uNeowBrLp764bdQjSknfbHcXK1ZWZz9QwbXKfe9dW22076jAGcsMvLh51CJKAG2+v66rqbut6WzwPaL1N7sVGhz1r1GEM5Mnf+OCoQ5CWvK9e0u/bgzXXttpuW97zmX8bdRgDOeOZe4w6BEnAmT9YdUW//Q7bkCRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOrJ4liRJkjqyeJYkSZI6sniWJEmSOlpUxXOSFyf5wKR9+yT5zqhikiQNX5K3JHnrqOOQtPSMTfGc5ElJvjTDacuBFX32LZ+bqCRJw5bkCUmumrTdkuTDPaetwNwuaQTWH3UAA7AIlqQloKrOBbbr3Zfk34HPjyYiSfqjsel5XgcFZNRBSJJmJ8mbgJ9X1ScnHXp12yv92VHEJWlpGqee59m6FNg+yTXTnHNlVe09XwFJkmaWZGPg74HNgYcnOb2q/qvnlJOq6s2jiU7SUrXoi+equgrYctRxSJK6SbIBcCjwP4G/qqovJDkQOLsdvvGWkQYoaUlbjMM2npfkmiQXDqvBJEcnuSDJBXX77cNqVpI0SZL1gPOBXYAnVdUXANr/fRhwJbByhjbuzNk3X3/jXIcsaYlZjD3Pn6iqI9tl66YbqtHP16rqsMk7q+pU4FSAZffdqoYRpCTp7qpqbZJHVtWaPsduBP4OIMlp07RxZ87eZc+HmrMlDdU4Fc9rgeVJAmwI3BPYCnggsBvw9d6Tq+pDwIfmO0hJ0rqpqjVJ1gd+DCyb4rQ7gNcAV8xbYJLEeBXPP6YZu3w9sAq4AbiGJnH+BPhtv4uSbAPct6q+P09xSpLWUVWtpukY6SvJ8cDjgS/OW1CSxBgVz1V1NbDrdOc0ndJ382fAAcDz5yAsSZIkLSFjUzxLkpaWJBcBmwF3G/9MM2nwVfMbkSQtjeLZL0mRpPH0UGCjqnKZI0kLxmIrnu9ot14/AU5IctU01xWwV1X9Zs4ikyQN6kfAT5KsnuL4T6vqoPkMSJIWVfHcb4WNqvoP4D6jiUiSNFtVtceoY5CkyRbjl6RIkiRJc8LiWZIkSerI4lmSJEnqyOJZkiRJ6sjiWZIkSerI4lmSJEnqyOJZkiRJ6sjiWZIkSerI4lmSJEnqyOJZkiRJ6sjiWZIkSerI4lmSJEnqyOJZkiRJ6mj9UQcwbtb+/lZuveC7ow5jIP916JtHHcKs7HnG/zvqECSNuU3WX58DNt9i1GEM5IBvXDrqEGbliCfuPOoQpHlhz7MkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLU0YIsnpO8Mck7pjiWaa67JMlOffbfJ8l7kvwwydVJfp3kqiRnJzlyujYlSbOT5Ngk17Xbb5Js3u5/bpIPtP99aJJ/GrDdbybZZy5ilqSZzHvxnGT/JDe3xWvvdlmS9dvTlgMrpmjiFUlOnuLYivba3vttDvwQuAM4qKq2rar7ATsDJwGvA/oW6pKk2UmyAjgF2K7dHgDclmQZd83xd8nbSTZP8vEkNya5Icn72rZ6LWdSrpek+TKKnucdgU9X1XaTtp2qanWH67cBbhrgfvsA11bVm6vqyomdVbWyqr4EvB44eKB3IEmaUttp8Qfg9j7bVJ0fEz4N3EqT63cFdgM+OmfBStKAFuSwjRk8lsGK54uAnZI8Y/KBJFsCxwDfG1JskrTkVdX1wAbAhsB9aArgjdrXr5zquiSPAnYCjqmq29t2ngcclGTPOQ9ckjpYyMXzK5Nck+SciR1JdgYeDbwoSafYq+pyYD/g1e3wkPOSfD3JfwEXAD8HXjX88CVp6aqqlcDf0OTZf6YZPrdvVa1pT3lOkquAf+y5bH/g81W1qqed64GvA4+bl8AlaQbrz3zK0BVw5wS9drLeFsD9gd1pilmAk6vqDZOufRvwXmB74K+Av+50w6ofAE9KshHNo8AVwE1VdfU6vA9J0hSSHELT4/zAqlqTZFfg60l2aE/5VFUdmeRImieK0OTny/s09zPg3Une0r6+z1zFLUkzGUXxfDHw90me2t5/LXA9cBVNgrys30VJDqVJsA+nKb6/neSmqnrfFOc/BfjwdIH0WWTjF1X1xD7nHQ0cDcCKe0zXpCSpsTvwrYme5qr6eZLbgC2nuaam2L8e8Laq+geAJN+d7sa9Ofv+999u0LglaVrzXjxX1beSbAos63001yvJAZNevwx4E/BnVXVLu+9JwGeTPKKqXtrnPl+hmeE9jJhPBU4FyL02miq5S5L+6AvAR5N8lqZT5FDg5qr6dZKbgev6XHMVzUpIk+3ettdJb85+5CMfbs6WNFSj6HmmqtbS9DhP5cPAMoAk9wD2BvaZtFrGtUn2pZmNPaUkewEfn+aUNcBpVfX33aKXJM2kqr6f5FXAe4CtgfOBg9pjnwU+2556Hn984vg54JwkK9ox0yS5H/Bg4KvzGL4kTWlkEwaTrJ/kZe0Evl+1k/l+meRMYNequhSgqlZV1Uuq6sokGyd55EQbVbW6qn7S0+xKmvWc6TnngqraZaoN+J/AX8zDW5akJaWqvlpVT6uqRwIPBO7ZezzJ2cA9q+qb7fk/olkh6UPtes/3Bz4FHD9RTEvSqI1ytY33A88CXlFV21fVdjSP5v4VOC3JC/tcsxcw5TdRVdVuVdV3zPQ01qNnAqMkabjaoXpPpCmgJ/ZtCDwB2GPS6c+lWQ/6IuBc4JNTzW2RpFEYybCN1kHA06vqhxM72p6FzyTZGngmcPq63qSd2f0fwC1TnLKGdmycJGlOvAG4FHhdki+3Q/deQ7PG/muTfLKq7gBo57XcbR6LJC0UoyyezwbenuS1VfVTuHN8837AX9IsSzfZXZa562hb4JdV9eh1CVaSNJgky4E3A38O/ClwIvCpJOcCh9Cs3fx64HNJjqqqq0YWrCR1NMri+RjgZcAHk2xLM0FwFXAhzbdLfbnPNVcAO7cL60/lw1X15p7XV9F8w+Dl01yzFtizqm4d5A1Ikqb1DzTjnPetqt+1azq/FXgq8JSq+j3wjiQvoRnb/Kcd272DSfNbJGm+jKx4rqrVwMnt1vWay2m+UGWQ+1zB9OuKSpLmQFW9atLrNfR5qlhVHwQ+OEC7j1/36CRpdhby13NLkiRJC4rFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktSRxbMkSZLUkcWzJEmS1JHFsyRJktTR+qMOYNzce80fePKtPx51GAO5bbtXjjqEWXn/xz446hAGdsxhLxl1CJJ63L62+PEf7hh1GAP5k2cdPOoQZuXZow5Amif2PEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR0tquI5yRuTvGPAaw5L8i9zFZMkLRZJLk5yS5IzR3T/E5LclOS2UdxfkmBMi+ckZyZ5bp9Dy4EVPec9KMlVk7Zrk3xxqmskSVPaCNizqp4NkGRlkpvbgvaaJOcm2XfyRUnemWRVkt37NZpkdU87NyQ5P8lrkmzQe15VHVdVm7VxSNJIjGXxDNwTWDnTSVX1k6rarncD9gUeMucRStLitxzYrS1otwGOB85KsuPECUmWAS8GzgOOmKKdZT3tbAkcB+wPnJtkkzmMX5IGNq7F89bAdV1OTLJzkr0mNuDhwNo5jU6SlpiqWltV5wBnA8/rObQ/cCXwLuDwJNP+3qmqNVX1HeAA4FqaglySFoz1Rx3AoJLcA9gV+EWHc/cCvgJ8o2f3WuCESacenOQa4IaqevCQQpWkpegyYIee10cBH6HJw+sB+wFfnamRqqokrwMuTPL6qrp9+KFK0uDGrngGHkMzbGNX4DcznLsF8IOqesYM551VVc8fRnCStMTtAlwMkGRz4EDgmKpam+QM4EV0KJ4BquqSJNcDDwO+O0fxStJAxnHYxv7A1cArpzj+ynbiyjk0vcwrkqyXZEWSLdpJhAcmOSbJC7rcMMnRSS5IcsHK1TWktyFJi0eSTZO8Angi8IF29+HA16pqYpjdGcAhSe45QNNX0wzVGySWO3P2TddfP8ilkjSjsSqek2wEvAT4C+BR/WZ1AydX1dZVtT/wY2AD4KfA94DPAScDLwDuD9zY5b5VdWpV7VVVe61YP0N4J5K0aFyS5Hc0+fYJwH5VNZFbJ4ZsAFBVFwJXAYcM0P4yYNUgAfXm7M0233yQSyVpRuM2bONtwJeq6vtJXgKcnmTvqurbtVBVVwOPmK7BJA+nw8odkqS+dquqaybvTPIIYCfgM5MOfYxm6MbpHdvfEbhinSKUpCEam+I5yTNoZnDvBVBV30xyKvDlJE/u6emY6voDaR4hPgi4N3ADcBFwelWdMafBS9LScxSwCXB7crcndmuTbNt2cEwpyWOB26vqR3MUoyQNbKBhG0nuP1eBzHDfHYB/Ag7uGTtHVf0tTa/G9jNc/zrg74CPAk8FdgOeBvw7cHKS185J4JK0QMxn/k6yHDgM2LuqMnmjWfN52jknSTYGTsKl6iQtMIOOeT4vybeTvDzJFnMSUR9VdTmwR1X9sM+xt/fbP8mzgHdU1Req6oaqWlVV11bVWcCxNElekhaz+czfBwO/qqoLpjh+KvDCfgfaid2HARcA362qU+YoRkmalUGL5x2BdwB7Az9N8rkkh7cT+ebUVOOaO/oqzSocD53YkcajgLcAX1jX+CRpgZuL/L2S/pP5XgicNs11ZwJbJXlY+3oNzcTDG4HzaZa3O7qqXr4OsUnSnBhozHNVrQW+BHypfSx3IPB84MQkX6KZCPKF9rxRuIP+fxC8lWaVjvcl2Z7mfa+h+aKV0+g+cUWSxtJc5O+q2mCK/U+f4bqVwFY9r8dm/o0krctSdfcFdqbpzbiNZi3OF9L0Hjx5CLENrKqOr6q39tm/tqpOq6onVNWOVXX/qtqhqp5SVf9aVS7eLGkpmW3+XgNclORTcx/i3SU5McnNNGv4S9JIDPTXfttr+2zguTRLEJ0JvL6qvtVzzoNoejdGMrlQknR3w8jfVbXzPIQ6pao6lmaeiiSNzKCPyi4Evgi8Czinqlb3OeenwK3rGpgkaajM35I0BIMWz+8E3jvdMIf22IPWKSpJ0rCZvyVpCAYd8/xWxwdL0lgyf0vSEAxaPP+fJEfMSSSSpLlk/pakIRh02MZ3gVclOYbmG6Kup5l9PWFlVf3vYQUnSRoa87ckDcGgxfOfABe1/715u/W6fZ0jkiTNBfO3JA3BoF+S8rK5CkSSNHfM35I0HLP6Vqckm9PMyN6sqs4ebkiSpLli/pakdTPol6TcCzgZ+Avgl8BuwMbtsf2AvavqPcMOUpK0bszfkjQcg6628T5gNbB9VT0KWNVz7PvAy4cVmCRpqMzfkjQEgw7bOAjYrqp+376+c83QqropyWZDi0ySNEzmb0kagkF7nlcDm/Y7kGQHnK0tSQuV+VuShmDQ4vnDwKeSPKB3ZzsB5TTgU0OKS5I0XB/G/C1J62zQYRt/BRwPXJTkl8DGSc4HHgx8HvjLIce34NzvgXvytq9+adRhDOSy4/cZdQizcubnbxh1CAP7l6//fNQhDOyI/XYddQiaH0syf//00p/zJ4c8bdRhDOR7L9pu1CHMyrv/6UejDkGaF4Ou87wG+F9J/hbYE9gGuBW4sKqunoP4JElDYP6WpOGY1TrPVXUjcO6QY5EkzTHztyStm0HXeT4RWD7NKSur6rh1C0mSNGzmb0kajkF7nn8NbNDzOsD9gCcBt9GsIypJWnjM35I0BIOOeX53v/1JAvwNsMswgpIkDZf5W5KGY9Cl6vqqqgLeBDx7GO1JkuaH+VuSBjOU4rm1MbDRENuTJM0P87ckdTTohMGHASv6tPEA4FiatUIlSQuM+VuShmPQCYOf4O7Jdw3wG5rE+3fDCEqSNHTmb0kagkEnDO4+V4FIkuaO+VuShmOYY54lSZKkRa1zz3OSLYEjgMcD2wF3AFcD3wI+XlXX9Jy7XlWtHXKskqRZMH9L0vB06nlO8lzgF8CBwJeBt9OsC3oucDDw8yRHtuduCJwzB7FKkgZk/pak4Zqx5znJo4GTgD+vqvP6nHJSkscBZyb5PfA/gJ8NN0xJ0qDM35I0fF16nt8MvHKKxAtAVX0LeAXwMeAnVXXMkOKTJM2e+VuShqzLmOdHA8/ocN6ngT9U1SvXLSRJ0pCYvyVpyLr0PK9XVWtmOqk9544uN03ymCTXTdr27jn+xiTv6NLWpHa/mWSfAa/ZKcklg95LksbAUPN3kouT3JLkzKFEN6AkJyS5Kclto7i/JEG34vnHSZ4w00lJnghcNsM590iyC3Ad8JhJ241JdkmyEbCcSYv5J7kiybYzhLG83Sau2SnJz5Jc02fbo+eayV8cIEmLwdDyd2sjYM+qenZ73cokN7cF7TVJzk2yb5/235lkVZK+a00nWd3Tzg1Jzk/ymiQb9J5XVcdV1Wb4VeKSRqjLsI1/BE5N8uSquqrfCUnuD5wC/P0MbT2Cmb8C9o1T7F8B3GOGayfbFbi2qh444HWStBgMM3/3sxx4QFVdk2Q94KnAWUn2rqpftu0vA14MnEezXN5f9WlnGbBb284ymuEmbwOel+SpVXXLLGKTpDkxY89zVZ0FnAFcnOTkJE9pe4h3TfLUJO8DLga+VlX/PENb/1FVW1TVFsCewFHA0cBjJvZX1anr/rbuFGD1ENuTpLExzPzd4V5rq+oc4GzgeT2H9geuBN4FHN4W2dO1s6aqvgMcAFwLHL8ucUnSsHVa57mq3g48meZR2SnARe12GrA58MyqennXmyZ5A/Bt4Jk0CfLsJKckSc9pr2wfA7rmqCTN0rDzdweXATv0vD4K+AjwDZrfOft1aaSqCngdcFS7/rQkLQidv2Gwqr4HfG9db5hkU+AvgR0mHsUluQfwn8DjgG+2p55cVW9Y1/tJ0lI3rPzd0S40vdkk2Zzmy1mOqaq1Sc4AXgR8tUtDVXVJkuuBhwHfnaN4JWkgnXqeh+y2dntUz74HAZvRfF3sKNy/7eX+dZIdJh9McnSSC5JccOP1189/dJK0wCXZNMkrgCcCH2h3H04zJOS69vUZwCFJ7jlA01cDWw8Yy505m1WO3JM0XPNePFfVauBpwOuT/CLJpcD7gCOr6tL2tF8BV8xjWFdW1dZVdb+qunzywao6tar2qqq97r355vMYliQteJck+R3wY+AJwH5VdWN7bGLIBgBVdSFwFXDIAO0vA1YNElBvzuYenR+wSlInI8kqVXVRkiOq6rdTnPLxPvvWctdl6O4F3BfYnqbn+ox+t2LSHwhJNqYZj7cb8Hvgl4PGL0m6025Vdc3knUkeAewEfGbSoY/RDN04vWP7OzK/nSmSNK1R/kn+30k2q6p+i92/HtiAuy5p9EXg20lW06ygcSvwW5pZ3D+jWVljsp8BeyS5hqb4rva6XwG/AD47pPciSbqro4BNgNvvOhccgLVJtq2qaYfqJXkscHtV/WiOYpSkgY2yeA5wWZLqc2xj4OTeHVX14hkbnJSgq+qXSe4DrF9VfR/7TbVovyRpdpIsBw4D9q6qC/ocPxd4AfDuadrYGDgJl6qTtMCMYsJgr53bscZ32ZjdYv19VWOg8XKSpHVyMPCrfoVz61Tghf0OJNkiyWHABcB3q+qUOYpRkmZllMVz0UwE6Weq/ZKkhWMl/SfzvZBmHempnAlsleRh7es1NBMPbwTOp1ne7ughrz8tSUMxymEbl9AM2+iXeDcGjp1Fm3e02yBWzeIaSVryqmqDKfY/fYbrVgJb9bx2SQxJY2NkCauqHjoHbT5+FtdcCjxw2LFI0iK0BrgoyRer6jnzffMkJ9JMRFw73/eWpAn+tS9J6qSqdh7x/Y9ldk8lJWloRj1hUJIkSRobFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkcWz5IkSVJHFs+SJElSRxbPkiRJUkfrjzqAcbMscK9l4/XP9qzbHjfqEGblvBP+etQhDOyWKz4z6hAk9XjIfe/JJ1+z96jDGMgeX1w56hBm5ZBRByDNE3ueJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI5GWjwnuTjJLUnOHNH9T0hyU5LbRnF/SRon5mxJGn3P80bAnlX1bIAkK5Pc3CbHa5Kcm2TfyRcleWeSVUl279doktU97dyQ5Pwkr0myQe95VXVcVW3WxiFJmp45W9KSN+riebLlwG5tctwGOB44K8mOEyckWQa8GDgPOGKKdpb1tLMlcBywP3Bukk3mMH5JWkrM2ZKWnIVWPN+pqtZW1TnA2cDzeg7tD1wJvAs4PMm076Gq1lTVd4ADgGtpkrskaYjM2ZKWigVbPPe4DNih5/VRwEeAb9DEv1+XRqqqgNcBRyXZcLghSpJa5mxJi9o4FM+7AJcDJNkcOBD4RFWtBc4AXtS1oaq6BLgeeNjww5QkYc6WtMgt2OI5yaZJXgE8EfhAu/tw4GtVdV37+gzgkCT3HKDpq4GtB4zl6CQXJLnghuuvH+RSSVoSFmzOvuUPg1wqSTNaiMXzJUl+B/wYeAKwX1Xd2B6bePwHQFVdCFwFHDJA+8uAVYMEVFWnVtVeVbXXfTbffJBLJWmxW9g5e5MNZr5Akgaw/qgD6GO3qrpm8s4kjwB2Aj4z6dDHaB4Dnt6x/R2BK9YpQknSBHO2pCVlIRbPUzkK2AS4PcnkY2uTbFtVV0/XQJLHArdX1Y/mKEZJUsOcLWlRWojDNu4myXLgMGDvqsrkjWb90BfM0MbGwEm47JEkzSlztqTFbCyKZ+Bg4FdVdcEUx08FXtjvQJItkhwGXAB8t6pOmaMYJUkNc7akRWuhFc8r6T8x5IXAadNcdyawVZKJ5YzW0ExiuStIXWEAABP4SURBVBE4n2appKOr6uXDDFaSljhztqQlZ0GNea6qvtOiq+rpM1y3Etiq5/WCel+StBiZsyUtRaPueV4DXJTkU6O4eZITk9wMrB3F/SVpzJizJS15I/1rv6p2HvH9jwWOHWUMkjQuzNmSNPqeZ0mSJGlsWDxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdWTxLkiRJHVk8S5IkSR1ZPEuSJEkdrT/qAMbNqiquueMPow5jIFe+7NGjDmFW7rXRpqMOYWDZeo9RhyCpx9qNt+H2x71j1GEM5HePv2HUIczKEU85Y9QhSPPCnmdJkiSpI4tnSZIkqSOLZ0mSJKkji2dJkiSpI4tnSZIkqSOLZ0mSJKkji2dJkiSpI4tnSZIkqSOLZ0mSJKkji2dJkiSpI4tnSZIkqSOLZ0mSJKkji2dJkiSpI4tnSZIkqSOLZ0mSJKmjkRbPSS5OckuSM0d0/xOS3JTktlHcX5LGiTlbkkbf87wRsGdVPRsgycokN7fJ8Zok5ybZd/JFSd6ZZFWS3fs1mmR1Tzs3JDk/yWuSbNB7XlUdV1WbtXFIkqZnzpa05I26eJ5sObBbmxy3AY4Hzkqy48QJSZYBLwbOA46Yop1lPe1sCRwH7A+cm2STOYxfkpYSc7akJWehFc93qqq1VXUOcDbwvJ5D+wNXAu8CDk8y7XuoqjVV9R3gAOBamuQuSRoic7akpWLBFs89LgN26Hl9FPAR4Bs08e/XpZGqKuB1wFFJNhxuiJKkljlb0qI2DsXzLsDlAEk2Bw4EPlFVa4EzgBd1baiqLgGuBx42/DAlSZizJS1yC7Z4TrJpklcATwQ+0O4+HPhaVV3Xvj4DOCTJPQdo+mpg6wFjOTrJBUkuuOn66we5VJKWhIWas280Z0sasoVYPF+S5HfAj4EnAPtV1Y3tsYnHfwBU1YXAVcAhA7S/DFg1SEBVdWpV7VVVe222+eaDXCpJi92Cztn3NmdLGrL1Rx1AH7tV1TWTdyZ5BLAT8JlJhz5G8xjw9I7t7whcsU4RSpImmLMlLSkLsXieylHAJsDtSSYfW5tk26q6eroGkjwWuL2qfjRHMUqSGuZsSYvSQhy2cTdJlgOHAXtXVSZvNOuHvmCGNjYGTsJljyRpTpmzJS1mY1E8AwcDv6qqC6Y4firwwn4HkmyR5DDgAuC7VXXKHMUoSWqYsyUtWguteF5J/4khLwROm+a6M4GtkkwsZ7SGZhLLjcD5NEslHV1VLx9msJK0xJmzJS05C2rMc1VtMMX+p89w3Upgq57XC+p9SdJiZM6WtBSNuud5DXBRkk+N4uZJTkxyM7B2FPeXpDFjzpa05I30r/2q2nnE9z8WOHaUMUjSuDBnS9Loe54lSZKksWHxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdWTxLEmSJHVk8SxJkiR1ZPEsSZIkdZSqGnUMYyXJb4Er5qDpLYDr5qDduTaOcRvz/BjHmGHu4n5AVW05B+1qGubsuxnHuI15/oxj3HMZc9+8bfG8QCS5oKr2GnUcgxrHuI15foxjzDC+cWt+jevPyTjGbczzZxzjHkXMDtuQJEmSOrJ4liRJkjqyeF44Th11ALM0jnEb8/wYx5hhfOPW/BrXn5NxjNuY5884xj3vMTvmWZIkSerInmdJkiSpI4vnEUry4iQfmLRvnyTfGVVMbQxvTPKOKY5lmusuSbJTn/33SfKeJD9McnWSXye5KsnZSY6crs1hmO79THPNYUn+ZQj3fkyS6yZte69LbO1130yyz4DX7JTkko7nXpzkliRnDhrbMCQ5IclNSW4b4Jqxi1mLR5K3JHnriO59bE9++U2Szdv9z534HZPk0CT/NGC7A+eZSdeP5WdyHOM25sGtS862eJ4jSZ6U5EsznLYcWNFn3/K5iQqS7J/k5rZ47d0uS7L+NHFNeEWSk6c4toJJsbdJ/IfAHcBBVbVtVd0P2Bk4CXgdMHDx2E+SM5M8t8+hu7yfJA/q8/6vTfLFqa6ZRSz3SLILzdqTj5m03ZhklyQb9btPkiuSbDvDLe7yc9IWxj9Lck2fbY9ZvKeNgD2r6tlt+yvbn5ub2jbPTbJvn/f9ziSrkuzer9Ekq3vauSHJ+Ulek2SD3vOq6riq2qyNo6txjFljIMkT+uSMW5J8uOe0u+W/eYptBXAKsF27PQC4Lcky7vqZv0t8STZP8vEkN7Y/1+9r2+q1rr+PxvUzOY5xG/P8xAxYPM+lOS2C18GOwKerartJ205VtbrD9dsANw1wv32Aa6vqzVV15cTOqlpZVV8CXg8cPNA7mNo9gZUznVRVP5n8/oF9gYcMKQ6ARwDfnWF7wRTXrgDuMeD9dqX5d966z3bRrN7BXS0HdmsTzTbA8cBZSXacOKH9Zf1i4DzgiCnaWdbTzpbAccD+wLlJNhlCnOMesxagqjq3T874GvD5UcaVpnPiD8DtfbapOjkmfBq4leazsSuwG/DROQu2Ma6fyXGM25jnMGaL54WngDkdxrCOHstgxfNFwE5JnjH5QJItgWOA7w0ptq3p+C1DSXZOstfEBjwcWDukOKiq/6iqLapqC2BP4CjgaOAxE/urapgzhAN0+eNnnVXV2qo6BzgbeF7Pof2BK4F3AYcnmTa/VNWaqvoOcABwLU2inBPjGLMWriRvAn5eVZ+cdOjVba/0Z+cjjqq6HtgA2BC4D00BvFH7+pVTXZfkUcBOwDFVdXvbzvOAg5LsOeeBM76fyXGM25iHH7PF88JzKbB9+j9+n9iGVWxO55Xtvc6Z2JFkZ+DRwItm+oGdUFWXA/vxx18q5yX5epL/Ai4Afg68al2DTXIPmt6TX3Q4dy/gP4E392yHAidMOvXg9t/g4nWI6w3At4Fn0nx4z05ySnKXcd53+7ceE5cBO/S8Pgr4CPANmtyyX5dGqlny53XAUUk2HG6IdzOOMWuBSLJxklNo/th+Rp9C86S2Z/ov5iumqloJ/A1NPv1nmmFy+1bVmvaU5yS5CvjHnsv2Bz5fVat62rke+DrwuHkJ/I/G9TM5jnEb85BitnheYKrqqqracorH7xPb3jO3NPUt6OnZTmPLJI9MM0luou2T23vt33Pt24D3Aj8F/mqA9/SDqnoS8EDgSJoekQOr6gFV9dqqun0d3s+Ex9AM29i1w7lbAD+oqmf0bIdU1YmTzjur/Td48GwCSrIp8JfAw6rqxVV1NLAHzVCW3l9Q/f6tx8EuwOVw5+PjA4FPVNVa4AzgRV0bqqpLgOuBhw0/zLsYx5g1Ykk2SHIUzaPis6rqOcCraf4YPqn9rI8qtkNoepwfWFWPB/4c+Jf8cQ7Lp9phJq/puWwb2s/BJD8D3j3RUQM8cu4iv9O4fibHMW5jHlLMFs+j97w2UV04T/e7GDgkyX8nuRb4Nc1fcH8D/CnNWKG7SXIozZCNdwL/D3BokldMdZMkT8mkCTY0ifkbwDnA+ZOPJ/nGOryv/YGrmfpRZW/v7lpgRZL1kqxIskWaSYQHJjkmyVRjkQd1W7s9qmffg4DN2lhH4f7tv8Ovk+wwmwaSbNr+f/9EYGK1mMOBr1XVxLCZM2h+zu45QNNX0wy9GbpxjFkLQ/uU7XyaX+JPqqovALT/+zCaR8gzzrWYQ7sD35roaa6qn9PknS2nuWaqL3hYD3jbREcNMGe/l8b1MzmOcRvzXQwl5vVnPkVz7BNVdWSaZeuuGfDar1XVYYNcUFXfantJlvU+suuV5IBJr18GvAn4s6q6pd33JOCzSR5RVS/tc5+v0Mz8nnNpVq14CfA04JNJ9q2qb0867eSqekN7/rY04wR/yh8n29wKXANcQTPMYrpfPJ1U1eokT6PpyTmNpsf/v4Ejq+rS9rRfsQ6reszClVW1wyyvvaQtJG4GvgPsV1U3tseOovkDDICqurD9g+kQ4PSO7S8D+v5MroNxjFkLSFWtTfLInmEQvcduBP4OoP2Mj8IXgI+mGWd9Gc0QtJur6tdJbqb/PJCraFY8mmz3tr25NK6fyXGM25jvbigxWzzPnbXA8nZs64Y0Qwq2ohm6sBvN2LI7VdWHgA/NR2Dt447pJsd9mLYHuh1LvDewT911tYxr0ywhM+0wiXZ88cenOWUNcFpV/X236Pt6G/Clqvp+kpcApyfZux3DdzdVdTXNahhTSvJwhtCbVFUXJTmiqn47xSn9/m3Wctclpe4F3BfYnqbn+ox+t2LSk6QkG9OMFdsN+D3wy0Hjn2S3qrrbH3hJHkEz+egzkw59jOaRWtektiPNHy/DNI4xa4GpqjXtMIgfM8XTOZrlOF/DPP88tHnvVcB7aHrUzgcOao99FpiYvHgeTXEN8DngnCQr2jHTJLkf8GDgq3Mc8rh+JscxbmO+u6HEbPE8d35M03t5Pc1fOTfwx57NnwB9i6kk2wD3rarvz2Vw7S+Cl9Isl7Y9TeG1imYi3Qeq6ssAbe/0S9prNqYZV3dhe2x1+14mrKT5BXKnqrqA5nHnVHEcQDN+elbFc5pVPJ4H7NXe75tJTgW+nOTJPX+xTnX9gTSPgh4E3Jvm/6eLgNOrql+ROhv/nWSzquq3EPvraXrBe8eQfxH4dpLVNCto3Erz83IlzdCXfqux/AzYo316sZammL6Vpmf7F/zxF+hcOArYBLg9d/++m7VJtm3/YJlSkscCt1fVj+YoxsnGMWaNUJvvdpvqeJLjgcfTfH7nVVV9lbboTfJVms6a3tjOBt5UVd9sz/9RkouADyV5Nc0KHWcAx08U0yMwrp/JcYzbmNeRxfMcaf9PnKlXtt/uP6NZleH5cxBWr/fT9Eq+oqp+2Mazgmbs8GlJ3lJVk/+S2wv4W5rJeXdTVVP+YpnGesxyab52zO4/0Uw+vPPRZFX9bZrF0LcHpiyek7yOZgLj62l6a35HU0D/KXBykg9W1T/MJrbJtwIuS9JvnOHGTFqPtapePGODk352quqXSe4DrD/NcJy+C8qviyTLgcOAvds/lCYfP5fmD7R3T9PGxjRfmDMvy76NY8xSF+2QvCfSPOH8ZbtvQ+AJNJOVf9hz+nNpVuC4iGb42olV9b75jHfCuH4mxzFuY3apOq2bg4A3ThTOcOcXl3yG5ofrmcO4SZId0nx73y/6bTTLw501m7arWQZvj9730HPs7f32T/Is4B1V9YWquqGqVlXVtVV1FnAszYd1WHauPiunMMse936qMd/jbw8GftUvobVOBV7Y70CaiZqH0Syx9d2qOmWOYpxsHGPWApDkoiRXJrl88kaTT74x2gh5A81yp6/LH5cTfQ3NWvqvbYsQAKrqlqp6aVVtU82XZJ00gngnjOtnchzjNuYhsOd54ZmvL0k5G3h7ktdW1U/hzvHN+9Esr/a2IcW2LfDLqnr0ugQ7lanGNXf0VZpVOH4y8RinHaP+SOAtDG/iTDH1OMmp9i80K+k/yeKFwHQTpc4ETkjysPaPmTU0E0LW0gyR+Q5w9MTj5CEbx5i1sD0U2KiGs7zm0LRF8Ztplqn7U+BE4FNtj9whNEtjvh74XJKjquqqEYU6rp/JcYzbmOcwZovn0bqDSWOEacYQn5BmBulUCtirqn6zDvc+BngZ8ME0q09MzEC9kOZbp77c55orgJ1niO3DVfXmntdX0XzD4OXTXLOW5vvtbx3kDQzgDvo/ZXkrzXju9yXZnubzsIZmjPBpdJ+AMJNLaIZt9EsKG9P0cg+q38/OTFbN4hoAqmqDKfY/fYbrVtJMlJ14PW85Zxxj1oL3I+An7XyEfn5aVQfNZ0Ctf6AZ57xvVf0uyZE0+e2pwFOq6vfAO9JMqP4UTYHdxWzyzJTG9TM5jnEb89xK1VTLPUpaqpJcSpOMvljNF0LM9/1PpJkgsnFVdeqdH8eYpcVsXD+T4xi3MQ9uXXK2xbMkSZLUkRMGJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4snqUZJDk0yVmT9m2Q5NtJpvxGI0nS/DNna65ZPEszW9FuvU6kWf/0TfMfjiRpGuZszSkX/5cGlOT5NN/k9aiqmurLEiRJC4A5W8Nmz7M0gCQPBN4LPKuqru3Zf98k/5bkliTXJHlrz7EvJzl0UjtnJDli/iKXpKXHnK25YPEsdZRkA+CTwBur6j8mHT4L+DWwLfAo4JlJXtwe+xRweE87mwAHAv8+50FL0hJlztZcsXiWujsJeAhwbu/OJAcAWwCvrqrfVdXVwOuAV7ennAk8Icmm7etDgK9U1c3zE7YkLUnmbM0Ji2epm8cAewP/DJyaJJOOfaGq1vbs+x7wkCTrVdUNwNeBg9tjzwc+Ng8xS9JSZc7WnLF4lrpZBTyDpndiB+ClPce2AV6a5KaJDfgVsBqY6Ln4GPCcJFvQPCL83HwFLklLkDlbc8biWermP6vqiqq6DXgZ8J4k92uP3QqcXFWbTdo2rKob23M+A+wDvAT4TFWtnP+3IElLhjlbc8biWRpQVX0R+DxwcrvrP4HHz3DN79tr3gZ8dE4DlCTdyZytYbN4lmbnWJoJJc8APg3cN8k/JtkMIMlOSR4x6ZqPAzcC35jXSCVJ5mwNjcWzNLOV7XanqvotTTJ+LxDgycBOwBXt+LnPAltPamd74OOTJqlIkobLnK05laoadQzSopZkPeCewLeB51fVxSMOSZI0BXO2ZmLPszT3TqWZyf0xk7AkLXjmbE3LnmdJkiSpI3ueJUmSpI4sniVJkqSOLJ4lSZKkjiyeJUmSpI4sniVJkqSOLJ4lSZKkjv4v4o5t1+4eAo4AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x360 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5UZen0UQPYZE"
      },
      "source": [
        "마스킹된 그림을 보면 셀프 어텐션에 대한 마스킹 때문에 상삼각행렬에 해당하는 요소가 모두 마스킹되었고 타겟에 있는 [PAD]에 대해서 열방향으로 마스킹 된 것을 확인할 수 있습니다.\n",
        "\n",
        "이제 마지막으로 디코더에서 일어나는 크로스 어텐션에 대한 마스크를 확인하도록 하겠습니다. 이를 위해 적당히 (nbatches=2, head=1, dv=5) 크기를 가지는 벨류 텐서를 만들고 앞서 소스 마스크에서 만들어 놓은 `masked_encoder_self_attn_scores` 텐서와 곱해 헤드를 만듭니다.\n",
        "그 다음 임의로 초기화 된 $W^o$행렬과 곱해서 최종 인코딩을 만듭니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NtYp3hvB17pN",
        "outputId": "fb7ce230-debf-4d25-d5ed-fe4c9e798205"
      },
      "source": [
        "V = torch.rand((2, model_const['h'], model_const['src_len'], model_const['dv']))\n",
        "# head가 하나 뿐이라 concat 할 필요없이 squeeze()합니다.\n",
        "head = torch.matmul(masked_encoder_self_attn_scores, V).squeeze() \n",
        "Wo = torch.rand(model_const['dv']*model_const['h'], model_const['d_model'])\n",
        "encoding_src = torch.matmul(head, Wo)\n",
        "\n",
        "print(encoding_src.shape)\n",
        "print(encoding_src)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2, 5, 7])\n",
            "tensor([[[1.3157, 0.8231, 1.8169, 2.2399, 0.7467, 0.7838, 0.9017],\n",
            "         [1.0317, 0.6436, 1.4306, 1.7503, 0.5839, 0.5992, 0.7001],\n",
            "         [1.7735, 1.1149, 2.3128, 2.7675, 1.1070, 0.8919, 1.2505],\n",
            "         [2.4297, 1.5170, 3.2814, 3.9383, 1.4432, 1.2764, 1.6642],\n",
            "         [0.5630, 0.3511, 0.7676, 0.9250, 0.3292, 0.3033, 0.3835]],\n",
            "\n",
            "        [[0.7225, 0.4850, 0.8106, 1.1491, 0.5061, 0.5566, 0.6293],\n",
            "         [0.7500, 0.5038, 0.8479, 1.2112, 0.5195, 0.5941, 0.6536],\n",
            "         [0.7592, 0.5096, 0.8507, 1.2044, 0.5327, 0.5821, 0.6612],\n",
            "         [0.7580, 0.5088, 0.8508, 1.2065, 0.5306, 0.5848, 0.6602],\n",
            "         [0.3948, 0.2652, 0.4466, 0.6385, 0.2731, 0.3136, 0.3440]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lvg7w2qh2DLM"
      },
      "source": [
        "이렇게 만들어진 인코딩에서 4행과 5행은 [PAD]가 인코딩된 정보임을 상기 합시다. 이제 디코더 쪽에서 타겟에 대한 인코딩을 동일한 방식으로 만듭니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GwQw-5dG2aSW",
        "outputId": "3599cdb3-0686-421a-ce00-ad64bb47cd2b"
      },
      "source": [
        "V = torch.rand((2, 1, model_const['target_len'], model_const['dv']))\n",
        "head = torch.matmul(masked_decoder_self_attn_scores, V).squeeze()\n",
        "Wo = torch.rand(model_const['dv']*model_const['h'], model_const['d_model'])\n",
        "encoding_trg = torch.matmul(head, Wo)\n",
        "\n",
        "print(encoding_trg.shape)\n",
        "print(encoding_trg)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "torch.Size([2, 6, 7])\n",
            "tensor([[[0.1092, 0.2553, 0.2908, 0.3739, 0.3702, 0.1855, 0.0366],\n",
            "         [0.6836, 1.0122, 1.1891, 1.4366, 1.6190, 0.8272, 0.2952],\n",
            "         [0.2467, 0.5165, 0.5903, 0.7477, 0.7585, 0.3868, 0.0951],\n",
            "         [1.0650, 1.6200, 1.7648, 1.9914, 2.1176, 1.4748, 0.8864],\n",
            "         [1.5561, 2.6366, 2.8497, 3.2805, 3.3495, 2.3318, 1.3270],\n",
            "         [0.8810, 1.3566, 1.5207, 1.7734, 1.9178, 1.1775, 0.5928]],\n",
            "\n",
            "        [[0.5224, 0.7097, 0.7051, 0.6843, 0.7026, 0.7583, 0.6666],\n",
            "         [0.3033, 0.5964, 0.5850, 0.6280, 0.5492, 0.5731, 0.4405],\n",
            "         [0.5860, 0.8012, 0.7957, 0.7740, 0.7920, 0.8543, 0.7493],\n",
            "         [0.2773, 0.5113, 0.5024, 0.5316, 0.4760, 0.4994, 0.3928],\n",
            "         [1.1601, 1.9902, 1.9602, 2.0378, 1.8777, 1.9823, 1.6005],\n",
            "         [0.7322, 1.0746, 1.0643, 1.0591, 1.0460, 1.1204, 0.9576]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ab_t0pHC2oRj"
      },
      "source": [
        "이제 `encoding_src`를 키와 벨류로 사용하고 `encoding_trg`를 쿼리로 사용해서 어텐션합니다. 이때 어텐션 마스크는 소스 마스크인 `src_mask_enc_reshape`를 사용합니다.\n",
        "\n",
        "`encoding_src`와 `encoding_trg`가 `MultiHeadedAttention.forward()`로 입력되면 앞서 알아본과정을 통해 이 인코딩들도 멀티헤드 쿼리, 키, 벨류로 바뀌게 됩니다. 여기서는 이 과정을 거쳤다고 가정하고 바로 텐서를 (nbatches, h, n_seq, d_k)로 변환하여 사용하겠습니다.\n",
        "\n",
        "변환된 두 인코딩 텐서를 어텐션 연산하고 마스크를 적용합니다.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nw_GcUc070oN",
        "outputId": "262f7f98-5772-4eda-c3eb-5af9e8f52198"
      },
      "source": [
        "multiheaded_encoding_src = encoding_src.unsqueeze(1)\n",
        "multiheaded_encoding_trg = encoding_trg.unsqueeze(1)\n",
        "\n",
        "cross_QKt = torch.matmul(multiheaded_encoding_trg, \n",
        "                         multiheaded_encoding_src.transpose(3,2))\n",
        "\n",
        "print('cross_QKt shape =>', cross_QKt.shape)\n",
        "print(cross_QKt)\n",
        "\n",
        "masked_cross_QKt = cross_QKt.masked_fill(src_mask_enc_reshape == 0, 0)\n",
        "print(masked_cross_QKt)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "cross_QKt shape => torch.Size([2, 1, 6, 5])\n",
            "tensor([[[[ 2.1746,  1.7004,  2.8068,  3.9115,  0.9124],\n",
            "          [ 9.2343,  7.2198, 11.9659, 16.6396,  3.8791],\n",
            "          [ 4.4522,  3.4812,  5.7513,  8.0110,  1.8685],\n",
            "          [13.9381, 10.8922, 18.0558, 25.0925,  5.8496],\n",
            "          [22.2684, 17.4027, 28.8159, 40.0696,  9.3428],\n",
            "          [11.9004,  9.3017, 15.4144, 21.4298,  4.9960]]],\n",
            "\n",
            "\n",
            "        [[[ 3.2766,  3.4272,  3.4387,  3.4388,  1.8053],\n",
            "          [ 2.5783,  2.6983,  2.7056,  2.7060,  1.4214],\n",
            "          [ 3.6942,  3.8640,  3.8769,  3.8770,  2.0354],\n",
            "          [ 2.2324,  2.3361,  2.3427,  2.3429,  1.2306],\n",
            "          [ 8.7947,  9.2023,  9.2293,  9.2301,  4.8475],\n",
            "          [ 4.8854,  5.1106,  5.1270,  5.1272,  2.6920]]]])\n",
            "tensor([[[[ 2.1746,  1.7004,  2.8068,  0.0000,  0.0000],\n",
            "          [ 9.2343,  7.2198, 11.9659,  0.0000,  0.0000],\n",
            "          [ 4.4522,  3.4812,  5.7513,  0.0000,  0.0000],\n",
            "          [13.9381, 10.8922, 18.0558,  0.0000,  0.0000],\n",
            "          [22.2684, 17.4027, 28.8159,  0.0000,  0.0000],\n",
            "          [11.9004,  9.3017, 15.4144,  0.0000,  0.0000]]],\n",
            "\n",
            "\n",
            "        [[[ 3.2766,  3.4272,  0.0000,  0.0000,  0.0000],\n",
            "          [ 2.5783,  2.6983,  0.0000,  0.0000,  0.0000],\n",
            "          [ 3.6942,  3.8640,  0.0000,  0.0000,  0.0000],\n",
            "          [ 2.2324,  2.3361,  0.0000,  0.0000,  0.0000],\n",
            "          [ 8.7947,  9.2023,  0.0000,  0.0000,  0.0000],\n",
            "          [ 4.8854,  5.1106,  0.0000,  0.0000,  0.0000]]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "od8KVLXs7-al"
      },
      "source": [
        "적용된 결과를 그려보면..."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 362
        },
        "id": "vyMHxfrFPtRf",
        "outputId": "9d36a37c-0716-424a-bcf4-c94c51f10fec"
      },
      "source": [
        "fig, ax = plt.subplots(figsize=(12,5), nrows=1, ncols=2)\n",
        "\n",
        "ax[0].imshow(masked_cross_QKt[0][0], cmap='BrBG')\n",
        "ax[0].set_xticks([0, 1, 2, 3, 4])\n",
        "ax[0].set_xticklabels(['i', 'love', 'you', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[0].set_xlabel('Key', fontsize=15)\n",
        "ax[0].set_yticks([0, 1, 2, 3, 4, 5])\n",
        "ax[0].set_yticklabels(['나는', '당신을', '사랑', '합니다', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[0].set_ylabel('Query', fontsize=15)\n",
        "ax[0].set_title('Sample 1', fontsize=15)\n",
        "\n",
        "ax[1].imshow(masked_cross_QKt[1][0], cmap='BrBG')\n",
        "ax[1].set_xticks([0, 1, 2, 3, 4])\n",
        "ax[1].set_xticklabels(['go', 'job',  '[PAD]', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[1].set_xlabel('Key', fontsize=15)\n",
        "ax[1].set_yticks([0, 1, 2, 3, 4, 5])\n",
        "ax[1].set_yticklabels(['잘', '했어', '[PAD]', '[PAD]', '[PAD]', '[PAD]'], fontsize=15)\n",
        "ax[1].set_ylabel('Query', fontsize=15)\n",
        "ax[1].set_title('Sample 2', fontsize=15)\n",
        "\n",
        "plt.show()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqIAAAFZCAYAAABDvE20AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhkZXn38e+PgWEVUBYRRNkUNYoSIRo1Ii5BjYohuIGoqC9xDxhN3OL6xsTERIL4ihgTExRQSAyIC+4oGtEJ0aAoLsiqIzIwbOIwy/3+cU6Toqju6Zqp7lPV/f1c17noOud5nnPXMH3PXec556lUFZIkSdJ826TrACRJkrQ4WYhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaiUp8kz0lyVtdxSJLWz5w92SxE1Yk0Xpnk+0muT/KrJG/rOq7W5u22UZJskeTMJBeNICZJ6sxCztlJ9kjy/iSXte/twiRPGWF8msGmXQegRestwKOBP6iqy5JsA9yj45hGJsndgLOBW4HNOg5HkjbWQs7Z9wS+A7wJuA54MnB6kkdUlRcS5piFqLrySuBxVXUZQFXdDPy404hG6znAR4EfACd1HIskbawFm7Or6nzg/J5dn0ry78ChgIXoHHNqXl0pYJfpDia5X5LPtNM/1yW5IMnDe44/LskXkrwhyS/aNv+QZJMkx7ZTLMuT/E2SJT39Tk7yoiT/kuSatt/7k2wxQyybtuP8KsnKJB9Jsu2Mb67qfVX1/iH/TCRpXC3onD3ADcB2Q/bRBrAQVVfeCnwkyeOnOb4F8C5gN2CH9uePJZm6ir8WeATwAOC+wL2BBwGnAo9vf74vcCDwgp5xlwJvB/6LJqk+oG3zzhli/UvgUcD+NFM4twEnzvaNStIC8FYWSc5uC+GnAOfOto82QlW5uXWyAS+m+dR5CrDzLNovB/Zpf34MsArYtuf4M9p9d+/b9289rz8MfKFv3IcANwJbtK9fAHy2/fluwE3AXj3t7wKsBHaaRcyPAX7Y9Z+1m5ub28ZuiyFnt+1fBnyj6z/vxbJ5RVSdqap/BH6L5pP095L8ztSxJJslOS7J+e00zg3AzjSftKdcWVU39rz+FXBJVf2yb19vH4DP9cXxHWA1sMeAMPcDLquqS3va3wRcCtx/du9UkibfYsjZSe5Dc/X3j9fXVqPhw0rqVFVdBTwjyXHAmUn2rqrVwHtpplbeSHMT+XXAFUB6uq8ZMOR1szjtygH7bmTw/UC7AvdL0t9nKbD9LM4lSQvGQs7ZSe4C/Afw5+XT8vPGQlRjoarek+S1wAOT/AB4IXD/qvop3H7PzrQ3yg/p7gP27QxcPWD/zcB3qurAEZ1bkibeQsvZbbynA1+sqn8ePkRtKKfmNRaSbEVzb89Kmk+5mwKX9zR5HKP74HSHm+2THARcC/x8QNsLgfsn2WlE55akibcAc/bxNDXRcRsUoTaYhag6keTIqUSR5O7AvwIfr6qfAdfQJLSXtt/m8SDgb4Gfjuj0D0zy6iSbJ9mN5mnKv6+qdf0N22moz9AsbrxnG+9dkzx2RLFI0thbyDk7ySuAxwLPqqq1I4pZs2Qhqq78Ps3N7jcAXwf+BzgGoJrHFp9Gsyj8dcBHgD8FLgOm1pf7Dc3Tlr1WzXLf24CH0nya/g7NNyCdMEOf59MsTP+fSW4Evgv8DrMz6PySNGkWcs5+FbAXcEW77ujU9o0Z+mhE0i5VIC0KST4MfKWqPtxxKJKk9TBnL3xeEdVis4ZmcWNJ0vgzZy9wXhGVJElSJ7wiKkmSpE5YiEqSJKkTLmg/pK2WprbfMutvOEbWTejdF5tO4MekX3sn07y45bZi1ZqarF9EdWKHu96ldt9tspYBTiYw+QGX//TS9TfSojRTzrYQHdL2W4YXP2LzrsMYyq2rJ7MS3XGbyUvGyy4f9A12GrUvXuKfs2Zn99124otnvqPrMIayZOnWXYewQf7PHz2z6xA0pmbK2ZP3L70kSZIWBAtRSZIkdcJCVJIkSZ2wEJUkSVInLEQlSZLUCQtRSZIkdcJCVJIkSZ2wEJUkSVInLEQlSZLUCQtRSZIkdcJCVJIkSZ2wEJUkSVInLEQlSZLUCQtRSZIkdcJCVJIkSZ1YUIVokhcm+UDfvkck+UZXMUmSRi/JXyR5c9dxSNo4E1OIJnlsks+tp9lSYPMB+5bOTVSSpFFLclCSq/q2G5N8uKfZ5pjbpYm3adcBDMGCUpIWgao6D7hn774k/wF8upuIJM2VibkiuhEKSNdBSJI2TJI3Aj+uqo/3HXpVe7X0k13EJWnjTdIV0Q31U+BeSZbP0ObKqjpwvgKSJK1fkm2AdwM7AA9JckpV/U9PkxOq6k3dRCdpFBZ8IVpVVwE7dR2HJGl2kmwBPAf4E+D1VfWZJE8Czmmn6P+i0wAljcxCnJp/VpLlSS4c1YBJjkmyLMmyX99WoxpWktQnySbABcA+wGOr6jMA7X8fDFwJrFrPGLfn7BXX3zjXIUvaCAvxiujHquoF7VJOM03HD/Klqjqif2dVnQycDLDrdptYiUrSHKmqdUl+u6rWDjh2PfC3AEk+OMMYt+fshzxwL3O2NMYmqRBdByxNEmBLYGtgZ+C+wL7Al3sbV9U/Af8030FKkjZOVa1NsinwfWDJNM1uA14NXD5vgUkauUkqRL9Pc6/nCmA1cB2wnCYJ/QD41aBOSXYF7l5V/z1PcUqSNlJVraG5yDBQkncCjwY+O29BSRq5iSlEq+pq4D4ztWkult7J7wNPBJ49B2FJkiRpA01MISpJWlySXARsD9zpflGaB5ZeOb8RSRq1xVCIuqC9JE2mBwJbVdWtXQciaW4stEL0tnbr9QPgPUmumqFfAQdU1S/nLDJJ0rC+B/wgyZppjv+wqp4ynwFJGq0FVYgOelK+qr4F3K2biCRJG6qqHtR1DJLm1kJc0F6SJEkTwEJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1YtOuA5g0O+66G8e8/diuwxjKtT+8oOsQNsh9n/jarkMY2vN//5FdhyCpx5pVN3PtT/6z6zCGstnWd+k6BGneeEVUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1YiwL0SRvSPK2aY5lhn6XJNlrwP67JfmbJN9NcnWSXyS5Ksk5SV4w05iSpA2T5Ngk17bbL5Ps0O5/ZpIPtD8/J8n/G3LcryZ5xFzELGl+zXshmuSQJDe0hWDvdmmSTdtmS4HNpxni5UlOnObY5m3f3vPtAHwXuA14SlXtVlX3APYGTgBeAwwseiVJGybJ5sBJwD3b7d7ALUmWcMccf4e8nWSHJKcnuT7JdUne147Vayl9uV7SZOriiuiewCeq6p59215VtWYW/XcFVg5xvkcA11TVm6rqyqmdVbWqqj4HvBY4dKh3IEmaVnsB4DfArQO26S4kTPkEcDNNrr8PsC/w0TkLVlKnxnJqfj0exXCF6EXAXkme3n8gyU7AS4Fvjyg2SVr0qmoFsAWwJXA3mmJyq/b1K6brl+ShwF7AS6vq1nacZwFPSbLfnAcuad6NcyH6iiTLk5w7tSPJ3sDDgOclmVXsVXUZcDDwqvYWgPOTfDnJ/wDLgB8Drxx9+JK0eFXVKuCvaPLsP9LcIvXIqlrbNnlGkquAv+/pdgjw6apa3TPOCuDLwO/NS+CS5tWm628ycgXc/nBQ+6DQjsDuwP1oCkOAE6vqdX193wK8F7gX8HrgL2d1wqrvAI9NshXNdM/mwMqqunoj3ockaRpJDqO5Enrfqlqb5D7Al5Ps0TY5o6pekOQFNDNd0OTnywYM9yPgXUn+on19t7mKW9L86qIQvRh4d5IntOdfB6wArqJJNpcO6pTkOTTJ6iE0hezXk6ysqvdN0/7xwIdnCmTAw/I/qarHDGh3DHAMwG673HWmISVJjfsBX5u6AlpVP05yC7DTDH1qmv2bAG+pqr8DSPLNmU7cm7N33WmbYeOWNI/mvRCtqq8l2Q5Y0jv90ivJE/tevwR4I/D7VXVju++xwCeT7F9VLx5wni/QPKk5iphPBk4G2O/+u0+XKCVJ/+szwEeTfJLmAsNzgBuq6hdJbgCuHdDnKpoVTfrdrx1vVnpz9gPvs7M5WxpjXVwRparW0VwJnc6HgSUASTYDDgQe0ffU+zVJHknzVOW0khwAnD5Dk7XAB6vq3bOLXpK0PlX130leCfwNsAtwAfCU9tgngU+2Tc/nf2fCPgWcm2Tz9h5TktwDeADwxXkMX9I86exhpSSbJnlJ+/DQFe2DRD9LciZwn6r6KUBVra6qF1XVlUm2SfLbU2NU1Zqq+kHPsKto1gulp82yqtpnug34E+Cp8/CWJWlRqaovVtWTq+q3gfsCW/ceT3IOsHVVfbVt/z2alU7+qV1PdHfgDOCdU4WppIWly6fm3w/8EfDyqrpXVd2TZvrlX4EPJjlqQJ8DgGm/gaOq9q2qgfeYzmATeh6ekiSNVns71mNoitGpfVsCBwEP6mv+TJr1Ri8CzgM+Pt2zAJImXydT862nAE+rqu9O7Wg/8Z6dZBfgD4FTNvYk7ROa3wJunKbJWtp7iSRJc+J1wE+B1yT5fHt71qtp1nD+0yQfr6rbANrnAO5037+khanLQvQc4K1J/rSqfgi33w96MPBnNEs19bvD0k+ztBvws6p62MYEK0kaTpKlwJuAPwB+FzgeOCPJecBhNGuDvhb4VJKjq+qqzoKV1IkuC9GXAi8BPpRkN5qHk1YDF9J8q8bnB/S5HNi7XQR5Oh+uqjf1vL6K5puVLpuhzzpgv6q6eZg3IEma0d/R3Bf6yKq6qV0z9M3AE4DHV9WvgbcleRHNvaC/O8txb6PveQBJk6mzQrT9XvkTWf/3Dvf2uYxm8fthznM5M69bJ0maA1X1yr7Xaxkw21VVHwI+NMS4j9746CSNg3H+ik9JkiQtYBaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6sSmXQcwaW5e8UvO//DxXYcxlOuuv7nrEDbIyp/9sOsQJE24pVvdjXsd+OyuwxhKbbJl1yFsoL/rOgBNIK+ISpIkqRMWopIkSeqEhagkSZI6YSEqSZKkTliISpIkqRMWopIkSeqEhagkSZI6YSEqSZKkTliISpIkqRMWopIkSeqEhagkSZI6YSEqSZKkTliISpIkqRMWopIkSeqEhagkSZI6saAK0SRvSPK2IfsckeRf5iomSVooklyc5MYkZ3Z0/vckWZnkli7OL2n0JrIQTXJmkmcOOLQU2Lyn3f2TXNW3XZPks9P1kSRNaytgv6o6HCDJqiQ3tMXh8iTnJXlkf6ckb0+yOsn9Bg2aZE3PONcluSDJq5Ns0duuqo6rqu3bOCQtABNZiAJbA6vW16iqflBV9+zdgEcCvzXnEUrSwrcU2LctDncF3gmclWTPqQZJlgAvBM4Hnj/NOEt6xtkJOA44BDgvybZzGL+kjk1qIboLcO1sGibZO8kBUxvwEGDdnEYnSYtMVa2rqnOBc4Bn9Rw6BLgSeAdwZJIZ/92pqrVV9Q3gicA1NMWtpAVq064DGFaSzYD7AD+ZRdsDgC8AX+nZvQ54T1/TQ5MsB66rqgeMKFRJWowuBfboeX008BGaPLwJcDDwxfUNUlWV5DXAhUleW1W3jj5USV2buEIUeDjN1Px9gF+up+2OwHeq6unraXdWVT17FMFJ0iK3D3AxQJIdgCcBL62qdUlOA57HLApRgKq6JMkK4MHAN+coXkkdmsSp+UOAq4FXTHP8Fe1N8+fSXP3cPMkmSTZPsmP7ANOTkrw0yXNnc8IkxyRZlmTZTbc6qy9J/ZJsl+TlwGOAD7S7jwS+VFVTt1KdBhyWZOshhr6a5nasYWK5PWdfe93KYbpKmmcTVYgm2Qp4EfBU4KGDns4ETqyqXarqEOD7wBbAD4FvA58CTgSeC+wOXD+b81bVyVV1QFUdcJctJ+qPTJLm2iVJbqLJtwcBB1fVVG6dmpYHoKouBK4CDhti/CXA6mEC6s3ZO95t+2G6SppnkzY1/xbgc1X130leBJyS5MCqWjGocVVdDew/04BJHsIsnsCXJA20b1Ut79+ZZH9gL+DsvkOn0kzPnzLL8fcELt+oCCWNrYkpRJM8neZJzAMAquqrSU4GPp/kcT2fwKfr/ySaaaL7A3cFrgMuAk6pqtPmNHhJWnyOBrYFbk3Sf2xdkt3aiwXTSvIo4Naq+t4cxSipY0PNMyfZfa4CWc959wD+H3Boz71GVNVf03zavtd6+r8G+Fvgo8ATgH2BJwP/AZyY5E/nJHBJGhPzmb+TLAWOAA6sqvRvNGuKzniPfpJtgBNw+SZpQRv2hsfzk3w9ycuS7DgnEQ1QVZcBD6qq7w449tZB+/v8EfC2qvpMVV1XVaur6pqqOgs4liZhStJCNp/5+1DgiqpaNs3xk4GjBh1oHyo9AlgGfLOqTpqjGCWNgWEL0T2BtwEHAj9M8qkkR7YPEc2p6e4DnaUv0jxN/8CpHWk8FPgL4DMbG58kjbm5yN+rGPwg0VHAB2fodyawc5IHt6/X0jz0dD1wAc2ST8dU1cs2IjZJE2Coe0Srah3wOeBz7dTLk4BnA8cn+RzNTeifadt14TYGF9dvpnna/n1J7kXzvtfSLIr/QWZ/07wkTaS5yN9VtcU0+5+2nn6rgJ17Xk/M8wqSRmtj1iK6O7A3zafsW2jWejuK5lPt40YQ29Cq6p1V9eYB+9dV1Qer6qCq2rOqdq+qParq8VX1r1VVXcQrSR3Z0Py9FrgoyRlzH+KdJTk+yQ34Nc3SgjHUp9D2auLhwDNpluU4E3htVX2tp839aT51d/JgkyTpzkaRv6tq73kIdVpVdSzNff2SFohhp0MuBD4LvAM4t6rWDGjzQ+DmjQ1MkjRS5m9JY2fYQvTtwHtnmspuj91/o6KSJI2a+VvS2Bn2HtE3ez+lJE0k87eksTNsIfrvSZ4/J5FIkuaS+VvS2Bl2av6bwCuTvJTmmzFW0DxFOWVVVf3DqIKTJI2M+VvS2Bm2EP0dmu9nB9ih3XrdutERSZLmgvlb0tgZdkH7l8xVIJKkuWP+ljSONujbLJLsQPNk5fZVdc5oQ5IkzRXzt6RxMuyC9ncBTgSeCvwM2BfYpj12MHBgVf3NqIOUJG0c87ekcTTsU/PvA9YA96qqhwKre479N/CyUQUmSRop87eksTPs1PxTgHtW1a/b17evSVdVK5NsP7LIJEmjZP6WNHaGvSK6Bthu0IEke+BTl5I0rszfksbOsIXoh4Ezkty7d2d78/sHgTNGFJckabQ+jPlb0pgZdmr+9cA7gYuS/AzYJskFwAOATwN/NuL4xs7293wAT/vrL3YdxlBWr/hG1yFskK8tmcCvvP7EA7qOQJrOoszfv7lhORd/6t1dhzGUJZst7ToEad4Mu47oWuDPk/w1sB+wK3AzcGFVXT0H8UmSRsD8LWkcbdA6olV1PXDeiGORJM0x87ekcTLsOqLHAzPNGayqquM2LiRJ0qiZvyWNo2GviP4C2KLndYB7AI8FbqFZp06SNH7M35LGzrD3iL5r0P4kAf4K2GcUQUmSRsv8LWkcDbt800BVVcAbgcNHMZ4kaX6YvyV1aSSFaGsbYKsRjidJmh/mb0mdGPZhpQcDmw8Y497AsTRr0UmSxoz5W9I4GvZhpY9x50S2FvglTRL721EEJUkaOfO3pLEz7MNK95urQCRJc8f8LWkcjfIeUUmSJGnWZn1FNMlOwPOBRwP3BG4Drga+BpxeVct72m5SVetGHKskaQOYvyWNq1ldEU3yTOAnwJOAzwNvpVl37jzgUODHSV7Qtt0SOHcOYpUkDcn8LWmcrfeKaJKHAScAf1BV5w9ockKS3wPOTPJr4I+BH402TEnSsMzfksbdbK6Ivgl4xTRJDICq+hrwcuBU4AdV9dIRxSdJ2nDmb0ljbTb3iD4MePos2n0C+E1VvWLjQpIkjYj5W9JYm80V0U2qau36GrVtbpvNSZM8PMm1fduBPcffkORtsxmrb9yvJnnEkH32SnLJsOeSpAkw0vyd5OIkNyY5cyTRDSnJe5KsTHJLF+eXNHqzKUS/n+Sg9TVK8hjg0vW02SzJPsC1wMP7tuuT7JNkK2ApfQsvJ7k8yW7rCWNpu0312SvJj5IsH7A9qKdP/yLPkrQQjCx/t7YC9quqw9t+q5Lc0BaHy5Ocl+SRA8Z/e5LVSQauZZpkTc841yW5IMmrk2zR266qjquq7fHrSKUFYzZT838PnJzkcVV11aAGSXYHTgLevZ6x9mf9XyP3hmn2bw5stp6+/e4DXFNV9x2ynyQtBKPM34MsBe5dVcuTbAI8ATgryYFV9bN2/CXAC4HzaZaQev2AcZYA+7bjLKG5peAtwLOSPKGqbtyA2CRNgPVeEa2qs4DTgIuTnJjk8e2Vy/skeUKS9wEXA1+qqn9cz1jfqqodq2pHYD/gaOAY4OFT+6vq5I1/W7cLsGaE40nSxBhl/p7FudZV1bnAOcCzeg4dAlwJvAM4si1YZxpnbVV9A3gicA3wzo2JS9J4m9U6olX1VuBxNNMhJwEXtdsHgR2AP6yql832pEleB3wd+EOaZHNOkpOSpKfZK9qpHte0k6QNNOr8PQuXAnv0vD4a+AjwFZp/cw6ezSBVVcBrgKPb9U0lLUCz/malqvo28O2NPWGS7YA/A/aYmm5JshnwX8DvAV9tm55YVa/b2PNJ0mI3qvw9S/vQXGUlyQ40C+m/tKrWJTkNeB7wxdkMVFWXJFkBPBj45hzFK6lDXXzX/C3t9tCeffcHtqf5yrku7N5eff1Fkj36DyY5JsmyJMuuvXbF/EcnSWMuyXZJXg48BvhAu/tImmn/a9vXpwGHJdl6iKGvBnYZMpbbc/b1N89qMRdJHZn3QrSq1gBPBl6b5CdJfgq8D3hBVf20bXYFcPk8hnVlVe1SVfeoqsv6D1bVyVV1QFUdsOOOO8xjWJI09i5JchPwfeAg4OCqur49NjUtD0BVXQhcBRw2xPhLgNXDBNSbs++6zdL1d5DUmVlPzY9SVV2U5PlV9atpmpw+YN867rg0012AuwP3ormietqgU9FXbCfZhub+pX2BXwM/GzZ+SdLt9q2q5f07k+wP7AWc3XfoVJrp+VNmOf6ezO+FCUnzqJNCtPXzJNtX1aCFiV8LbMEdl/n4LPD1JGtonoS/GfgVzdOYP6J5Qr7fj4AHJVlOU8hW2+8K4CfAJ0f0XiRJd3Q0sC1w6x2fQwVgXZLdqmrG27GSPAq4taq+N0cxSupYl4VogEuT1IBj2wAn9u6oqheud8C+ZFdVP0tyN2DTqho4tTPdAsuSpA2TZClwBHBgVS0bcPw84LnAu2YYYxvgBFy+SVrQunhYqdfe7b2Zd9jYsIWVB6rGUPcXSZI2yqHAFYOK0NbJwFGDDiTZMckRwDLgm1V10hzFKGkMdFmIFs1N6INMt1+SND5WMfhBoqNo1imdzpnAzkke3L5eS/PQ0/XABTRLPh0z4vVNJY2hLqfmL6GZmh+UxLYBjt2AMW9rt2Gs3oA+krToVdUW0+x/2nr6rQJ27nnd5b9FkjrU2S9/VT1wDsZ89Ab0+Sngd9FL0vqtBS5K8tmqesZ8nzzJ8TQPQa2b73NLmht+CpUkzUpV7d3x+Y9lw2bLJI2prh9WkiRJ0iJlISpJkqROWIhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6sSmXQcwaX66ciWHf/I/ug5jKG983CFdh7BBnnr4U7sOYWiHdx2ApDv4wcp1HHDWzV2HMZzrVnQdwQYx/2lDeEVUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1wkJUkiRJnbAQlSRJUicsRCVJktQJC1FJkiR1otNCNMnFSW5McmZH539PkpVJbuni/JI0SczZkkat6yuiWwH7VdXhAElWJbmhTTTLk5yX5JH9nZK8PcnqJPcbNGiSNT3jXJfkgiSvTrJFb7uqOq6qtm/jkCTNzJwtaaS6LkT7LQX2bRPNrsA7gbOS7DnVIMkS4IXA+cDzpxlnSc84OwHHAYcA5yXZdg7jl6TFxJwtaaOMWyF6u6paV1XnAucAz+o5dAhwJfAO4MgkM76HqlpbVd8AnghcQ5MoJUkjZM6WtCHGthDtcSmwR8/ro4GPAF+hif/g2QxSVQW8Bjg6yZajDVGS1DJnS5q1SShE9wEuA0iyA/Ak4GNVtQ44DXjebAeqqkuAFcCDRx+mJAlztqQhjG0hmmS7JC8HHgN8oN19JPClqrq2fX0acFiSrYcY+mpglyFjOSbJsiTLbrvppmG6StKiMK45m9tWD9NV0jwbx0L0kiQ3Ad8HDgIOrqrr22NTUzwAVNWFwFXAYUOMvwQYKjNV1clVdUBVHbD0LncZpqskLXRjnbNZutkwXSXNs027DmCAfatqef/OJPsDewFn9x06lWaq55RZjr8ncPlGRShJmmLOlrTBxrEQnc7RwLbArUn6j61LsltVXT3TAEkeBdxaVd+boxglSQ1ztqT1Gsep+TtJshQ4AjiwqtK/0axP99z1jLENcAIuBSJJc8qcLWm2JqIQBQ4FrqiqZdMcPxk4atCBJDsmOQJYBnyzqk6aoxglSQ1ztqRZGbdCdBWDb0o/CvjgDP3OBHZOMrXEx1qaG+ivBy6gWT7kmKp62SiDlaRFzpwtaaOM1T2iVbXFNPuftp5+q4Cde16P1fuSpIXInC1pY3V9RXQtcFGSM7o4eZLjkyxCxFUAAA9KSURBVNwArOvi/JI0YczZkkaq00+hVbV3x+c/Fji2yxgkaVKYsyWNWtdXRCVJkrRIWYhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhKkiSpExaikiRJ6oSFqCRJkjphISpJkqRObNp1AJNm72235MzHP7DrMIZy3eUf6zqEDXL1p77cdQhD+5ODdu46BEk99ttzb8495d+6DmMoN61d3XUIG+QNj9216xA0gbwiKkmSpE5YiEqSJKkTFqKSJEnqhIWoJEmSOmEhKkmSpE5YiEqSJKkTFqKSJEnqhIWoJEmSOmEhKkmSpE5YiEqSJKkTFqKSJEnqhIWoJEmSOmEhKkmSpE5YiEqSJKkTFqKSJEnqRKeFaJKLk9yY5MyOzv+eJCuT3NLF+SVpkpizJY1a11dEtwL2q6rDAZKsSnJDm2iWJzkvySP7OyV5e5LVSe43aNAka3rGuS7JBUlenWSL3nZVdVxVbd/GIUmamTlb0kh1XYj2Wwrs2yaaXYF3Amcl2XOqQZIlwAuB84HnTzPOkp5xdgKOAw4Bzkuy7RzGL0mLiTlb0kYZt0L0dlW1rqrOBc4BntVz6BDgSuAdwJFJZnwPVbW2qr4BPBG4hiZRSpJGyJwtaUOMbSHa41Jgj57XRwMfAb5CE//Bsxmkqgp4DXB0ki1HG6IkqWXOljRrk1CI7gNcBpBkB+BJwMeqah1wGvC82Q5UVZcAK4AHjz5MSRLmbElDGNtCNMl2SV4OPAb4QLv7SOBLVXVt+/o04LAkWw8x9NXALkPGckySZUmWXbti5TBdJWlRGNecvWLFimG6Sppn41iIXpLkJuD7wEHAwVV1fXtsaooHgKq6ELgKOGyI8ZcAq4cJqKpOrqoDquqAHXfYfpiukrTQjXXO3mGHHYbpKmmebdp1AAPsW1XL+3cm2R/YCzi779CpNFM9p8xy/D2ByzcqQknSFHO2pA02joXodI4GtgVuTdJ/bF2S3arq6pkGSPIo4Naq+t4cxShJapizJa3XOE7N30mSpcARwIFVlf6NZn26565njG2AE3ApEEmaU+ZsSbM1EYUocChwRVUtm+b4ycBRgw4k2THJEcAy4JtVddIcxShJapizJc3KuBWiqxh8U/pRwAdn6HcmsHOSqSU+1tLcQH89cAHN8iHHVNXLRhmsJC1y5mxJG2Ws7hGtqi2m2f+09fRbBezc83qs3pckLUTmbEkbq+sromuBi5Kc0cXJkxyf5AZgXRfnl6QJY86WNFKdfgqtqr07Pv+xwLFdxiBJk8KcLWnUur4iKkmSpEXKQlSSJEmdsBCVJElSJyxEJUmS1AkLUUmSJHXCQlSSJEmdsBCVJElSJyxEJUmS1AkLUUmSJHXCQlSSJEmdsBCVJElSJyxEJUmS1AkLUUmSJHUiVdV1DBMlya+Ay+dg6B2Ba+dg3Lk2iXEb8/yYy5jvXVU7zdHYWkDM2XcyiXEb8/yZq7inzdkWomMiybKqOqDrOIY1iXEb8/yYxJil2ZrUv9+TGLcxz58u4nZqXpIkSZ2wEJUkSVInLETHx8ldB7CBJjFuY54fkxizNFuT+vd7EuM25vkz73F7j6gkSZI64RVRSZIkdcJCdEwk+VCSozuO4QNJjusyhoUmycVJbkxyZkfnf0+SlUluGbLfRMYtLVZJHpzkgiSbzbL965O8fo5jmsg8MolxT2LMUyxEx0RVvaiq/rnjMDZvN43OVsB+VXU4QJJVSW5of2GXJzkvySP7OyV5e5LVSe43aNAka3rGua79B+jVSbbobVdVx1XV9m0ciyFuaVGqqu9W1cOqavUsu8xHvp/UPDKJcU9izICFqDTflgL7tr+wuwLvBM5KsudUgyRLgBcC5wPPn2acJT3j7AQcBxwCnJdkW+OWNIYmNY9MYtwTE7OFqNSRqlpXVecC5wDP6jl0CHAl8A7gyCQz/p5W1dqq+gbwROAamoQzZyY1bknjY1LzyCTGPe4xW4iOiSRnJXlO13FMSbJdkve3l/RXtpfjH99z/FtJHtvX5xlJPt3z+s+SXJ3mvpVPJdl1jmJ9Y5IP9O374yT/3P78/CTfb9/H5e1UxKY9bV+f5B/6+u+a5Pq5iHeAS4E9el4fDXwE+ArN7+jBsxmkmiUwXgMcnWTL0YY40KTGLc1Kkk2S/N8kv0hyU5IvJXlckkt62syYX+Yx1v374npykmVtXL9IckKSbfq6LUnyjiRXtu/vs71XzObJpOaRSYx7LGO2EB0f43Z/5r8BdwUe2P737cBpSQ5sj38aeHpfn8OAswGSvBR4EfB4YGfgQuD0OYr1DOAZueNN+i8ATk/yXJpPey9upxZ+r93e3dN20J/9UmC+iqJ9gMsAkuwAPAn4WFWtA04DnjfbgarqEmAF8ODRh3knkxq3NFuvobmC9PvA9sB7gVNp88Us88t8uT2PJTkIOAV4C03+fhBwd5qio9dL2uP70Ezf/hr45PqujI3YpOaRSYx7PGOuKrcx2IDPAi/oOIYPA6+j+VS0HNii7/ifAZ9uf94f+EnPsU3bv5S70NxTcjVwUM/xTYDLgYfMUezLgCe1P+/Zxr8E+Bnwh31tdwNuBu7Zvn4rcFJfmz2A34wgrsuAPXpeF7BL+/N2wMuBK4C7tvteBZzd0/63gZuArfvGvX2cAef8T+Dp/e0XQ9xubqPcgJ8CT+7b9/ft78cms8kv8xjrw4HL2p/PA47rO74lcC3wO+3rt069j542dwFuAR4xopgmMo9MYtyTGPPU5hVRDfJomr+gv+nbf2Z7jKr6b2CzJPv29PlBVS0HdgfuUlXnTXWs5hPXfwH7zVHMpwLPaH9+FvBx4J40hfFZvQ2r6mqaK7S/O0exrM8lSW4Cvg8cBBxcVVO3AUxNlQBQVRcCV9FcbZ6tJcBsn5wdxqTGLQ2tncbeC/ha36FPtv/dnTHML+0DKI8APtYX1600M1kH9ez+XJubp9rcBHyd5grqXJnUPDKJcU9EzPN+H4smwq7Azwfs/zmwdZK7tn+ZzwaeDFwCPBX4957+WydZ2dd/M+DLcxMypwPfbafnnw38cRvHL3sTbY+f0xSq08noQ7zdvm3BfscTJvvT/MN3dt+hU2mmTE6Z5fh70lx9HrVJjVvaENsDa9rirNcv2v9uTH6ZSzvTFAh3+l3lznGtGNDmWmDrOYhryqTmkUmMeyJithDVICtokmy/XYFVNJfvobkS8DrgPTSF6NTDTDcD11XVTnMc5+2q6udJLgJeAWxTVRckuS+wS5JNBvxjsSvwq/bntdz5d+EecxvxQEcD2wK3Jneqg9cl2a292jKtJI8Cbq2q781RjINMatzSTG4GNk2yfVX1fqjeuf3vCmaXX+bbSmAdzdXa/gsKuwI/6nm9+4D+OwK/nJvQZjSpeWQS4x6rmJ2a1yCfAp424Gm4w4HPVNWa9vVXgAcleRhwY1Vd1u6/BNgyyQPmI9geHwX+kuZTHcCPae6JObS3UZLdgIcAn293/RK4d99Yj2ceJVkKHAEcWFXp32jWeXvuesbYBjiBeVwGaVLjltanLT4vAp7Qd+hpNIXebPPLvGqn4L/MHZfpIclWNA+nnNOz+8nt/qk229BM6397HkLtjW0i88gkxj2OMVuI6k6q6uvAd4B/SbJjGs8E/pzmCdGpdmuALwDH87/T8lTVKuD9wKlJHgSQZOskT5zj0P+NZvr/o20cBbwZOCHJ77Zx7Al8AvhAVU196v8S8HtJHt22eRR3/sdnrh0KXFFVy6Y5fjJw1KAD7f+jI2ge2PpmVZ00RzEOMqlxS7PxFuDvkjw0yaZJnk3zj/SKIfJLF94GvCnJH7T5e2ea1UW+UlXf6Wm3EvjnNj9vA/wz8IWq+tGAMefSpOaRSYx77GJ2an58rGq3cYnhcJpPOxfRPG15MXBYe0NzrzNoHgw6um//64BbgXOS3I1mmuvjNKsDzJV7AN+tZlkJAKrq9DRzDycn2Z0m8f4j8Fc9bX6S5P8A/9ReBf4xcAzwjTmIcRWDb+4+CvjgDP3OBN6T5MFV9V2a2wkuSbIOuI4m1mOq6qujDrg1qXFLG6yqPpHmqwzPpJmyvgD4EM3T5bPKL/NoE5rfL6rq/LZo/r80H8x/TXMf/Zt62q+iWSklNNP1W9PcbvWSOYxxUvPIJMY9MTGnfdxemlg9a959CPhWVb2/y3h6JbkMeEzPbQtdxVHttMts21/GBMYtjVKS36KZZfkuzQNABwEnAYdU1aVdxtYvyXHAU6vqsettPE8mNY9MYtyTGPMUp+a1EBwF3EBzleIfO46l31rgoiRndHHyJMcnuYHmnrZhTGrc0ijtDPwLzZXOFcAbgGePYRF6Kc13gL+141D6TWoemcS4JzHmpq9XRCVJktQFr4hKkiSpExaikiRJ6oSFqCRJkjphISpJkqROWIhqQUjynCRn9e3bIsnXk7yrq7gkSXdmztYUC1EtFJu3W6/jgduAN85/OJKkGZizBfjNSlqg2m8V+QPgoe1XkUqSxpQ5e/HyiqgWnCT3Bd4L/FFVXdOz/+5J/i3JjUmWJ3lzz7HPJ3lO3zinJXn+/EUuSYuPOXtxsxDVgtJ+L/THgTdU1bf6Dp8F/ALYDXgo8IdJXtgeOwM4smecbYEnAf8x50FL0iJlzpaFqBaaE4DfAs7r3ZnkicCOwKuq6qaquhp4DfCqtsmZwEFJtmtfHwZ8oapumJ+wJWlRMmcvchaiWkgeDhxI833zJydJ37HPVFXv9+B+G/itJJtU1XXAl4FD22PPBk6dh5glabEyZ8tCVAvKauDpNJ+a9wBe3HNsV+DFSVZObcAVwBpg6hP1qcAzkuxIMw30qfkKXJIWIXO2LES1oPxXVV1eVbcALwH+Jsk92mM3AydW1fZ925ZVdX3b5mzgEcCLgLOratX8vwVJWjTM2bIQ1cJUVZ8FPg2c2O76L+DR6+nz67bPW4CPzmmAkqTbmbMXLwtRLWTH0tzM/nTgE8Ddk/x9ku0BkuyVZP++PqcD1wNfmddIJUnm7EXIQlQLxap2u11V/Yomsb0XCPA4YC/g8vZ+o08Cu/SNcy/g9L4b5CVJo2XOFgCpqq5jkDqXZBNga+DrwLOr6uKOQ5IkTcOcvXB4RVRqnEzzROapJjRJGnvm7AXCK6KSJEnqhFdEJUmS1AkLUUmSJHXCQlSSJEmdsBCVJElSJyxEJUmS1AkLUUmSJHXi/wPz6QahteK3+QAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 864x360 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-6jm3HXj8DN-"
      },
      "source": [
        "드디어 😲 타겟의 쿼리 '나는', '당신을', '사랑', '합니다', '[PAD]', '[PAD]'가 소스쪽 키 'i', 'love', 'you'와 어텐션되는 것을 확인할 수 있습니다! 인코더에서 [PAD]가 인코딩되어 온 정보는 디코더쪽 크로스 어텐션에서 비로소 패딩되어 사라지는 것을 확인할 수 있습니다.\n",
        "\n",
        "<a name=\"cell-id-targetmask\"></a>\n",
        "그럼 쿼리로 쓰이는 타겟에 포함된 [PAD]는 언제 마스킹 될까요? 이 [PAD]는 로스를 계산할 때 마스킹되게 됩니다. 그 과정이 아래쪽 [Label Smoothing](#cell-id-labelsmoothing) 절에 나와 있습니다.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ZjtZBy_-Pah"
      },
      "source": [
        "## Coffe Break\n",
        "\n",
        "트랜스포머를 이해하는데 있어 매우 귀찮은 마스킹을 완전히 분석했습니다. 여기까지 읽고 내용을 머리에 그릴 수 있다면 이제 글을 더 읽지 말고 커피나 음료수를 마시면서 쉬도록 합시다. 나머지는 내일 읽어주셔도 됩니다. 😁"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RJcN5-zm_MiL"
      },
      "source": [
        "## 장난감 데이터 생성함수"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WoncHyDrbpUM"
      },
      "source": [
        "원문에서는 아래 함수처럼 입력과 출력이 동일한 무작위 숫자를 열개 생성하여 입력이 들어갔을 때 제대로 출력이 나오는지 체크합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qffH6rN0lpl"
      },
      "source": [
        "def data_gen(V, batch, nbatches):\n",
        "    \"Generate random data for a src-tgt copy task.\"\n",
        "    for i in range(nbatches):\n",
        "        data = torch.from_numpy(np.random.randint(1, V, size=(batch, 10)))\n",
        "        data[:, 0] = 1\n",
        "        src = data; src.requires_grad=False\n",
        "        tgt = data; tgt.requires_grad=False\n",
        "        yield Batch(src, tgt, 0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h78nlMpkd6Fl"
      },
      "source": [
        "입력과 동일한 출력이 나오는 예제는 너무 재미가 없으니 여기서는 이 예제를 살짝 바꿔서 길이 10짜리 무작위 숫자를 입력으로 사용하는 것은 똑같으나 출력은 5번부터 10번 숫자 즉 뒤쪽 절반에 +1을 한 것을 출력으로 설정하겠습니다.\n",
        "\n",
        "예를 들면 다음과 같습니다.\n",
        "\n",
        "```\n",
        "- 입력: 1 2 3 4 5 6 7 8  9 10\n",
        "- 출력: 1 2 3 4 5 7 8 9 10 11\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hKIKnFeQBFqV"
      },
      "source": [
        "간단하게 데이터를 생성하는 함수를 만듭니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Ck0GGgOTSiA"
      },
      "source": [
        "def data_gen2(V, batch, nbatches):\n",
        "    \"\"\"\n",
        "    무작위 데이터를 생성하는데\n",
        "    데이터 길이의 뒷 절반에 +1 한 것이 정답 벡터인 데이터를 생성한다.\n",
        "    V: 단어장에 단어수\n",
        "    batch: 한 미니배치에 샘플 수\n",
        "    nbatches: 한 에폭에 있는 미니배치 수\n",
        "    \"\"\"\n",
        "    for i in range(nbatches):\n",
        "        # 샘플하나당 시퀀스 길이는 10으로 고정\n",
        "        data = torch.from_numpy(np.random.randint(1, V-1, size=(batch, 10)))\n",
        "        data.requires_grad = False\n",
        "        data[:, 0] = 1\n",
        "        \n",
        "        src = data.clone() \n",
        "        tgt = data.clone()\n",
        "        # 뒤에 다섯개는 +1\n",
        "        tgt[:, V//2:] +=1 \n",
        "        \n",
        "        yield Batch(src, tgt, 0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nf-ZS8TsThrC",
        "outputId": "07ad4297-36df-4792-d32b-a7b045a3601b"
      },
      "source": [
        "# data_gen2가 제대로 작동하는지 확인\n",
        "# 단어수: 11, 미니배치사이즈:1, 에폭당 미니배치:10\n",
        "for data in data_gen2(11, 1, 10) :\n",
        "    # 입력, teach_forcing의 입력, teach_forcing의 출력\n",
        "    print('data.src: ', data.src)\n",
        "    print('data.src_mask: ', data.src_mask)\n",
        "    print('data.trg: ', data.trg)\n",
        "    print('data.trg_y: ', data.trg_y)\n",
        "    print('data.trg_mask\\n', data.trg_mask)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "data.src:  tensor([[1, 8, 3, 4, 5, 2, 6, 1, 6, 4]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[1, 8, 3, 4, 5, 3, 7, 2, 7]])\n",
            "data.trg_y:  tensor([[8, 3, 4, 5, 3, 7, 2, 7, 5]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 5, 4, 1, 2, 9, 4, 8, 6, 7]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[ 1,  5,  4,  1,  2, 10,  5,  9,  7]])\n",
            "data.trg_y:  tensor([[ 5,  4,  1,  2, 10,  5,  9,  7,  8]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 5, 2, 1, 7, 3, 7, 3, 8, 7]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[1, 5, 2, 1, 7, 4, 8, 4, 9]])\n",
            "data.trg_y:  tensor([[5, 2, 1, 7, 4, 8, 4, 9, 8]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 3, 8, 3, 8, 6, 5, 3, 7, 7]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[1, 3, 8, 3, 8, 7, 6, 4, 8]])\n",
            "data.trg_y:  tensor([[3, 8, 3, 8, 7, 6, 4, 8, 8]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 2, 7, 4, 8, 8, 7, 9, 1, 5]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[ 1,  2,  7,  4,  8,  9,  8, 10,  2]])\n",
            "data.trg_y:  tensor([[ 2,  7,  4,  8,  9,  8, 10,  2,  6]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 3, 3, 4, 4, 2, 3, 1, 3, 9]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[1, 3, 3, 4, 4, 3, 4, 2, 4]])\n",
            "data.trg_y:  tensor([[ 3,  3,  4,  4,  3,  4,  2,  4, 10]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 4, 2, 9, 9, 9, 9, 7, 1, 2]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[ 1,  4,  2,  9,  9, 10, 10,  8,  2]])\n",
            "data.trg_y:  tensor([[ 4,  2,  9,  9, 10, 10,  8,  2,  3]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 4, 2, 7, 4, 6, 2, 1, 5, 6]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[1, 4, 2, 7, 4, 7, 3, 2, 6]])\n",
            "data.trg_y:  tensor([[4, 2, 7, 4, 7, 3, 2, 6, 7]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 4, 4, 4, 1, 6, 9, 3, 4, 9]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[ 1,  4,  4,  4,  1,  7, 10,  4,  5]])\n",
            "data.trg_y:  tensor([[ 4,  4,  4,  1,  7, 10,  4,  5, 10]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n",
            "data.src:  tensor([[1, 8, 8, 5, 3, 3, 4, 3, 4, 9]])\n",
            "data.src_mask:  tensor([[[True, True, True, True, True, True, True, True, True, True]]])\n",
            "data.trg:  tensor([[1, 8, 8, 5, 3, 4, 5, 4, 5]])\n",
            "data.trg_y:  tensor([[ 8,  8,  5,  3,  4,  5,  4,  5, 10]])\n",
            "data.trg_mask\n",
            " tensor([[[ True, False, False, False, False, False, False, False, False],\n",
            "         [ True,  True, False, False, False, False, False, False, False],\n",
            "         [ True,  True,  True, False, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True, False, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True, False, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True, False, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True, False, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True, False],\n",
            "         [ True,  True,  True,  True,  True,  True,  True,  True,  True]]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WmQJGmokef8j"
      },
      "source": [
        "미니배치 내 샘플들을 최대 길이로 잘라주는 함수입니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "egTWNxnQ0LRX"
      },
      "source": [
        "global max_src_in_batch, max_tgt_in_batch\n",
        "def batch_size_fn(new, count, sofar):\n",
        "    \"Keep augmenting batch and calculate total number of tokens + padding.\"\n",
        "    global max_src_in_batch, max_tgt_in_batch\n",
        "    if count == 1:\n",
        "        max_src_in_batch = 0\n",
        "        max_tgt_in_batch = 0\n",
        "    max_src_in_batch = max(max_src_in_batch,  len(new.src))\n",
        "    max_tgt_in_batch = max(max_tgt_in_batch,  len(new.trg) + 2)\n",
        "    src_elements = count * max_src_in_batch\n",
        "    tgt_elements = count * max_tgt_in_batch\n",
        "    return max(src_elements, tgt_elements)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JoTf84tRMNLq"
      },
      "source": [
        "## Learning Rate Warm Up"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uakqnXcbJbJM"
      },
      "source": [
        "$$\n",
        "lrate = d^{-0.5}_{\\text{model}}\\cdot \\min \\left({step\\_num}^{-0.5}, {step\\_num} \\cdot  {warmup\\_steps}^{-1.5} \\right)\n",
        "$$\n",
        "\n",
        "학습중 학습률을 위 식처럼 변경합니다. 식에서 $step\\_num$은 옵티마이저가 파라미터를 한번 업데이트할 때마다 1 증가하는 진행 스탭수를 나타냅니다. 학습 초기에는 $step\\_num$이 작기 때문에 다음과 같습니다.\n",
        "\n",
        "$$\n",
        "{step\\_num} \\cdot  {warmup\\_steps}^{-1.5} = \\min \\left({step\\_num}^{-0.5}, {step\\_num} \\cdot  {warmup\\_steps}^{-1.5} \\right)\n",
        "$$\n",
        "\n",
        "즉 $step\\_num$이 증가함에 따라 선형적으로 학습률이 증가합니다. $step\\_num$이 계속 증가하다가 $step\\_num =  warmup\\_steps$가 되면 $\\min$ 안에 두 항은 같아지게 되고 그 다음 스탭부터 \n",
        "\n",
        "$$\n",
        "{step\\_num}^{-0.5} = \\min \\left({step\\_num}^{-0.5}, {step\\_num} \\cdot  {warmup\\_steps}^{-1.5} \\right)\n",
        "$$\n",
        "\n",
        "가 되어 $step\\_num$의 제곱에 반비례하게 학습률이 천천히 줄어들게 됩니다. 코드를 보면 전체적으로 학습률의 크기를 조정하기 위해 모델 사이즈와 별도의 factor를 곱하는 것을 확인할 수 있습니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s_LTh2Cm0NGb"
      },
      "source": [
        "class NoamOpt:\n",
        "    \"Optim wrapper that implements rate.\"\n",
        "    def __init__(self, model_size, factor, warmup, optimizer):\n",
        "        self.optimizer = optimizer\n",
        "        self._step = 0\n",
        "        self.warmup = warmup\n",
        "        self.factor = factor\n",
        "        self.model_size = model_size\n",
        "        self._rate = 0\n",
        "        \n",
        "    def step(self):\n",
        "        \"Update parameters and rate\"\n",
        "        self._step += 1\n",
        "        rate = self.rate()\n",
        "        for p in self.optimizer.param_groups:\n",
        "            p['lr'] = rate\n",
        "        self._rate = rate\n",
        "        self.optimizer.step()\n",
        "        \n",
        "    def rate(self, step = None):\n",
        "        \"Implement `lrate` above\"\n",
        "        if step is None:\n",
        "            step = self._step\n",
        "        # 초기에는 min( , )에서 뒷부분이 작동하여 step에 선형적으로 lr이 증가\n",
        "        # 그렇게 뒷 부분이 자꾸 커지다 step에 self.warmup과 같아지면\n",
        "        # 뒷부분이 step*step**(-1.5)=step**(-0.5)가 되고 \n",
        "        # step = self.warmup+1부터는 앞부분이 작아져서\n",
        "        # 어느 순간 step의 제곱근에 반비례하게 lr이 줄어듬\n",
        "        return self.factor * \\\n",
        "            (self.model_size ** (-0.5) * \n",
        "             min(step ** (-0.5), step * self.warmup ** (-1.5)))\n",
        "        \n",
        "def get_std_opt(model):\n",
        "    return NoamOpt(model.src_embed[0].d_model, 2, 4000,\n",
        "            torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Pbnvo2ye7xl"
      },
      "source": [
        "모델 사이즈 512, 256, 웜업스탭 4000, 8000인 경우에 대해서 학습률이 어떻게 변화하는지 그래프로 그리면 아래와 같습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "V-y_OLgv0PQQ",
        "outputId": "f100d12f-eb36-4c49-f9d0-df70097ee20a"
      },
      "source": [
        "opts = [NoamOpt(512, 1, 4000, None), \n",
        "        NoamOpt(512, 1, 8000, None),\n",
        "        NoamOpt(256, 1, 4000, None)]\n",
        "plt.plot(np.arange(1, 20000), [[opt.rate(i) for opt in opts] for i in range(1, 20000)])\n",
        "plt.legend([\"512:4000\", \"512:8000\", \"256:4000\"])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f77f70dbe10>"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAD3CAYAAADyvkg2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3iUVdrA4d876aT3XiAkoYTeQw1FQYoUERGwomLDdS2764q7q6hYFl1dy6qfoIhIERAElDahhB5KgJDQ0jsJ6X3mfH8MRpBAJnVmknNfF5fMecs8g5BnznnPeY4ihECSJEmSAFSGDkCSJEkyHjIpSJIkSbVkUpAkSZJqyaQgSZIk1ZJJQZIkSaplbugAmsLNzU0EBQUZOgxJkiSTEhMTc0UI4V7XMZNOCkFBQRw7dszQYUiSJJkURVGSb3VMDh9JkiRJtWRSkCRJkmrJpCBJkiTVMulnCpIktW3V1dWkpaVRUVFh6FBMkrW1NX5+flhYWOh9jV5JQVGUOcAsQAMcFEK8q8/x27SbAa8D/YQQ46+7z1jgeaAUSBNC/FnvTyJJUpuTlpaGvb09QUFBKIpi6HBMihCCvLw80tLS6Nixo97X1Tt8pCiKPTAPuFsIMQ3ooShKSH3H67luErCJ65KSovs//jdguhDiXqBMUZRxen8SSZLanIqKClxdXWVCaARFUXB1dW1wL0ufZwoRwA7xeznVn4BIPY7f8johxE9CiMN/eJ9QIE4IUXnt9cY/vA8AiqI8rijKMUVRjuXm5uoRviRJpkwmhMZrzJ+dPknBFci/7nX+tbb6jtd3XUPfBwAhxBdCiP5CiP7u7nWuvZDqUFJVwtrza6nR1hg6FEmSjJg+zxTygO7XvXa51lbf8fquq+t9nBtwvtQA38R9w+enPie/PJ8nej1h6HAkyWT06dOHQYMGAWBubs7HH39c+w186dKlrFixghMnTtSe/8EHH3D+/Hmqq6txdnbmvffeu+W9ly9fzquvvkpaWhoAhYWFPP744yiKQmlpKf/9738JDAxscHuTCCFu+wtwArYCyrXX3wJd6jte33XX2nZe93szYDdgde31G8Cdt4utX79+QtLPvK3zRPjycNFvRT+RXJhs6HAkSS9xcXGGDkGMGTOmzvbo6Gjx888/3/K4EELMnz9fnD17VgghxDvvvCO+/vrr2mOHDh0SH3744Q3Xv/rqq2LHjh1CCCESEhLE3LlzG9V+vbr+DIFj4hY/V+vtKQghChRFWQGsUhSlBjgphIjX5/jtrrum+rr7aBRFeQNYqShKCZALbK8/rUn1ySvP42TOSWaEzODXpF/518F/8dUdX8mxWsmk/GvzWeIyipr1nt18HPjH5O63PUej0fC3v/2NlJQUZs6cydSpUwGIiIgAdD2DWykuLsbb2xuAZ599FnNz3Y/c7OxsVq9ezdKlS9m8eXPt+UeOHOGNN94AIDQ0lOzs7Ea1N4VeU1KFEKuAVde3KYryI3CvEEJT1/FbXfeH4xP+8FoNqPWJSdLf3rS9CASzwmbR3a07rx98nQ0XNzA9ZLqhQ5Mko6dW634kVVdXM3PmTLp3705ISEg9V8GGDRvo168fzs66UXEbG5va+yxevJh33nnnpmu0Wu0Nr8W1eToNbW+KRi9eE0LMaPK7S61id+puvG296eLShTCXMLZc3sL7x95nuO9w3DvIh/WSaajvG31Ls7CwYNy4cZw9e7bepLBv3z727t1bZy/i0KFDZGVl8ec/65ZhxcfH8+STT/Lee++hUt049+e33nxD25tClrlo48pryjmUcYhR/qNQFAWVouKfQ/5JZU0lrx96vVm+WUhSe3Hw4EF69+5923MOHz7MqlWr+Pe//31D+7vvvsuyZcsYPnw4a9eu5fPPP+fzzz+nS5cufPbZZ9jZ2dGvXz92794NwIULF/hthmVD25tClrlo4w5mHKRCU0Gk/+9LPoIcg1jYdyHvH3ufjRc3Mi1kmgEjlCTj9uCDD2JjY0NJSQlTp07lj3u4XF9Cory8nClTpjBlyhSeeuopAObNm8fQoUNZuHAhZmZmN93fysqq9vcvv/wyCxcu5LvvvqO4uLg2sTS0vSkUU/6m2L9/fyH3U7i916JfY2fyTvbctwcL1e9/ebVCy2PbH+PMlTOsm7IOf3t/A0YpSXU7d+4cXbt2NXQYJq2uP0NFUWKEEP3rOl8OH7VhGq2GPWl7GOY37IaEAKBSVCweuhgzxYy/7/87Gq3GQFFKkmRMZFJow2KvxJJfkX/D0NH1vO28eWXwK5zIOcGys8taOTpJkoyRTAptmDpFjbnKnGG+w255zsSOE7kz6E4+OfEJsbmxrRidJEnGSCaFNkydqmaA5wDsLe1veY6iKCwavAhPW09e2vMShZWFrRihJEnGRiaFNupy4WWSipKIDKh76Oh6jlaOvDfiPXLKc3g1+lU5TVWS2jGZFNqoqNQogFs+T/ijHu49eKHfC0SlRrEibkULRiZJkjGT6xTaKHWKmq4uXfGy9dL7mjld53As+xgfxHxAL49e9HLv1YIRSpJpaIkqqadOneKNN97A1dWVzMxMXnnlFQYPHmwaVVKN+Zesklq33LJc0WN5D/HpiU8bfG1hZaG4c92dYvTq0SKnNKcFopMk/bXVKqlTp04V2dnZQgghsrKyxNSpU4UQJlIlVTI9vxXA0+d5wh85WDrwn8j/MG/bPJ6Pep6v7/waSzPLFohSkhpo218h63Tz3tOrB0xYcttTWqJK6rRp09i1axezZ89m27ZtzJ07FzCOKqnymUIbpE5R42PrQ5hzWKOuD3MJ481hb3Iq9xRvHn5TPniW2jW1Ws3bb7/N8uXLWb58ORcuXNDrurqqpP5WEmPSpEmcOHGC9evXc+bMGcaMGQOYeJVUyTiVVZdxMPMgM0JmNKli4rjAcTze83G+iP2CMOcw7u96fzNGKUmNUM83+pbWXFVSAR577DFWrlyJtbU148aNY/78+axevVpWSZWa36HMQ1RqKhs1dPRHT/d+mlF+o3j36LsczDjYDNFJkmlrjiqpAJmZmbXf6i0tLUlNTQUaXg21JaqkyoJ4bcyi6EXsSt51UwG8xiqpKmHetnlklWbxzYRvCHUObYYoJUk/xlAQ749VUu+5554bjk+YMIFt27YBuiqpQUFBTJkypbYi6m9VUisqKjAzM8PCwoKdO3fy+eef4+bmxpUrV5g/fz7jx4+noKCAhQsXYm5uXlv1NCAgoMHt12toQTyZFNoQjVbD6LWjGeQ9iHdHvNts980qzWLOljkoisLKu1biaevZbPeWpNsxhqRg6mSV1HbsVO4p8ivyGe0/ulnv62XrxadjP6WkuoSndj1FSVVJs95fkiTjIZNCG6JOrb8AXmOFuYSxdORSLhdc5s9Rf6ZaU93s7yFJkuHJpNBGCCFQp6oZ6DUQO0u7FnmPCN8IXhvyGgczD/LK/lfkHgyS1AbJKaltRGJRIslFycztOrdF32dayDQKKwv5d8y/sbWw5R9D/tEs0+AkSTIOMim0EeoUNQCj/Ee1+Hs9FP4QxdXFfBH7BR0sOvBS/5dkYpCkNkImhTZCnaqmm2u3BhXAa4pnej9DaXUpK+JWYG9hz5O9n2yV95UkqWXJpNAGXCm/QmxubKv+YFYUhZcHvExpdSmfnvoUCzML5veY32rvL0mt5cknn0SlUpGfn8/EiROZO3cuY8eOpXPnzrXnLFmyBCcnJ8rKynjllVcoKCjAysqKWbNmMXp03bMBlyxZwrp16/htWn1KSgrPPvssHTp0oKamhi+//BInJ6cGtzfZrSrlmcIvWSVVZ13COhG+PFzE58W3+nvXaGrEX/b+RYQvDxefnmx4VVZJuh1jqJL6G61WK4YNGyaEuHXl1Oeff16cOHHipvann35a7Ny5s/b1hg0bxOrVq2+4zwMPPCASEhKEEELs2LFDvPLKK41q/yNZJbUdUqfqCuAZYrWxmcqMN4e+iZlixqcnP0Wj1fB076flMwap2b1z5B3i8+Ob9Z5dXLrwl4F/0evcyspKXFxcALCzs2PRokUkJSUxYsQIHnvsMYQQJCYmsm7dOpYuXYq3tzdvvPEGlpaWLFmyhA4dOgC6xWSnT59m0aJFfPHFF7X3z8zMJDRU9294zJgxtfswNLS9qWRSMHFl1WUcyjzEPaH3GOwHsZnKjDeGvoG5ypz/xf4PjdCwsM9CmRikNuXVV1/l5ZdfBmDjxo2AbqTlySefJDg4mPDwcA4fPszHH3+Mn58fP/zwA++88w6LFi3Czk43TbyoqIjPPvuszkJ54rrqEoqi1FZAbWh7U8mkYOIOZh7UFcDTc9vNlqJSVPxjyD8wU8z46vRXVNRU8NKAl1ApcimM1Dz0/UbfEj744AP69OnD0KFDb2hXFIXJkycTGxvLwIED6dOnD35+fgDcfffdzJ49+4bzt23bRl5eHk8//TQA8fHx/OlPf+LDDz+8oeKpEKL2dUPbm0omBROnTlFjb2lPX8++hg4FlaJi0eBFWJlZ8d2577haeZU3It7AwqzphfkkyVA+/fRTbG1tmTNnTp3H9+7dy5QpU7Czs6OmpobS0lJsbW05fPgwPXv2BOCZZ55h2rRpzJo1i1mzZtVeO3bsWD788EMA3N3duXjxIp07d2b37t307du3Ue1NJZOCCdNoNexN28tw3+HNUhG1Ofw2K8nF2oWPTnxEQUUBS0ctpYNFB0OHJkkNduDAAZYsWcJdd93FggULAHjjjTdYsmQJJSUlVFRUMGjQoNoexOuvv868efPw8PBAq9WydOlSAN55553aZwrXs7Kyqv3922+/zYsvvoitrS1VVVV8/PHHjWpvKlkl1YTFZMfw0C8P8d7I9xgfNN7Q4dzkx/M/8vqh1wl3DeeTMZ/gZN0M0+WkdkVWSW26FqmSqijKHEVRNimKskFRlJf1Pd6I9ucVRflOUZSvFUX5RlEU+fXyNtQp1wrg+TR/AbzmMCN0BktHLSU+P5552+aRWpRq6JAkSapHvUlBURR7YB5wtxBiGtBDUZSQ+o43ot0JGCuEmCuEeASIA8Y18+dtM8S1AniDvAa1WAG85jAmYAz/G/c/8ivyuX/r/RzPPm7okCRJug19egoRwA7x+zjTT0CkHscb2l4IZCqK4q0oig0QCOz/YzCKojyuKMoxRVGO5ebm6vs525zEwkRSilMMPutIH/29+vP9xO9xtHJk/vb5bL602dAhSSbElIe4Da0xf3b6JAVXIP+61/nX2uo73qD2a0liGfAU8AQQLYTI+2MwQogvhBD9hRD9m2M/UlO1O1W3L+tI/5EGjkQ/gQ6BrLxrJX08+vDK/lf46PhHaEXzzKuW2i5ra2vy8vJkYmgEIQR5eXlYW1s36Dp9Zh/lAd2ve+1yra2+4w1qVxSlJzBJCPE3AEVRpiuKMl8I8ZWen6Vdae0CeM3B0cqRz8d+zuLDi/ny9JckFiayeNhibC1sDR2aZKT8/PxIS0ujPY8KNIW1tXXtugl96ZMUDgPPKYrywbVv81OAt/Q4ntXA9kDg+iWw5UBQgz5NO3Gl/Aqnc0/zVO+nDB1Kg1mYWfDPIf+kk2MnPoj5gNlbZvNh5Id0cuxk6NAkI2RhYUHHjh0NHUa7Um9SEEIUKIqyAlilKEoNcFIIEa/P8Ya0K4qSAIxQFOVboBLoACxs3o/bNkSlRiEQJvE8oS6KovBg9wfp5tqNF/e8yOyfZ7N42GLGBcp5BZJkaI1ep6Aoyo/AvUIIg+3J2F7XKTy962kuFVxi2/RtJl9fKKs0ixeiXiD2SiwPhz/Mwj4LMVfJNZWS1JKavE6hLkKIGYZMCO1VWXUZhzIOEekfafIJAcDL1otl45cxM3Qmy84sY/72+WSVZhk6LElqt2S1MhNzMOMgVdoqkx06qoulmSWvDXmNt4a9RVxeHPdsvqd2e1FJklqXTAomZnfqbuwt7enj2cfQoTS7ycGTWTNpDT62PixUL+Stw29Rqak0dFiS1K7IpGBCarQ17E3bywi/EUZTAK+5BTkG8d1d3zG361xWxa9izpY5XC64bOiwJKndkEnBhJzMOUlBZUGbGjqqi6WZJX8Z+Bc+GfMJOWU5zNw8k2/OfoNGKx9hSVJLk0nBhKhT1VioLBjma5wF8JrbCL8RrL97PRG+Ebx/7H0e+fURWVRPklqYTAom4rcCeAO9B7arFcBuNm58FPkRbw57kwtXLzBj8wxWx6+WZQ8kqYXIpGAiLhdeJrU4ldH+ow0dSqtTFIUpwVNYf/d6+nj0YfHhxTy24zHZa5CkFiCTgolQp+qmaI70M40CeC3By9aLz8d+zqLBizh75SzTNk3jq9NfUa2tNnRoktRmyKRgItQparq7dsfT1tPQoRiUoijcG3YvG+/eyHDf4fzn+H+Y9fMsTuWeMnRoktQmyKRgAnLLcom9EtvmZx01hKetJx9EfsBHkR9RVFnEvK3zWHxoMcVVxYYOTZJMmkwKJiAqLQqAyACZFP4oMiCSn6b+xP1d72dNwhombZjEhgsb5F4NktRIMimYAHWKGl87X0KcQuo/uR2ytbDlrwP/yqpJq/C39+e1A68xZ8scYnNjDR2aJJkcmRSMXFl1GYczDze5AF5eSSWfqC9SWlnTjNEZl+6u3VkxYQVvDXuL7LJs5mydw6v7X+VK+RVDhyZJJkMmBSN3IOMAVdoqRgc0bSrqBzvP896vCTyxIoaqmrY7tKIoCpODJ7N52mYeCX+ELYlbmLRhEl+d/orymnJDhydJRk8mBSOnTlXjYOlAH4/GF8Arqqhmw/F0APZfvMKLa0+h1bbtxV+2FrY83+95Nt69kQGeA/jP8f/UPm+Q5TIk6dZkUjBiNdoa9qTtYYTfiCZtPLPmaCqlVRo2PzOMl+4MY9OpDN7YEtcuVgUHOgTy8ZiPWT5+OV4dvHjtwGvcs/ke9qTuaRefX5IaSiYFI3Yi5wSFlYVNmoqq0Qq+OZjEgCBnevg58tSoYB4eGsSy6CQ+UV9svmCNXD/Pfnx313csHbWUam01z+x+hod/fZiTOScNHZokGRWZFIzYbwXwhvoObfQ9dp7LJjW/nIeH6jY/VxSFRRO7Ma2PL+9vP8//9lxqrnCNnqIojAscx4a7N/DqoFdJLExk3rZ5LNi5gNO5pw0dniQZBZkUjJQQgqjUKAZ5D2pSAbxl0Yn4OtlwR7ffV0KrVArv3dOTST29eXtbPF/ubV/7FVioLJjVZRbbpm/jT33/xNkrZ7l/6/08vetpzuadNXR4kmRQMikYqUsFl0gtTm3S0NHZjEIOXc7ngSGBmJvd+L/a3EzFh7N6M7GHN29uPcdX+9pXYgDoYNGBR3s8yi8zfuG5vs9xKvcU9/18H8/uepa4vDhDhydJBiGTgpH6rQDeKP9Rjb7H8ugkbCzMuG9AQJ3Hzc1UfHhfbyaEe7F4y7l212P4ja2FLfN7zOeX6b/wTO9niMmJYdbPs3hq51PEZMfIB9JSuyKTgpFSp6oJdw3Ho4NHo66/UlLJTyczmNHPF8cOt96608JMxUez+9T2GN77Nb7d/hC0s7TjiV5P8OuMX3m2z7OczTvLQ788xAPbHiAqNUqWzpDaBZkUjFBOWQ6nr5xuUq2j7w+nUKXR8lBEx3rP/S0xzB7ozyfqS7y68QyaNr6O4XbsLe15vOfj/DLjF14Z9Ao5ZTk8u/tZZmyaweZLm2WpbqlNk0nBCEWlRgE0+nlCVY2WFYeSGRnqTmcPO72uMVMpvDWtB0+OCmbl4RSe++FEm175rA8bcxtmd5nNz9N/5u3hbwPwyv5XmLh+IsvPLKewstDAEUpS85NJwQhFpUbhZ+dHZ6fOjbp+y+kMcosreWRY/b2E6ymKwl/Gd+GvE7rwc2wmD359hMIy+a3YQmXBpE6TWD9lPZ+M+QRfO1/+HfNvxq0bx+JDi7lc2D6fxUhtk0wKRqa2AF5A4wrgCSH4en8Swe62jAhxa1QMC0YGs/TeXhxLzmfaZ9Ek55U26j5tjaIojPAbwbLxy1g7eS13BN7B+gvruXvj3SzYuYD96fvlcwfJ5MmkYGSiM6Kp0lY1eugoJvkqp9MLeXhoxyZVVZ3e14/vHh1EfmkV0z49wLGk/Ebfqy3q4tKFxcMWs+OeHTzd+2kS8hN4cueTTP1pKivPraSoqsjQIUpSo8ikYGTUKWocrRwbXQDv6+hEHKzNmd7Xt8mxDOrkyoanhuJoY8H9Xx1m/fG0Jt+zrXG1cWVBrwVsn7Gdt4a9ha25LUuOLGHMmjG8uv9VYnNj2+1sLsk0yaRgRGq0NexN38sI38YVwEu7WsYvZ7KYPSiADpaNL6B3vY5utqx/MoK+AU78ec0pXvvpTLt/AF0XCzMLJgdPZtWkVayetJpJwZPYnrydOVvnMHPzTFbHr6akqsTQYUpSvWRSMCK1BfAaORV1xcFkFEXhgSFBzRqXs60l3z06iPnDOvLtwWRmf3mI7KKKZn2PtqSbazf+MeQf7J65m0WDF6EoCosPL2b02tH888A/OZlzUvYeJKMlk4IRUaeqsVRZMtSn4QXwyqpqWHUkhfHdvfB1smn22MzNVLw6qRsfz+7DucwiJn60n8OX85r9fdoSO0s77g27lzWT1vD9Xd8zPmg8WxO3Mm/bPKZsnMJXp78iqzTL0GFK0g0Ufb6xKIoyB5gFaICDQoh39TneiPZg4O/XbqsB/iGEyLhVXP379xfHjh3T/9MaMSEEd62/i46OHfl07KcNvn7FoWQWbTzDugVD6B/k0gIR/u58djFPrIghOa+UhWNCeCay8021laS6lVSVsCN5Bz9d+omY7BgUFAZ7D+buznczOmA0NubNn9Al6Y8URYkRQvSv61i9A8+KotgD84AJQgihKMoKRVFChBAXbnccyGpg+0VgCfCEEKLdTXW5WHCRtJI0HunxSIOv1WoFy6MT6ennSL9A5xaI7kahnvZsemYoizae4cOdFzhwMY8P7uvdIj2UtsbO0o5pIdOYFjKN1KJUNl3exOZLm/nrvr9iZ2HHnUF3MrHTRPp59kOlyEQrtT59/tZFADvE712Kn4BIPY43tH0AkAq8pSjKSkVR5tcVjKIojyuKckxRlGO5ubl6fUhT8FsBvJF+Ixt87d4LuVzKLeXhoUFNmobaEPbWFnx4Xx+W3tuLsxmFTPhwL9tOZ7bKe7cV/g7+PN37abZO38rXd37N6IDRbE3cyiO/PsK4deN47+h7nLlyRj5/kFqVPknBFbj+m3v+tbb6jje0PQgIB/4khJgD9FMUZfgfgxFCfCGE6C+E6O/u7q5H+KZBnaKmh1uPRhXAWxadhLu9FRN7+LRAZLc3va8fWxYOp6ObLU+uPM4La05RWC5XQTeESlExwGsAbw57k6h7o3h3xLt0c+3G9/HfM3vLbCZtmMTHJz7m4tX2s1OeZDj6JIU84PoxCZdrbfUdb2h7GboexG/TWjYB/fSIz+TllOVwJu9MoxasXcwpYc/5XOYNDsTS3DDDDUFutqxdEMEzkZ3ZeDKdOz7Ygzo+xyCxmLoOFh2Y0HECH4/+mKh7o3g94nV87Hz46vRXTNs0jembpvNl7JckFSYZOlSpjdLnp8hhYKzy+7jEFGCvHscb2h4DDLzuvoOA2IZ9HNPUlAJ4yw8kYmmu4v5Bde+Z0FoszVW8eGcYG56KwMnGkoeXH+XFtbLX0BSOVo5MC5nGl3d8ya6Zu/jbwL9ha27LRyc+YvLGyUz7aRr/PfFfEvIT5BCT1Gz0nX00G7gbqAFOCiHe1+d4I9ofA0YDpUCSEGLx7eJqK7OPntz5JMlFyWyZtqVBzwQKy6oZ/PYuJvX05r2ZvVowwoaprNHw8a6LfLbnEm52lvxzcnfGh3u12vOOti6rNItdKbvYlbKLmOwYtEKLn50fYwPHMiZgDD3de8qH1NJt3W72kV5J4RY3/RG4VwihaUpwTdEWkkJpdSnDfxjO7C6zeWnASw269n97LvH2tni2LhxONx+HFoqw8U6nFfKXH2OJyyxiVJg7r08JJ8C1g6HDalPyyvOISo1iZ8pODmUeokZbg4eNB5EBkUT6RzLAawCWZpaGDlMyMi2SFIxBW0gK25O288KeF/j6zq8Z4DVA7+tqNFpGvKsmwLUDPzw+pAUjbJoajZZvDiazdHsCNVrBM5GdeXxkJ6zMzQwdWptTXFXM3rS97Ezeyf70/VRoKrAxtyHCJ4KRfiMZ7jccN5vGVc6V2pYmrVOQWpY6tXEF8LbHZZNRWME/p3Rvociah7mZikeHdWRiD29e//ks/95xng0n01k0qRuRYY3balSqm72lPRM7TWRip4lU1FRwJOsIe9P2sidtD7tSdgEQ7hrOCP8RjPQbSVeXrnJIT7qJ7CkYULW2mlGrRzHKfxRvDnuzQdfe89kBsosriHoxEjOV6fzDjkrI4Z+bzpKUV8aIUHf+fldXwrzsDR1WmyaE4PzV87UJIjY3FoHAw8aD4X7DifCJYJD3IBytHA0dqtRKZE/BSJ3IPkFRVVGDZx3FphVwLPkqiyZ1M6mEADAqzIPtz7vx7cEkPtp1gQn/2cvsgQH8eVwornZWhg6vTVIUhTCXMMJcwnis52PkV+SzP30/e1L38GvSr/x44UdUiopwt3AifCIY6jOUcLfwRlXqlUyf7CkY0DtH3mFNwhr23bePDhb6P4B9fvVJtp/N4uArY3CwtmjBCFvW1dIqPtx5nu8Op9DBwowFo4J5eGhQs5X9lupXo63h9JXTHMg4wIGMA5y5cgat0GJnYccg70FE+EQQ4ROBn72foUOVmpF80GyEhBBMWD+BYKdgPhnzid7X5RRVMPSd3cwZFGj0zxP0dTGnhLe3nmNXfA5udlY8HRnM/YMC5MNoAyisLORw5uHaJJFZqitdEmAfwGDvwQzwHsAAzwG42rjWcyfJmMnhIyN0oeAC6SXpPNrj0QZd992hZGq0gociglomMAPo7GHH/z00gJjkfN77NYF/bY7jy72XWTgmhBn9/LCQFVhbjaOVI3cE3cEdQXcghCCpKKk2QWxJ3MKa82sA6OzUmQFeAxjoNZD+nv1xsnYycORSc5E9BQP536n/8d+T/2X3zN24d9CvhlNFtYahS3bTJ8CJrx7Uf/qqKRFCEH0xjymZpXQAACAASURBVPe2J3AqtYAg1w48HdmZqX18ZXIwsBptDXF5cRzJOsLRrKOcyDlBeU05AGHOYbVJop9XPxwsjW/djPQ7OXxkhO77+T7MFDNWTlyp9zVrjqXy8rpYVs4fxNDObXu+uRCCnedyWLrjPOcyi/B1smHByE7M7O+PtYUcVjIG1ZpqzuSd4UjmEY5mH+VkzkkqNZUoKHRx6UIfjz708exDX4++jSr0KLUcmRSMTHZpNmPXjeW5vs8xv0edFcJvIoTgro/2o9UKfvnT8HYzv1wIgTohh//uvsjxlALc7KyYP7wjcwcHYmclRz+NSZWmitjcWI5mHSUmO4bYK7G1PQlfO1/6evStTRIdHTvKUhwGJJ8pGJk9aXuAhhXAO3Q5n3OZRSyZ3qPdJATQTacc3cWTyDAPDl3O59OoiyzZFs9nUZeYMyiAB4YE4eVobegwJcDSzJL+Xv3p76X7WVOtrSYhP4Hj2cc5mXuS6IxoNl/eDOieXfRx/70n0c21myzHYSRkT8EAFuxcQGpRKj9P+1nvH/CPf3uMo0n5HPzbmHY/fHIytYDPoi6yPS4bM0VhUk9vHh3WiR5+cvGVMRNCkFqcyvGc45zIOcHx7OMkFSUBYKGyoItLF3q49aCHew96ufXCz96vXX0Bak1y+MiINKYAXkpeGSPfV/PUqGBeurNLC0doOlLyylh+IIk1x1IpqaxhYJALjwwLYlw3L5Nb1Nde5VfkcyLnBLG5scTmxnI272ztkJOzlTPhbuG1SSLcPVw+wG4mcvjIiOxP30+1trpBQ0ffHEzCTFGYNzioxeIyRQGuHXhtcjeeHxfCmmNpLD+QyILvjuPnbMPsgQHM7O+Hh70cWjJmLtYujAkYw5iAMYBuhtOlgkvEXonldO5pTl85zf70/Qh0X16DHILo6d6THm496O7anVCXUKzM5Er45iR7Cq3sr/v+SnR6NOp71XqVESiprGHIW7uI7OLBR7MbVjSvvdFoBTvislhxKJnoi3l0NUtjoccp/PpPpPvgCajklFaTVFJVwpm8M5zOPa3rUVyJJb9Ct6OvuWJOsFMw3Vy71f4KdQ7F2lx+Gbgd2VMwEtXaavam7SXSP1LvujLrjqVSXFnDI8M6tnB0ps9MpTA+3Jvx4d5kHV6Ly6//wPJqOexYSdpOb9I73kPIHY/j4mXYXeqkhrGztGOw92AGew8GdM8mMkoziMuLq/2lTlWz4eIGAMwUs5sSRZhzmEwUepJJoRWdyD5BcVUxo/1H63W+VitYfiCJPgFO9PaXK0b1otXC3nfxinobfPpSefcXnDm8E+vTKxl0+WNqPvuEk7aD0faZS/jImVhayhkvpkZRFHztfPG182Vc4DhAlygySzM5l3eOs3lnicuPY2/aXjZe3AjoEkUnp050delKqHOorkCgcxjO1s63e6t2SSaFVqROVWNlZsUQH/02xVEn5JCUV8YLd4S1cGRtRGUxbFgA8T9Dr/th0gdYWVjTb0oITHmSxIRTZKi/JCxrM27RT5Eb/QoXPMbjHjGPzj0jUFRyeMlUKYqCj50PPnY+jAnUPZ8QQpBdlq1LEtd6FAcyDrDp0qba6zxsPAh1CSXMOaw2UQQ4BLTrCrHymUIr+a0AXmenzvx3zH/1umbOV4e4lFPKvr9EyhIP9cm/DKvuhyvn4Y7FMPhJuMV0xpqqSuL2/Yg4/h1dSw5hqWhIVvmTHTiZgFEP4hUoZ3i1ZXnleZy/ep7zV8+TkJ9AwtUELhdepkZbA4CVmRXBTsG1iSLUOZRQ59A2td+EfKZgBM5fPU96SbreK5gTsoqJvpjHy+PDZEKoz6XdsPZh3e/n/gjBt5/ZZW5pRc8x98OY+ynKy+H47hXYn1/PwMRPIfFT4i26kR88lZDIubh7+rbCB5Bak6uNK0NshtzQY6/WVHO58DIJVxNqE0VUalTtcwoAjw4edHbqTLBTcO1/gx2DsbO0M8THaDEyKbQSdaoaBYVR/qP0On9ZdCLWFipmD5APRW9JCDj0KWx/Fdy7wH0rwaVTg27h4OrB4JkvAC+QejmBtH3f4p28iYj4t6g5t4RYq16Ud55IyMj7cPH0b5nPIRmchZlF7UZEBOvahBBcKb9SmyguFVziYsFF1iaspUJTUXutt633DYmis1NnOjl2atAeKcZEDh+1klk/z8JcZc7Ku+ovgJdfWsWQt3cxva8fb0/v0QrRmaDqctj8J4j9AbpOhqmfg1UzfWMTgpRzR8g8sAqf9F/xFxlohUKCdQ+KO00kaNh9ePgGNc97SSZHo9WQUZLBhYILtYniUsElEgsTqdJW1Z7na+er601cSxRBDkEEOQYZxQI8OXxkYFmlWcTlxfFc3+f0On/VkRQqa7Q8PDSoZQMzVYXpsHoOZJyAyL/D8BehOR8SKwoB3QYR0G0QQqvlUtwxsg6uxidzO13PvQ3n3uaceVfyAsbjP3gaASE9ZTmGdsRMZYa/gz/+Dv6MDvh9JmGNtoa04rQbEsWFggscyDhQ+7wCwM3GjSCHIDo6dqz9b0fHjnjbemOmMnwJG5kUWsGeVF0BPH2molZrtHx7MInhIW6EesoN7W+SchhWz4XqMrjve+gysUXfTlGpCA4fSHD4QIQQJJ8/SebBNXikbmPY5Q/g8gekKL5keo7Eqc8UOvcbi5m56W6RKjWeucqcIEddb+C3GVCgW5+UVpxGUmESiUWJJBYmklSYxK9Jv1JUVVR7nqXKkkDHQDo6dCTI8VqyuPZ7Wwvb1vscrfZO7Zg6VU2gQyAdHetfgLb1dCbZRZVy2KguMd/AlhfA0Q8e3AQeXVv17RVFITCsD4FhfYC3yUlJIOnAeqyTdtAnczWWWd9TuM2OCw6DIXQ8wUOm4uyq3wZKUttlobKo7Q1E8vskCCEEVyuv6pJFYSJJRbr/xufHszNlJ1qhrT3Xw8aDIMcgAh0Ca391durcIntny2cKLaykqoThq4czp8scXhzwYr3nT/0kmsLyanb9eSQqWdRNR1MNv/wNjn4JwaNhxv9BBxdDR3WDwoJ8EqJ/gvO/EFJ4AGeKqBEqEiy7UugzArdeE+jccygqc/k9TKpflaaK1OLUG5JFUlESKUUpFFQWADAucBxLRy1t1P3lMwUD2p+xnxptDZEB9RfAO55ylZOpBfxrSneZEH5TegXWPAjJ+yHiWRjzTzAzvr+2jk4uDJz4MEx8GE1NDedP7qHg5CZcsvYTkfwZJH9GwSZ7Eh0GoO0UScCASbj7NmymlNR+WJpZ1j6k/qOCigKSi5OxVLXManzj+9fVxqhT1DhbOdPbvXe95y6LTsLe2px7+jV/l9AkZcbCD/dDSQ5M+wJ6zTJ0RHoxMzcntP8Y6K8bV87PTuPykS1oLuyiU9Fh3E/uhpOLSFL5k+k6BMuwsXTuPw5HJ+Pq/UjGycnaCSfrlit7I5NCC6rWVrMvfR+j/UfXO6sgs7CcraczeTgiCFu5zSSc+RE2Pq0bJnrkF/Dta+iIGs3F0w+XyU8AT6DVaLkQd5S8U1uxS9tL35wNWOWuoWafigSLEK56DMIubBSd+4/F2rbtrKCVTIf86dOCjmcfp7iqWK+hoxUHkxFC8GBEUMsHZsy0Gti9GPYvBf/BcO+3YO9p6KiajcpMRUiPQYT0GARAVXkJ8SfUFMbtxiH7EP3SV2KR8S01u1Wctwwl32MQNiEj6dR3DPYOsiii1PJkUmhBtQXwvG9fAK+8SsP3R1IY180TfxfTXAXZLCoK4cf5cGE79HsIJrwH5m27iqmljR1dIiZDxGQASooLORuzi9KEKFxyj9Av7Tss0r+hWm1GvEVn8l37Yt0pgsDekbjKFdZSC5BJoYUIIVCnqBnsPbje5e4bT6ZTUFbNI0Pb8Z4Juefhh9lwNQkmLoUBjxo6IoOws3ek96jpMGo6AGUlhZw/vpvSBDUOucfol7UOq+xVcBDSFG9ynHqh9RuER/eR+IX0RmVm+MVPkmnTKykoijIHmAVogINCiHf1Od7Q9mvHzIFvgWIhxBNN+3iGc/7qeTJKM3i85+O3PU8IwbLoRLp5OzCwYzt90Hj+V10PwcwSHtgEQUMNHZHR6GDnSPcR02DENACqKsqJj43masI+rDKPEnT1AC5Xf4HTUIQtidbdKPHsj11wBEE9huLo7GrgTyCZmnqTgqIo9sA8YIIQQiiKskJRlBAhxIXbHQeyGtL+2/2AV4HlwL3N/WFb0+7U3SgojPQfedvzoi/mcT67hPdn9mp/pRKE0D072PUGePXQrVB2kkMit2NpbUOXgWNh4FgAtBotyZdOk312L6QewrPgFL2uTYHV7lJINvMl1747wrcfrqFDCOg6AHMrGwN/CsmY6dNTiAB2iN9Xuf0ERAIX6jme3MD2C4qi3A8cA87fKhhFUR4HHgcICDDeCqLqFDU93XviZuN22/O+jk7Ezc6Syb28WykyI1FVCj89A2fXQ/gMmPJfsGzHz1MaSWWmIjC0F4GhvYBnASgpyCUldh/Flw9jnX2SjoWHcC38FeKgaoM5Fyw6ke8UjuLXD7ewCAJCemIuF9VJ1+jzN8EVyL/udT4Qosfxkoa0K4rSB/ASQnyvKErQrYIRQnwBfAG6Fc16xN/qskqzOJd/jj/1/dNtz0u8Usru+ByeGxOClXk7Ggu+mgw/zIHsMzD2XzD0uVtuiCM1nJ2TO91GTIcRuucSQqslPeUi6Wf3U50Sg+PVWMJzt2J7ZT2chBJhQ7JlJ4qduqHy6YV7yAD8Q/tgbmll4E8iGYI+SSEP6H7da5drbfUdb2j7fYCToiifA/ZAX0VRnhJCfKrfRzEeUalRAPVORV0enYiFmcKcwcbb42l2SfthzQOgqYE5ayFknKEjavMUlQrfoFB8g0KBRwDQ1NSQfDGWvIQDaNJicCg4R8+cTXTIXQunoEqYc8kiiKsOXRBePbEP6od/1/7Y2stpsW2dPknhMPCcoigfXBvymQK8pcfxrIa0X+sBAHCtp/CqKSYEuK4AnsOtZxMVllezNiaNyT198LC3bsXoDEQIOPoV/PJX3UY4960Ct86GjqrdMjM3J7BLXwK7/L4oUFtTQ/KlM+ScP0J12knsrsYRnL8H5/yfIQ60WxRSVD5k24ZS5doNG78eeIX0xTsgRO5v3YbUmxSEEAWKoqwAVimKUgOcFELE63O8oe3X0QA1mKDiqmKOZB1hbte5t31wvPZYKmVVGh5uD9NQayp11U1PrIDQ8TD9C7CWq3WNjcrcnMCw3gSG/V6SRWi1ZKZdIjvhKBWpJ7DJO4t/6Rm8StS6p4PRuuGndMsgihxCUTy6Yh/YC+/Qfji4tJ1Fh+1Jo6ukKoryI3CvEELTvCHpzxirpP6S+Asv7X2Jb8Z/Q1/PukszaLSCke+p8Xa0Zu2CiFaOsJUVZ8HqeZB2RLcZTuTfm3dDHMkgyorySUuIoSDpFNrsszgUnse3KhFHpbT2nFycybLqSIljKIpnNxwCe+Ib3AtH53Y69dqItEiVVCHEjMaH1HbtTt2Ni7ULvdx73fKcHXHZpF0t5+93te5+AK0uPQZ+mAsVBTBzOXSfZuiIpGbSwcGF0AHjYMDvz4SEVkt6WhK5l05QlnYa8yvxOJdeICT7R6xzfoDTuvOycSXHKoAyh2AU9zAc/LvjHdwTR3c/OeHACMh5aM2oWlvN/rT9jAkcc9sCeMuiE/F1smFctzbcvT65CjY/B3ae8Oh23ToEqU1TVCp8AzrhG9AJ+P07o7amhsyUeK5cPklZxjnM8i7gUJpIx5wt2OWugzjdeUXYkmnhT5FtRzQuIVh6d8U1sDteQV2wkjOhWo1MCs0oJjuG4upiIv1vPevobEYhhxPzeeWuLpibtcFhFE0N7HgNDn0CQcNh5jdgK1fVtmcqc3O8O4Xj3Sn8hnatRktGRiI5l2MpTY9DdeUCdiWJBBUcwr1gG1wGoqFSmJOk8iLf2p9K+yAUt2DsvUNxC+qGh28nFCPY17gtkUmhGalTrhXA87l1Abxl0UnYWJgxq38bnIZalg/rHobLUTDwCbjzTTCT+xVLdVOZqfDxD8bHPxi4cWixqOAK2ZdOU5R2Bm12AhZFSTiXpeBddgzrnOra3kWFsCDLzJsCmwAqHQJRuQZj4x2Ka0A3PH2CZC2oRpBJoZkIIVCnqhniPQQb87rLCOQWV7LpZAazBvjj2KGN/bDMjtMVtCvK0K1O7jvP0BFJJszByQ2HfpHQ78Zet1ajISsjiSvJcZRkJKC5cgnr4iRcypLxKTmMVWY1nNGdWy4syTDzpsDKj0r7AHAOooNnJ5x9Q/EMCMHaxtYAn8z4yaTQTBKuJpBZmsmCXgtuec73h1Oo0mh5aGhQ6wXWGs5thvVPgJUdPLQF/AcaOiKpjVKZmeHlH4yXfzAw+YZjNTU1ZKRdIi81nvKs84i8S1gXJeFekYxH2RFdDyPh9/NzceGKhTdltn5oHAMxdw3C1jMYZ99Q3LwD220vQyaFZqJOUaOgMMJvRJ3HK2s0rDiUzKgwd4Ld7Vo5uhai1cKed2DPEvDtB7O+AwcfQ0cltVPm5ub4BIXhExR20zGh1ZCXk05uSgIlWRepupKIeWEyHcrS8S2IwePqdlTJv0/PrxLmZKk8KLD0odTWD41jAJaugdh5dMTVtxNuXgGYtdGkIZNCM1Gnqunl3uuWBfC2xGZypaSy7SxWqyyGDQsg/mfodT9M+gAs2sHKbMkkKSozXL0CcPUKAG4urVJWVkpu2kWuZlykIucyXE3GsjgFh4p0AvLjccovgcTfz68SZmSp3Ciw8KS8gzcae3/Mnf2xcQ/C0SsIN79grGzsW+8DNiOZFJrBbwXwnu/3fJ3HhRB8HZ1IZw87RoTcvmqqSci7pCtod+U8jF8CgxbI+eWSSevQwfa6arM3qyi+Sm76RQoyEynPTUJbkIp5cTq2FZn4FcTgfnU7Zqk3LgS+igN5Zu4UW3lRYeuD1sEfCxd/bN0DcfYMxM07AEtL49tZUCaFZqBOVQPccirqseSrnEkvYvHUcNPfM+HiLt0MI0UF89ZDp1GGjkiSWpy1vTP+XQbg32VAnccrKipIT0+iMDuRspxENFdTMStOx7osA5eKFNzLjtEhtxIu/X6NRijkKM4UmLtRYuVJdQdPcPDBwtmPDm7+OHkF4eIZiGUrPxCXSaEZqFPUBDkE0dGx7qGhZdGJONpYML2vbytH1oyEgIOfwI5F4N5FtyGOSxsZCpOkJrK2tiYguAsEd6n7BCEoKcwjP+MiRdnJVOSloilMR1WchXV5Fi7lSbiWHMM+t/ymSwuwJ9/MlRJLDypsPNHY+WDm6INTx96E9h3V7J9FJoUmKq4q5mj2UeZ1rXsKZtrVMn45k8VjIzrRwdJE/7iry3Wrk2NXQ9cpMPUz3UwjSZL0oyjYOblh5+QG3QbXeYoQgqKiq+RlJFOYo0scNQUZmJdmYFWWjV1VDj7l53HLLwAgJmk0yKRgfPan76dGW3PLvRNWHExGURQeGBLUuoE1l8J0WD0HMk7oitkNf1EWtJOkFqAoCg6OLjg4ukDXPrc8r7KynLzMFFpqr0aZFJpInaLGxdqFnm49bzpWVlXDqiMpjO/uha+TCe6Lm3JIV+G0ukw3XNRloqEjkqR2z8rKps5pt81FfuVrgmpNNfvT9zPSb2SdBfB+PJ5OUUUNjwwLav3gmipmOSyfpBsmmr9TJgRJaidkT6EJjmUfu2UBPK1WsCw6kZ5+jvQNcDZAdI2kqdbtjnb0KwgeDfd8DTYmFL8kSU0iewpNoE5VY21mzWCfmx8c7b2Qy+XcUh4Z2tF0pqGW5MK3d+sSQsRCmLNOJgRJamdkT6GRfiuAN9hncJ0F8L6OTsLD3oq7erTU46BmlnlKtyCtNBemfwk97zV0RJIkGYDsKTRSfH48WaVZjPYffdOxiznF7D2fy7zBgViam8Af8Zkf4f/uBKGFR36RCUGS2jHZU2gkdeqtC+Ati07C0lzF/YOMfM8ErQZ2vwH7PwD/wTBrBdh5GDoqSZIMSCaFRopKjaK3R29cbW7cVaygrIr1x9OZ2tsHVzsj3kKwvAB+nA8Xd0C/h2DCe2BufHVYJElqXSYwtmF8MksyOZd/rs5ZRz8cTaW8WmPc1VBzz8NXY+CyGiYuhcn/kQlBkiRA9hQa5bcCeKP8R93QXqPR8u2BJIZ0cqWrt4MBItPD+V91PQQzS3hgEwQNNXREkiQZEdlTaAR1at0F8H49m01GYQUPG+POakLA3vfh+1m6QnaPR8mEIEnSTWRPoYGKqoo4lnWMed1vLoC3LDqRAJcOjOnqaYDIbqOqFH56Gs5ugPB7YMrHYNnB0FFJkmSEZFJooP1p+6kRNTdNRY1NK+BY8lUWTeqGmcqIFqtdTdatP8g+A2P/BUOfkxviSJJ0SzIpNFBUahQu1i70cOtxQ/uy6CTsrMy5t7+fgSKrQ+I+WPOAburpnLUQcvM2hJIkSdeTzxQaoFpTzb70fYzyH3VDAbycogp+js3gnn5+2FtbGDDCa4SAw1/oSlbYusFju2VCkCRJL7Kn0ABHs49SUl1y01TU7w4lU6MVPBQRZJjArldTCVtegBMrIHS8rmSFtZHOhJIkyejIpNAA6hRdAbxB3oNq2yqqNaw8nMKYLh4EubXuXqo3Kc7S7X+QdkS3GU7k3+WGOJIkNYhMCnoSQhCVFsUQnyE3FMDbdCqDvNIqHjH0YrW0GN0OaRWFMHM5dJ9m2HgkSTJJ8mukns7lnyOrNOuGoSMhBF/vTyTM054hwa63ubqFnVwFyyaAmQU8ul0mBEmSGk2vnoKiKHOAWYAGOCiEeFef441o/wzQAi7AFiHEd03+hM1EnapGpagY6T+ytu3Q5Xzis4p5Z0YPw+yZoKmBHYvg0KcQNBxmfgO2BkxOkiSZvHqTgqIo9sA8YIIQQiiKskJRlBAhxIXbHQeyGtIuhLgghHjy2j0VYC9gNEkhKjWK3u69cbF2qW37OjoR5w4W3N3bt/UDKsuHtQ9B4h4Y+ATc+aaupyBJktQE+gwfRQA7hBDi2uufgEg9jje0/XpWQH5dwSiK8riiKMcURTmWm5urR/hNl1GSQXx+/A1DRyl5Zew8l82cQYFYW9y8P3OLyo6DLyMh5SDc/Qnc9a5MCJIkNQt9koIrN/6Azr/WVt/xhrZfbzHwLnUQQnwhhOgvhOjv7u6uR/hNV1cBvOUHkjBTFOYNCWyVGGrFbYKvxkJ1BTy0FfrMbd33lySpTdMnKeQB12/U63Ktrb7jDW0HQFGU54ETQohoPWJrFepUNR0dOxLkGARAcUU1a46lMrGnN54O1q0ThFYL6rdgzTzw6KIraOc/oHXeW5KkdkOfpHAYGKv8/iR1Crrx/vqON7QdRVGeAkqFECsb+XmaXVFVETFZMTcMHa2LSaOksqb19kyoLIbVc2HPO9Drfl0PwcFE9n6WJMmk1PugWQhRoCjKCmCVoig1wEkhRLw+xxvSrihKBPBXYKuiKJ9fu/0iIUTrPDi4hX1p+6gRNbVJQasVLD+QRN8AJ3r7O7V8AHmX4If74coFGL8EBi2QBe0kSWoxek1JFUKsAlZd36Yoyo/AvUIITV3Hb3XdrdqFEAcAo9vUOCo1CldrV3q69wRgd3wOyXllvHhHWMu/+cVdsO5hUFQwbz10GtXy7ylJUrvW6MVrQogZQghNcwZjbKo11exP388o/1GoFN0f1bIDiXg7WjM+3Kvl3lgIOPAxrLwHHPzgMbVMCJIktQpZ5uI2jmbpCuD9NusoPquI6It5vDw+DAuzFloMXl0Om5+D2NXQdQpM/Qys7FrmvSRJkv5AJoXb2J26GxtzGwZ7DwZgeXQS1hYqZg9ooVGuwjTdhjiZJ3XF7Ia/KAvaSZLUqmRSuAUhBFGpUQzxHoK1uTX5pVVsOJHO9L5+ONtaNv8bphzSzTCqroD7VkGXu5r/PSRJkuohv4beQlx+HNll2UQG6GYdrTqSQmWNlkeGBjX/m8Ush+WTwMoe5u+UCUGSJIORPYVbiEqN0hXA8xtJtUbLtweTGB7iRoinffO9SU0V/PJXOPZ/EDwa7vkabJzrv06SJKmFyJ7CLahT1PR2742ztTNbT2eSXVTZvHsmlOTCiqm6hBCxEOaskwlBkiSDk0mhDukl6SRcTahdsPZ1dBKd3GwZGdpMtZYyT8EXoyA9Rrdd5h1vgKqVi+pJkiTVQSaFOkSlRgEQGRDJ8ZSrnEot4KGhQahUzbCS+PQ6+L87AQGP/AI97236PSVJkpqJTAp1UKeo6eTYiUCHQL7en4i9tTkz+vo17aZaDez4B/z4KPj01hW08+nTHOFKkiQ1G5kU/qCwspBj2ceI9I8ks7CcbWeyuG+AP7ZWTXgmX14A38+C6A+h38PwwCaw82i+oCVJkpqJnH30B/vT96MRGiIDIvn2YDJCCB4YEtT4G+aeh1X3QUEyTFwKAx5ttlglSZKam0wKf6BOVeNq7Upnh26sOqLmjm5e+Lt0aNzNEn6B9Y+BmSU8uBkCI5o3WEmSpGYmh4+uU6Wpqi2A99PJTArKqnm4MYvVhIC97+t6CC4ddc8PZEKQJMkEyJ7CdY5mHaW0upRR/qN4c20i3X0cGNjRpWE3qSqFjU9B3EYIvwemfAyWjexpSJIktTLZU7iOOlWNjbkNmtLOXMgp4eGhHVEasqHN1WT4vzsg7icY9zrM+EomBEmSTIrsKVwjhECdqibCJ4KVhzJxs7Nkcq8GbHmZuA/WPKCbejpnHYSMbblgJUmSWojsKVwTlx9HTlkO4U5D2B2fw5xBgViZ67HKWAg4/D/49m6wdYPHdsuEIEmSyZI9hWvUKWpUioqLSf5YmhUyZ7AeeybUVMKWP8OJ7yB0Akz/AqwdWj5YSZKkFiKTwjXqVDU93Xqz6VARk3p542FvffsLirN0+x+kHYURL8GoV+SGOJIkmTz5UwxIK07j/NXz2Gt6UValqb8arOursAAAB7xJREFUalqMrqBd9lmY+Q2MflUmBEmS2gTZU+D3AngnEnwZGORCuK/jrU8++T1s/hPYe8KjO8ArvHWClCRJagXy6y26oSNP60Ayr9jxyLCguk/S1MAvf4ONT4L/QHgsSiYESZLanHbfUyisLCQmOwan6jvwdbJhXDevm08qy4e1D0HiHhi0AO5YDGYWrR6rJElSS2v3SWFf+j40QkNKakf+NjoIsz/umZB9FlbNhuJMuPsT6DPXMIFKkiS1gnafFNQpaixxRKMN5N4B/jcejNsEGxaAlT08tBX8BxgmSEmSpFbSrpNClaaKfen7KS8I555+ATjaXBsS0moh6m3Y+y749odZ34FDA1Y3S5Ikmah2nRSOZB2hvKaMyuJuPBgRpGusKIINT0DCVug9R7cHgkU9axYkSZLaiHadFHYm7watJcN8BxPsbgd5l/j/9u4/RI6zjuP4+2NsLthUMDGCUKyIF2JbDNLD0kBaoimt/tEEi1UsBbGmglpMVTCK/mOiYq1QFJoaUVviEamWNtSq+VGlLWJTryagjcG2aFEhcGkQe2kqSe7rHzO7mVxubmd2nttNOp8XBGa/M3vzydzz7HMzs7sPP/sYHHkOrv82XPkpqPOFeGZm57nWDgrTMc3uv/+WE1Oj3Hrdcnh+L/ziE6DXwS0PwTuuGXZEM7OBa+3nFA4eOcjLJ19iqd7D6skdMP5heOPFsOF3HhDMrLVae6Yw/pdfEyHuWfgM2rML3nUDrN8KI4uHHc3MbGgqDQqSbgY+ApwC/hARd1ZZn6o+Hx5/cTeXvhqsPLwL1nwVrv6i7x+YWev1HBQkXQTcAnwgIkLSdkmjEfHcXOuBwynqnf2ktP/AI7ysw1x3fAo+ugNWfDD1LszMzktV7imsAvZEROSPdwJrKqxPVT+DpNskTUiamJycrPJ/PMvUoqWsOj7CFdfe6wHBzKygyuWjpcDRwuOjwGiF9VOJ6meIiG3ANoCxsbGYub6K1StWsXrFRD9PNTN7TatypvAS8KbC4yV5rdf6VHUzMxuQKoPCPmCt1L0LewPwRIX1qepmZjYgPS8fRcR/JG0Hdkg6CRyIiENV1qeqm5nZYOj0fd2aT5QeBG6KiFNpI1U3NjYWExO+N2BmVoekZyJibLZ1fX94LSJu7D+SmZmdi1r7NRdmZnY2DwpmZtblQcHMzLr6vtF8LpA0CbzY4Ee8GTiSKE5KzlWPc9XjXPW8FnNdEhHLZltxXg8KTUmaKLsDP0zOVY9z1eNc9bQtly8fmZlZlwcFMzPravugsG3YAUo4Vz3OVY9z1dOqXK2+p2BmZmdq+5mCmZkVeFAwM7Ouvr/76Hw2yLmgC/vcCkyTzRPxaET8VNJe4PnCZpvyb51dCXyTbOKhV4DbIuJEWb1hrv1kX1sOcBK4PZ8OdS1wB3AM+FdEfD7fvla9z0wrgI2F0lXABuAHKbL2kWcB8HXgioi4fq6fPcjjVpJrC1kbuxD4c0Tcldd/BCzM9wPwnYh4QdLbgO+TtafXAxvyNjhrvUGuJG29SR+YmUvSMmBzYZPLge9FxAOD7pslrw/DaWMR0ap/wEXAbzh9P2U7MDrA/Qt4Ml/eW7LNo8CSfPmTZB2ytN4wz1kZ8oyPASP54y3AtXXriY7XAuCX+T4aZ+0zwzrgys7+Ux2fphln5ppl/S7gwnz5PuDiWba5H1ieL68FvjFXvd9cqdp6kz5Q4Xg9CLwhZd4+2pqAJ4fZxtp4+ajSXNDzaITT045OSdosabukDQCSFgEnI6KzzcPAmrJ6gjwLJH1L0rik9XltOXAwIv43Y1916yncCOzMf18pstYWETsjYl+hlOr4NMo4S66ufLKqaeB4XjoGbJR0n6RNkjp9/60R8bd8+TFgrEe931yN23rTPtDjeL0X+GtEvJIqb9VcM3ReH4bWxtp4+ajXnNPzbQtwJ0BErIduB94q6QXgEFA8TT9Kdkq5pKTeSESsyTNcAPxc0rPMfoyW9lFP4ePAhxJmTSHV8ZnPjJ8DfhIR0wAR8ZnOCklfITuuPyb7S5J8mygMFmX1viRq6/PSB3Ibge5llSH2zc7rw9DaWBvPFIY2F7SkO4D9EfH7Yj3/K/gR4N0l+Y7OUU8isuufe4DLSvY18Pm1Jb0feCoiXk2YNYVzel5ySTcBCyPigZJNdpK1NcjOJjrPU+FxWb2Rhm19XvqApFHgWEQcTpy3bo7i68PQ2lgbB4WhzAUt6dNkDW+8ZJOrgT/mp3kXSOr8ItcBj5fVE8e8CjhAdoPtckkjM/ZVt97UZ4F7EmdNIdXxSZ5R0jrg0pj7zRPXAE/ny5OS3pkvvw/4U496Cn219XnsA18A7k6dt06AWV4fhtbGWnf5KHrMOT0fJK0CNgG/knRvXv5aXlsMLAL2Fc4gvgT8UNJ/yd9l06PeJNv9ZNedFwMPR8Q/8vpmYFzSFDAJ7M4vI1SuN8y1Evh3RBwp1BpnbZIJOAEQEadSHJ+EGU8ASLqE7FOuDxXa2d0RcSi/ZPR2shv3/4yIzmD7ZeAuScfI3p10e4967Vx5tu+Spq2n6APFXG8BlkXEs8UNEubtaY7Xh6G0MX+i2czMutp4+cjMzEp4UDAzsy4PCmZm1uVBwczMujwomJlZlwcFMzPr8qBgZmZd/wc7B+B44TmJKQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZsOLpNcrKSH_"
      },
      "source": [
        "<a name=\"cell-id-labelsmoothing\"></a>\n",
        "## Label Smoothing\n",
        "\n",
        "예측을 할 때 정답이라고 예측된 토큰에 거의 확률 1로 예측하는 것을 막기위해 레이블을 깍아주는 작업을 하면 예측 성능이 더 좋아지는 것으로 알려져 있습니다.\n",
        "\n",
        "아래 원문처럼 0, 1로 구성된 확률 분포를 정답 분포로 쓰지 않고 정답자리 확률을 0.8정도로 깍아내리고 깍아낸 0.2를 나머지 오답 자리에 고루 분배하여 정답 분포를 마치 '신뢰도'정도로 해석할 수 있게 하는 방식입니다.\n",
        "\n",
        "> We implement label smoothing using the KL div loss. Instead of using a one-hot target distribution, we create a distribution that has `confidence` of the correct word and the rest of the `smoothing` mass distributed throughout the vocabulary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_uYddYnl0R24"
      },
      "source": [
        "class LabelSmoothing(nn.Module):\n",
        "    \"Implement label smoothing.\"\n",
        "    def __init__(self, size, padding_idx, smoothing=0.0):\n",
        "        super(LabelSmoothing, self).__init__()\n",
        "        \n",
        "        # size_average and reduce are in the process of being deprecated, \n",
        "        # and in the meantime, specifying either of those two args will override reduction.\n",
        "        # self.criterion = nn.KLDivLoss(size_average=False)\n",
        "        \n",
        "        # smoothing을 적용한 타겟과 로스를 구하므로 NLLLoss 대신 KLDivLoss 사용\n",
        "        self.criterion = nn.KLDivLoss(reduction='sum') # input: log-probabilities \n",
        "        self.padding_idx = padding_idx\n",
        "        self.confidence = 1.0 - smoothing\n",
        "        self.smoothing = smoothing\n",
        "        self.size = size\n",
        "        self.true_dist = None\n",
        "        \n",
        "    def forward(self, x, target):\n",
        "        # x: model.generator에서 출력한 log_softmax 값\n",
        "\n",
        "        assert x.size(1) == self.size\n",
        "        true_dist = x.data.clone()\n",
        "        true_dist.fill_(self.smoothing / (self.size - 2)) # 정답자리와 패딩자리 두자리 빼고\n",
        "        true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n",
        "        # 여기까지 스무딩 시켰고...\n",
        "        \n",
        "        # 패딩 토큰 위치는 확률을 0으로 지정\n",
        "        true_dist[:, self.padding_idx] = 0\n",
        "        \n",
        "        # target이 패팅토큰 번호라면 그 데이터에 대해서는 로스를 구할 필요\n",
        "        # 없으므로 모든 확률분포자리를 0으로 만들어 버림\n",
        "        mask = torch.nonzero(target.data == self.padding_idx)\n",
        "        if mask.dim() > 0:\n",
        "            # index_fill_(dim, index, val): dim차원을 따라 index가 지정된 위치에 val을 채움\n",
        "            true_dist.index_fill_(0, mask.squeeze(), 0.0)\n",
        "        self.true_dist = true_dist\n",
        "\n",
        "        # loss 계산\n",
        "        return self.criterion(x, true_dist)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QcTg7E9WhaFS"
      },
      "source": [
        "늘 그렇듯이 위 코드를 보면 또 이해가 쉽지 않습니다. 그래서 하나씩 뜯어서 알아보도록 하겠습니다. 간단한 예제를 위해 단어장 크기가 5라고 가정하겠습니다. `smoothing=0.4`로 두면 다음처럼 시작할 수 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0RCnP1R-h-fj"
      },
      "source": [
        "size = 5\n",
        "smoothing = 0.4\n",
        "# 정답자리가 가져갈 확률 원래는 1인데 (1-smoothing)으로 깍아버리고\n",
        "# 깍은 만큼(smoothing)을 오답자리에 동일하게 나눠 주는 작업을 시작\n",
        "confidence = 1- smoothing \n",
        "padding_idx = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ofZSXUIGh_iG"
      },
      "source": [
        "총 세 개 샘플에 대해서 다음처럼 예측이 나왔다고 하겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5cgZHQnTiGmy"
      },
      "source": [
        "# 예측은 2번, 2번, 2번 토큰으로 예측\n",
        "# 총 다섯개의 토큰이 있고 0번 토큰은 [PAD], 1~4번 토큰은 일반 토큰\n",
        "#              토큰위치 0    1    2    3   4  \n",
        "x = torch.FloatTensor([[0, 0.2, 0.7, 0.1,  0],\n",
        "                       [0, 0.2, 0.7, 0.1,  0], \n",
        "                       [0, 0.2, 0.7, 0.1,  0]])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mAZUZHbhiHmi"
      },
      "source": [
        "세 개 샘플에 대해서 모두 2번 단어가 정답이라고 예측한 상황입니다. 그리고 정답 타겟은 (2, 0, 0)이라고 가정하겠습니다. 측 첫 샘플만 2번 단어가 정답이고 나머지 뒤 두개는 패딩 토큰인 상황입니다. 각 샘플에 대한 정답 분포르 만듭니다. 원래 정답분포는 다음과 같아야 하지만\n",
        "\n",
        "$$\n",
        "\\begin{bmatrix}\n",
        "0 & 0& 1& 0& 0 \\\\\n",
        "1& 0& 0& 0& 0 \\\\\n",
        "1& 0& 0& 0& 0 \\\\\n",
        "\\end{bmatrix}\n",
        "$$\n",
        "\n",
        "smoothing을 통해 어떻게 변하는지 확인해보기로 합시다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bMrGF-bzif03",
        "outputId": "3c3a61dc-bd1e-4227-c810-fb446121f835"
      },
      "source": [
        "# 타겟은 2, 0, 0\n",
        "target  = torch.LongTensor([2,0,0])\n",
        "true_dist = x.clone()\n",
        "\n",
        "# 정답1개, 패딩1개 해서 -2\n",
        "# 여기서 smoothing을 정답과 패딩 토큰 위치를 제외한 나머지 토큰들에게\n",
        "# 나눠준다.\n",
        "true_dist.fill_(smoothing / (size-2)) \n",
        "\n",
        "# 정답자리에는 confidence, 여기선 0.6을 대입\n",
        "#                  1번축을 따라\n",
        "#                     이 인덱스에 해당하는 위치에\n",
        "#                                               confidence값을 대입\n",
        "true_dist.scatter_(1, target.data.unsqueeze(1), confidence)\n",
        "print(true_dist)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([[0.1333, 0.1333, 0.6000, 0.1333, 0.1333],\n",
            "        [0.6000, 0.1333, 0.1333, 0.1333, 0.1333],\n",
            "        [0.6000, 0.1333, 0.1333, 0.1333, 0.1333]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MHYVZagonvcG"
      },
      "source": [
        "계산된 `true_dist`는 원래 정답분포가 0인 자리에 `(1-smoothing) / 3`값이 들어가고 1인 자리에 `smoothing`값이 들어간것을 확인할 수 있습니다.\n",
        "\n",
        "이제 나머지 작업을 수행합니다. 패딩 토큰에 0.6이라는 확률이 부여 되어 있는데 패딩 토큰은 예측할 일이 없으니 0으로 만듭니다. 그리고 정답이 패딩토큰인 샘플은 모든 자리를 0으로 만들어 버립니다. \n",
        "\n",
        "[Target mask] 절에서 최종적으로 마스킹된 결과에 디코더에서 입력된 [PAD] 토큰 위치는 마스킹하지 않고 그대로 둔 것이 기억 나시나요? 기억이 잘나지 않으면 [여기](#cell-id-targetmask)를 클릭해서 다시 돌아가봅시다. \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MnMJSkfLVVm5",
        "outputId": "5dc23e08-14e2-493e-fc3c-d5e8b0b1749c"
      },
      "source": [
        "# 예측 벡터의 길이는 단어장 단어 개수와 같은데 \n",
        "# 패딩 토큰은 예측할 일이 없으니 패딩 토큰 자리는 확률을 0으로 지정\n",
        "true_dist[:, padding_idx] = 0 \n",
        "print(true_dist)\n",
        "\n",
        "# target이 패딩되어 있어서 (타겟도 시퀀스니까 미니배치 안에서 패딩될 수 있음)\n",
        "# [2, padding_idx, padding_idx]처럼 target이 생겼는데\n",
        "# target에서 padding_idx가 나타나는 위치에 예측 벡터는 모조리 0으로 채움\n",
        "# 이렇게 하면 2번, 3번 데이터는 로스 계산에서 빠짐\n",
        "# torch.nonzero(..., as_tuple=False) \n",
        "# (default) returns a 2-D tensor where each row is the index for a nonzero value.\n",
        "print(target.data == padding_idx)\n",
        "mask = torch.nonzero(target.data == padding_idx) # 정답이 패딩인덱스와 같은 데이터 번호\n",
        "print(mask)\n",
        "print(mask.squeeze())\n",
        "\n",
        "# index_fill_(dim, index, val): dim차원을 따라 index가 지정된 위치에 val을 채움\n",
        "true_dist.index_fill_(0, mask.squeeze(), 0.0)\n",
        "print(true_dist)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([[0.0000, 0.1333, 0.6000, 0.1333, 0.1333],\n",
            "        [0.0000, 0.1333, 0.1333, 0.1333, 0.1333],\n",
            "        [0.0000, 0.1333, 0.1333, 0.1333, 0.1333]])\n",
            "tensor([False,  True,  True])\n",
            "tensor([[1],\n",
            "        [2]])\n",
            "tensor([1, 2])\n",
            "tensor([[0.0000, 0.1333, 0.6000, 0.1333, 0.1333],\n",
            "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
            "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zn2mlSOmKccL"
      },
      "source": [
        "마지막에 출력된 결과 텐서를 보면 1행만 smoothing된 확률분포가 들어있고 [PAD] 토큰에 해당하는 2, 3행은 모두 0으로 채워진 것을 볼 수 있습니다. 이렇게 해서 마스킹에 대한 세부 사항이 모두 정리가 되었습니다. 🥳\n",
        "\n",
        "위 `LabelSmoothing` 클래스는 이 코드들을 하나로 묶에 놓은 것입니다.  `LabelSmoothing`클래스를 이용해서 실제로 smoothing하는 예제가 아래 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 403
        },
        "id": "NnWsqjC20UyS",
        "outputId": "ff3b8712-b37e-4cca-9e2f-a2398c773d1e"
      },
      "source": [
        "# Example of label smoothing.\n",
        "# 단어장 크기가 5인데 0번 단어는 패딩토큰\n",
        "crit1 = LabelSmoothing(5, 0, 0.0)\n",
        "crit2 = LabelSmoothing(5, 0, 0.4)\n",
        "\n",
        "# 예측은 2번, 2번, 2번 토큰으로 예측\n",
        "# 총 다섯개의 토큰이 있고 0번 토큰은 [PAD], 1~4번 토큰은 일반 토큰\n",
        "#                    토큰위치 0    1    2    3   4  \n",
        "predict = torch.FloatTensor([[0, 0.2, 0.7, 0.1,  0],\n",
        "                             [0, 0.2, 0.7, 0.1,  0], \n",
        "                             [0, 0.2, 0.7, 0.1,  0]])\n",
        "# 정답은 2번, 1번, 0번, 즉 마지막 단어는 패딩토큰이 정답이라면\n",
        "v1 = crit1(predict.log(), torch.LongTensor([2, 1, 0]))\n",
        "v2 = crit2(predict.log(), torch.LongTensor([2, 1, 0]))\n",
        "\n",
        "# Show the target distributions expected by the system.\n",
        "# 결과를 보면 패딩토큰 자리는 모두 0 (1번 열)\n",
        "# target에서 패딩된 단어에 해당하는 예측 3행은 모든 자리가 0\n",
        "fig, ax = plt.subplots(figsize=(10,5), nrows=1, ncols=2)\n",
        "\n",
        "ax[0].set_title(\"Distribution of target without smoothing\")\n",
        "print(crit1.true_dist)\n",
        "ax[0].imshow(crit1.true_dist)\n",
        "\n",
        "ax[1].set_title(\"Distribution of target with smoothing\")\n",
        "print(crit2.true_dist)\n",
        "ax[1].imshow(crit2.true_dist)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor([[0., 0., 1., 0., 0.],\n",
            "        [0., 1., 0., 0., 0.],\n",
            "        [0., 0., 0., 0., 0.]])\n",
            "tensor([[0.0000, 0.1333, 0.6000, 0.1333, 0.1333],\n",
            "        [0.0000, 0.6000, 0.1333, 0.1333, 0.1333],\n",
            "        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000]])\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 8722 missing from current font.\n",
            "  font.set_text(s, 0.0, flags=flags)\n",
            "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 8722 missing from current font.\n",
            "  font.set_text(s, 0, flags=flags)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlsAAADFCAYAAABjNjdrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAX5ElEQVR4nO3df7RdZXng8e9DEsAIWAIufxQt1iJOq1aHVKcUhFioYlHa2gGXARu1RqRSkI4CU6AuwapUq9YZkEARi0ys4EhAkAiSEPmVAoXVVhYiVUZtgSIRK6ghCc/8sd+T7BzO/b3fe25uvp+17rp773ef9333j/PcZ++zz3sjM5EkSVIdOwy7A5IkSbOZyZYkSVJFJluSJEkVmWxJkiRVZLIlSZJUkcmWJElSRdtlshUR+0XEcWOsc0xE7DpK+dsi4oAyfUFvehJ9OTAiXtqaf39E7DuZuiYrIk6MiNsj4u0Dyp4XEW+Y5v7sGhHHdFDPRyNizzK91XZExNERcdpU2xjQZid971p/vyLigIi4YIR1d4mIT01f71SDce4pfTDOzVDt8zQi9oqIa0dZ9zMRMXd6etadWZlsRcR9EbE6Im6MiDsi4qKIeEGvPDPvyMxzxqjmHcAeIxVm5mcz88YyO7f8TMZvA/u16j07M781ybom6z3AqzPzwgFlLwTeNM392YNm/09JZp6cmT8ss/3bMZVjNppO+l5Bf79G3P7MfCwzT5iWXmnSjHMTZpybud7fmp4LzBtpxcw8NjM31u9St7aFgzAZczPzYICICOD3gJURcVBmPjDUns1MO2TmT4fdCUkTYpybGOOchiczZ90PcP+AZccBZ5XpA4ALyvSBwM3AKuBaYKfy+1HgVuCEst7tNFdG3wDeAJwGHF3KLgL+Avg6sLb8/uX+tlp9+Vb5/UngfuAe4LNl2QXAAWX6WcAXgDXADcA5wK6lbAnwaWB1KV8DvGCE/fErwFdKHWuAs2iuHF5ZXv/z8vvAvtctAe4CHiz75xeAfcv+WVX2z0mt9fv30e6l/zcC1wAnAle11n9PaffrwMXAbsDrSr2PlrJ9+/r0OWD/1vxfADe05ncG1pTp64C9RtiOJcAVwMrSzk29Y1Zeu5jmvFhdfg5tld3T16cLynEeq++7AJeVY3AdcHhZfjNwZjk+twK/Afxt2Y8rgWeNs1/7lX15Q3ntiUAM6hdwcFnvSuB64DZg4YBzNMprLyp13gm8tbXeS8o+vaEc49OA04YdA7aHH4xzxrmcWpxr1TcHWEZzjlwHvKNV98mlvtuA3wE+Wrb1BuDFrToOLftkdfn9lnEcm5f3HZuDgL2BfwYuLe3fCRzWf16V6W8AHy91/iNwcqvsecBVpWwl8F76ztFpfb8OO2BU2ajBQehXgavL9MHARWX6S8B/G7D+amDv1vx9wLtb8x8AlpTpi4CrgXll/q2D2hrUv3Y9rboObp3of9gqOwk4v0wvAe4Anlbmjx50ItHcvfwnyhuX5o/nJ4E/H21/tcq26j/Nm3xOmd4BuBt4+gj76FN9J/9HgNVl+jU0QTTK/GLgQ2V67956A/rzu8CnW/PXAP+HEoCB/w6c0n8MB2zHEpqAsVNr/jNl+kCaoLRbmX8W8C9s+cNyf1+f2sdstL6/oddG//kAvKFM/xrwI7YkYq8HPjxWv2gC673Ai0rZjjSJ3eJB/Sr74z7gGa35a0Y4R5+kvEeAZwA/YMu5fifwm61zYzXwgWG997enn/7zsCwzzqVxrvW6JYwQ5/raeymt93/f+XF8mV4APAC8p3WuLS/TL6SJRc8u87uWdg+Y6LEp++VHwF6tuu8eYd3vAkeV6XnAN4HnlfkVrbIdyv67qH8bp+tnVj6zNYK5wKDPeT8CHBsRh5db8aO9/gujlC/LzA1l+mKaE3HSImIXmhP3stbiT9JcPfSszMyflelbaf7o9nsx8L3MvBkgmzPvwzQfOUzG04G/iogbaO6IPAfYs5T176MDaa5gez7Rmn498GpgVUSsBt4N/OI42l8JHBQRO0TEq2iusC8Fjizlb6Z5U43H9Zm5vkzfRHP1BXAE8InM/E+AzHyI5pgeNs56R/J1YENEvC8ifqGv7OrS1jeBTZn5lbL8bpp9PFa/DgC+lpn3lrIngL9i9ON8c2b+uEy3t7/fg5l5a6n3xzTJ4XMiYjeaj7JuKWU/B84dfReoMuMcxrk+I8W5truBOyPiAxHxnL6yXmxaB6ynSZTJzHZsOowmkXmwlP2EJsk8gskdm3/OzB+U9f+V5ngMEsCXy3obaC7+es8tviIz/76UPUmTFA/N9pRs/S7NLcytZOZtwNto9sVVEbHjSBVk5o9Gqb//gb5s/Z7TW1j+QI1XDlj2ZGv6idb0RgYfz0F19NczERfS3EH5nWyeF/kOzQnfNLb1PgpgQ2u+3d8daK7wDi4/B2TmkrEaz+bByBtpAtxbgOXAV4HXRcQzgF0y83vj3Jb2PtjAlmcYx9pnGRFzWsvHdUwz86eZeTzNHafPRcT+rbJNrVUfG6mKUfo1meO8uawEqpGe4Xyib753rvUf30HranoZ50auZyK2hzjXbm9TZp5Kc7H00Yj4g1bxpr51B8WnarFpjPp7F5Y97fNjRsWmWZ9sRcS8iHgbTRb9lG/mRMQO2bgCmE/zWT00GfzuE2jqPRExv0wfTXMFBvB94Ncjorevl7D1lefAdsoJ/UBEHNlafBLNFc9E3APs1fr6dgB/Dvzfcb6+v3+/DKzIzPUR8Urgv4zy2quAP23Nv58tb6JVwPHlypZyBdf7AzDWvr+EJgDtm5n3lDsq/w78D5pEZjzbMZrLgZNKUCMing0cQxPsAL4HvKKUPY/mOYMx2+mdA5n5XZqrsYl++2m0ft0IHBoRLy5lO9Ls795xnuj5PKZyl+snEXFQaXM+zfMpk/0Dp0kyzhnnxlnnQK3Y9BBNorl4glV8FVjSuytWhhP5U5o4N55js3Pr3OnKLRFxdGlzLvBnDDE2zdZvI26KiFVlOmhuAx/auhW9kS2B4LJygswF/oHmdio0zzj8fUTclpmLaU7itnYdG2keQrwmIp5G87n2HwNk5nciYgVwfUQ8QnM7+KFWPV8r7byeLQGqV+9i4FMR8Sdl/ps0b7T+9qG5+njKxweZualcpXwyIs6iufpcDfx1a7X+bWv7F+AXI+IbNLd+TwWuiIif0VztXcWWK77+es4CPh4RN5Z2vwH8W+nXlRHxq8ANEfGTUseflPYeBP4jIm4FzsvMz/bVezPwWZoHOnu+QBOcfqm1bCNbrsr6t2PE/ZeZN0XE39DcAdhQ+nZCZt5f1v2fwN9ExAM0V0uXteoare+LI+Iktty56n3tu3+/tec39bZhrH5FxJuBT0fEPJr9fWnvNnp/v4Bv89TzZf04pmHr/Xo08L8j4szSn3spx1jVGecK49zE41yfg6IZW+/HNM97njig7kHb/kTZzu9ExAnAFyKid5fz073HD8ZxbL4K3BERve2bSmzqvfZ44H9FxFKaO7K3le0bit5De1LnImInYENmPlmuWs6hef7iy0PumjoSEfOzfJ2+3HG7guah/B8Mt2fS9DDOzUwR8bTehUc5Rl8EPpiZdwyjP7P1zpZmhl+nuWLtXX1cbgCadY6JiLfSXOE+CZxpoqXtjHFuZjo0Ik6lGVYiaO4eDiXRAu9sSZIkVTWuO1vR/G+lQwa9PjMPiIjn03zN86elzndm5qN9dZxOM+jig2XRhZn5D5PuuSSNg/FL0rCN92PEHwNnZOaa3oKIeC5wepk9E3hfZt4bEYcA76P5tkHbHOBjueX/bEnSdDB+SRqq8X7V8gHguxGxN2z+ivcLaAZpBHhObzDFsmzhgDp+DhwdzX+O/3D5Nosk1Wb8kjRU435APjO/HxF7lkC1V/ka+h+W4vZgbzlovIzM/EhvOiLeQvPV2jP61ytf01wKMIc5+80f33iRmiYvetns+T+u9/7T/LFX0rT6OY/zRK4fbYTzSTF+TU3uNjveK/vu/cNhd6Ez37p/z7FX0rR77D//7YeZ+cz+5RP9NuITNONV9I8gu3mgsDJg2VgDh61gy78d2EpmLqOMK7JbLMhXxW9PsIuqaeXKu4bdhc689rkvH3YX1Gdtfn3slSbP+DVJT/zWbwy7C51YdeH5w+5CZxa9/Z3D7oIGWPPVk//foOXjHrE1mn+/8MIyavT6KCNVFw9HRO//Lb2G5r9vj+YgmoH1JKk645ekYRrvna35wGsy83KAzHwkIp5JM77IZTS31D8WEY/TjD57fH8FEfFO4JU0/+PoZzQPoUpSbcYvSUM13mRrEfDCiDixb/mLgdMz8/vAUf0vioh3AP+RmVdm5vnA7LmHK2lbYfySNFTjSrYy822TqTwz/3Yyr5Okrhi/JA1b1/9lW5IkSS0mW5IkSRWZbEmSJFVksiVJklSRyZYkSVJFJluSJEkVmWxJkiRVZLIlSZJUkcmWJElSRSZbkiRJFZlsSZIkVWSyJUmSVJHJliRJUkUmW5IkSRWZbEmSJFVksiVJklSRyZYkSVJFc7uqKCIWA0cBm4BbMvPsiZRL0jAZwyTV0smdrYjYFTgGOCIzfx94aUTsM95ySRomY5ikmrr6GHF/4NrMzDK/Alg0gfLNImJpRNweEbdvYH1H3ZOkUXUSw4xfkgbpKtnaA1jXml9Xlo23fLPMXJaZCzNz4Tx26qh7kjSqTmKY8UvSIF0lW48Au7fmF5Rl4y2XpGEyhkmqpqtkay1wSEREmX8jsGYC5ZI0TMYwSdV08m3EzHw0Ii4GlkfERuCuzLxnvOWSNEzGMEk1dTb0Q2YuB5a3l0XEl4AjM3PToHJJmimMYZJq6SzZGiQz31SzfkmqyRgmqQuOIC9JklSRyZYkSVJFJluSJEkVmWxJkiRVZLIlSZJUkcmWJElSRSZbkiRJFZlsSZIkVWSyJUmSVJHJliRJUkUmW5IkSRWZbEmSJFVksiVJklSRyZYkSVJFJluSJEkVmWxJkiRVNLeriiJiMXAUsAm4JTPP7iu/E1hbZjcCx2dmdtW+JE2FMUxSLZ0kWxGxK3AMcFhmZkRcHBH7ZOa3W6s9kpnHdtGeJHXJGCappq7ubO0PXNu6ylsBLALagWpORHwYeD5waWZePqiiiFgKLAXYmfkddW/4Vv77XcPuQide+9yXD7sLUg2dxLDZGr9WXXj+sLvQiUVvf+ewu6DtVFfJ1h7Autb8OmCf9gqZuQggIuYBl0bEN/uuGnvrLQOWAewWC7xFL2k6dBLDjF+SBunqAflHgN1b8wvKsqfIzA3AtcCvddS2JE2VMUxSNV0lW2uBQyIiyvwbgTWjrP+bwOz4XE3SbGAMk1RNJx8jZuajEXExsDwiNgJ3ZeY97XUi4nPAz4BdgMsz8/4u2pakqTKGSaqps6EfMnM5sLy9LCK+BByZmZsy84+6akuSumYMk1RLZ8nWIJn5ppr1S1JNxjBJXXAEeUmSpIpMtiRJkioy2ZIkSarIZEuSJKkiky1JkqSKTLYkSZIqMtmSJEmqyGRLkiSpIpMtSZKkiky2JEmSKjLZkiRJqshkS5IkqSKTLUmSpIpMtiRJkioy2ZIkSarIZEuSJKkiky1JkqSKOku2ImJORHwoIq4ZofyQiLgqIr4YEX/dVbuSNFXGL0k1dXln63DgCmBuf0FEBHAq8AeZeSTw04g4tMO2JWkqjF+Squks2crMFZm5doTiFwF3Z+b6Mn85sGjQihGxNCJuj4jbN7B+0CqS1Cnjl6SapuuZrT2Ada35dWXZU2TmssxcmJkL57HTtHROkkZh/JI0JdOVbD0C7N6aX1CWSdJMZ/ySNCXTlWzdB7wkInqXekcAN0xT25I0FcYvSVPylIdBO7Chf0FmboqIM4FLIuIx4GHgaxXalqSpMH5J6lznyVZmHtabjojzgDMy86HMXAWs6ro9SeqK8UtSDTXubG2Wme+qWb8k1WL8ktQVR5CXJEmqyGRLkiSpIpMtSZKkiky2JEmSKjLZkiRJqshkS5IkqSKTLUmSpIpMtiRJkioy2ZIkSarIZEuSJKkiky1JkqSKTLYkSZIqMtmSJEmqyGRLkiSpIpMtSZKkiky2JEmSKprbVUURMQf4ILBfZr5uQPl1wH2tRadk5qNdtS9Jk2X8klRTZ8kWcDhwBfCqkVbIzGM7bE+SumL8klRNZ8lWZq4AiIiRVnksIs4E9gbWZOb5g1aKiKXAUoCdmd9V94butc99+bC7IGkExq/RzZb4tSO3DbsL2k51eWdrVJn5ewDRRLNzI+JfM/P6AestA5YB7BYLcrr6J0kjMX5Jmoppf0A+MxO4EnjZdLctSVNh/JI0GcP6NuKrwfu5krZJxi9JE1LjY8QNgxZGxMeBXYCdgbWZeVOFtiVpKoxfkjrXebKVmYf1piPiPOCMzHwoM/+s67YkqUvGL0k1VH1APjPfVbN+SarF+CWpK44gL0mSVJHJliRJUkUmW5IkSRWZbEmSJFVksiVJklSRyZYkSVJFJluSJEkVmWxJkiRVZLIlSZJUkcmWJElSRSZbkiRJFZlsSZIkVWSyJUmSVJHJliRJUkUmW5IkSRWZbEmSJFVksiVJklTR3K4qiohzgSeBBcBVmfn5vvJDgPcCjwM/yMyTumpbkqbKGCapls6Srcx8N0BEBLAG2ByoyrJTgddn5vqIOCsiDs3Ma/vriYilwFKAnZnfVfckaVRdxDDjl6RBanyMuBOwrm/Zi4C7M3N9mb8cWDToxZm5LDMXZubCeexUoXuSNKpJxzDjl6RBaiRbZwFn9y3bg62D17qyTJJmGmOYpE51mmxFxHuBOzPzpr6iR4DdW/MLyjJJmjGMYZJq6CzZiojjgMcz85IBxfcBL4mI3n31I4AbumpbkqbKGCaplk4ekI+I/YFTgKsj4jNl8emZ+TBAZm6KiDOBSyLiMeBh4GtdtC1JU2UMk1RTJ8lWZt4MPL9/eUScB5yRmQ9l5ipgVRftSVKXjGGSaups6IdBMvNdNeuXpJqMYZK64AjykiRJFZlsSZIkVWSyJUmSVJHJliRJUkUmW5IkSRWZbEmSJFVksiVJklSRyZYkSVJFJluSJEkVmWxJkiRVZLIlSZJUkcmWJElSRSZbkiRJFZlsSZIkVWSyJUmSVJHJliRJUkVzu6ooIs4FngQWAFdl5uf7yq8D7mstOiUzH+2qfUmaLOOXpJo6S7Yy890AERHAGuDzA9Y5tqv2JKkrxi9JNXWWbLXsBKwbsPyxiDgT2BtYk5nnD3pxRCwFlvZec11e9q0KfWzbE/hh5Tami9sy88yW7YDp2ZZfqlz/WIxfw+O2zEyzZVumazsGxrDIzE5biYiPAV/OzJtGKA/gXOCLmXl9p41PQkTcnpkLh92PLrgtM89s2Q6YXdsyEuPX8LgtM9Ns2ZZhb0enD8hHxHuBO0cKVADZZHdXAi/rsm1Jmgrjl6RaOku2IuI44PHMvGQcq78auK2rtiVpKoxfkmrq5JmtiNgfOAW4OiI+UxafnpkPt9b5OLALsDOwdrSrx2m2bNgd6JDbMvPMlu2A2bUtmxm/Zgy3ZWaaLdsy1O3o/JmtrSqPOA84IzMfqtaIJFVg/JLUlarJliRJ0vauxtAP24yIWAwcBWwCbsnMs4fcpUmJiDnAB4H9MvN1w+7PVIw1uOS2JCLOAeYBTwfuzcwPDLdHkxcRc4G/A36Sme8adn80e+IXzJ4YZvyauYYdw7bbZCsidgWOAQ7LzIyIiyNin8z89rD7NgmHA1cArxp2R6ZqPINLbisy87jedER8LiL2zcza4y7VchpwEXDkkPshZl38glkSw4xfM9pQY9h2m2wB+wPX5pbPUVcAi4BtLlhl5gqA5v09a4w0uOQ2JyJ2B54JbJPP/kTEW4DbgXuH3RdtNmviF8zKGGb8mkFmQgzbnv8R9R5s/WZYV5ZpZjgL2GY/FgGIiF+JiEuAfwSWbYv/Sy8iXgE8OzO/Muy+aCvGr5nN+DVDzJQYtj0nW48Au7fmF5RlGrLxDC65LcjM+zJzMbAPsDginj3sPk3Cm4F9y5AIHwJ+q4xJpeEyfs1Qxq8ZZ0bEsO35Y8S1wAkR8YlyK/6NwF8OuU/bvQkOLrlNyMyN5QHgHYfdl4nKzJN70xGxN3BaZp4ztA6px/g1Axm/Zp6ZEsO222QrMx+NiIuB5RGxEbgrM+8Zdr+maMOwOzAV4xlcclsREf8VOAl4DNgN+FJmfm+4vZqyTcDGYXdCszZ+wTYcw4xf24ShxTDH2ZIkSapoe35mS5IkqTqTLUmSpIpMtiRJkioy2ZIkSarIZEuSJKkiky1JkqSKTLYkSZIqMtmSJEmq6P8DC8DF7F4/3OgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 720x360 with 2 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fTVzkFiaKfPV"
      },
      "source": [
        "`LabelSmoothing` 클래스에서 정답 분포를 smoothing하고 나서 로스 까지 계산하므로 정답자리에 대한 확신이 점점 1에 가까워지면 로스가 어떻게 계산되는지 확인해볼 수 있습니다.\n",
        "\n",
        ">Label smoothing actually starts to penalize the model if it gets very confident about a given choice.\n",
        "\n",
        "아래 코드셀에서 `x`가 커지면 `x/d`가 점점 1에 가깝게 되고 그렇게 되면 로스 값이 조금씩 증가하는 것을 볼 수 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "mH9l5qAF0WSn",
        "outputId": "7178fa3a-648a-4fdf-c4db-df203d074a6d"
      },
      "source": [
        "crit = LabelSmoothing(5, 0, 0.1)\n",
        "def loss(x):\n",
        "    d = x + 3 * 1\n",
        "\n",
        "    predict = torch.FloatTensor([[0, x/d, 1/d, 1/d, 1/d]])\n",
        "    \n",
        "    return crit(predict.log(), torch.LongTensor([1]))#.data[0]\n",
        "\n",
        "plt.plot(np.arange(1, 100), [loss(x) for x in range(1, 100)])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f77f6f47d90>]"
            ]
          },
          "execution_count": 57,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD3CAYAAADv7LToAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAX2ElEQVR4nO3dfXBldX3H8ff3nvuYm6x5ZBfch6AsPvBU2EXbrTplioqtwlAcsFCmnY5dKu1YaWdqW3Xsg7UO02k7fbRIa8cVo21piy2Vgq0PgHTrogiKi0h3l0UWmk3Y3Twn995v/7jnJichIXc3yd6ccz6vcSc551yS7zF3P/vL9/x+55i7IyIi8ZZpdQEiIrJyCnMRkQRQmIuIJIDCXEQkARTmIiIJkG3FN+3t7fX+/v5WfGsRkdh6+OGHj7p732LHWhLm/f397Nu3rxXfWkQktszs0FLH1GYREUkAhbmISAIozEVEEkBhLiKSAApzEZEEUJiLiCSAwlxEJAFiFeZHjk/wR/c+wf8Ojra6FBGRdSVWYX50ZJo//a/v89TgWKtLERFZV2IV5sVcvdzJmWqLKxERWV9iFuYBABMKcxGReWIZ5lMKcxGReWIV5qW8RuYiIouJVZgXs42eea3FlYiIrC+xCvNskCEXmEbmIiILxCrMAYrZQLNZREQWiF+Y5xXmIiILxS7MS7mAiWmFuYhIVOzCvJjL6AKoiMgCsQvzUi7QBVARkQViF+aFnHrmIiILxS7MSwpzEZEXiV2Yq2cuIvJisQtz9cxFRF4sdmFeVJtFRORFYhnmGpmLiMwXyzCfUs9cRGSe2IV5KRcwXa1RqSrQRUQa4hfm+fA2uBWFuYhIQ+zCvPG0IV0EFRGZE9sw1822RETmxDbMpyoKcxGRhtiFeWl2ZK6euYhIQ+zCvJhrXADVyFxEpCF2YV5Sz1xE5EViF+aazSIi8mKxDXMt6RcRmRPDMA975gpzEZFZ2WZeZGY3ANcBVeAhd791wfFbgB3ANBAA73H38VWuFZjrmeue5iIic5YdmZtZB3AjcJW7Xw1cYGbbI8c7gcvd/Wfc/eeBx4E3r1XBpbzaLCIiCzXTZtkF3OfuHm7fBVwWOX4cOGJmZ5pZCdgGPLDwi5jZbjPbZ2b7BgcHT7ngYlYXQEVEFmomzHuA4cj2cLgPgDDkPwncDNwEPOjuQwu/iLvf5u473X1nX1/fqRecMfLZjEbmIiIRzYT5ENAV2e4O9wFgZhcCb3f3D7n7nwATZvbu1S1zvmI2o3uai4hENBPme4HLzczC7SuBr0aOnwlYZHsC6F+V6pZQygdaNCQiErHsbBZ3P2Zme4ABM6sAj7j7/shL7gXeZGafAqaANuC9a1JtqJgLtJxfRCSiqamJ7j4ADET3mdmdwLXuXgU+sAa1LamU08hcRCSqqTBfjLtfs5qFnIxCLtCThkREImK3AhSglMswqZG5iMisWIZ5MRdoaqKISEQsw7yUC7RoSEQkIrZhrpG5iMicWIZ5IRfoRlsiIhGxDHO1WURE5otlmBdzGYW5iEhELMO8lAuo1JyZqlotIiIQ0zDXc0BFROaLZ5jrARUiIvPEM8yz4XNAp9VmERGBmIZ549FxunOiiEhdLMO88eg43TlRRKQulmE+OzJXz1xEBIhpmDdms+gCqIhIXUzDPLwAqiX9IiJATMO8pHnmIiLzxDLMtWhIRGS+WIZ5ST1zEZF5YhnmcyNz9cxFRCCmYV4IV4BqZC4iUhfLMM9kjEJWt8EVEWmIZZhDfeGQwlxEpC62YV7MBlrOLyISim2Yl/IBkxVdABURgRiHeSGb0chcRCQU2zAv5QOmdAtcEREgzmGeU89cRKQhtmFezAV6OIWISCi2Ya6RuYjInNiGeSGX0XJ+EZFQbMO8lNOiIRGRhtiGeTEX6N4sIiKhbDMvMrMbgOuAKvCQu9+64PgrgQ+Em1Xgw+7+7GoWulBjZO7umNlafisRkXVv2TA3sw7gRuBt7u5mtsfMtrv7k+FxAz4G3OTuwy/xdXYDuwG2bt264sKLuQw1h+lqjUI2WPHXExGJs2baLLuA+9zdw+27gMsixy8FDgMfNbM7zOzdi30Rd7/N3Xe6+86+vr4VFQ26p7mISFQzYd4DREfcw+G+hn7gfOB97n4DsMPM3rhqFS5Bj44TEZnTTJgPAV2R7e5wX8M49ZH7ZLj9eWDH6pS3ND3UWURkTjNhvhe43OauMl4JfDVy/GHgdZHt1wOPrk55Syvl9RxQEZGGZS+AuvsxM9sDDJhZBXjE3fdHjh8xs3vNbAAYAw66+3+tXcl1xVz93yH1zEVEmpya6O4DwEB0n5ndCVzr7lV3/wTwiTWob0mNnrmW9IuINBnmi3H3a1azkJOlC6AiInNiuwJUF0BFRObENsxn2ywKcxGR+IZ5WzibZVw9cxGR+IZ5Z1sOgBfGpltciYhI68U2zAvZgPZCliGFuYhIfMMcoLucZ1hhLiKiMBcRSYJYh3lPOa82i4gIMQ/z+sh8qtVliIi0XLzDvL3eZpm71bqISDrFOsx7ynlmqs7IVKXVpYiItFSsw7y7XAA011xEJNZh3lPOA+giqIikXqzDvDsM8+FRhbmIpFsywlwjcxFJuViHeU+72iwiIhDzMG/LZynmMpprLiKpF+swB+hu0ypQEZH4h3m77s8iIhL/MC8XFOYiknqxD/Oecp4hTU0UkZSLfZjrNrgiIgkJ84mZKhN6FqiIpFjsw3xuSb+mJ4pIesU+zLUKVEQkAWGuVaAiIgkI88ZtcHWzLRFJswSEudosIiKxD/MNxSy5wNRmEZFUi32YmxldbXqws4ikW+zDHLRwSEQkEWHeo5ttiUjKNRXmZnaDmX3ezP7ZzH59iddkzewzZvbXq1vi8nSzLRFJu2XD3Mw6gBuBq9z9auACM9u+yEs/CPwdEKxqhU3oKeue5iKSbs2MzHcB97m7h9t3AZdFX2Bm1wP7gO8t9UXMbLeZ7TOzfYODg6da76K6y3lGJitMV2qr+nVFROKimTDvAYYj28PhPgDM7GJgk7v/20t9EXe/zd13uvvOvr6+Uyp2KY255i+Ma3QuIunUTJgPAV2R7e5wX8O7gFeZ2ceB3wd+1MxuXr0Slzd7sy2tAhWRlMo28Zq9wK+Y2R+HrZYrgY82Drr7+xufm1k/8EF3/8tVrvMlaRWoiKTdsmHu7sfMbA8wYGYV4BF337/Ey6tAZTULbMbczba0cEhE0qmZkTnuPgAMRPeZ2Z3Ate5ejbzuMPCLq1phE2ZvtqWRuYikVFNhvhh3v2Y1C1mJzlKObMYYHNHIXETSKRErQDMZY3NXiUND460uRUSkJRIR5gDbesocHBprdRkiIi2RmDA/u7fMwaNjzK1tEhFJj8SE+baeNsamqxzVXHMRSaHEhHl/bxlArRYRSaXEhPnZPWGYH1WYi0j6JCbMX95VIsiYRuYikkqJCfNckGFLV4mDmp4oIimUmDCHcHqi2iwikkKJCvOze8scGhrX9EQRSZ1Ehfm2njZGpyqanigiqZOoMG9MTzyki6AikjLJCvNweuIB9c1FJGUSFeabw+mJuuGWiKRNosI8F2TY3FXigNosIpIyiQpzqLda1DMXkbRJYJi3cfCopieKSLokL8x7y4xOVRjSI+REJEWSF+a64ZaIpFDywnz2Vria0SIi6ZG4MG9MT9TIXETSJHFhngsybOtpY/9zI60uRUTktElcmANcvKWLbz79gma0iEhqJDLML9nWydDYNE8Pq28uIumQzDDf2gXAN55+ocWViIicHokM83M3dlDOB3zj0LFWlyIiclokMsyDjHHRlk6NzEUkNRIZ5lBvtex/boTx6UqrSxERWXPJDfNtnVRrzrcOH291KSIiay6xYX7xlvpF0G8eVqtFRJIvsWHeVc7zit6yLoKKSCokNswBLt6qxUMikg6JDnMtHhKRtMg28yIzuwG4DqgCD7n7rQuO/xVQA7qBu93906td6KmILh7aFt4aV0QkiZYdmZtZB3AjcJW7Xw1cYGbbo69x9/e4+y8B1wM3LfF1dpvZPjPbNzg4uAqlL0+Lh0QkLZpps+wC7vO5xvNdwGVLvLYADC92wN1vc/ed7r6zr6/v5Cs9BUHG2NHfzf1PDqpvLiKJ1kyY9zA/oIfDfYv5CHDrEsda4q3nbeTg0DhPPK9b4opIcjUT5kNAV2S7O9w3j5ndAnzT3R9cpdpWxVteuwkz+MJjz7W6FBGRNdNMmO8FLjczC7evBL4afYGZ3QyMufsdq1zfivV1FLi0v5t7vq0wF5HkWjbM3f0YsAcYMLNPA4+6+/7GcTPbBfwGcImZfTz8c3qa4k264rxNPPH8CP87ONrqUkRE1kRTUxPdfQAYiO4zszuBa939a8DWNaht1Vxx/iZ+998e557vPMfNP3ZOq8sREVl1p7xoyN2vcffqahazVs7qLHHR5pfxH2q1iEhCJXoFaNQV55/Jt545zg+OTbS6FBGRVZeiMN8EoAuhIpJIqQnzs3vLvHpTB3c/+myrSxERWXWpCXOAd+7YzDeePsajz2h5v4gkS6rC/LpLt9BeyHL7/QdaXYqIyKpKVZh3FHO869It3P3YEV0IFZFESVWYA/zcj/YD8HcPanQuIsmRujDf3NXG287fxGf/5zAjkzOtLkdEZFWkLswBfuGNr2BkqsLnvn641aWIiKyKVIb5RVs6eV1/N3/7wAEmZ2KxiFVE5CWlMswBbnnzuTx7fJK//PJTrS5FRGTFUhvmP/LKHq76obP4+Fee4uDRsVaXIyKyIqkNc4Df+onXkA8y/Pa/fkePlRORWEt1mG/cUOR9l2/ny08Mct/jz7e6HBGRU5bqMAf42V39nLuxnd/518c5PqGpiiIST6kP81yQ4Q9+6kKePzHJr/39I9RqareISPykPswBdmzr4oM/+Rq++N3/4y++9P1WlyMictIU5qGf3dXP1Re/nD/64vf40hP/1+pyREROisI8ZGZ89OoLePWmDbzvs4/w3SMnWl2SiEjTFOYRpXzAbTfuoJQLuP4T/83jzyrQRSQeFOYLbOlu47O7f5hiLuD62/+bb//geKtLEhFZlsJ8Ef29ZT63+0co57PccPtevvbU0VaXJCIx5e5MzlQZHpvm8PA4z5+YXJPvY61Y+bhz507ft2/faf++J+vw8Dg/98n/4cDRMd5/xavZ/aZXYGatLktE1lCt5oxNVxifrjI6VWFsqsLYVLX+cbr++fh0hdGpxV8TPTYWfqxEpjy/46Kz+LOfvviUajOzh91952LHsqd2uumwpbuNu375Dfz6P36LP/jCfh45fIyPXXMhLyvlWl2aiITqI9/abKhGP45GQnY2dKcrjEb2jYcB3Tg+Pt38nVRLuYByIUu5ENCWz9JeCOhsy/PyrhLlfJZyIUtbPnxNPqCtkOUVveU1+f9BYb6M9kKWv7j+Em6//wAfu2c/Xz/4FT709tdw5UVnaZQucorcnalKjZHJuZAdmZwfwqNTFUYno6E8f/9sQE9XqTa52K8Rvu2FRghn2dhRpK033BcGcLnQeF2Wcj5LWyGofx5uN8I7yKyfDFCb5SR8+wfH+a1/foxHnznOG7f38uF3vJZzzuhodVkip02jBdEI1JFo4M7bnmE0DOiFodzYrjQRwBljLlTDjx3F7GzoNkK5vZiloxG2814/F9rldRa+p+Kl2iwK85NUrTl7HjrIH977PcamK7zjwrN474+fo1CXdc3dmZipMjpZ4UQYrCOTM/UADkM4uj061QjmmbntyXqLopnIKOUCOoqR8A0DtSMSutEAbp/dzlEuBLSH/20pF+g34AiF+RoYGp3iE/cf4FMPHWRipspbXruRG16/jTec00sm5v/6y/pSrXkYwtFgrX8+MjnDidkAbuyLHp8bDTfTiijn60HaUczNGwV3zO4L6h+Lc+G8YfbzHO1hCyIbaKLcWlCYr6Gh0Sn+5oEDfO7rhxkam2Zrdxvv3LGZn7jgTM45o73V5UmLVaq12VA9EQngeR+n5odv9HWjk/We8HJygdFRzM2OhtvDcN1QjARxZKS8IdyeHT0X6ttxb0MkncL8NJiqVPmP7zzPZ/YeYu+BYdzh3I3tvOW1m3jD9l4u2dpFPqvRSpzMVGuzbYfFgvjEghFwNJRPTNQ/TjTxjNliLkN7YUHwFrJsKGVnA7rxcUMxS3uhsW9ufyGbUTsiBRTmp9nzJya559vPcfdjR3j40AtUa05bPmBnfzeXbO3k4q1dXLT5ZXS25VtdaiJF+8MjU/PbDo12xejUi9sRIwvaF5MztWW/VzGXmRe4s4FcmB/C84+HwVyqh7b+kZdmKcxb6MTkDA89NcQDTx7l6weHeeL5kdkLSGe+rMirNnXwqo0d9PeW6e8pc3ZvmTM6CqnruzemqjXm+UZnPowtOUWtOtuKmDdz4iT7w42WRKP9sHDUGw3kDZF9CmI53Va8aMjMbgCuA6rAQ+5+68kcT7MNxRxvPW8Tbz1vEwAjkzM89sxxHv3BcZ54boTvHjnB174/xHR1bhSYzRgbNxQ5q7NIX0eBvvYCve0FOst5Oks5Ottysxej2gs5SvmAYi5DPlibX7UbQTtdrTFdqTFVqTE1U2WqUmNypsrkTI3JSpWpmSoTM1UmpmuMT1eYmK5vj09XmZiuMhbua6yuawT3YqvkXkopF8zr/7YXsmwpt83OlFh4Aa9jYWtC/WFJoGXD3Mw6gBuBt7m7m9keM9vu7k82c1zm6yjm2HVOL7vO6Z3dV605zx6b4ODQGIeGxnn22ARHjk/y7LEJ9j83wgMjRzkxWVn2a2cMCtmAXGDksxlyQYaMGdnACMyg/j/MDHfHHTz8/jV3ajWnEv6ZqdaoVMOPK3j6UjZjlHIBbeEii1Kuvviiu5xn8yKr5NrDz6PT2WbnDGumhMiSmhmZ7wLu87l+zF3AZcCTTR4HwMx2A7sBtm7dusKykyXIGFu629jS3cYbty/+mqlKleMTMxwfn+HYxMy8BRoTM1UmZ+qj38boebpaY6ZSoxoJaQdwcBwzmw32wCCTMTJm5AIjyBjZTCb8B2Hu88Lsn4BC+JtAMVf/vJgLKGYD2vJB/TeFbP2j2hAip0czYd4DDEe2h4HtJ3EcAHe/DbgN6j3zk6405QrZgDM6As7oKLa6FBFZh5oZNg0BXZHt7nBfs8dFRGSNNRPme4HLbe7K2pXAV0/iuIiIrLFl2yzufszM9gADZlYBHnH3/c0eFxGRtdfU1ER3HwAGovvM7E7gWnevLnZcREROn1O+n7m7X7OahYiIyKnTvDERkQRQmIuIJIDCXEQkAVpyoy0zGwQOncR/0gscXaNy1ru0nrvOO1103s3Z5u59ix1oSZifLDPbt9SdwpIureeu804XnffKqc0iIpIACnMRkQSIS5jf1uoCWiit567zThed9wrFomcuIiIvLS4jcxEReQkKcxGRBDjle7OcLml6vqiZ/RVQo35P+Lvd/dNmdjlwCzAGPOPuv9rKGteKmWWBTwEj7n5TGs7bzF4JfCDcrAIfpv6UrkS/383sFmAHMA0EwHuoP7EscT9vMwuA3wV2uPsV4b5F39srfs/XnwW5Pv8AHcA9zPX29wDbW13XaThvA+4PP/4nUAj3fwR4c6vrW6Nz/m3gLcDtaTjv8Bz/AeiO7Ev8+x3opD5QaWy/H7gqqT/v8NxeD3wx8nN/0bmuxnt+vbdZlnq+aNIVqD9+71zgcXefCvf/Cwk8fzO7HtgHfC/clYbzvhQ4DHzUzO4ws3eTjvf7ceCImZ1pZiVgG/AcCf15u/td7r43smup9/aK3/Prvc3S1PNFE+gjwK0sfv49LalojZjZxcAmd/+MmfWHuxN/3kA/cD5wpbtPhi22lwNPR16TuPe7u7uZfRK4mfrjJR+k3mpJ+s+7Yan39orf8+s9zIeA8yLbiX++aNhP/Ka7P2hmryL5z1d9F9BpZh+n3ma4BHiM5J/3OPVR+GS4/XngQhJ+3mZ2IfB2d//NcPungAtI+HlHLPXM5BU/S3m9t1lS9XxRM7sZGHP3O8Jd3wfON7NCuH0V8JWWFLdG3P397n6Tu/8i9YuBDwJ/TsLPG3gYeF1k+/XAkyT//X4m9f5wwwThbykJ/3k3LPV3esV/19f1yNxT9HxRM9sF/Abw7+EoFeBDwO8Bd5jZKDAI3NuiEk+HKlBx96qZJfq83f2Imd1rZgPUZy8cdPd/Cv8yJ/n9fi/wJjP7FDAFtAHvpf5bSWJ/3sAMwFLv7bD9tKL3vFaAiogkwHpvs4iISBMU5iIiCaAwFxFJAIW5iEgCKMxFRBJAYS4ikgAKcxGRBPh/5MTcQIJSi/0AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FF3cdO9M0HM4"
      },
      "source": [
        "## Loss 계산\n",
        "\n",
        "`LabelSmoothing`에서 smoothing 뿐만 아니라 `nn.KLDivLoss`를 사용해서 로스값을 계산합니다. 원래 정답분포는 정답자리만 1이고 나머지는 모두 0입니다. 이럴 때 주로 사용하는 손실함수는 `nn.NLLLoss`입니다. 하지만 `LabelSmoothing`이 정답분포를 부드럽게 깍았으므로 모든 오답자리에도 확률이 할당되어 있습니다. 이럴때 출력분포와 정답분포의 차이를 계산하기 위해 Kullback–Leibler divergence[[7](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence)]를 사용합니다. `SimpleLossCompute`는 `model.generator`를 호출해서 모델의 출력값을 계산한다음 그 값을 `LabelSmoothing`에 넘겨 로스값을 받아오고 백워드까지 수행합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFQKwCOT1MLH"
      },
      "source": [
        "class SimpleLossCompute:\n",
        "    \"A simple loss compute and train function.\"\n",
        "    def __init__(self, generator, criterion, opt=None):\n",
        "        # 여기서 generator는 model.generator\n",
        "        self.generator = generator\n",
        "        \n",
        "        # 여기서 criterion은 LabelSmoothing \n",
        "        self.criterion = criterion\n",
        "\n",
        "        self.opt = opt\n",
        "        \n",
        "    def __call__(self, x, y, norm):\n",
        "        # norm은 batch에서 토큰 수\n",
        "        # self.ntokens = (self.trg_y != pad).data.sum() # 패딩 토큰이 아닌 토큰 수\n",
        "        \n",
        "        x = self.generator(x)\n",
        "        \n",
        "        loss = self.criterion(x.contiguous().view(-1, x.size(-1)), \n",
        "                              y.contiguous().view(-1)) / norm\n",
        "        loss.backward()\n",
        "        if self.opt is not None:\n",
        "            self.opt.step()\n",
        "            self.opt.optimizer.zero_grad()\n",
        "        # return loss.data[0] * norm\n",
        "        return loss * norm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q3trxirYgDG0"
      },
      "source": [
        "## 학습\n",
        "\n",
        "모든 설명이 끝이 났습니다! 이제 적당히 학습 함수를 정의하고 학습을 하면 됩니다.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ylQqmx3R0J3X"
      },
      "source": [
        "def run_epoch(data_iter, model, loss_compute):\n",
        "    \"Standard Training and Logging Function\"\n",
        "    start = time.time()\n",
        "    total_tokens = 0\n",
        "    total_loss = 0\n",
        "    tokens = 0\n",
        "    # data_iter를 만들 때 설정한 nbatches 만큼 루프를 돈다. 즉 1에폭\n",
        "    for i, batch in enumerate(data_iter):\n",
        "        out = model.forward(batch.src, batch.trg, \n",
        "                            batch.src_mask, batch.trg_mask)\n",
        "        \n",
        "        # 여기서 loss_compute()는 SimpleLossCompute 임\n",
        "        loss = loss_compute(out, batch.trg_y, batch.ntokens)\n",
        "        total_loss += loss\n",
        "        total_tokens += batch.ntokens\n",
        "        tokens += batch.ntokens\n",
        "        \n",
        "        if i % 50 == 1:\n",
        "            elapsed = time.time() - start\n",
        "            print(\"Epoch Step: %d, Loss: %f, Tokens per Sec: %f\" %\n",
        "                    (i, loss / batch.ntokens, tokens / elapsed))\n",
        "            start = time.time()\n",
        "            tokens = 0\n",
        "    return total_loss / total_tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NK_IZIRp1N0O",
        "outputId": "a9e1fcbd-fb6e-4c17-d4ae-b5b541d7bff3"
      },
      "source": [
        "# Train the simple copy task.\n",
        "V = 11\n",
        "criterion = LabelSmoothing(size=V, padding_idx=0, smoothing=0.0)\n",
        "model = make_model(V, V, N=2)\n",
        "#                   model_size, factor, warmup, optimizer\n",
        "model_opt = NoamOpt(model.src_embed[0].d_model, 1, 1200,\n",
        "        torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9))\n",
        "\n",
        "for epoch in range(18):\n",
        "    model.train()\n",
        "    print(f'{epoch} epoch train')\n",
        "    run_epoch(data_gen2(V, 30, 20), model, # 미니배치에 샘플 30개씩 20배치가 한에폭 \n",
        "              SimpleLossCompute(model.generator, criterion, model_opt))\n",
        "    print('eval')\n",
        "    model.eval()\n",
        "    eval_loss = run_epoch(data_gen2(V, 30, 5), model, # 미니배치에 샘플 30개씩 5배치가 한 에폭\n",
        "                    SimpleLossCompute(model.generator, criterion, None))\n",
        "    print('eval_loss:', eval_loss,'\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0 epoch train\n",
            "Epoch Step: 1, Loss: 2.899037, Tokens per Sec: 210.496277\n",
            "eval\n",
            "Epoch Step: 1, Loss: 2.265778, Tokens per Sec: 514.014282\n",
            "eval_loss: tensor(2.2115, grad_fn=<DivBackward0>) \n",
            "\n",
            "1 epoch train\n",
            "Epoch Step: 1, Loss: 2.333732, Tokens per Sec: 419.434052\n",
            "eval\n",
            "Epoch Step: 1, Loss: 2.035715, Tokens per Sec: 516.446472\n",
            "eval_loss: tensor(2.0386, grad_fn=<DivBackward0>) \n",
            "\n",
            "2 epoch train\n",
            "Epoch Step: 1, Loss: 2.167701, Tokens per Sec: 412.256683\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.881836, Tokens per Sec: 512.943665\n",
            "eval_loss: tensor(1.9103, grad_fn=<DivBackward0>) \n",
            "\n",
            "3 epoch train\n",
            "Epoch Step: 1, Loss: 2.069123, Tokens per Sec: 419.231018\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.850417, Tokens per Sec: 517.300476\n",
            "eval_loss: tensor(1.8184, grad_fn=<DivBackward0>) \n",
            "\n",
            "4 epoch train\n",
            "Epoch Step: 1, Loss: 2.078735, Tokens per Sec: 397.969116\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.755276, Tokens per Sec: 524.911316\n",
            "eval_loss: tensor(1.7278, grad_fn=<DivBackward0>) \n",
            "\n",
            "5 epoch train\n",
            "Epoch Step: 1, Loss: 2.029374, Tokens per Sec: 416.710968\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.571162, Tokens per Sec: 505.926727\n",
            "eval_loss: tensor(1.5942, grad_fn=<DivBackward0>) \n",
            "\n",
            "6 epoch train\n",
            "Epoch Step: 1, Loss: 1.775693, Tokens per Sec: 419.271454\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.534954, Tokens per Sec: 525.713135\n",
            "eval_loss: tensor(1.5013, grad_fn=<DivBackward0>) \n",
            "\n",
            "7 epoch train\n",
            "Epoch Step: 1, Loss: 1.771013, Tokens per Sec: 416.784576\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.503360, Tokens per Sec: 520.307434\n",
            "eval_loss: tensor(1.4806, grad_fn=<DivBackward0>) \n",
            "\n",
            "8 epoch train\n",
            "Epoch Step: 1, Loss: 1.663535, Tokens per Sec: 420.530945\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.283393, Tokens per Sec: 508.099396\n",
            "eval_loss: tensor(1.2777, grad_fn=<DivBackward0>) \n",
            "\n",
            "9 epoch train\n",
            "Epoch Step: 1, Loss: 1.491704, Tokens per Sec: 416.922150\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.156100, Tokens per Sec: 513.559937\n",
            "eval_loss: tensor(1.1623, grad_fn=<DivBackward0>) \n",
            "\n",
            "10 epoch train\n",
            "Epoch Step: 1, Loss: 1.273780, Tokens per Sec: 414.009247\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.057899, Tokens per Sec: 520.631042\n",
            "eval_loss: tensor(1.0376, grad_fn=<DivBackward0>) \n",
            "\n",
            "11 epoch train\n",
            "Epoch Step: 1, Loss: 1.345537, Tokens per Sec: 416.217438\n",
            "eval\n",
            "Epoch Step: 1, Loss: 1.030028, Tokens per Sec: 518.225952\n",
            "eval_loss: tensor(1.0957, grad_fn=<DivBackward0>) \n",
            "\n",
            "12 epoch train\n",
            "Epoch Step: 1, Loss: 1.337737, Tokens per Sec: 419.506073\n",
            "eval\n",
            "Epoch Step: 1, Loss: 0.926710, Tokens per Sec: 515.558350\n",
            "eval_loss: tensor(0.9208, grad_fn=<DivBackward0>) \n",
            "\n",
            "13 epoch train\n",
            "Epoch Step: 1, Loss: 1.186046, Tokens per Sec: 415.672577\n",
            "eval\n",
            "Epoch Step: 1, Loss: 0.690210, Tokens per Sec: 521.995117\n",
            "eval_loss: tensor(0.8092, grad_fn=<DivBackward0>) \n",
            "\n",
            "14 epoch train\n",
            "Epoch Step: 1, Loss: 0.863369, Tokens per Sec: 414.650177\n",
            "eval\n",
            "Epoch Step: 1, Loss: 0.636962, Tokens per Sec: 507.883820\n",
            "eval_loss: tensor(0.5867, grad_fn=<DivBackward0>) \n",
            "\n",
            "15 epoch train\n",
            "Epoch Step: 1, Loss: 0.743255, Tokens per Sec: 409.956757\n",
            "eval\n",
            "Epoch Step: 1, Loss: 0.382606, Tokens per Sec: 521.047974\n",
            "eval_loss: tensor(0.3791, grad_fn=<DivBackward0>) \n",
            "\n",
            "16 epoch train\n",
            "Epoch Step: 1, Loss: 0.585855, Tokens per Sec: 408.838745\n",
            "eval\n",
            "Epoch Step: 1, Loss: 0.263854, Tokens per Sec: 514.657593\n",
            "eval_loss: tensor(0.2577, grad_fn=<DivBackward0>) \n",
            "\n",
            "17 epoch train\n",
            "Epoch Step: 1, Loss: 0.558738, Tokens per Sec: 419.449036\n",
            "eval\n",
            "Epoch Step: 1, Loss: 0.356468, Tokens per Sec: 507.473694\n",
            "eval_loss: tensor(0.3535, grad_fn=<DivBackward0>) \n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RUrupippgGIZ"
      },
      "source": [
        "## 예측"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T19efGH6gKZI"
      },
      "source": [
        "트랜스포머를 학습 시킬 때 디코더에 정답이 모두 입력되고 출력도 한번에 나오게 됩니다. 하지만 학습이 끝나고 예측을 할 때는 그렇게 하지 않고 다음과 같은 순서로 하게 됩니다.\n",
        "\n",
        "1. 디코더에 [START] 토큰을 입력한다.\n",
        "\n",
        "2. [START] 토큰 다음에 나올 토큰을 트랜스포머가 예측한다.\n",
        "\n",
        "3. 예측된 [TKN1]을 붙여서 [START], [TKN1]을 디코더에 입력한다.\n",
        "\n",
        "4. 이런 식으로 계속 진행한다.\n",
        "\n",
        "이 과정에서 트랜스포머의 예측은 (nbatches, n_seq, vocab) 사이즈로 나오게 되는데 현재 스텝의 예측을 결정하기 위해 (nbatches, -1, vocab)에서 확률값이 가장 큰것을 고르게 됩니다. 이렇게 각 타임 스탭에서 적합한 단어를 고르는 전략을 탐욕 탐색greedy search라고 합니다. 하지만 이런 전략이 전체 시퀀스에 대해서 꼭 좋은 것은 아니라 빔서치같은 더 복잡한 방법을 사용하기도 합니다. 여기서는 가장 간단한 방법인 탐욕 탐색을 사용하였습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JsKP-ZSX1QI8"
      },
      "source": [
        "def greedy_decode(model, src, src_mask, max_len, start_symbol):\n",
        "    memory = model.encode(src, src_mask)\n",
        "\n",
        "    # 시작은 [START]로 시작한다.\n",
        "    ys = torch.ones(1, 1).fill_(start_symbol).type_as(src.data)\n",
        "    \n",
        "    # 생성할 시퀀스의 최대 길이만큰 순환하면서\n",
        "    for i in range(max_len-1):\n",
        "        print('ys.shape:', ys.shape)\n",
        "        out = model.decode(\n",
        "                    memory, src_mask, ys, \n",
        "                    subsequent_mask(ys.size(1)).type_as(src.data)\n",
        "                )\n",
        "        print('out.shape:', out.shape)\n",
        "        print('out[:, -1].shape:', out[:, -1].shape)\n",
        "        \n",
        "        # 마지막 타임스탭의 결과를 단어들로 바꾼다.\n",
        "        prob = model.generator(out[:, -1])\n",
        "        print('prob.shape:', prob.shape)\n",
        "        \n",
        "        # 가장 확률이 높은 단어를 선택한다.\n",
        "\n",
        "        _, next_word = torch.max(prob, dim = 1)\n",
        "        next_word = next_word.data[0]\n",
        "\n",
        "        # 예측된 단어를 추가하고 루프 처음으로 돌아가 다시 ys를 디코더로 입력한다.\n",
        "        ys = torch.cat([ys, \n",
        "                        torch.ones(1, 1).type_as(src.data).fill_(next_word)], dim=1)\n",
        "        print('\\n')\n",
        "    return ys"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0k0G9Z7W3hc6",
        "outputId": "337213bd-3f90-49bd-9b58-97daa4241ff9"
      },
      "source": [
        "model.eval()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "EncoderDecoder(\n",
              "  (encoder): Encoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (norm): LayerNorm()\n",
              "  )\n",
              "  (decoder): Decoder(\n",
              "    (layers): ModuleList(\n",
              "      (0): DecoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (src_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "      (1): DecoderLayer(\n",
              "        (self_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (src_attn): MultiHeadedAttention(\n",
              "          (linears): ModuleList(\n",
              "            (0): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (1): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (2): Linear(in_features=512, out_features=512, bias=True)\n",
              "            (3): Linear(in_features=512, out_features=512, bias=True)\n",
              "          )\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): PositionwiseFeedForward(\n",
              "          (w_1): Linear(in_features=512, out_features=2048, bias=True)\n",
              "          (w_2): Linear(in_features=2048, out_features=512, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (sublayer): ModuleList(\n",
              "          (0): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (1): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (2): SublayerConnection(\n",
              "            (norm): LayerNorm()\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (norm): LayerNorm()\n",
              "  )\n",
              "  (src_embed): Sequential(\n",
              "    (0): Embeddings(\n",
              "      (lut): Embedding(11, 512)\n",
              "    )\n",
              "    (1): PositionalEncoding(\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "  )\n",
              "  (tgt_embed): Sequential(\n",
              "    (0): Embeddings(\n",
              "      (lut): Embedding(11, 512)\n",
              "    )\n",
              "    (1): PositionalEncoding(\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "  )\n",
              "  (generator): Generator(\n",
              "    (proj): Linear(in_features=512, out_features=11, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "execution_count": 62,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "18Za-Ur6fz1J"
      },
      "source": [
        "아무 샘플 데이터나 무작위로 만들어 모델에 입력하고 입력 숫자들 중 뒤 다섯개가 1큰 숫자로 출력되는지 확인해봅시다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2qw9mZ013v0Q",
        "outputId": "c8767a2f-2193-4a7c-925e-4b007135c5bf"
      },
      "source": [
        "src = torch.LongTensor([[1, 3, 4, 5, 6,   8, 7, 2, 9,  7]])\n",
        "# 정답                   1, 3, 4, 5, 6,   9, 8, 3, 10, 8\n",
        "                         \n",
        "\n",
        "src_mask = torch.ones(1, 1, 10)\n",
        "print(greedy_decode(model, src, src_mask, max_len=10, start_symbol=1))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "ys.shape: torch.Size([1, 1])\n",
            "out.shape: torch.Size([1, 1, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 2])\n",
            "out.shape: torch.Size([1, 2, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 3])\n",
            "out.shape: torch.Size([1, 3, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 4])\n",
            "out.shape: torch.Size([1, 4, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 5])\n",
            "out.shape: torch.Size([1, 5, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 6])\n",
            "out.shape: torch.Size([1, 6, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 7])\n",
            "out.shape: torch.Size([1, 7, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 8])\n",
            "out.shape: torch.Size([1, 8, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "ys.shape: torch.Size([1, 9])\n",
            "out.shape: torch.Size([1, 9, 512])\n",
            "out[:, -1].shape: torch.Size([1, 512])\n",
            "prob.shape: torch.Size([1, 11])\n",
            "\n",
            "\n",
            "tensor([[ 1,  3,  4,  5,  6,  9,  8,  3, 10,  8]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hw06A7PTf7PR"
      },
      "source": [
        "예상처럼 잘 출력되는 것을 확인할 수 있습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jX-2_n7pf-nY"
      },
      "source": [
        "## 마무리\n",
        "\n",
        "처음 별것 아니라고 하고 시작한 글이 적고보니 엄청 복잡해진 듯한 느낌이 듭니다. 작성하는데 시간도 꽤 많이 걸린 듯 합니다. 부디 트랜스포머를 처음 공부하는 누군가에게 도움이 되길 바랍니다. 혹시 글에 대해서 이해하기 어려운 부분이나 수정이 필요한 부분이 있다면 언제든 연락주세요. "
      ]
    }
  ]
}