F2B-18-Seq2Seq(Attention).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 576
    },
    "colab_type": "code",
    "id": "g1lbXXSDgync",
    "outputId": "e1ff3a6e-a10e-4ed5-b337-16ebc63b3e60"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/rnn.py:38: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.5 and num_layers=1\n",
      "  \"num_layers={}\".format(dropout, num_layers))\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:78: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0400 cost = 0.000539\n",
      "Epoch: 0800 cost = 0.000178\n",
      "Epoch: 1200 cost = 0.000089\n",
      "Epoch: 1600 cost = 0.000053\n",
      "Epoch: 2000 cost = 0.000035\n",
      "Epoch: 2400 cost = 0.000024\n",
      "Epoch: 2800 cost = 0.000017\n",
      "Epoch: 3200 cost = 0.000013\n",
      "Epoch: 3600 cost = 0.000009\n",
      "Epoch: 4000 cost = 0.000007\n",
      "ich mochte ein bier P -> ['i', 'want', 'a', 'beer', 'E']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEzCAYAAABE0wr4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFfdJREFUeJzt3X1U1vX9x/EX97RSK+8Woo7KKE8g\noIjOuwY2HXk8mCtHw7xZyrS51I6mrmVrzptz7GZK8rPZJtlmZZo70rKD2d3UqaQdQQ0n4bjR1LzJ\nMJTg+v7+8Nf1k9QEJ9+v8n4+/uK6fX8+QE+/3+s6cQU4juMIAAwL9HoBAOA1QgjAPEIIwDxCCMA8\nQgjAPEIIwDzTIayoqFBMTIz27t37nfdLTk7Wyy+/7NKq3Ldw4ULde++9Xi/jv7J69Wr17dvX62U0\nSHl5uaKjo7Vnz55zbrsa93M1C/Z6AV5q166dCgoKvF7GFe/EiRN66623NGzYMK+XckFpaWlKS0vz\nehmXTVPbz9mSk5N18OBBBQaeOQ4LDQ1Vp06dNGHCBPXq1cuTNZk+IkT9bNq0Sa+++qrXy0ATMn36\ndBUUFKigoEAbNmzQPffco8zMzIuenTUW0yE8+9Tk6NGjmjhxorp27apevXpp7ty5qq2t9d+3qqpK\nkydPVlxcnPr27asNGzY0+vqio6O1Zs0aDR06VLGxsRo1apQOHDigzMxMxcfHa8iQISorK/Pff/36\n9UpLS1NcXJz69eun559/Xmf/j0NLly5VcnKy4uPjNWLECJWUlNSZt2LFCvXp00dxcXGaNWuWJCk3\nN1eTJk3Srl27FBMTo5KSEvl8PmVlZenuu+9Wly5dlJaWpk2bNjX690OSDhw4oHHjxqlHjx7q2rWr\nJk2apGPHjmnVqlVKSkqS9P8/1w0bNvi/H+np6frss89cWWND7dq1S4MHD1Z8fLwyMjJUUVFRZz+S\nVFRUpJEjRyoxMVFJSUl64okndPr0aUnSqlWrNHDgQM2fP1/x8fF1fieuBuHh4Ro+fLiioqL07rvv\nerIG0yE82+OPP66amhq99957ev3117Vu3TotXbrUf/vrr7+uBx98UJs3b1avXr301FNPubKu5cuX\na9GiRXrzzTf18ccfa+TIkXr44Yf14Ycfqqamxr/GPXv26Fe/+pUyMzO1detWPffcc8rJydHKlSsl\nSevWrVN2draysrK0efNm3XLLLZowYYJ/TkVFhQ4fPqx169YpKytLy5Yt07Zt2zRo0CCNGzdOnTt3\nVkFBgaKiovTSSy/p73//uxYvXqz8/Hylp6dr/PjxOn78eKN+LxzH0bhx49S6dWu98847ysvLU2Vl\npX7/+9+f9/45OTl64YUXtH79eh0/flx/+ctfGnV9l+qVV15Rdna23n//fYWEhOixxx6rc3tVVZUe\neughJSYmasOGDXrjjTdUWFiorKws/30+//xzBQQEaMuWLYqMjHR7C5dFbW2tgoO9ebWOEEo6fvy4\n3n33XY0dO1bNmjXTTTfdpGeeeUYJCQn++9x1112Ki4tTWFiYBg4cqP/85z/6+uuvG31t99xzj9q2\nbav27durU6dOuuOOOxQbG6vrrrtOiYmJ2rdvn6Qzoe7evbt+8pOfKCQkRPHx8UpNTdVbb70lSVq5\ncqVSU1PVuXNnhYaGasKECRo/frx/DwEBAcrMzFRYWJh69+6tli1bqri4+LxrWrFihUaMGKGbb75Z\nISEhGjZsmCIjI7V27dpG/V4UFBSoqKhIU6dO1bXXXqsbb7xREydO1Nq1a1VVVXXO/e+//361adNG\nN954o5KSki64H6+lp6erXbt2at68uUaPHq38/Hx99dVX/tvfe+89ff3113r44YcVGhqqiIgI/fKX\nv9Qbb7zhv09lZaXGjBmjkJAQBQQEeLGNS/bVV19p2bJl2r9/v/r37+/JGky/WfKNwMBA+Xy+Ov+S\nxsbG1rnP2beFh4fLcRxVV1crJCSkUdf2/e9/3/91WFiY2rZtW+dydXW1JKmsrEy33nprncd27NjR\nf8paVlambt26+W+74YYblJqa6r8cERGhoKAg/+Xw8HD/qde3lZaWau7cuZo3b57/OsdxdODAgUvZ\nYr2VlZXJ5/OpZ8+e59x2vp/D2T+za6655oL78drZP7cOHTrIcZw6+ykrK9Px48cVExNT53E+n8//\n87/uuuvUvHlzdxZ8GcyZM8f/+xMeHq7o6GgtWbJE7du392Q9hFDyv472XX+Ix6t/Zb95Z+1Cl7/x\nzX8Q3/bNugMCAuTz+S7LmsLDw/W73/2uTkjdEBYWprCwMO3YseOc21atWnXOdRf6Xl1pzl7nN7+D\nhw8f9l8XFhamqKgo/9H9+Zz9j9jVYPr06crIyPB6GX5Xx2+KCwIDA+u8eZCfn9/op3qXU4cOHc45\n9fv000/VsWNHSVL79u3r7O/EiRN68cUXdfLkyUuaVVRUVOe68vLyS1h1w3Ts2FGnT5/2vxwgnXn9\n7MiRI40+uzGd/XMpLS1VUFCQWrdu7b+uY8eOqqioUGVlpf+6L774Ql9++aWr62zKCKGkFi1aKCUl\nRc8//7yOHTumgwcPaubMmSotLfV6afU2ZMgQbd68WXl5eaqpqVF+fr5yc3M1ZMgQSdLQoUO1du1a\n5efnq7q6WtnZ2Vq5cqWuvfbaiz53WFiYPv/8cx07dkzV1dVKT0/X8uXLlZ+fr9raWr3zzjsaNGiQ\nPv3000bdY6dOndStWzfNmjVLR48e9b9R8utf/7pR5za25cuX69ChQ6qsrFROTo769etX59S4d+/e\nat26tWbPnq0vv/xSR48e1ZQpUy74JhEajhD+n7lz5+r6669XcnKyhg4dqj59+mj06NFeL6veYmNj\nNWfOHC1YsECJiYmaOXOmHn/8cQ0cOFCSlJKSoilTpmjy5MlKSkrS7t2767zr+F369++vwMBA/ehH\nP9KOHTs0dOhQPfjgg5o0aZISEhK0YMECPf3007r55psbc4uSpPnz5ys4OFgpKSlKSUnRiRMn9Mwz\nzzT63MaUnp6u0aNHq0+fPqqpqdGTTz5Z5/bg4GAtWrRIZWVl6t27twYNGqSWLVvqiSee8GbBTVAA\nf6EagHUcEQIwjxACMI8QAjCPEAIwjxACMI8QAjCPEAIwjxACMI8QAjDvqvvrMwOuG+HarMVb/qDM\n7r9xZdZbeze6MkeSAlq+KefIPa7NGxAR59qsF3Y8rbGxj7o2zy1NdV+Su3vL86047/UcEX6HH3S+\nOv/S78UEhNzm9RIaTdSdHbxeQqNoqvuSroy9EUIA5hFCAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFC\nAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFCAOYRQgDmEUIA5hFCAOZdUSGsqKhQ\nTEyM9u7d6/VSABhyRX2cZ7t27VRQUOD1MgAYc0UdEQKAF66oEJaXlys6Olp79uzxeikADLmiQggA\nXghwHMfxehHfKC8vV0pKitasWaPbbrvtvPfZt6tcP+gc6fLKADRlV9SbJfWR2f03rs16uzJHA64b\n4cqst/ZudGWOJAV+/9/yfdbJtXkDIuJcm5XnW6G7A+9zbZ5bmuq+JHf3ludbcd7rOTUGYB4hBGAe\nIQRgHiEEYN4V9WZJZGSkioqKvF4GAGM4IgRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4h\nBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYN4V9eFN9eH76qsm\nOW9ARJwrcyQpz+fuvKYqqFXLJjtv+ce5rs2SpNfKN7k679s4IgRgHiEEYB4hBGAeIQRgHiEEYB4h\nBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEE\nYB4hBGAeIQRg3hUTwlWrVunIkSNeLwOAQVdECGtrazVnzhxCCMATFw3hXXfdpby8PP/lhx56SKmp\nqf7Ln3zyiWJiYlRcXKzMzEwlJSUpMTFR48aN06FDh/z3i46O1ttvv6309HTFxcVp8ODBKioqkiR1\n7dpVJ06c0L333qvnnnvucu4PAC7qoiFMSkrStm3bJJ05ctu5c6dOnTqlY8eOSZLy8/MVHx+vp556\nSs2aNdOHH36o9evXq7KyUvPmzavzXEuWLNHs2bO1ceNGtWjRQgsXLpQk5ebmSjpzejxx4sTLukEA\nuJiLhrBHjx7avn27JGnnzp3q2LGjYmNj9dFHH0k6E8KePXtq8eLFmjVrlkJDQ9WsWTMlJyersLCw\nznMNGjRIUVFR+t73vqe+ffuquLi4EbYEAA0TfLE79OjRQ7/97W91+vRpbd26Vd26dVObNm300Ucf\nqX///srPz9eoUaNUWFioZ599Vp988omqq6vl8/nUtm3bOs8VGRnp//qaa67R6dOnG7zgF3Y8rag7\nOzT4cZcqz7fCtVluaqr7kpru3tYe+h+vl9Bobogo93T+RUN40003KSIiQgUFBdq6dauGDRumVq1a\nKTc3V6WlpTp16pQ6dOiglJQU3XfffcrOzlbz5s2Vk5OjnJycOs8VGPjfvzczNvbR//o56ivPt0J3\nB97n2jy3NNV9Se7uLahVS1fmSGciOLDNL12bt/zjXNdm3RBRrmP7Iy9+x8s063zqVaYePXooPz9f\n27dvV0JCgu644w6VlJTon//8p7p37659+/bp5MmT+sUvfqHmzZtLOnMaDQBXg3qHcPXq1WrTpo1a\ntGih4OBg3X777frrX/+qnj17KiIiQoGBgdq+fbuqqqr06quvqqSkRF988YVOnTp10ecPDw+XJO3b\nt0+VlZX/3Y4AoIHqFcKkpCTt27dPXbt29V+XkJCgvXv36oc//KHatm2rqVOnaubMmerXr5+Ki4u1\nYMECXX/99frxj3980edv1aqVBgwYoMmTJ2v+/PmXvhsAuAQBjuM4Xi+iIdx8baupvpbWVPcl8Rrh\n5cJrhABgDCEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEE\nYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGBesNcLAK5abdz7FDu35xVWh7k2q4+L8/pc4HqO\nCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcI\nAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZhHCAGYRwgBmEcIAZjnaQh37typ4cOHKzExUT169NDU\nqVNVWVnp5ZIAGORpCCdOnKguXbroX//6l3Jzc1VYWKg//elPXi4JgEEBjuM4Xg0/efKkQkJCFBoa\nKkmaNWuWSkpK9OKLL17wMSWFpYq6s4NbSwRgQLCXwzdt2qRFixappKRENTU1qq2tVdeuXb/zMWNj\nH3VpdVKeb4XuDrzPtXluaar7ktzdW1Dn21yZI0lrC/+ggXf+xrV5v8l9xbVZfX5QrA/33eLarPPx\n7NS4uLhYjzzyiAYNGqSNGzeqoKBAGRkZXi0HgGGeHRHu3r1bQUFBGjVqlAICAiSdefMkMJA3sgG4\ny7PqtG/fXtXV1SosLFRlZaWysrJUVVWlw4cPq7a21qtlATDIsxB26dJFI0eO1KhRozRgwACFhIRo\n9uzZOnHiBKfIAFzl6Zsl06ZN07Rp0+pct3HjRo9WA8AqXpADYB4hBGAeIQRgHiEEYB4hBGAeIQRg\nHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGAeIQRgHiEEYB4hBGCe\npx/eBFzNHJc/g9vNeV877qbB7XnfxhEhAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA\n8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8+odwvLyckVH\nR2vPnj2NuR4AcB1HhADMI4QAzGtwCHft2qXBgwcrPj5eGRkZqqiokCRt2bJFP/vZz5SQkKDevXvr\n2Weflc/n8z/ub3/7m1JTU9WlSxcNGDBA//jHP/y3DR8+XPPmzVNaWppGjBhxGbYFAPXX4BC+8sor\nys7O1vvvv6+QkBA99thj+uyzz5SZmamf/vSn2rJli5YuXao1a9botddekyStW7dOf/zjHzV37lxt\n27ZN06ZN09SpU1VcXOx/3jfffFMzZ87U0qVLL9vmAKBenHoqKytzbrvtNmf16tX+6z744AMnOjra\nycrKcoYMGVLn/n/+85+d+++/33EcxxkzZowzd+7cOrdnZmY68+fPdxzHcTIyMpzx48fXax2fFvyn\nvksGgHoJbmg4b731Vv/XHTp0kOM42rx5s3bv3q2YmJizA6tWrVpJkkpLS7Vhwwa9/PLLdW5v1qyZ\n/3JERES95o+NfbShS75keb4VujvwPtfmuaWp7ktyd2+Bd97uyhxJenvH7zUg9reuzXvs76+5Nis5\nqkjrS6Jdm3U+DQ5hYOD/n007jiNJateunUJDQ7VkyZLzPiY8PFyPPPKIxo4de8HnDQ5u8FIA4LJo\n8GuEJSUl/q9LS0sVFBSk22+/Xf/+97/rvDly5MgRnTp1StKZI8eiorol3r9/f537A4BXGhzC5cuX\n69ChQ6qsrFROTo769euntLQ0VVZWauHChaqqqtL+/fs1ZswYLV68WJKUnp6ut99+W+vWrVNNTY22\nbdumtLQ0bd68+bJvCAAaqsEhTE9P1+jRo9WnTx/V1NToySefVIsWLZSdna0PPvhASUlJGjZsmBIT\nEzV+/HhJUs+ePTVjxgzNmTNHCQkJmjFjhqZMmaKePXte9g0BQEPV+4W5yMhI/+ltamrqObd3795d\nK1euvODjH3jgAT3wwAPnvW3ZsmX1XQYAXHb8nyUAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOE\nAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMwjhADMI4QAzCOEAMzjU9WvEEGt\nWzfZeS9tW+3aLEn6a9kGV+b8vL0rY/x8hZ+4NmvOLbGuzUr2uTcv+QIfpc4RIQDzCCEA8wghAPMI\nIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wgh\nAPMIIQDzCCEA8wghAPMIIQDzPPk4z+TkZB08eFCBged2eMaMGUpPT/dgVQCs8uxzjadPn66MjAyv\nxgOAH6fGAMwjhADM8+zUeM6cOZo3b94513/88ccKCgryYEUArApwHMdxe2hycrJGjx59Sa8RlhSW\nKurODo2wKgBWeXZEeKnGxj7q2qw83wrdHXifK7OCWrd2ZY4krT24SAPbjndt3kvbVrs2q027/TpU\nEeHKrJ+37+XKHMnd30W3ubm3PN+K817Pa4QAzCOEAMy74t4s6devn7KysjxYEQCrPAnh+vXrvRgL\nAOfFqTEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzCCEA8wgh\nAPMIIQDzCCEA8wghAPMIIQDzCCEA8wghAPMIIQDzAhzHcbxeBAB4iSNCAOYRQgDmEUIA5hFCAOYR\nQgDmEUIA5v0vuCX2qTfk+RcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "# S: Symbol that shows starting of decoding input\n",
    "# E: Symbol that shows starting of decoding output\n",
    "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n",
    "sentences = ['ich mochte ein bier P', 'S i want a beer', 'i want a beer E']\n",
    "\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "number_dict = {i: w for i, w in enumerate(word_list)}\n",
    "n_class = len(word_dict)  # vocab list\n",
    "\n",
    "# Parameter\n",
    "n_hidden = 128\n",
    "\n",
    "def make_batch(sentences):\n",
    "    input_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[0].split()]]]\n",
    "    output_batch = [np.eye(n_class)[[word_dict[n] for n in sentences[1].split()]]]\n",
    "    target_batch = [[word_dict[n] for n in sentences[2].split()]]\n",
    "\n",
    "    # make tensor\n",
    "    return Variable(torch.Tensor(input_batch)), Variable(torch.Tensor(output_batch)), Variable(torch.LongTensor(target_batch))\n",
    "\n",
    "class Attention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Attention, self).__init__()\n",
    "        self.enc_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)\n",
    "        self.dec_cell = nn.RNN(input_size=n_class, hidden_size=n_hidden, dropout=0.5)\n",
    "\n",
    "        # Linear for attention\n",
    "        self.attn = nn.Linear(n_hidden, n_hidden)\n",
    "        self.out = nn.Linear(n_hidden * 2, n_class)\n",
    "\n",
    "    def forward(self, enc_inputs, hidden, dec_inputs):\n",
    "        enc_inputs = enc_inputs.transpose(0, 1)  # enc_inputs: [n_step(=n_step, time step), batch_size, n_class]\n",
    "        dec_inputs = dec_inputs.transpose(0, 1)  # dec_inputs: [n_step(=n_step, time step), batch_size, n_class]\n",
    "\n",
    "        # enc_outputs : [n_step, batch_size, num_directions(=1) * n_hidden], matrix F\n",
    "        # enc_hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n",
    "        enc_outputs, enc_hidden = self.enc_cell(enc_inputs, hidden)\n",
    "\n",
    "        trained_attn = []\n",
    "        hidden = enc_hidden\n",
    "        n_step = len(dec_inputs)\n",
    "        model = Variable(torch.empty([n_step, 1, n_class]))\n",
    "\n",
    "        for i in range(n_step):  # each time step\n",
    "            # dec_output : [n_step(=1), batch_size(=1), num_directions(=1) * n_hidden]\n",
    "            # hidden : [num_layers(=1) * num_directions(=1), batch_size(=1), n_hidden]\n",
    "            dec_output, hidden = self.dec_cell(dec_inputs[i].unsqueeze(0), hidden)\n",
    "            attn_weights = self.get_att_weight(dec_output, enc_outputs)  # attn_weights : [1, 1, n_step]\n",
    "            trained_attn.append(attn_weights.squeeze().data.numpy())\n",
    "\n",
    "            # matrix-matrix product of matrices [1,1,n_step] x [1,n_step,n_hidden] = [1,1,n_hidden]\n",
    "            context = attn_weights.bmm(enc_outputs.transpose(0, 1))\n",
    "            dec_output = dec_output.squeeze(0)  # dec_output : [batch_size(=1), num_directions(=1) * n_hidden]\n",
    "            context = context.squeeze(1)  # [1, num_directions(=1) * n_hidden]\n",
    "            model[i] = self.out(torch.cat((dec_output, context), 1))\n",
    "\n",
    "        # make model shape [n_step, n_class]\n",
    "        return model.transpose(0, 1).squeeze(0), trained_attn\n",
    "\n",
    "    def get_att_weight(self, dec_output, enc_outputs):  # get attention weight one 'dec_output' with 'enc_outputs'\n",
    "        n_step = len(enc_outputs)\n",
    "        attn_scores = Variable(torch.zeros(n_step))  # attn_scores : [n_step]\n",
    "\n",
    "        for i in range(n_step):\n",
    "            attn_scores[i] = self.get_att_score(dec_output, enc_outputs[i])\n",
    "\n",
    "        # Normalize scores to weights in range 0 to 1\n",
    "        return F.softmax(attn_scores).view(1, 1, -1)\n",
    "\n",
    "    def get_att_score(self, dec_output, enc_output):  # enc_outputs [batch_size, num_directions(=1) * n_hidden]\n",
    "        score = self.attn(enc_output)  # score : [batch_size, n_hidden]\n",
    "        return torch.dot(dec_output.view(-1), score.view(-1))  # inner product make scalar value\n",
    "\n",
    "input_batch, output_batch, target_batch = make_batch(sentences)\n",
    "\n",
    "# hidden : [num_layers(=1) * num_directions(=1), batch_size, n_hidden]\n",
    "hidden = Variable(torch.zeros(1, 1, n_hidden))\n",
    "\n",
    "model = Attention()\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# Train\n",
    "for epoch in range(2000):\n",
    "    optimizer.zero_grad()\n",
    "    output, _ = model(input_batch, hidden, output_batch)\n",
    "\n",
    "    loss = criterion(output, target_batch.squeeze(0))\n",
    "    if (epoch + 1) % 400 == 0:\n",
    "        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "# Test\n",
    "test_batch = [np.eye(n_class)[[word_dict[n] for n in 'SPPPP']]]\n",
    "test_batch = Variable(torch.Tensor(test_batch))\n",
    "predict, trained_attn = model(input_batch, hidden, test_batch)\n",
    "predict = predict.data.max(1, keepdim=True)[1]\n",
    "print(sentences[0], '->', [number_dict[n.item()] for n in predict.squeeze()])\n",
    "\n",
    "# Show Attention\n",
    "fig = plt.figure(figsize=(5, 5))\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "ax.matshow(trained_attn, cmap='viridis')\n",
    "ax.set_xticklabels([''] + sentences[0].split(), fontdict={'fontsize': 14})\n",
    "ax.set_yticklabels([''] + sentences[2].split(), fontdict={'fontsize': 14})\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Seq2Seq(Attention)-Torch.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}