F2B-19-Bi_LSTM(Attention).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 393
    },
    "colab_type": "code",
    "id": "0WS_AKj4hR22",
    "outputId": "dbc2130f-0972-4f66-edbc-fce1c94b65cd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost = 0.003327\n",
      "Epoch: 2000 cost = 0.000716\n",
      "Epoch: 3000 cost = 0.000275\n",
      "Epoch: 4000 cost = 0.000129\n",
      "Epoch: 5000 cost = 0.000067\n",
      "sorry hate you is Bad Mean...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAJkAAAEMCAYAAADXgBT+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlcVOX+B/DPAAKSxo9wY1MRBbxd\nRgaxgVwwCxeuK4YKUhHmlgviUloWXhUkveWCpZResSwMwS3NVErFBYhFERMXFg1IQBhQ2UGe3x9e\n5sU0DszMOUcWv+/Xy9cLDud5vmfwy1mf53tEjDEGQgSk09obQDo+SjIiOEoyIjhKMiI4SjIiOEoy\nIjhKMiI4SjIiOEoyIji91t4A8kRAQIDa627dulXALeEf7cnaCCMjI/k/AwMDxMXFIS8vD506dYKe\nnh7u3LmD8+fP46WXXmrtTdWYiJ5dtj1BQUFwcnLCpEmTFJZHR0cjLS0N69ata6Ut0w4lWRvk7OyM\nhIQE6Okpns3U1dXB1dUVycnJrbRl2qHDZRtkbGyMM2fOKC2Pi4tD165dW2GLuKET/zZo3rx5WLx4\nMezs7GBpaYn6+noUFBTg5s2bWL16dWtvnsbocNlG5eTkIDY2FoWFhaitrUWPHj0wYsQIiMXi1t40\njVGStUHbt2/HwoULW3szeEPnZG1QVFQUZDJZa28Gb2hP1gZFRETg1KlT8PDwgJmZmdJVppubWytt\nmXYoydoge3t7lT8TiUTIyMh4hlvDHSUZERzdwmijGGO4evUq8vPzAQB9+/bFP/7xj1beKu1QkrVB\nmZmZmDdvHvLy8vDCCy8AACoqKmBnZ4ddu3ahe/furbyFmqHDZRv07rvvwtzcHMuWLZM/EC8sLMSm\nTZtQV1fX7kZhUJK1QRKJBPHx8TA0NFRYXl5ejjFjxuDixYuttGXaoftkbVDXrl1RWVmptLyurg4i\nkagVtogbOidrg4YOHYolS5YgMDAQNjY2AICsrCxs3boVQ4YM4SVGVVWV2ut27tyZUyw6XLZBjx49\nwscff4zTp0/LlzHG4ObmhtDQUJiYmHCOYW9vr/Zeket9OUqyNqi2thb6+vp4+PAh8vPzUVtbCysr\nK15Hxf7+++/yr7OzsxEZGYlp06bB2toaDQ0NyMzMxMGDBzFr1iylwZMaY6TNcXBwYO+88w7buXMn\nS0tLYw0NDYLGe/PNN9ndu3eVlmdmZjJPT0/O/dM5WRsUFRWFlJQUJCcnY9++faipqcErr7wCV1dX\nuLq6ol+/frzGy8zMRM+ePZWWm5ubIzs7m3P/dLhsRmZmptrr9u/fX7DtyM3NxYULF/D9998jKyuL\n92eX3t7eMDMzw5w5cxQGSe7atQt//vknoqKiOPVPSdaMxpNjxpjCSfLfvwe4nxz/3YMHD5CSkiLf\no+Xk5MDOzg5OTk4IDAzkNdbdu3fx4Ycf4sqVK/LPxRhD//79sXXrVvkVrrYoyZrR+NwQAFJTUxET\nEwM/Pz+Fk+PvvvsOb7/9Nt544w3e4v7rX/9CTU0NxGIxnJyc4OjoiIEDB0JXV5e3GE9TXFysMBLX\nwsKCl37pnKwZTX/Jc+fORUREBLp16yZfZm1tDQcHB/j7+/OaZGZmZrh+/Tru3buHoqIilJSUoLKy\nUrBJJGPGjMHJkyfRrVs3hc/HF7rjr6a//voLBgYGSsuNjIxQUFDAa6xdu3bhwoUL+Oijj/B///d/\n2L9/P9zd3TFp0iRB5lyamZk9dXYUX+hwqab33nsP1dXVePfdd2FhYYHHjx/j3r17+Pbbb9GpUyfs\n3r1bkLi1tbVIT09HcnIyDh8+jLt37+L69eu8xli1ahXOnDkDc3NzmJubKx2WuT6Qp8OlmjZu3Ijg\n4GAsW7YM1dXVAAA9PT24uroiODiY11ixsbG4fPkyUlNT8ccff8DMzAwuLi5YtGgRXFxceI3V6LXX\nXhOkX4D2ZGprekVZVlaG2tpavPTSS0rj7/nw2muvQSqVwsXFBS4uLujVq9dT17t16xZsbW15j883\nSjI1SSQSXL58ubU3Q8GgQYOQlpbGS1+JiYn4+eefkZ+fD5FIhD59+mDy5Mn45z//yblvOvFX0+uv\nv44ffvihtTdDAV/7hx9//BGzZs3CvXv30LdvX/Tp0wc5OTmYMWMG4uLiOPdP52RqKisrw9atWxEW\nFoZevXopnRxHR0c/823ia2zZf//7X2zfvh0jR45UWH769Gls2bIFI0aM4NQ/JZmaHB0d4ejo2Nqb\nIYiCgoKnJtKoUaOwcuVKzv1TkqmpI5UN+DtLS0ukpKQoDYhMS0vjZdIKJZkGDh48qHBy3Lt3b0yd\nOhXu7u6tvWmc+Pn5Yc6cORg/frz8OWV2djaOHTuGxYsXc+6fkkxNX331Ffbu3QsPDw8MGzYMwJP/\niJUrV6KiogKTJ09u5S3UnpeXF7p3747o6GgcOnQItbW16N27N0JCQjB27FjuATiPSHtOjBw5kl2+\nfFlpeVJSEhs/fnwrbBFjzs7OvPQTExPD7t27x0tfT0P3ydQkkUiQnJysdFX5+PFjDBkyBKmpqZz6\nP3funNrr8l1wZcqUKbh16xYsLS3x6quvwtXVFS4uLnjxxRd56Z+STE1TpkzB/PnzMXr0aIXlsbGx\n2LZtG44ePcqp/78XWWkcx9aUjo4OOnfujJSUFE6xnqa8vBwpKSlITU1FcnIyrl+/jn79+mHo0KFY\nunQpp74pydT022+/YdGiRZBKpQonx4mJidiwYQMmTJjAW6zY2FicOHEC8+bNUxi79vXXX8PDw0Mp\n0fmWnZ2N+Ph4REZG8jISl5JMA7dv30Z0dDRyc3PlJ8eTJ0/mvcSmu7s7Dh06hC5duigsLysrw9Sp\nU/Hrr7/yGi89PV0+AvfKlSswMTGBRCKBRCKBk5MT+vTpw6l/urpUU1hYGF599VWsWLFCkIfiTZWV\nlaGiokIpyWpqavDgwQPe43l5eaFfv37w8fFBaGioUlyuaE+mpoULFyIlJQU1NTVwdnaWzxxqrmCd\ntpYtW4aMjAxMnz4dlpaW8rFrBw4cgI2NDe8FV37++WckJSUhKSkJDx48gJOTE5ydneHs7KzRJGBV\nKMk0lJWVJT+0pKSkoKqqCq6urvj88895i1FVVYUdO3YgNjYWBQUFCtWvly9fzvuepqnS0lIkJycj\nMTERFy5cQElJCZKSkjj1SUmmhYqKCqSmpiIlJQWxsbG4c+cOrl271tqbxdndu3eRmpoq/2xFRUUY\nPHgwwsPDOfVLSaamX375RT5FrbCwEA4ODvKTY7FYrFTmiQvGGM6ePYusrCz5KNym+H6O+v777yMt\nLQ0VFRVwdHSUD5Z0cHDgZYYUJZma7O3t5SfHnp6eMDIyEizWihUrcOLECfTp00cpeUUiEe/DirZu\n3QoXFxdIJBLo6+urXG/Tpk1YsWKFxv1Tkqnp6tWr+P3335GUlIS0tDRYWlrKT46dnJx4LYYyePBg\nfP/994JcVHCh7UhcSjItMMaQkZGB+Ph4REdH486dO7zOIB81ahR+/vlnXg/BfBCLxbh69arG7eg+\nmQYqKiqQlpYmPzm+cuUKevXqBW9vb17jBAQE4D//+Q+WLFki6JWkprS9lUF7MjVNmjQJmZmZMDMz\nU5hJJEQl6okTJ+Kvv/5CRUUFXnzxRejoKE7FiI+P5z2mOrQ9XNKeTE1vvfUWXFxcYGlp2ex6gYGB\n2Lx5M6dY7777Lqf2bQ3tyXjG5zS1tob2ZG0EH3+z9fX12LFjx1OHevv5+XHfSC1p+9koyXjGxzS1\nzz77DL/99hu8vb3lIyCysrKwZ88ePH78GLNmzeIcQxvTpk3Tqh0dLnnGx+Fy2LBh2Lt3r1LxuRs3\nbiAgIAAnT57k1D/w5ApWXVwfyNMM8jaoqqoKvXv3Vlrev39/lJSU8BLDyMhI/s/AwABxcXHIy8tD\np06doKenhzt37uD8+fO83GSmw2UbNGDAAERGRuLtt99WWL5//35YW1vzEmPDhg3yr4OCgrBmzRql\nUurR0dG8XMTQ4ZJn2t4Vb+ry5cvw9/dHjx49FIZ6FxQU4Msvv8TQoUP52FQ5Z2dnJCQkKA3GrKur\ng6urK5KTkzn1T4dLNW3btu2pyysqKrB27Vr590uWLOEcSyKR4Ndff8XMmTNhYWGB7t27w8vLCydO\nnOA9wQDA2Nj4qZUW4+LieCkhSnuyFshkMpSUlGDq1Kk4dOiQ0mX8nTt3sHTpUs57r78rLCyErq6u\nvIZrdnY2DA0NYW5uzmscADhw4AA+/fRT2NnZKZRYv3nzJlavXo2ZM2dy6p+SrAWHDh3Chg0b8OjR\nI5X3iUaPHq1yT6eNc+fOISAgAKGhofIZ3D/++CNCQ0Oxbds2DB8+nLdYjXJychAbG6tQ/XrEiBH8\nTJIRbNpwB1JfX88cHR1ZXl6e0r/i4mLe402YMIEdO3ZMafnp06fZxIkTeY8XFhbGe59NUZJp4PHj\nxwpf//HHH0wmk/Eex9HRkdXX1ystr62tZY6OjrzHGz58OCspKeG930Z0C0NNCQkJ+OCDDxAXF4f6\n+nr4+vriypUr0NfXR1hYGK+lA/r27YuTJ0/Cw8NDYXl0dHSLD+i14e/vj4ULF8LDwwNmZmZKV5lc\nPxudk6lp6tSpmDFjBry8vHDkyBFs2rQJUVFRuHLlCnbv3o2YmBjeYl28eBELFy6EpaUlLCwswBhD\nTk4OioqKsGfPHkgkEt5iAcolEpoSiUQ0g/xZcXJyQkpKCkQiEQIDA9G9e3d89NFHYIzB2dmZ9/oU\nhYWFOHHiBHJzcyESiWBlZYXx48fD1NSU1zjPAh0u1WRoaIiHDx/CwMAAly5dwpYtWwA8KVQixHvB\ne/bsiRkzZqCoqOipj5i4qq6ulg/vbulV0VxfD01JpiY3Nze888470NXVhYmJCVxcXFBTU4Pg4GAM\nHjyY11hlZWVYt24dfvnlF4hEIly7dg0ymQwBAQH4/PPP0aNHD84xpFKp/JGRRCJ56h8K+9+7C7ge\nLinJ1BQUFISIiAg8evQIPj4+EIlEaGhowP379xESEsJrrLVr16KiogJRUVHy+QNGRkawsrLC+vXr\nebkn1/Q1Pd9++y3n/pol2HVrByP0vaSmnJ2dWWlpKWOMMbFYLF/+6NEjJpVKn9l28IX2ZGqKioqC\nj48Pr/MrVdHT03vqdLja2lrU1NTwHu/GjRvYtGkTsrKynto/14krlGRqEvpeUlMSiQSfffYZli9f\nLl/2559/Yv369XB1deUtTqMlS5bAwsICc+fOFWSuJ93CUJPQ95KaKigowPz583Hr1i08fvwYhoaG\n8pJVmzZtUvlCL20NGjQICQkJnK8iVaE9mZpu3LjxzGL16tULGzduRHV1NXJzc1FcXIzKykrY2try\nnmDAk/FkRUVFnCsqqkJ7smY8y3tJTe3btw9hYWFITExEWVkZPDw80LVrV5SVlWHOnDm8TCRpWm27\nuLgYUVFRmDhxIiwsLJRuZ9BjJQE1nRSiquIg4+leUlPu7u744osv4ODggD179uDgwYM4fPgwsrOz\nsWDBApw6dYpzjKaH/6dV2m76M7pPJqCmV1rdunXD5s2beXv9X3OKi4vh4OAAALhw4QI8PDygq6uL\nAQMG4P79+7zEaHr4f+utt/Ddd98prdN4T5ArSrJmmJmZYdGiRbCyskJZWVmzL4N/5ZVXeItrYmKC\nzMxMGBoaIjExER988AEAID8/n9e6aOnp6bh69SouX76MH374QekPKDc3F3l5eZzjUJI1Y+PGjdi7\ndy+uXbuGx48fIz09/anr8f3s0tvbG1OnToVIJMKwYcNgZ2eHR48eYcGCBfy86+h/qqqqcP78edTX\n12PXrl1KPzc0NNRofqZKrXUXuL3x9fV9pvFSU1PZ2bNnWU1NDWPsyejc8PBwVltby3us2bNn895n\nU3TiTwRHU+KI4CjJiOAoyYjgKMmI4CjJiOCe+/tk7jpeWrX7+urnmCNeplEbHS3rSoTHr8Vc1081\nbnfi5nmN24hMj4OV/Evjdjq9bqv+mca9EQCA9T/5n9yhSt9/8D/XUhVRJ1ve+6QkI4KjJCOCoyQj\ngqMkI4KjJCOCoyQjgqMkI4JrMcny8vJgZ2eHW7duCbohBw8ehFQqFTQGaR2C78kiIiJQW1srdBgA\nTwqVLFy4EHZ2drwMGyb8EDTJZDIZQkNDUVdXJ2QYAEBmZiamTJnCS0lwwi+1k+z69euYOHEiJBIJ\nfH19kZ+fDwA4fvw4JkyYAIlEAjc3N+zcuRPAkyJuI0aMAGMMLi4uOHDgAADg6NGjGDt2LBwdHeHl\n5aX0xoszZ87A3d0dDg4OCAwMVHsvWFxcjNDQ0A73rsgOoaXx2bm5uczW1pZNnz6d5eXlsQcPHjA/\nPz82c+ZMlpubywYOHMjOnDnDGGMsLS2Nvfzyy+zChQuMMcYSEhKYra0tKy8vZ4wxlp6ezsRiMbt0\n6RKrq6tj4eHhTCqVsqqqKhYTE8PEYjELDg5mFRUVLCMjgzk4OLCjR49qNJ785s2bzNbWluXm5qq1\nfnb6XY36J5pTexSGt7c3LCwsADwpPjJ79myYmJggPj4exsbGAJ688sXa2hrXrl176pszDh8+DKlU\nKi8a4ufnB3Nzc/nhtKamBosWLYKRkRHs7e1hZ2eHzMxMzn9IzdF0JEWj0w0HNB7Boe0ojJMP/osx\nxv4at9NmFIZOr9toKBigVTtV1E6y/v37y7/u3bs3GGMoKirCyZMnERMTg8LCQjDGUFdXp/IQl5ub\nq1C9WV9fH+PHj5d/b2xsrHBOZWho+MwuGohw1E6ypi9bZ/+b4HT8+HHs3r0bYWFhcHFxgZ6eHiZP\nnqyyj8bqhOT5ovaJf05OjvzrP//8E7q6usjPz4eTkxOGDRsGPT09lJeX4+7duyr7sLKyUuinoaEB\ne/bsQWFhoZabT9oDtZMsMjISRUVFKC8vx969e+Hm5gZra2vk5OSgtLQUBQUF+OSTT2BmZiZPmsaK\nODk5OaisrISnpydSUlIQGxuLuro67Nu3D+Hh4ejSpYswn460CRqd+Pv7+yM/Px9isRghISEwMjJC\nUlISRo0ahR49emDVqlWQyWRYu3YtTE1NsWDBAjg5OWHGjBlYvHgx5syZgy1btiA0NBTLly+Hra0t\nwsPD8cILL3D+IKtXr8aRI0fkh/KxY8dCJBJh3bp1zR7CifCe+xnk2o7xp6tL5XYqf6Zxb4RoqF3M\nVpo3bx4uXryo8ucffvghfH19n+EWEU20iyRrfFRF2ic6XBLBUZIRwbWLw6WQ9Hr1fGZt86bbaB3r\nL38HjdsMOKt5vKwZwICzflq1U4X2ZERwlGREcJRkRHCUZERwlGREcJRkRHCUZERwlGREcJRkRHBU\npoAIrsOUKaiursaGDRvg5uaGwYMHw8fHB1euXBE8LmlZhylTsGnTJsTHx2Pfvn24dOkSBg0ahPnz\n56O6ulrw2KR5HaZMQZcuXbBq1SpYWVnBwMAA/v7+kMlkyMrKUvcjEqG0NMW8vZUpaJSWlsbs7e1Z\nYWFhs+vlZORr1T9RX4csU1BZWYmgoCB4eXmhR48eza47f1SIxv0DwIm/tmOc+UKN2mg71Cd9cyAc\nAjdr3K5SWqlxm6wZH8Nmf7BW7VTpcGUKZDIZ5s6dC1NTU6xevVqjtkQYap+TqSpTEB4ejqCgIKSm\npiI9PR329vYq+xC6TEFeXh6mT58OGxsb7NixA/r6+oLFIurrMGUKSktLMWvWLIwePRqhoaHo1KkT\n5z4JPzpMmYLNmzdjwIABWLFiBee+CL86TJmCAwcOQFdXFw4OimPhqUxB63vuyxRoeoXYiK4uldup\nQg/IieDaxZQ4KlPQvrWLJKMyBe0bHS6J4CjJiOAoyYjg2sU5mZBub9G+FoambU2PaP9IzahI87Zm\nm7QYxzcDsNGynSq0JyOCoyQjgqMkI4KjJCOCoyQjgqMkI4KjJCOCoyQjgqMyBURwHaZMQVFREQID\nAyGVSjFkyBD4+vri6tWrgsclLeswZQoCAgKgq6uL06dP4/z583j55Zcxd+7cZxKbNK9DlCloaGjA\nm2++iZUrV+LFF1+EoaEhpkyZAplMhqKiIrV/GUQYaifZ/v37sWPHDpw7dw6dOnXChx9+iLy8PKxY\nsQLLli3D5cuXERYWhu3bt+PixYvo2bMndu/eDQBISEiAl5cXrl27hk8++QRBQUFITk6Gu7s75s6d\nKy+KUl1djfj4eBw5cgQHDhzAr7/+ipMnT7b8IXR0MHXqVHTr1g0AUFJSgj179kAikcDc3Fyb3wvh\nU0t1DBprYRw+fFi+LC4ujtnZ2bHy8nJWVlamsP748ePZzp07GWPKtTDWrVvHZs+eLV+3pqaG/fTT\nT+zhw4csJiaG2dnZsYcPH8p//uabb7IvvvhCo7oLrq6uzNbWlvn6+rKioqIW179Z1nytDMJdhytT\ncOnSJRQXF2Pnzp3w9vbGkSNHmp1y53EyXKP+G2VO+wT9o9Zp1Mb0iJFWsRK/XwbpzM81bmd845HG\nbU6lrMHowWu0aqdKhypT0Khbt25YuXIlSkpKcPbsWcHjkeZ1iDIF+fn5GDlyJHJzc+XLdHR0wBiD\nnt5zPy6z1XWIMgXm5uYwNTVFSEgIZDIZqqursXXrVujr68PZ2ZlT34S7DlGmQCQS4auvvsKGDRvw\nxhtvQEdHB/b29vj6669hamrKqW/C3XNfpkDTk/dGdOKv3E4VekBOBNcuzoqpTEH71i6SjMoUtG90\nuCSCoyQjgqMkI4JrF+dkQhqwrFi7htM0b7s0ruURJU+3DOvXf6Nxq89dRmkVTSeP3+FRtCcjgqMk\nI4KjJCOCoyQjgqMkI4KjJCOCoyQjgqMkI4KjMgVEcB2mTEFTsbGxsLOzQ2Ji4jONS56uw5QpaFRZ\nWYnQ0FAYGWk3CpXwr0OUKWgqLCwMrq6uMDEx0agdEU6HKFPQ6ObNm/jpp5+wdOlSDX8NRFAtTTFv\nL2UKGhoa2PTp01l0dDRjjLHXXnuNJSQktNgu58ZfavVPtNdhyhQ0vrnX09NT3Y8EAJj/RqhG6zc6\nkbsV46wCNGqj7VAfd+sbOJ2jema+KtoM9fml8CuM7fm+Vu1UUTvJVJUp2L17N8LCwuDi4gI9Pb1m\nX8UsVJkCmUyGbdu2ISIiAiKRiPf+CTcdokzBuXPnUFpaipkzZ0IqlUIqleLevXt4//33sW6ddvMq\nCX/U3pNFRkbC2dkZRkZGCmUKEhMTUVpaipqaGnz22WcqyxT069cPnp6e8PLyQmxsLNzc3BAZGYnw\n8HBMmzaN04cYO3YsXF1dFZZNnz4dK1euxKuvvsqpb8JdhyhT0LlzZ3Tu3Flhma6uLl566SUYGxtz\n6ptw99yXKdD05L0Rnfgrt1OFHpATwbWL2UpUpqB9axdJRmUK2jc6XBLBUZIRwVGSEcG1i3MyId1e\n1OeZtX1//xzt4qzSrq2NcYFW8WDMrYbv39GejAiOkowIjpKMCI6SjAiOkowIjpKMCI6SjAiOkowI\njpKMCI5qYRDBCf5YKSIiAj4+PtDX1xc0zltvvYWUlBTo6urKl/Xu3RvHjx8XNC5pmaBJ1lgLw8vL\nS/AkA4D58+dj0aJFgschmulwtTBIG9TSFPPGMgXTp09neXl57MGDB8zPz4/NnDmT5ebmsoEDB7Iz\nZ84wxhhLS0tjL7/8Mrtw4QJjTLlMQXp6OhOLxezSpUusrq6OhYeHM6lUyqqqqlhMTAwTi8UsODiY\nVVRUsIyMDObg4MCOHj2q1lR4X19f5uPjwyZMmMAcHR2Zj48Py8zMbLHdzaL7avVPtKfRlDgLCwsA\ngL+/P2bPng0TExPEx8fLp52JxWJYW1vj2rVrGDp0qFIfhw8fhlQqlc+R9PPzg7m5uby0VE1NDRYt\nWgQjIyPY29vDzs4OmZmZam2fjY0NGhoaEBAQAD09PQQHB2P27Nk4ceIEDAwMVLbz2POtur8CBZkf\nLEX/jV9o1Eb0WKtQuL1qKQZs0CwWANjs0Xyozy+3NmKs7QdatVOlw9TCWLNmjcL3n376KaRSKX7/\n/XcMHz5crT6IMDpELYyn6dKlC4yNjVFUxO97gojmOkQtjPLycqxZs0ahH5lMBplMBisrK059E+7U\nTrLIyEgUFRWhvLxcoRZGTk4OSktLUVBQgE8++URlLYzKykp4enoiJSUFsbGxqKurw759+xAeHo4u\nXbgN9+3SpQvS0tKwfv16lJWV4cGDB/j3v/8NW1tbODs7c+qbcKd2kjXWwhg+fDjq6+uxZs0aeHt7\nw8bGBqNGjcI777yDSZMm4b333sOxY8ewefNmDBw4UF4LY9++fRg4cKC8FsaQIUNw7NgxXmphAMCX\nX34JxhjGjBmDkSNHoq6uDt98843CYZ60jue+FoamV4iN6OpSuZ0q9GdOBNcupsRRLYz2rV0kGdXC\naN/ocEkER0lGBEdJRgTXLs7JhNRgVf3M2hpc69zySirolWteOp7d0+6RmrbtVKE9GREcJRkRHCUZ\nERwlGREcJRkRHCUZERwlGREcJRkRHJUpIIITfE8WERHxzCboxsXFYcKECRCLxRg3bhxOnTr1TOKS\n5gmaZI1lChrnVQrpxo0bWLZsGZYsWYKkpCQsXrwYX375JSoqKgSPTZrXYcoU7N27F+PGjcPrr78O\nAwMDjBs3DkeOHOFl/gDhqKUp5u2lTIG7uzsLCwtjfn5+TCKRME9PT5aUlNRiuxulRWr1T7TXYcoU\nFBQU4MCBA9i2bRv69++P8PBwzJs3D6dPn4aJiYnKduNOfKPur0BBtvdH6BcZolEbbUdhZAQHYuDH\nmzVu13tbWssr/c3JRxEY09VPq3aqqH24VFWmIDIyEu7u7hCLxXBwcMDt27c1LlPQWJqAS5kCxhgm\nTJiAQYMG4YUXXsDixYuho6ODuLg4dT8iEUiHKVPQvXt3hfeN6+npoWfPnlSmoA3oEGUKgCd72oyM\nDPn39fX1KCgokB/iSevpEGUKAMDX1xenTp1CbGwsqqursX37dujr62PkyJGc+ybcaHTi7+/vj/z8\nfIjFYoSEhMDIyAhJSUkYNWoUevTogVWrVkEmk2Ht2rUwNTXFggUL5GUKFi9ejDlz5sjLFCxfvhy2\ntra8lSkYMWIEgoKCEBISgvsC/tmLAAAAZElEQVT378Pe3h67d++GkZER574JN899mQJNrxAb0dWl\ncjtV6AE5EVy7mK1EZQrat3aRZFSmoH2jwyURHCUZERwlGRHcc38LgwiP9mREcJRkRHCUZERwlGRE\ncJRkRHCUZERw/w+BHfskq/G6tgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "dtype = torch.FloatTensor\n",
    "\n",
    "# Bi-LSTM(Attention) Parameters\n",
    "embedding_dim = 2\n",
    "n_hidden = 5 # number of hidden units in one cell\n",
    "num_classes = 2  # 0 or 1\n",
    "\n",
    "# 3 words sentences (=sequence_length is 3)\n",
    "sentences = [\"i love you\", \"he loves me\", \"she likes baseball\", \"i hate you\", \"sorry for that\", \"this is awful\"]\n",
    "labels = [1, 1, 1, 0, 0, 0]  # 1 is good, 0 is not good.\n",
    "\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "vocab_size = len(word_dict)\n",
    "\n",
    "inputs = []\n",
    "for sen in sentences:\n",
    "    inputs.append(np.asarray([word_dict[n] for n in sen.split()]))\n",
    "\n",
    "targets = []\n",
    "for out in labels:\n",
    "    targets.append(out) # To using Torch Softmax Loss function\n",
    "\n",
    "input_batch = Variable(torch.LongTensor(inputs))\n",
    "target_batch = Variable(torch.LongTensor(targets))\n",
    "\n",
    "class BiLSTM_Attention(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BiLSTM_Attention, self).__init__()\n",
    "\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim, n_hidden, bidirectional=True)\n",
    "        self.out = nn.Linear(n_hidden * 2, num_classes)\n",
    "\n",
    "    # lstm_output : [batch_size, n_step, n_hidden * num_directions(=2)], F matrix\n",
    "    def attention_net(self, lstm_output, final_state):\n",
    "        hidden = final_state.view(-1, n_hidden * 2, 1)   # hidden : [batch_size, n_hidden * num_directions(=2), 1(=n_layer)]\n",
    "        attn_weights = torch.bmm(lstm_output, hidden).squeeze(2) # attn_weights : [batch_size, n_step]\n",
    "        soft_attn_weights = F.softmax(attn_weights, 1)\n",
    "        # [batch_size, n_hidden * num_directions(=2), n_step] * [batch_size, n_step, 1] = [batch_size, n_hidden * num_directions(=2), 1]\n",
    "        context = torch.bmm(lstm_output.transpose(1, 2), soft_attn_weights.unsqueeze(2)).squeeze(2)\n",
    "        return context, soft_attn_weights.data.numpy() # context : [batch_size, n_hidden * num_directions(=2)]\n",
    "\n",
    "    def forward(self, X):\n",
    "        input = self.embedding(X) # input : [batch_size, len_seq, embedding_dim]\n",
    "        input = input.permute(1, 0, 2) # input : [len_seq, batch_size, embedding_dim]\n",
    "\n",
    "        hidden_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden]\n",
    "        cell_state = Variable(torch.zeros(1*2, len(X), n_hidden)) # [num_layers(=1) * num_directions(=2), batch_size, n_hidden]\n",
    "\n",
    "        # final_hidden_state, final_cell_state : [num_layers(=1) * num_directions(=2), batch_size, n_hidden]\n",
    "        output, (final_hidden_state, final_cell_state) = self.lstm(input, (hidden_state, cell_state))\n",
    "        output = output.permute(1, 0, 2) # output : [batch_size, len_seq, n_hidden]\n",
    "        attn_output, attention = self.attention_net(output, final_hidden_state)\n",
    "        return self.out(attn_output), attention # model : [batch_size, num_classes], attention : [batch_size, n_step]\n",
    "\n",
    "model = BiLSTM_Attention()\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# Training\n",
    "for epoch in range(5000):\n",
    "    optimizer.zero_grad()\n",
    "    output, attention = model(input_batch)\n",
    "    loss = criterion(output, target_batch)\n",
    "    if (epoch + 1) % 1000 == 0:\n",
    "        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))\n",
    "\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "# Test\n",
    "test_text = 'sorry hate you'\n",
    "tests = [np.asarray([word_dict[n] for n in test_text.split()])]\n",
    "test_batch = Variable(torch.LongTensor(tests))\n",
    "\n",
    "# Predict\n",
    "predict, _ = model(test_batch)\n",
    "predict = predict.data.max(1, keepdim=True)[1]\n",
    "if predict[0][0] == 0:\n",
    "    print(test_text,\"is Bad Mean...\")\n",
    "else:\n",
    "    print(test_text,\"is Good Mean!!\")\n",
    "    \n",
    "fig = plt.figure(figsize=(6, 3)) # [batch_size, n_step]\n",
    "ax = fig.add_subplot(1, 1, 1)\n",
    "ax.matshow(attention, cmap='viridis')\n",
    "ax.set_xticklabels(['']+['first_word', 'second_word', 'third_word'], fontdict={'fontsize': 14}, rotation=90)\n",
    "ax.set_yticklabels(['']+['batch_1', 'batch_2', 'batch_3', 'batch_4', 'batch_5', 'batch_6'], fontdict={'fontsize': 14})\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Bi-LSTM(Attention)-Torch.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}