NLP/Transformer/transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dquM-bfUIW-J"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from tqdm import tqdm, tqdm_notebook, trange\n",
    "import sentencepiece as spm\n",
    "import wget\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "8pjohaiGId95",
    "outputId": "7e5d8ddc-d2f3-4059-b53b-d1d1304aa746"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "config.json\n",
      "config_half.json\n",
      "kowiki.csv.gz\n",
      "kowiki.model\n",
      "kowiki.txt\n",
      "kowiki.vocab\n",
      "kowiki_bert_0.json\n",
      "kowiki_gpt.json\n",
      "ratings_test.json\n",
      "ratings_test.txt\n",
      "ratings_train.json\n",
      "ratings_train.txt\n",
      "save_bert_pretrain.pth\n",
      "save_gpt_pretrain.pth\n"
     ]
    }
   ],
   "source": [
    "data_dir = \"../data\"\n",
    "for f in os.listdir(data_dir):\n",
    "    print(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "u69gqsv_IkrL",
    "outputId": "3609e01b-9c36-4009-9e0d-d5d9ac0a7e7e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# vocab loading\n",
    "vocab_file = f\"{data_dir}/kowiki.model\"\n",
    "vocab = spm.SentencePieceProcessor()\n",
    "vocab.load(vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IHGknKGNI9SU"
   },
   "outputs": [],
   "source": [
    "\"\"\" configuration json을 읽어들이는 class \"\"\"\n",
    "class Config(dict): \n",
    "    __getattr__ = dict.__getitem__\n",
    "    __setattr__ = dict.__setitem__\n",
    "\n",
    "    @classmethod\n",
    "    def load(cls, file):\n",
    "        with open(file, 'r') as f:\n",
    "            config = json.loads(f.read())\n",
    "            return Config(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "x3UQU0e8JM0b",
    "outputId": "202d1af5-c2c5-421f-bb0c-fc6ba16320eb",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_enc_vocab': 8007, 'n_dec_vocab': 8007, 'n_enc_seq': 256, 'n_dec_seq': 256, 'n_layer': 6, 'd_hidn': 256, 'i_pad': 0, 'd_ff': 1024, 'n_head': 4, 'd_head': 64, 'dropout': 0.1, 'layer_norm_epsilon': 1e-12}\n"
     ]
    }
   ],
   "source": [
    "config = Config({\n",
    "    \"n_enc_vocab\": len(vocab),\n",
    "    \"n_dec_vocab\": len(vocab),\n",
    "    \"n_enc_seq\": 256,\n",
    "    \"n_dec_seq\": 256,\n",
    "    \"n_layer\": 6,\n",
    "    \"d_hidn\": 256,\n",
    "    \"i_pad\": 0,\n",
    "    \"d_ff\": 1024,\n",
    "    \"n_head\": 4,\n",
    "    \"d_head\": 64,\n",
    "    \"dropout\": 0.1,\n",
    "    \"layer_norm_epsilon\": 1e-12\n",
    "})\n",
    "print(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sinusoid_encoding_table(n_seq, d_hidn):\n",
    "    def cal_angle(position, i_hidn):\n",
    "        return position / np.power(100000, 2 * (i_hidn // 2) / d_hidn)\n",
    "    def get_posi_angle_vec(position):\n",
    "        return [cal_angle(position, i_hidn) for i_hidn in range(d_hidn)]\n",
    "    \n",
    "    sinusoid_table = np.array([get_posi_angle_vec(i+seq) for i_seq in range(n_seq)])\n",
    "    sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2])\n",
    "    sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2])\n",
    "    \n",
    "    return sinusoid_table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### size() 함수를 쓰기 위해서 torch로 q와 k를 만들어준다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['▁겨울', '은', '▁추', '워', '요', '.']\n",
      "['▁감', '기', '▁조', '심', '하', '세', '요', '.']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 8])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = [\n",
    "  \"겨울은 추워요.\",\n",
    "  \"감기 조심하세요.\"\n",
    "]\n",
    "\n",
    "inputs = []\n",
    "for line in lines:\n",
    "    pieces = vocab.encode_as_pieces(line)\n",
    "    ids = vocab.encode_as_ids(line)\n",
    "    inputs.append(torch.tensor(ids))\n",
    "    print(pieces)\n",
    "# 입력 길이가 다르므로 입력 최대 길이에 맟춰 padding(0)을 추가 해 줌\n",
    "k = torch.nn.utils.rnn.pad_sequence(inputs, batch_first=True, padding_value=0)\n",
    "k.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['▁겨울', '은', '▁추', '워', '요', '.']\n",
      "['▁감', '기', '▁조', '심', '하', '세', '요', '.']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 8])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = [\n",
    "  \"겨울은 추워요.\",\n",
    "  \"감기 조심하세요.\"\n",
    "]\n",
    "\n",
    "inputs = []\n",
    "for line in lines:\n",
    "    pieces = vocab.encode_as_pieces(line)\n",
    "    ids = vocab.encode_as_ids(line)\n",
    "    inputs.append(torch.tensor(ids))\n",
    "    print(pieces)\n",
    "# 입력 길이가 다르므로 입력 최대 길이에 맟춰 padding(0)을 추가 해 줌\n",
    "q = torch.nn.utils.rnn.pad_sequence(inputs, batch_first=True, padding_value=0)\n",
    "q.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_attn_pad_mask(seq_q, seq_k, i_pad):\n",
    "    batch_size, len_q = seq_q.size()\n",
    "    batch_size, len_k = seq_k.size()\n",
    "    pad_attn_mask = seq_k.data.eq(i_pad).unsqueeze(1).expand(batch_size, len_q, len_k)\n",
    "    return pad_attn_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False]],\n",
       "\n",
       "        [[False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False]]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#torch.Size([2, 8]), torch.Size([2, 8])\n",
    "get_attn_pad_mask(q, k , 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False]],\n",
       "\n",
       "        [[False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False],\n",
       "         [False, False, False, False, False, False, False, False, False]]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#torch.Size([2, 9]), torch.Size([2, 8])\n",
    "get_attn_pad_mask(q, k , 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 함수를 통해서 위와 같이 크기를 맞줘줘서 나중에 내적을 쉽게 할 수 있다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IJbvIXsNjloN"
   },
   "source": [
    "### Decoder Mask를 구하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------------------\n",
    "\n",
    "unsqueeze(-1) : 가로 새로 바구기\n",
    "```\n",
    "torch.unsqueeze(x, 0)\n",
    "tensor([[ 1,  2,  3,  4]])\n",
    "torch.unsqueeze(x, 1)\n",
    "tensor([[ 1],\n",
    "        [ 2],\n",
    "        [ 3],\n",
    "        [ 4]])\n",
    "```\n",
    "-----------\n",
    "\n",
    "torch.ones_like : 가로안 만큼 1 만들기\n",
    "\n",
    "\n",
    "------------------\n",
    "\n",
    "triu(diagonal=1) : 뭉쳐 있는 배열들을 보기 쉽게 아래로 내려주기\n",
    "```\n",
    "np.triu([[1,2,3],[4,5,6],[7,8,9],[10,11,12]], -1)\n",
    "array([[ 1,  2,  3],\n",
    "       [ 4,  5,  6],\n",
    "       [ 0,  8,  9],\n",
    "       [ 0,  0, 12]])\n",
    "```\n",
    "--------------------\n",
    "\n",
    "torch.triu(a, diagonal=1) # 아래는 다 0으로 조지기\n",
    "```\n",
    "tensor([[ 0.0000,  0.5207,  2.0049],\n",
    "        [ 0.0000,  0.0000,  0.6602],\n",
    "        [ 0.0000,  0.0000,  0.0000]])\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dttn_decoder_mask(seq):\n",
    "    # 빈 마스크를 만들어주고 (크기만큼만)\n",
    "    subsequent_mask = torch.ones_like(seq).unsqueeze(-1).expand(seq.size(0), seq.size(1), seq.size(1))\n",
    "    # 행렬의 위쪽 삼각형 부분(2-D)\n",
    "    subsequent_mask = subsequent_mask.triu(diagonal=1)\n",
    "    return subsequent_mask"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "B7rndBaMj16n"
   },
   "source": [
    "###### ScaledDotProductAttention\n",
    "![](https://raw.githubusercontent.com/paul-hyun/paul-hyun.github.io/master/assets/2019-12-19/scale_dot_product_attention.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[3091, 3604,  206, 3958, 3760, 3590,    0,    0],\n",
       "         [ 212, 3605,   53, 3832, 3596, 3682, 3760, 3590]]),\n",
       " tensor([[3091,  212],\n",
       "         [3604, 3605],\n",
       "         [ 206,   53],\n",
       "         [3958, 3832],\n",
       "         [3760, 3596],\n",
       "         [3590, 3682],\n",
       "         [   0, 3760],\n",
       "         [   0, 3590]]))"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q, q.transpose(-1, -2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# masked_fill_ 역할은 위치에 있는 것들 제거하는거"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "In [3]: mask = torch.tensor([[[1, 0]], [[0, 1]]], dtype=torch.uint8)\n",
    "\n",
    "In [4]: mask.shape\n",
    "Out[4]: torch.Size([2, 1, 2])\n",
    "\n",
    "In [5]: x = torch.randn(2, 2)\n",
    "\n",
    "In [6]: x, mask = torch.broadcast_tensors(x, mask)\n",
    "\n",
    "In [7]: mask.shape\n",
    "Out[7]: torch.Size([2, 2, 2])\n",
    "\n",
    "In [8]: x.is_contiguous()\n",
    "Out[8]: False\n",
    "\n",
    "In [9]: x.masked_fill(mask, 0)  # This should be the correct behavior\n",
    "Out[9]: \n",
    "tensor([[[ 0.0000,  0.9482],\n",
    "         [ 0.0000,  0.0719]],\n",
    "\n",
    "        [[ 0.6267,  0.0000],\n",
    "         [-0.6296,  0.0000]]])\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ScaledDotProductAttention(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "        self.dropout = nn.Dropout(config.dropout)\n",
    "        self.scale = 1 / (self.config.d_head ** 0.5)\n",
    "    \n",
    "    def forward(self, Q, K, V, attn_mask):\n",
    "        scores = torch.matmul(Q, K.transpose(-1, -2)).mul_(self.scale)\n",
    "        scores.masked_fill_(attn_mask, -1e9)\n",
    "        \n",
    "        attn_prob = nn.Softmax(dim=-1)(scores)\n",
    "        attn_prob = self.dropout(attn_prob)\n",
    "        \n",
    "        context = torch.matmul(attn_prob, V)\n",
    "        return context, attn_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zJMLVMjkj7_1"
   },
   "source": [
    "###### MultiHeadAttention\n",
    "![](https://raw.githubusercontent.com/paul-hyun/paul-hyun.github.io/master/assets/2019-12-19/multi_head_attention.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "self.config.n_head * self.config.d_head 만큼 들어오고 \n",
    "\n",
    "\n",
    "self.config.d_hidn 만큼 멀티해드 어텐션 저 위에 h가 이것이여"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "        \n",
    "        self.W_Q = nn.Linear(self.config.d_hidn, self.config.n_head * self.config.d_head)\n",
    "        self.W_K = nn.Linear(self.config.d_hidn, self.config.n_head * self.config.d_head)\n",
    "        self.W_V = nn.Linear(self.config.d_hidn, self.config.n_head * self.config.d_head)\n",
    "        \n",
    "        self.scaled_dot_attn = ScaledDotProductAttention(self.config)\n",
    "        self.linear = nn.Linear(self.config.n_head * self.config.d_head, self.config.d_hidn)\n",
    "        self.dropout = nn.Dropout(config.dropout)\n",
    "\n",
    "    def forward(self, Q, K, V, attn_mask):\n",
    "        batch_size = Q.size(0)\n",
    "        # (bs, n_head, n_q_seq, d_head) / view는 크기를 조절하는 거 일단 batch는 고정하고 seq는 자유롭게 r그리고 n_head 고정\n",
    "        # 마무리로 늘려주기 까지\n",
    "        q_s = self.W_Q(Q).view(batch_size, -1, self.config.n_head, self.config.d_head).transpose(1,2)\n",
    "        # (bs, n_head, n_k_seq, d_head)\n",
    "        k_s = self.W_K(K).view(batch_size, -1, self.config.n_head, self.config.d_head).transpose(1,2)\n",
    "        # (bs, n_head, n_v_seq, d_head)\n",
    "        v_s = self.W_V(V).view(batch_size, -1, self.config.n_head, self.config.d_head).transpose(1,2)\n",
    "        # (bs, n_head, n_q_seq, n_k_seq)\n",
    "        # unsqueeze(1) 살려두는 것들은 1로 한 줄로 만드는건가\n",
    "        attn_mask = attn_mask.unsqueeze(1).repeat(1, self.config.n_head, 1, 1)\n",
    "        # (bs, n_head, n_q_seq, d_head), (bs, n_head, n_q_seq, n_k_seq)\n",
    "        context, attn_prob = self.scaled_dot_attn(q_s, k_s, v_s, attn_mask)\n",
    "        # (bs, n_head, n_q_seq, h_head * d_head)\n",
    "        context = context.transpose(1, 2).contiguous().view(batch_size, -1, self.config.n_head * self.config.d_head)\n",
    "        # (bs, n_head, n_q_seq, e_embd)\n",
    "        output = self.linear(context)\n",
    "        output = self.dropout(output)\n",
    "        # (bs, n_q_seq, d_hidn), (bs, n_head, n_q_seq, n_k_seq)\n",
    "        return output, attn_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "sizes (torch.Size or python:int...) – The number of times to repeat this tensor along each dimension\n",
    "\n",
    "Example:\n",
    "```\n",
    ">>> x = torch.tensor([1, 2, 3])\n",
    ">>> x.repeat(4, 2)\n",
    "tensor([[ 1,  2,  3,  1,  2,  3],\n",
    "        [ 1,  2,  3,  1,  2,  3],\n",
    "        [ 1,  2,  3,  1,  2,  3],\n",
    "        [ 1,  2,  3,  1,  2,  3]])\n",
    ">>> x.repeat(4, 2, 1).size()\n",
    "torch.Size([4, 2, 3])\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "df1dKmZSkEiV"
   },
   "source": [
    "###### PoswiseFeedForwardNet\n",
    "![](https://raw.githubusercontent.com/paul-hyun/paul-hyun.github.io/master/assets/2019-12-19/feed-forward.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PoswiseFeedForwardNet(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "\n",
    "        self.conv1 = nn.Conv1d(in_channels=self.config.d_hidn, out_channels=self.config.d_ff, kernel_size=1)\n",
    "        self.conv2 = nn.Conv1d(in_channels=self.config.d_ff, out_channels=self.config.d_hidn, kernel_size=1)\n",
    "        self.active = F.gelu\n",
    "        self.dropout = nn.Dropout(config.dropout)\n",
    "        \n",
    "    def forward(self, inputs):\n",
    "        # (bs, d_ff, n_seq)\n",
    "        output = self.active(self.conv1(inputs.transpose(1, 2)))\n",
    "        # (bs, n_seq, d_hidn)\n",
    "        output = self.conv2(output).transpose(1, 2)\n",
    "        output = self.dropout(output)\n",
    "        # (bs, n_seq, d_hidn)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KQaP14UDkgpV"
   },
   "source": [
    "#### 7. Encoder\n",
    "![](https://raw.githubusercontent.com/paul-hyun/paul-hyun.github.io/master/assets/2019-12-19/encoder.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4HxrA0mdkoFu"
   },
   "source": [
    "###### EncoderLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EncoderLayer(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "\n",
    "        self.self_attn = MultiHeadAttention(self.config)\n",
    "        self.layer_norm1 = nn.LayerNorm(self.config.d_hidn, eps=self.config.layer_norm_epsilon)\n",
    "        self.pos_ffn = PoswiseFeedForwardNet(self.config)\n",
    "        self.layer_norm2 = nn.LayerNorm(self.config.d_hidn, eps=self.config.layer_norm_epsilon)\n",
    "    \n",
    "    def forward(self, inputs, attn_mask):\n",
    "        # (bs, n_enc_seq, d_hidn), (bs, n_head, n_enc_seq, n_enc_seq)\n",
    "        att_outputs, attn_prob = self.self_attn(inputs, inputs, inputs, attn_mask)\n",
    "        att_outputs = self.layer_norm1(inputs + att_outputs)\n",
    "        # (bs, n_enc_seq, d_hidn)\n",
    "        ffn_outputs = self.pos_ffn(att_outputs)\n",
    "        ffn_outputs = self.layer_norm2(ffn_outputs + att_outputs)\n",
    "        # (bs, n_enc_seq, d_hidn), (bs, n_head, n_enc_seq, n_enc_seq)\n",
    "        return ffn_outputs, attn_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "2OqU2L7IlDO_"
   },
   "source": [
    "###### Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "\n",
    "        self.enc_emb = nn.Embedding(self.config.n_enc_vocab, self.config.d_hidn)\n",
    "        # 멀티헤드 어텐션으로 위치 찾아주기\n",
    "        sinusoid_table = torch.FloatTensor(get_sinusoid_encoding_table(self.config.n_enc_seq + 1, self.config.d_hidn))\n",
    "        # 포지션 임베딩 때리고\n",
    "        self.pos_emb = nn.Embedding.from_pretrained(sinusoid_table, freeze = True)\n",
    "        # 레이어 수만큼 반복\n",
    "        self.layers = nn.ModuleList([EncoderLayer(self.config) for _ in range(self.config.n_layer)])\n",
    "        \n",
    "    def forward(self, inputs):\n",
    "        positions = torch.arange(inputs.size(1), device=inputs.device, dtype=inputs.dtype).expand(inputs.size(0), inputs.size(1)).contiguous() + 1\n",
    "        pos_mask = inputs.eq(self.config.i_pad)\n",
    "        positions.masked_fill_(pos_mask, 0)\n",
    "        \n",
    "        # (bs, n_enc_seq, d_hidn)\n",
    "        outputs = self.enc_emb(inputs) + self.pos_emb(positions)\n",
    "        # (bs, n_enc_seq, n_enc_seq)\n",
    "        attn_mask = get_attn_pad_mask(inputs, inputs, self.config.i_pad)\n",
    "        \n",
    "        attn_probs = []\n",
    "        for layer in self.layers:\n",
    "            # (bs, n_enc_seq, d_hidn), (bs, n_head, n_enc_seq, n_enc_seq)\n",
    "            outputs, attn_prob = layer(outputs, attn_mask)\n",
    "            attn_probs.append(attn_prob)\n",
    "        # (bs, n_enc_seq, d_hidn), [(bs, n_head, n_enc_seq, n_enc_seq)]\n",
    "        return outputs, attn_probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tYxaY_hYlYcv"
   },
   "source": [
    "#### 8. Decoder\n",
    "![](https://raw.githubusercontent.com/paul-hyun/paul-hyun.github.io/master/assets/2019-12-19/decoder.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DecoderLayer(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "        \n",
    "        self.self_attn = MultiHeadAttention(self.config)\n",
    "        self.layer_norm1 = nn.LayerNorm(self.config.d_hidn, eps=self.config.layer_norm_epsilon)\n",
    "        self.dec_enc_attn = MultiHeadAttention(self.config)\n",
    "        self.layer_norm2 = nn.LayerNorm(self.config.d_hidn, eps=self.config.layer_norm_epsilon)\n",
    "        self.pos_ffn = PoswiseFeedForwardNet(self.config)\n",
    "        self.layer_norm3 = nn.LayerNorm(self.config.d_hidn, eps=self.config.layer_norm_epsilon)\n",
    "\n",
    "    def forward(self, dec_inputs, enc_outputs, self_attn_mask, dec_enc_attn_mask):\n",
    "        # (bs, n_dec_seq, d_hidn), (bs, n_head, n_dec_seq, n_dec_seq)\n",
    "        self_att_outputs, self_attn_prob = self.self_attn(defc_inputs, dec_inputs, dec_inputs, self_attn_mask)\n",
    "        self_att_outputs = self.layer_nrom1(dec_inputsn + self_att_outputs)\n",
    "        # (bs, n_dec_seq, d_hidn), (bs, n_head, n_dec_seq, n_enc_seq)\n",
    "        dec_enc_att_outputs, dec_enc_attn_prob = self.dec_enc_attn(self_att_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n",
    "        dec_enc_att_outputs = self.layer_norm2(self_att_outputs + dec_enc_att_outputs)\n",
    "        # (bs, n_dec_seq, d_hidn)\n",
    "        ffn_outputs = self.pos_ffn(dec_enc_att_outputs)\n",
    "        ffn_outputs = self.layer_norm3(dec_enc_att_outputs + fnn_outputs)\n",
    "        # (bs, n_dec_seq, d_hidn), (bs, n_head, n_dec_seq, n_dec_seq), (bs, n_head, n_dec_seq, n_enc_seq)\n",
    "        return ffn_outputs, self_attn_prob, dec_enc_attn_prob"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FVw019y4lkxM"
   },
   "source": [
    "###### Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5d7mwC7clnG5"
   },
   "outputs": [],
   "source": [
    "\"\"\" decoder \"\"\"\n",
    "class Decoder(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "\n",
    "        self.dec_emb = nn.Embedding(self.config.n_dec_vocab, self.config.d_hidn)\n",
    "        sinusoid_table = torch.FloatTensor(get_sinusoid_encoding_table(self.config.n_dec_seq + 1, self.config.d_hidn))\n",
    "        self.pos_emb = nn.Embedding.from_pretrained(sinusoid_table, freeze=True)\n",
    "\n",
    "        self.layers = nn.ModuleList([DecoderLayer(self.config) for _ in range(self.config.n_layer)])\n",
    "    \n",
    "    def forward(self, dec_inputs, enc_inputs, enc_outputs):\n",
    "        positions = torch.arange(dec_inputs.size(1), device=dec_inputs.device, dtype=dec_inputs.dtype).expand(dec_inputs.size(0), dec_inputs.size(1)).contiguous() + 1\n",
    "        pos_mask = dec_inputs.eq(self.config.i_pad)\n",
    "        positions.masked_fill_(pos_mask, 0)\n",
    "    \n",
    "        # (bs, n_dec_seq, d_hidn)\n",
    "        dec_outputs = self.dec_emb(dec_inputs) + self.pos_emb(positions)\n",
    "\n",
    "        # (bs, n_dec_seq, n_dec_seq)\n",
    "        dec_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs, self.config.i_pad)\n",
    "        # (bs, n_dec_seq, n_dec_seq)\n",
    "        dec_attn_decoder_mask = get_attn_decoder_mask(dec_inputs)\n",
    "        # (bs, n_dec_seq, n_dec_seq)\n",
    "        dec_self_attn_mask = torch.gt((dec_attn_pad_mask + dec_attn_decoder_mask), 0)\n",
    "        # (bs, n_dec_seq, n_enc_seq)\n",
    "        dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs, self.config.i_pad)\n",
    "\n",
    "        self_attn_probs, dec_enc_attn_probs = [], []\n",
    "        for layer in self.layers:\n",
    "            # (bs, n_dec_seq, d_hidn), (bs, n_dec_seq, n_dec_seq), (bs, n_dec_seq, n_enc_seq)\n",
    "            dec_outputs, self_attn_prob, dec_enc_attn_prob = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n",
    "            self_attn_probs.append(self_attn_prob)\n",
    "            dec_enc_attn_probs.append(dec_enc_attn_prob)\n",
    "        # (bs, n_dec_seq, d_hidn), [(bs, n_dec_seq, n_dec_seq)], [(bs, n_dec_seq, n_enc_seq)]S\n",
    "        return dec_outputs, self_attn_probs, dec_enc_attn_probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YevWC1osltnH"
   },
   "source": [
    "#### 9. Transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hvs28anxl3UX"
   },
   "outputs": [],
   "source": [
    "\"\"\" transformer \"\"\"\n",
    "class Transformer(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "\n",
    "        self.encoder = Encoder(self.config)\n",
    "        self.decoder = Decoder(self.config)\n",
    "    \n",
    "    def forward(self, enc_inputs, dec_inputs):\n",
    "        # (bs, n_enc_seq, d_hidn), [(bs, n_head, n_enc_seq, n_enc_seq)]\n",
    "        enc_outputs, enc_self_attn_probs = self.encoder(enc_inputs)\n",
    "        # (bs, n_seq, d_hidn), [(bs, n_head, n_dec_seq, n_dec_seq)], [(bs, n_head, n_dec_seq, n_enc_seq)]\n",
    "        dec_outputs, dec_self_attn_probs, dec_enc_attn_probs = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n",
    "        # (bs, n_dec_seq, n_dec_vocab), [(bs, n_head, n_enc_seq, n_enc_seq)], [(bs, n_head, n_dec_seq, n_dec_seq)], [(bs, n_head, n_dec_seq, n_enc_seq)]\n",
    "        return dec_outputs, enc_self_attn_probs, dec_self_attn_probs, dec_enc_attn_probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-fW_lLz_zONS"
   },
   "source": [
    "#### 10. Naver 영화 분류 모델"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8xM6W55dzPrZ"
   },
   "outputs": [],
   "source": [
    "\"\"\" naver movie classfication \"\"\"\n",
    "class MovieClassification(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.config = config\n",
    "\n",
    "        self.transformer = Transformer(self.config)\n",
    "        self.projection = nn.Linear(self.config.d_hidn, self.config.n_output, bias=False)\n",
    "    \n",
    "    def forward(self, enc_inputs, dec_inputs):\n",
    "        # (bs, n_dec_seq, d_hidn), [(bs, n_head, n_enc_seq, n_enc_seq)], [(bs, n_head, n_dec_seq, n_dec_seq)], [(bs, n_head, n_dec_seq, n_enc_seq)]\n",
    "        dec_outputs, enc_self_attn_probs, dec_self_attn_probs, dec_enc_attn_probs = self.transformer(enc_inputs, dec_inputs)\n",
    "        # (bs, d_hidn)\n",
    "        dec_outputs, _ = torch.max(dec_outputs, dim=1)\n",
    "        # (bs, n_output)\n",
    "        logits = self.projection(dec_outputs)\n",
    "        # (bs, n_output), [(bs, n_head, n_enc_seq, n_enc_seq)], [(bs, n_head, n_dec_seq, n_dec_seq)], [(bs, n_head, n_dec_seq, n_enc_seq)]\n",
    "        return logits, enc_self_attn_probs, dec_self_attn_probs, dec_enc_attn_probs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EDx-7ZE-0C64"
   },
   "source": [
    "#### 11. 네이버 영화 분류 데이터\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5hRUATPWRo1L"
   },
   "outputs": [],
   "source": [
    "\"\"\" 영화 분류 데이터셋 \"\"\"\n",
    "class MovieDataSet(torch.utils.data.Dataset):\n",
    "    def __init__(self, vocab, infile):\n",
    "        self.vocab = vocab\n",
    "        self.labels = []\n",
    "        self.sentences = []\n",
    "\n",
    "        line_cnt = 0\n",
    "        with open(infile, \"r\") as f:\n",
    "            for line in f:\n",
    "                line_cnt += 1\n",
    "\n",
    "        with open(infile, \"r\") as f:\n",
    "            for i, line in enumerate(tqdm(f, total=line_cnt, desc=f\"Loading {infile}\", unit=\" lines\")):\n",
    "                data = json.loads(line)\n",
    "                self.labels.append(data[\"label\"])\n",
    "                self.sentences.append([vocab.piece_to_id(p) for p in data[\"doc\"]])\n",
    "    \n",
    "    def __len__(self):\n",
    "        assert len(self.labels) == len(self.sentences)\n",
    "        return len(self.labels)\n",
    "    \n",
    "    def __getitem__(self, item):\n",
    "        return (torch.tensor(self.labels[item]),\n",
    "                torch.tensor(self.sentences[item]),\n",
    "                torch.tensor([self.vocab.piece_to_id(\"[BOS]\")]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DjDhfnnWR2hi"
   },
   "outputs": [],
   "source": [
    "\"\"\" movie data collate_fn \"\"\"\n",
    "def movie_collate_fn(inputs):\n",
    "    labels, enc_inputs, dec_inputs = list(zip(*inputs))\n",
    "\n",
    "    enc_inputs = torch.nn.utils.rnn.pad_sequence(enc_inputs, batch_first=True, padding_value=0)\n",
    "    dec_inputs = torch.nn.utils.rnn.pad_sequence(dec_inputs, batch_first=True, padding_value=0)\n",
    "\n",
    "    batch = [\n",
    "        torch.stack(labels, dim=0),\n",
    "        enc_inputs,\n",
    "        dec_inputs,\n",
    "    ]\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "qIJqTZswR_Q5",
    "outputId": "7a82edb4-54a5-4447-a0ff-32db6cf262a5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading ../data/ratings_train.json: 100%|███████████████████████████████| 149995/149995 [00:09<00:00, 16154.50 lines/s]\n",
      "Loading ../data/ratings_test.json: 100%|██████████████████████████████████| 49997/49997 [00:02<00:00, 18014.23 lines/s]\n"
     ]
    }
   ],
   "source": [
    "\"\"\" 데이터 로더 \"\"\"\n",
    "batch_size = 128\n",
    "train_dataset = MovieDataSet(vocab, f\"{data_dir}/ratings_train.json\")\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=movie_collate_fn)\n",
    "test_dataset = MovieDataSet(vocab, f\"{data_dir}/ratings_test.json\")\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=movie_collate_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YI3VfPuVS6s8"
   },
   "source": [
    "#### 11. 네이버 영화 분류 데이터 학습"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LewCUJDHTJjd"
   },
   "outputs": [],
   "source": [
    "\"\"\" 모델 epoch 평가 \"\"\"\n",
    "def eval_epoch(config, model, data_loader):\n",
    "    matchs = []\n",
    "    model.eval()\n",
    "\n",
    "    n_word_total = 0\n",
    "    n_correct_total = 0\n",
    "    with tqdm_notebook(total=len(data_loader), desc=f\"Valid\") as pbar:\n",
    "        for i, value in enumerate(data_loader):\n",
    "            labels, enc_inputs, dec_inputs = map(lambda v: v.to(config.device), value)\n",
    "\n",
    "            outputs = model(enc_inputs, dec_inputs)\n",
    "            logits = outputs[0]\n",
    "            _, indices = logits.max(1)\n",
    "\n",
    "            match = torch.eq(indices, labels).detach()\n",
    "            matchs.extend(match.cpu())\n",
    "            accuracy = np.sum(matchs) / len(matchs) if 0 < len(matchs) else 0\n",
    "\n",
    "            pbar.update(1)\n",
    "            pbar.set_postfix_str(f\"Acc: {accuracy:.3f}\")\n",
    "    return np.sum(matchs) / len(matchs) if 0 < len(matchs) else 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yc9AcaiYTKK2"
   },
   "outputs": [],
   "source": [
    "\"\"\" 모델 epoch 학습 \"\"\"\n",
    "def train_epoch(config, epoch, model, criterion, optimizer, train_loader):\n",
    "    losses = []\n",
    "    model.train()\n",
    "\n",
    "    with tqdm_notebook(total=len(train_loader), desc=f\"Train {epoch}\") as pbar:\n",
    "        for i, value in enumerate(train_loader):\n",
    "            labels, enc_inputs, dec_inputs = map(lambda v: v.to(config.device), value)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(enc_inputs, dec_inputs)\n",
    "            logits = outputs[0]\n",
    "\n",
    "            loss = criterion(logits, labels)\n",
    "            loss_val = loss.item()\n",
    "            losses.append(loss_val)\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            pbar.update(1)\n",
    "            pbar.set_postfix_str(f\"Loss: {loss_val:.3f} ({np.mean(losses):.3f})\")\n",
    "    return np.mean(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "62PrcR_qTs3G",
    "outputId": "51788523-7911-4cdb-b186-52a1671adb41"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_enc_vocab': 8007, 'n_dec_vocab': 8007, 'n_enc_seq': 256, 'n_dec_seq': 256, 'n_layer': 6, 'd_hidn': 256, 'i_pad': 0, 'd_ff': 1024, 'n_head': 4, 'd_head': 64, 'dropout': 0.1, 'layer_norm_epsilon': 1e-12, 'device': device(type='cuda'), 'n_output': 2}\n"
     ]
    }
   ],
   "source": [
    "config.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "config.n_output = 2\n",
    "print(config)\n",
    "\n",
    "learning_rate = 5e-5\n",
    "n_epoch = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mATLq-JjUAa5"
   },
   "outputs": [],
   "source": [
    "model = MovieClassification(config)\n",
    "model.to(config.device)\n",
    "\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "best_epoch, best_loss, best_score = 0, 0, 0\n",
    "losses, scores = [], []\n",
    "for epoch in range(n_epoch):\n",
    "    loss = train_epoch(config, epoch, model, criterion, optimizer, train_loader)\n",
    "    score = eval_epoch(config, model, test_loader)\n",
    "\n",
    "    losses.append(loss)\n",
    "    scores.append(score)\n",
    "\n",
    "    if best_score < score:\n",
    "        best_epoch, best_loss, best_score = epoch, loss, score\n",
    "print(f\">>>> epoch={best_epoch}, loss={best_loss:.5f}, socre={best_score:.5f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 621
    },
    "colab_type": "code",
    "id": "A0WVadcVJr58",
    "outputId": "1cf071c0-9da3-4413-eb8c-62d68b116dea"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loss</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.484429</td>\n",
       "      <td>0.802268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.403221</td>\n",
       "      <td>0.817009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.375795</td>\n",
       "      <td>0.812249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.355647</td>\n",
       "      <td>0.826450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.334026</td>\n",
       "      <td>0.829990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.313187</td>\n",
       "      <td>0.827850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.291726</td>\n",
       "      <td>0.834270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.268704</td>\n",
       "      <td>0.834070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.245113</td>\n",
       "      <td>0.834710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.220757</td>\n",
       "      <td>0.833990</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       loss     score\n",
       "0  0.484429  0.802268\n",
       "1  0.403221  0.817009\n",
       "2  0.375795  0.812249\n",
       "3  0.355647  0.826450\n",
       "4  0.334026  0.829990\n",
       "5  0.313187  0.827850\n",
       "6  0.291726  0.834270\n",
       "7  0.268704  0.834070\n",
       "8  0.245113  0.834710\n",
       "9  0.220757  0.833990"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtAAAAEGCAYAAABM2KIzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de3xcZ33v++9vbhrd5YskXyTHDjg3\nSIJBTgtsQhvIhVtCNm2ctNktl0N6oEkpcChwCu0u+5xX2WQfdulumjZls6EbaOKmoTvsBJJyKWm4\nRbJzw3biGEMs+SZZtm62ZiSNfuePNSONRjOSJtZ4RtLn/XrNa9Za86w1zzhj65tHv/U85u4CAAAA\nsDChcncAAAAAWEoI0AAAAEARCNAAAABAEQjQAAAAQBEI0AAAAEARIuXuQLHWrl3rmzdvLnc3AAAA\nsMzt2rXrhLs35x5fcgF68+bN6urqKnc3AAAAsMyZ2Yv5jlPCAQAAABSBAA0AAAAUgQANAAAAFIEA\nDQAAABSBAA0AAAAUgQANAAAAFIEADQAAABRhyc0DDQBARZqclFJJaSIhTSSzHomZz5k2qfE8F7Gc\nXXuJbeZ6PU/7+dqcq2vIJffgWZreLnhMBY753MfKem3N8X457202/eeU2Z561sz9vG2ynzXPa2dz\nfr73z7Nf1Pk5zy9/kyoJARoAsPS5S6mxucNrqkCgndE2O+TO1zYnFKfGyv2ngGXDNBWyIVlI+tNT\n5e7FDARoAOeeuzQ5EYzATU7MfqTGpclU1rGs/alzUunjmXMm8refdb2s8xf0/md5LZ+ULCyFwsEP\nAUs/h7K3wznHQ3nOsZx24ZzjueeE8lx7AefM6ucinmOh4M9sViDNN0Kbe3wsT5jNCcVny0JSpFqK\nxKRIXIpUTT+Hq4Lnmrqs4/G52844nqdtKDJz9M8XEJhmtfGzfL1SruEqOIo530ho3nNyj73Ea886\nVsJrz/WbBM8d9Z5nFHwho+tndX6+95+jzYLOzz6m2edXGAI0gMLcpbHTUmJAGj0ljQ7k2U7vZ7aT\nQ7PDZG649cnyfaZQVApHg/ASCgf7oUjwCEemt3Mf4agUrU7vR9PnRua/lkzyVPCZJ1PpH4Sp9PZk\n1rbntFvgOZMTwcjnjHMmc9pltrOOz3qfAuecS7lBdEYgjUvxxqzX8oXU3PC6kLZZx8L8SESFmi9g\n45zjXwtgJRhPzAy5cwbinO3JicLXtbBU3STFm4LnmtXSqs1SOLaAQJq9ny+Qztc+svAQO3UO900X\nbd7QnX18Af9DEI7lH6ENRwkIAJYMAjSwVKTGC4/6zrc956+3TYo3SNWrpoNww8ZgfyocF9iuqif0\nLHehkIIJm/hxAQAZ/IuIwORk8Kv30VN5HgOzj42fSY8kxYKRo6nn6MzjoWhOm9x2WcdDkTzXjAWj\niLnHQznvtVRGFidTUmJwgSPAgzOPj43Mfe1Y/czR4LUvzxN+m2YG5epVUlVDMHILAAAWhAC93KQm\nskLYfI+sdomBuetSY3XTQax6VVCLODkejIpOJKbrXlPjQT3m1PPYdI1mKe9Qt3CeUF4gbC96gI9m\n1QnnjADnhuPEkOa8ISJSPTPkNm2S1l+eP/zOCMeNQT8AAEDJEaAr1XiiiCCcFYiTQ3NfN96YDl3p\nx6rzZu7ne8SbgrvHz1b2DU8zwnZ6fzJP+E5NzGyTGpsO7jOOzxPcs9tMJKXk8ALeM/nSPmcoOnPU\nt26d1HzR3KUQme1I1dn/OQMAgJIqaYA2s+skfUFSWNIX3f2zOa9vkvQVSU3pNp9w94dL2adzKjOD\nQbGjwaOnpInRwte18MyAW7dOar54jhCcNWpczl/Vm02P/i4F7unpyRYQ2mM10yPD0RrqggEAWMZK\nFqDNLCzpLklXS+qR1GlmD7r73qxmn5K0093vNrNLJD0saXOp+vSSTU5KycECgXeeUeLJfCtNpYWr\nglkLMkF39RapelvWiGSBBzdunRtm6fINflEDAACmlTIZXCHpgLsflCQzu1fSDZKyA7RLakhvN0o6\nUsL+vHQvPi595R2FX8+tD265aP6yiOpVwZyyAAAAWFJKGaA3SurO2u+R9Cs5bf6jpEfN7A5JtZLe\nnO9CZnabpNskadOmTYve0XmtvUC69s8L1Ac3Lk59MAAAAJaEcv9u+hZJX3b3/8/MXivpf5rZK91n\nTgfh7vdIukeSOjo6zv2ajvXrpNd+8Jy/LQAAACpPKSfPPSypPWu/LX0s2/sk7ZQkd/+xpLiktSXs\nEwAAAHBWShmgOyVtNbMtZhaTdLOkB3PaHJL0Jkkys4sVBOi+EvYJAAAAOCslC9DuPiHpdkmPSNqn\nYLaNPWb2GTO7Pt3so5Leb2ZPS/oHSe9293NfogEAAAAsUElroNNzOj+cc+xPsrb3Snp9KfsAAAAA\nLKZSlnAAAAAAyw4BGgAAACgCARoAAAAoAgEaAAAAKAIBGgAAACgCARoAAAAoAgEaAAAAKAIBGgAA\nACgCARoAAAAoAgEaAAAAKAIBGgAAACgCARoAAAAoAgEaAAAAKAIBGgAAACgCARoAAAAoAgEaAAAA\nKAIBGgAAACgCARoAAAAoAgEaAAAAKAIBGgAAACgCARoAAAAoAgEaAAAAKAIBGgAAAChCSQO0mV1n\nZs+b2QEz+0Se1/+rmT2Vfuw3s4FS9gcAAAA4W5FSXdjMwpLuknS1pB5JnWb2oLvvzbRx9w9ntb9D\n0rZS9QcAAABYDKUcgb5C0gF3P+juY5LulXTDHO1vkfQPJewPAAAAcNZKGaA3SurO2u9JH5vFzM6T\ntEXS90rYHwAAAOCsVcpNhDdLut/dU/leNLPbzKzLzLr6+vrOcdcAAACAaaUM0IcltWftt6WP5XOz\n5ijfcPd73L3D3Tuam5sXsYsAAABAcUoZoDslbTWzLWYWUxCSH8xtZGYXSVol6ccl7AsAAACwKEoW\noN19QtLtkh6RtE/STnffY2afMbPrs5reLOled/dS9QUAAABYLCWbxk6S3P1hSQ/nHPuTnP3/WMo+\nAAAAAIupUm4iBAAAAJYEAjQAAABQBAI0AAAAUAQCNAAAAFAEAjQAAABQBAI0AAAAUAQCNAAAAFAE\nAjQAAABQBAI0AAAAUAQCNAAAAFAEAjQAAABQBAI0AAAAUAQCNAAAAFAEAjQAAABQBAI0AAAAUAQC\nNAAAAFAEAjQAAABQBAI0AAAAUAQCNAAAAFAEAjQAAABQBAI0AAAAUAQCNAAAAFAEAjQAAABQBAI0\nAAAAUISSBmgzu87MnjezA2b2iQJtbjKzvWa2x8y+Xsr+AAAAAGcrUqoLm1lY0l2SrpbUI6nTzB50\n971ZbbZK+qSk17v7KTNrKVV/AAAAgMVQyhHoKyQdcPeD7j4m6V5JN+S0eb+ku9z9lCS5e28J+wMA\nAACctVIG6I2SurP2e9LHsl0g6QIz+6GZ/cTMrst3ITO7zcy6zKyrr6+vRN0FAAAA5lfumwgjkrZK\n+jVJt0j6OzNrym3k7ve4e4e7dzQ3N5/jLgIAAADTShmgD0tqz9pvSx/L1iPpQXcfd/dfSNqvIFAD\nAAAAFamUAbpT0lYz22JmMUk3S3owp80/Kxh9lpmtVVDScbCEfQIAAADOSskCtLtPSLpd0iOS9kna\n6e57zOwzZnZ9utkjkvrNbK+k70v6mLv3l6pPAAAAwNkydy93H4rS0dHhXV1d5e4GAAAAljkz2+Xu\nHbnHy30TIQAAALCkEKABAACAIhCgAQAAgCIQoAEAAIAiEKABAACAIhCgAQAAgCLMG6DNrNXM/ruZ\nfSu9f4mZva/0XQMAAAAqz0JGoL+sYMGTDen9/ZL+sFQdAgAAACrZQgL0WnffKWlSmlphMFXSXgEA\nAAAVaiEB+rSZrZHkkmRmvyppsKS9AgAAACpUZAFtPiLpQUkvM7MfSmqW9Bsl7RUAAABQoeYN0O6+\n28zeKOlCSSbpeXcfL3nPAAAAgAo0b4A2s9/JOfRqM5O7/32J+gQAAABUrIWUcGzP2o5LepOk3ZII\n0AAAAFhxFlLCcUf2vpk1Sbq3ZD0CAAAAKthLWYnwtKQti90RAAAAYClYSA30N5Wewk5B4L5E0s5S\ndgoAAACoVAupgf4vWdsTkl50954S9QcAAACoaAupgf7BuegIAAAAsBQUDNBmNqzp0o0ZL0lyd28o\nWa8AAACAClUwQLt7/bnsCAAAALAULKQGWpJkZi0K5oGWJLn7oZL0CAAAAKhg805jZ2bXm9kLkn4h\n6QeSfinpWyXuFwAAAFCRFjIP9H+S9KuS9rv7FgUrEf5kIRc3s+vM7HkzO2Bmn8jz+rvNrM/Mnko/\n/o+ieg8AAACcYwsp4Rh3934zC5lZyN2/b2Z/Md9JZhaWdJekqyX1SOo0swfdfW9O0/vc/fbiuw4A\nAACcewsJ0ANmVifp3yR9zcx6FaxGOJ8rJB1w94OSZGb3SrpBUm6ABgAAAJaMgiUcZnaXmf07BaH3\njKQ/lPRtST+X9I4FXHujpO6s/Z70sVzvMrNnzOx+M2sv0JfbzKzLzLr6+voW8NYAAABAacxVA71f\n0p2S9kj6rKRL3f0r7v6X7t6/SO//TUmb3f0ySf8i6Sv5Grn7Pe7e4e4dzc3Ni/TWAAAAQPEKBmh3\n/4K7v1bSGyX1S/qSmT1nZn9iZhcs4NqHJWWPKLelj2W/R7+7J9O7X5T0mqJ6DwAAAJxj887C4e4v\nuvt/dvdtkm6RdKOkfQu4dqekrWa2xcxikm6W9GB2AzNbn7V7/QKvCwAAAJTNQuaBjpjZO8zsawrm\nf35e0r+f7zx3n5B0u6RHFATjne6+x8w+Y2bXp5v9gZntMbOnJf2BpHe/xM9RUscGE3rnXT/UV3/y\nooYS4+XuDgAAAMrI3D3/C2ZXKxhxfqukJyTdK+l/uftCZuAomY6ODu/q6jqn7/lU94A+fv8zev74\nsOLRkN566Xrt6GjXFVtWy8zOaV8AAABwbpjZLnfvmHV8jgD9PUlfl/RP7n6qxP1bsHIEaElydz3d\nM6j7Orv1zaePaCQ5oS1ra/WbHW36jVe3qaUhPv9FAAAAsGQUHaArVbkCdLYzYxN6+Nlj2tnZrSd+\neVLhkOnXLmjWTdvbddVFLYqGF7LAIwAAACoZAbpEDvaN6B939ej+XT3qG05qbV2V3vXqjbppe7te\n1lxX7u4BAADgJSJAl9hEalL/+nyf7uvq1vee61Vq0tVx3irdtL1db7t0vWqrFrLoIwAAACoFAfoc\n6h1O6IHdh7Wzs1sHT5xWbSysd1y+QTdtb9e29iZuPAQAAFgCCNBl4O7qevGU7uvs1kPPHNXoeEpb\nW+q0Y3u7bty2UWvqqsrdRQAAABRAgC6z4cS4/vczR3VfZ7ee6h5QNGx688Wtuml7u67c2qxwiFFp\nAACASkKAriD7jw/rvs5ufePJwzp5ekzrG+P6jde06Tdf065Na2rK3T0AAACIAF2RxiYm9Z19x3Vf\nZ7cee6FP7tLrXrZGO7a369pXrFM8Gi53FwEAAFYsAnSFOzIwqvt39WhnV7d6To2qIR7RO7dt1E0d\n7XrlxsZydw8AAGDFIUAvEZOTrh8f7Nd9nd369p5jGpuY1Cs2NGjH9nbdcPlGNdZEy91FAACAFYEA\nvQQNnhnXPz91WPd1dmvv0SHFIiG95ZXrdFNHu157/hqFuPEQAACgZAjQS9zPDg9qZ1e3/vnJwxpK\nTKh9dbV+8zXt+o3XtGlDU3W5uwcAALDsEKCXicR4So/sOab7Orv1o5/3K2TSG7Y2a8f2dr354lbF\nIqFydxEAAGBZIEAvQ4f6z+gfd3XrH7t6dGwoodW1Md24baN2bG/XBa315e4eAADAkkaAXsZSk67H\nXujTzs5ufWffcY2nXK9qb9KO7e16x+UbVFcVKXcXAQAAlhwC9ArRP5LUN54Mbjx8oXdE1dGw3nbZ\neu3Y3q6O81bJjBsPAQAAFoIAvcK4u57sHtDOzm598+kjOj2W0vnNtbqpo13//tUb1VIfL3cXAQAA\nKhoBegU7nZzQQ88e1c7ObnW9eErhkOmqi1q0o6Ndv3ZhsyJhbjwEAADIRYCGJOlA74j+cVe3/mnX\nYZ0YSaqlvkrvek2bbupo15a1teXuHgAAQMUgQGOG8dSkvv9cr3Z2dev7z/cpNem6Ystq7eho11sv\nXa/qWLjcXQQAACgrAjQKOj6U0D/t7tHOzm79sv+M6qsieserNmhHR7sua2vkxkMAALAiEaAxL3fX\nE784qfu6uvXws0eVGJ/URevqdVNHu965baNW18bK3UUAAIBzpiwB2syuk/QFSWFJX3T3zxZo9y5J\n90va7u5zpmMC9LkxlBjXN58+op2d3Xq6Z1CxcEhXX9Kqm7a369+9fK3CIUalAQDA8nbOA7SZhSXt\nl3S1pB5JnZJucfe9Oe3qJT0kKSbpdgJ05Xnu2JDu6+zWN548rIEz49rYVK1rX7FOW9bWqH118NjY\nVK14lLppAACwfBQK0KVcou4KSQfc/WC6A/dKukHS3px2/0nSf5b0sRL2BWfhonUN+tN3vEKfeMtF\n+pe9x3VfZ7e++tMXNTYxOaPduoa42ldXB6F6VY02rc4E7Gq11scVYtQaAAAsA6UM0BsldWft90j6\nlewGZvZqSe3u/pCZFQzQZnabpNskadOmTSXoKhaiKhLW2y/boLdftkGTk66+kaQOnTyj7pNn0s+j\n6j51Rj/+eb++MXRY2b/ciEVCamuqngrUm9IhOzOC3VgdLd8HAwAAKEIpA/SczCwk6fOS3j1fW3e/\nR9I9UlDCUdqeYSFCIVNrQ1ytDXFt37x61uvJiZQOnxpV96lRdadDdvepIGg/1T2gwdHxGe0b4hFt\nWjMzVLevCoL2xlXVqopQHgIAACpDKQP0YUntWftt6WMZ9ZJeKelf09OkrZP0oJldP18dNCpfVSSs\n85vrdH5zXd7XB0fH1X3yjHpOTY9eHzp5Rs8fH9Z39/VqLDVdHmKWLg9ZVaO2rNHrTOBuqa+iPAQA\nAJwzpQzQnZK2mtkWBcH5Zkm/lXnR3Qclrc3sm9m/Svq/CM8rQ2N1VI0bG/XKjY2zXpucdPUOJ4MR\n6/7pkeuek6P60YF+fWM4T3nIquqsuusgZLetojwEAAAsvpIFaHefMLPbJT2iYBq7L7n7HjP7jKQu\nd3+wVO+NpS0UMq1rjGtd49zlIYdOnplRInLo5Bk9eeiUhhITM9o3Vkfz1l23r6qmPAQAABSNhVSw\n7AyeGVf3qaybG0+lb3A8eUY9p0bzl4dMhevqqdlDNq2uUXMd5SEAAKxU5ZjGDiiLxpqoGmsKl4cc\nH05M1Vx3TwXsM/rhgRM6NpSY0T4WCal9Ve7UfNVTo9gNccpDAABYaQjQWFFCIdP6xmqtb6zWFVtm\nl4ckxlM6PDCarrkOSkQyddi7Xjyl4ZzykKaaqNpX1WhdY1zN9VVqqa9KP0/vr62rUiwSOlcfEQAA\nlBgBGsgSj4b1suY6vazQ7CHp8pAZ81+n67B3vXhKJ0+P5T1vVU10Rqhuznq01MfV0hBs11dFlJ6V\nBgAAVCgCNFCEucpDJGlsYlL9p5PqHUqqbzip3uHMc2Jq/xcnTqtvODmjFjsjHg1Nj2DXVQXBOvOc\nCdv1VVpdG1MkzKg2AADlQIAGFlEsEpoqEZmLu2twdLxgyO4dSupA34h+9PMTs2YVkYKbH9fUVuWU\njcwuH2lpqFJNjL/mAAAsJn6yAmVgZmqqiampJqatrfVztk2Mp9Q3nFTfSHpkeySpvqFEVvBO6vlj\nwzoxktTE5OxZdWpjYbU0BCPazdkj2nVVU8dbGqq0uibGjCMAACwAARqocPFoeGrWj7lMTrpOnRmb\nEawzI9uZ7b1HhtQ3nNRIcvaodjhkWlsXm1WrPV2zHZ/ajkeZOxsAsHIRoIFlIhQyramr0pq6Kl28\nfu62Z8YmZobsnBHtY4MJPdMzqP7TSeWbKr4hHplVLtKcLhlpqY+rtSEY3eamSADAckSABlagmlhE\n562J6Lw1tXO2m0hN6uTpsbx12pnnp7oH1DucUGJ89k2RNbGwWhuCket1jfFZ263pGUgY0QYALCUE\naAAFRcIhtTTE1dIQn7Odu2skOTF1A2TvcELHhxI6NpjU8eGEeocS2n3olI4PJTU2MTtoN9VEtS79\nPq3pgJ293doQ19q6KoWp0QYAVAACNICzZmaqj0dVH48WnENbCoL2wJlxHR9O6PhQUscHg6B9fDgI\n273DCT1/LKjTzr0fMmRSc31VMHLdEJSJtNbH1do4vb+uIa7G6ihlIwCAkiJAAzhnzEyramNaVRvT\nResKt5tITar/9Fh6FDuh4+k67cx298kz6vzlSQ2cGZ91biwSmgrTLQ1xrcuE7angHewzvR8A4KXi\nJwiAihMJh6bC7mVthdslxlPqHUqmR7SDgN07nJza3ntkSN/b16vR8dSsc+vjkemR7PR7ZcJ2Jng3\n11cpyoI1AIAcBGgAS1Y8GtamNTXatKbwFH/uruHkhHqHgrKRYBQ7od6s7Z8ePKnjQ4lZ82gHC9bE\nZpaN5NlmDm0AWFkI0ACWNTNTQzyqhnhUL28pvGjN5KTr5JmgbOR4VtjuHZ7efqZnQCdGxmadGw1b\nsMx6unSktSFnOz3lX0M10/oBwHJAgAYABfNor62r0tq6Kr1iQ2PBdmMTk+obCcpEsuuyM8H7hd4R\nPX7ghIbzLMFeFQnNnCs7Hbpb0ovUZMJ2Uw03QgJAJSNAA0ARYpGQNjZVa2NT9ZztTicngpCdni87\ns1hNELyD5df/bf8JDedZFTIWDk0tTNM6FbKD2uzsoL2K0hEAKAsCNACUQG1VROc31+n8Oab1k4JV\nIYO5s5NT5SK9wwn1pW+O/HnfiH708xMayjOiHQ2bmuuq1JyeM7vQ6PaaWoI2ACwmAjQAlFFNLKLN\nayPavHbuVSET4yn1ZUaws54zC9e82B9M7Xcqz9R+4VAQtAuNZGdC9xoWqwGABSFAA8ASEI+G1b66\nRu2rC884IknJiUzQTqpveGbZyPHhpHpOjerJQwPqPz37ZsiQSWvrZo5kN+fUZwerQsYUYXo/ACsY\nARoAlpGqSFhtq2rUtmruoD02MakTI8mc0ezE1Ih2MOvIoPpPJ+U5q0JmpvfLlInkq9NuaYirua5K\nsQhBG8DyQ4AGgBUoFglpQ1O1NsxzM+REalInRsZm1GdPjW6n67T3HhnSiZHZy69L0ura2Ixgnb1Y\nTWY+7bV1LFgDYGkhQAMACoqEQ1rXGNe6xvic7VKTrv5ZI9rJqUVr+oYT2n9sWH0jSaXyLlhTpXWN\nmdHs7IVqWLAGQOUpaYA2s+skfUFSWNIX3f2zOa//n5J+X1JK0oik29x9byn7BABYfOGQBaPMDXG9\ncmPhebRTk67+0+lwnV6wZnrxmoSODib0dIEFayIhmxrNbk0vVNOSuzIkC9YAOAfMc4vbFuvCZmFJ\n+yVdLalHUqekW7IDspk1uPtQevt6SR909+vmum5HR4d3dXXNODY+Pq6enh4lEolF/hSVLR6Pq62t\nTdFotNxdAYBFlanRPpZesOZ4VuDO1GgfH0rknd4vHg1NhemWnJHslvpgNL21oUo1MX4JC2BuZrbL\n3Ttyj5fyX48rJB1w94PpDtwr6QZJUwE6E57TaiW9pDTf09Oj+vp6bd68ecWMOri7+vv71dPToy1b\ntpS7OwCwqBZaoz06lpqqy84eyc7s7zkypO/u69XoeGrWufVVkayAnTOSnTWXdlUkXKqPCWCJKmWA\n3iipO2u/R9Kv5DYys9+X9BFJMUlX5buQmd0m6TZJ2rRp06zXE4nEigrPkmRmWrNmjfr6+srdFQAo\nm+pYWOetqdV5awrPo+3uGkmvDHk8p3QkE76f+MVJ9Q4nNJ6aPY6zqiY6K2C3pBevCUazg8VqmNoP\nWDnK/vsrd79L0l1m9luSPiXpd/O0uUfSPVJQwpHvOispPGesxM8MAMUyM9XHo6qPR/XylvqC7dxd\np86MT41i9w4FJSTZpSPPHRtS3/DsGUcyc2i3Zs0yMmPGkfS82iy/DiwPpQzQhyW1Z+23pY8Vcq+k\nu0vYHwAACjIzra6NaXVtTBevbyjYLjXpOjGSM5Kd2R5OqOfUqHYfGtDJPIvVRMM2tUhNa0NQj72+\nMa51jdXBc3qkm/mzgcpWygDdKWmrmW1REJxvlvRb2Q3MbKu7v5DefZukF7RE1dXVaWRkpNzdAACU\nWDhkUyUdc8leFbJ3KJEezQ62jw8ntP/4sH6wv09nxmbXZ6+tq9KGpiBQzwjY6cDd2hBXPEptNlAu\nJQvQ7j5hZrdLekTBNHZfcvc9ZvYZSV3u/qCk283szZLGJZ1SnvINAACWooWsCunuGk5O6NhgMIXf\nscHR9HOw/2L/Gf3kYH/e2UZW18ayAnb+oM1MI0BplPRvlrs/LOnhnGN/krX9ocV+zz/75h7tPTI0\nf8MiXLKhQX/6jlcsqK2764/+6I/0rW99S2amT33qU9qxY4eOHj2qHTt2aGhoSBMTE7r77rv1ute9\nTu973/vU1dUlM9N73/teffjDH17UvgMAKpeZqSEeVUM8qgtaC9dnn05O6NhQomDQfrI7f8lIY3V0\nZsBuqM4J3HHVx5kKFSgW/2u6yB544AE99dRTevrpp3XixAlt375dV155pb7+9a/r2muv1R//8R8r\nlUrpzJkzeuqpp3T48GH97Gc/kyQNDAyUufcAgEpUWxXRy5rr9LLmuoJtEuOp6YA9NDNgHxtM6GeH\ngyXXc9VVRbIC9syR7PVNca1vqGZxGiDHsgvQCx0pLpXHH39ct9xyi8LhsFpbW/XGN75RnZ2d2r59\nu9773vdqfHxc73znO/WqV71K559/vg4ePKg77rhDb3vb23TNNdeUte8AgKUrHg1r89pabV5beEq/\nsYlJHU/XY+cbyX7h+An1DidmzTJSHQ1PjVjPKBfJuhFydW2MkI0VY9kF6Ep15ZVX6rHHHtNDDz2k\nd7/73frIRz6i3/md39HTTz+tRx55RH/zN3+jnTt36ktf+lK5uwoAWKZikZDaV9eofXXhuuyJ1KT6\nRpI6MpAJ1qPBc7qE5KcHT+r4UEITOSk7FglpXVagXt84u1xkbW0V0/hhWSBAL7I3vOEN+tu//Vv9\n7u/+rk6ePKnHHntMd955pw3dznAAAA1fSURBVF588UW1tbXp/e9/v5LJpHbv3q23vvWtisViete7\n3qULL7xQt956a7m7DwBY4SLhUDr8Fl4FMjXp6h9J6mh2TXZWjfaThwb0rcFjGktNzrx2egaTQjc+\nbmisVnN9lcKEbFQ4AvQiu/HGG/XjH/9Yl19+ucxMn/vc57Ru3Tp95Stf0Z133qloNKq6ujr9/d//\nvQ4fPqz3vOc9mpwM/oH58z//8zL3HgCA+YVDppb0ioyXt+dv4+46eXosq0RkZrnIniND+s6+40qM\nT866dmt9ldY3BcF6Q1O11jXEtaEpPardxEg2ys/c8y7sV7E6Ojq8q6trxrF9+/bp4osvLlOPymsl\nf3YAwNLm7hocHQ/KRdI3Ph4dSOjI4KiODgS12kcGRpWcmBmyo+FgJHtDOlBnRq+nAndjsLw6Ndk4\nW2a2y907co8zAg0AAMrCzNRUE1NTTUyXbMi/+mNmifUjA6NTI9lHBhM6OhA8P3loQMcGE7PKRWKR\n0NTMIhuaMrOKBDc+rm8KAndTTZSQjZeEAA0AACpW9hLrr9zYmLfN5KSr//TYVJnI0YHg+Ui6PvuJ\nX+S/8TEeDU3d7Li+sTpY/bFxemR7fWO1GuJM4YfZCNAAAGBJC4VMzfVVaq6v0mVt+dukJl0nMjc+\nDkyPYh8dCp5/9PMTOj40ewq/mlh4qjQkc9PjhsxodvpGSBajWXkI0AAAYNkLp2cAaW2I61XtTXnb\nZE/hdzRdh30kPY3fkcGEnj/Wp76RpHJvH6uviqRrsdPhemoEe3pkm2XVlxf+awIAACh3Cr9VeduM\nTUyqdziYSeTIwPTMIpntvUcGdWKk8LLqmVrsDXlGs+PRcIk/IRYLARoAAGCBYpGQ2lbVqG1V4cVo\nkhMpHR9MBrOJDE7PLnJ0cFRHBhJ6umdQJ0/PDtmraqJTI9bBc7C9sala65uq1VpfpUg4VMqPhwUi\nQAMAACyiqkhYm9bUaNOawiE7MZ6accNj9uwiPaeCGx+HEhMzzgmZtK4hPYKdDtcbGmduM7PIuUGA\nrkATExOKRPhPAwDAchWPhrVlba22rK0t2GYkOaGjA6M6PDA6VZcdbI/qmZ4BPfKz2dP3VUfDQZhu\nqp4ZrtOhm1KRxbH8Utq3PiEde3Zxr7nuUuktn52zyenTp3XTTTepp6dHqVRKn/70p3X++efrQx/6\nkE6fPq2qqip997vfVTQa1Qc+8AF1dXUpEono85//vH79139dX/7yl/XAAw9oZGREqVRKP/jBD3Tn\nnXdq586dSiaTuvHGG/Vnf/Zni/u5AABAxaqrimhra722ttbnfX1y0nXidDK42TEdtDO12UcGRvXc\nsWH1DSdnnbemNjYVrNc3VmtjJlyny0Wa61jpcT7LL0CXybe//W1t2LBBDz30kCRpcHBQ27Zt0333\n3aft27draGhI1dXV+sIXviAz07PPPqvnnntO11xzjfbv3y9J2r17t5555hmtXr1ajz76qF544QU9\n8cQTcnddf/31euyxx3TllVeW82MCAIAKEQqZWurjaqmP6/ICM4skJ1Lpmxyng/WRdMg+2Hdaj79w\nQqfHUjPOmVrpsSkTrmcH7YYVPnXf8gvQ84wUl8qll16qj370o/r4xz+ut7/97WpqatL69eu1fft2\nSVJDQ7DC0uOPP6477rhDknTRRRfpvPPOmwrQV199tVavXi1JevTRR/Xoo49q27ZtkqSRkRG98MIL\nBGgAALBgVZGwzltTq/PW5C8VcXcNJSbSs4iM6nA6aB9Nl40UWoSmvioyFaYzQTszX/bGpmq1NsQV\niyzfGx6XX4AukwsuuEC7d+/Www8/rE996lO66qqrir5Gbe30l9vd9clPflK/93u/t5jdBAAAmGJm\naqyOqrE6qovX519OPTXp6htOpktE0qPYmRHtwVE9k2dWETOpua4qz82O0zXZa2pjS/aGRwL0Ijly\n5IhWr16tW2+9VU1NTfrrv/5rHT16VJ2dndq+fbuGh4dVXV2tN7zhDfra176mq666Svv379ehQ4d0\n4YUXavfu3TOud+211+rTn/60fvu3f1t1dXU6fPiwotGoWlpayvQJAQDAShQOmdY1BsucF5ofe3Qs\nNTVN35GpkB3UZD93bFjfe65XifGZNzzGIiFtaJy+wXHGdjpkV+oCNJXZqyXo2Wef1cc+9jGFQiFF\no1HdfffdcnfdcccdGh0dVXV1tb7zne/ogx/8oD7wgQ/o0ksvVSQS0Ze//GVVVVXNut4111yjffv2\n6bWvfa0kqa6uTl/96lcJ0AAAoOJUx8I6v7lO5zfX5X3d3TVwZnxqFpHMzY6ZGx9/eCD/UupN6bmx\n/9fvv76iSkLMc9ejrHAdHR3e1dU149i+fft08cUXl6lH5bWSPzsAAFg+xlOT6h1OTo1eHx4IllM/\ndWZMf/Vbry5Ln8xsl7t35B5nBBoAAABlFw2HtDF9E2Klq5yxcAAAAGAJKGmANrPrzOx5MztgZp/I\n8/pHzGyvmT1jZt81s/Ne6nsttVKUxbASPzMAAEC5lSxAm1lY0l2S3iLpEkm3mNklOc2elNTh7pdJ\nul/S517Ke8XjcfX396+oQOnu6u/vVzweL3dXAAAAVpRS1kBfIemAux+UJDO7V9INkvZmGrj797Pa\n/0TSrS/ljdra2tTT06O+vr6z6O7SE4/H1dbWVu5uAAAArCilDNAbJXVn7fdI+pU52r9P0rdeyhtF\no1Ft2bLlpZwKAAAAFKUiZuEws1sldUh6Y4HXb5N0myRt2rTpHPYMAAAAmKmUNxEeltSetd+WPjaD\nmb1Z0h9Lut7dk/ku5O73uHuHu3c0NzeXpLMAAADAQpQyQHdK2mpmW8wsJulmSQ9mNzCzbZL+VkF4\n7i1hXwAAAIBFUdKVCM3srZL+QlJY0pfc/f81s89I6nL3B83sO5IulXQ0fcohd79+nmv2SXqxZJ2e\n21pJJ8r03qhsfDdQCN8NFMJ3A3Ph+1EZznP3WeUPS24p73Iys658yzkCfDdQCN8NFMJ3A3Ph+1HZ\nWIkQAAAAKAIBGgAAACgCAbo495S7A6hYfDdQCN8NFMJ3A3Ph+1HBqIEGAAAAisAINAAAAFAEAjQA\nAABQBAL0ApjZdWb2vJkdMLNPlLs/qAxm1m5m3zezvWa2x8w+VO4+obKYWdjMnjSz/13uvqCymFmT\nmd1vZs+Z2T4ze225+4TKYGYfTv9M+ZmZ/YOZxcvdJ8xGgJ6HmYUl3SXpLZIukXSLmV1S3l6hQkxI\n+qi7XyLpVyX9Pt8N5PiQpH3l7gQq0hckfdvdL5J0ufieQJKZbZT0B5I63P2VChaiu7m8vUI+BOj5\nXSHpgLsfdPcxSfdKuqHMfUIFcPej7r47vT2s4AfgxvL2CpXCzNokvU3SF8vdF1QWM2uUdKWk/y5J\n7j7m7gPl7RUqSERStZlFJNVIOlLm/iAPAvT8NkrqztrvESEJOcxss6Rtkn5a3p6ggvyFpD+SNFnu\njqDibJHUJ+l/pEt8vmhmteXuFMrP3Q9L+i+SDkk6KmnQ3R8tb6+QDwEaOEtmVifpnyT9obsPlbs/\nKD8ze7ukXnffVe6+oCJFJL1a0t3uvk3SaUncXwOZ2SoFv+XeImmDpFozu7W8vUI+BOj5HZbUnrXf\nlj4GyMyiCsLz19z9gXL3BxXj9ZKuN7NfKij7usrMvlreLqGC9EjqcffMb6zuVxCogTdL+oW797n7\nuKQHJL2uzH1CHgTo+XVK2mpmW8wspqCY/8Ey9wkVwMxMQQ3jPnf/fLn7g8rh7p909zZ336zg34zv\nuTujSJAkufsxSd1mdmH60Jsk7S1jl1A5Dkn6VTOrSf+MeZO4wbQiRcrdgUrn7hNmdrukRxTcDfsl\nd99T5m6hMrxe0n+Q9KyZPZU+9n+7+8Nl7BOApeEOSV9LD8wclPSeMvcHFcDdf2pm90varWCmpyfF\nkt4ViaW8AQAAgCJQwgEAAAAUgQANAAAAFIEADQAAABSBAA0AAAAUgQANAAAAFIEADQBLiJmlzOyp\nrMeirWBnZpvN7GeLdT0AWK6YBxoAlpZRd39VuTsBACsZI9AAsAyY2S/N7HNm9qyZPWFmL08f32xm\n3zOzZ8zsu2a2KX281cy+YWZPpx+Z5YLDZvZ3ZrbHzB41s+qyfSgAqFAEaABYWqpzSjh2ZL026O6X\nSvorSX+RPvbfJH3F3S+T9DVJf5k+/peSfuDul0t6taTMCqtbJd3l7q+QNCDpXSX+PACw5LASIQAs\nIWY24u51eY7/UtJV7n7QzKKSjrn7GjM7IWm9u4+njx9197Vm1iepzd2TWdfYLOlf3H1rev/jkqLu\n/v+U/pMBwNLBCDQALB9eYLsYyaztlLhXBgBmIUADwPKxI+v5x+ntH0m6Ob3925L+Lb39XUkfkCQz\nC5tZ47nqJAAsdYwsAMDSUm1mT2Xtf9vdM1PZrTKzZxSMIt+SPnaHpP9hZh+T1CfpPenjH5J0j5m9\nT8FI8wckHS157wFgGaAGGgCWgXQNdIe7nyh3XwBguaOEAwAAACgCI9AAAABAERiBBgAAAIpAgAYA\nAACKQIAGAAAAikCABgAAAIpAgAYAAACK8P8DE8IZ4ac99SwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# table\n",
    "data = {\n",
    "    \"loss\": losses,\n",
    "    \"score\": scores\n",
    "}\n",
    "df = pd.DataFrame(data)\n",
    "display(df)\n",
    "\n",
    "# graph\n",
    "plt.figure(figsize=[12, 4])\n",
    "plt.plot(losses, label=\"loss\")\n",
    "plt.plot(scores, label=\"score\")\n",
    "plt.legend()\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Value')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "transformer-02.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}