DLL_Ass4.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Constantin-Jehn/DeepLearningLab/blob/main/DLL_Ass4.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EW6Ml8B63tPq",
        "outputId": "b5e339a9-e59d-4172-d4b9-b62b60b1e896"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "vpYuaKKh3PZk"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import math\n",
        "import time\n",
        "from tqdm import tqdm\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "from torch import Tensor\n",
        "import torch.nn as nn\n",
        "\n",
        "import torch.nn.functional as F\n",
        "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
        "\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "NoxHY5sa3eC0"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset\n",
        "\n",
        "\n",
        "class Vocabulary:\n",
        "\n",
        "    def __init__(self, pad_token=\"<pad>\", unk_token='<unk>', eos_token='<eos>',\n",
        "                 sos_token='<sos>'):\n",
        "        self.id_to_string = {}\n",
        "        self.string_to_id = {}\n",
        "        \n",
        "        # add the default pad token\n",
        "        self.id_to_string[0] = pad_token\n",
        "        self.string_to_id[pad_token] = 0\n",
        "        \n",
        "        # add the default unknown token\n",
        "        self.id_to_string[1] = unk_token\n",
        "        self.string_to_id[unk_token] = 1\n",
        "        \n",
        "        # add the default unknown token\n",
        "        self.id_to_string[2] = eos_token\n",
        "        self.string_to_id[eos_token] = 2   \n",
        "\n",
        "        # add the default unknown token\n",
        "        self.id_to_string[3] = sos_token\n",
        "        self.string_to_id[sos_token] = 3\n",
        "\n",
        "        # shortcut access\n",
        "        self.pad_id = 0\n",
        "        self.unk_id = 1\n",
        "        self.eos_id = 2\n",
        "        self.sos_id = 3\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.id_to_string)\n",
        "\n",
        "    def add_new_word(self, string):\n",
        "        self.string_to_id[string] = len(self.string_to_id)\n",
        "        self.id_to_string[len(self.id_to_string)] = string\n",
        "\n",
        "    # Given a string, return ID\n",
        "    # if extend_vocab is True, add the new word\n",
        "    def get_idx(self, string, extend_vocab=False):\n",
        "        if string in self.string_to_id:\n",
        "            return self.string_to_id[string]\n",
        "        elif extend_vocab:  # add the new word\n",
        "            self.add_new_word(string)\n",
        "            return self.string_to_id[string]\n",
        "        else:\n",
        "            return self.unk_id\n",
        "\n",
        "\n",
        "# Read the raw txt files and generate parallel text dataset:\n",
        "# self.data[idx][0] is the tensor of source sequence\n",
        "# self.data[idx][1] is the tensor of target sequence\n",
        "# See examples in the cell below.\n",
        "class ParallelTextDataset(Dataset):\n",
        "\n",
        "    def __init__(self, src_file_path, tgt_file_path, src_vocab=None,\n",
        "                 tgt_vocab=None, extend_vocab=False, device='cuda'):\n",
        "        (self.data, self.src_vocab, self.tgt_vocab, self.src_max_seq_length,\n",
        "         self.tgt_max_seq_length) = self.parallel_text_to_data(\n",
        "            src_file_path, tgt_file_path, src_vocab, tgt_vocab, extend_vocab,\n",
        "            device)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return self.data[idx]\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def parallel_text_to_data(self, src_file, tgt_file, src_vocab=None,\n",
        "                              tgt_vocab=None, extend_vocab=False,\n",
        "                              device='cuda'):\n",
        "        # Convert paired src/tgt texts into torch.tensor data.\n",
        "        # All sequences are padded to the length of the longest sequence\n",
        "        # of the respective file.\n",
        "        print(src_file)\n",
        "\n",
        "        assert os.path.exists(src_file)\n",
        "        assert os.path.exists(tgt_file)\n",
        "\n",
        "        if src_vocab is None:\n",
        "            src_vocab = Vocabulary()\n",
        "\n",
        "        if tgt_vocab is None:\n",
        "            tgt_vocab = Vocabulary()\n",
        "        \n",
        "        data_list = []\n",
        "        # Check the max length, if needed construct vocab file.\n",
        "        src_max = 0\n",
        "        with open(src_file, 'r') as text:\n",
        "            for line in text:\n",
        "                tokens = list(line)[:-1]  # remove line break\n",
        "                length = len(tokens)\n",
        "                if src_max < length:\n",
        "                    src_max = length\n",
        "\n",
        "        tgt_max = 0\n",
        "        with open(tgt_file, 'r') as text:\n",
        "            for line in text:\n",
        "                tokens = list(line)[:-1]\n",
        "                length = len(tokens)\n",
        "                if tgt_max < length:\n",
        "                    tgt_max = length\n",
        "        tgt_max += 2  # add for begin/end tokens\n",
        "                    \n",
        "        src_pad_idx = src_vocab.pad_id\n",
        "        tgt_pad_idx = tgt_vocab.pad_id\n",
        "\n",
        "        tgt_eos_idx = tgt_vocab.eos_id\n",
        "        tgt_sos_idx = tgt_vocab.sos_id\n",
        "\n",
        "        # Construct data\n",
        "        src_list = []\n",
        "        print(f\"Loading source file from: {src_file}\")\n",
        "        with open(src_file, 'r') as text:\n",
        "            for line in tqdm(text):\n",
        "                seq = []\n",
        "                tokens = list(line)[:-1]\n",
        "                for token in tokens:\n",
        "                    seq.append(src_vocab.get_idx(\n",
        "                        token, extend_vocab=extend_vocab))\n",
        "                var_len = len(seq)\n",
        "                var_seq = torch.tensor(seq, device=device, dtype=torch.int64)\n",
        "                # padding\n",
        "                new_seq = var_seq.data.new(src_max).fill_(src_pad_idx)\n",
        "                new_seq[:var_len] = var_seq\n",
        "                src_list.append(new_seq)\n",
        "\n",
        "        tgt_list = []\n",
        "        print(f\"Loading target file from: {tgt_file}\")\n",
        "        with open(tgt_file, 'r') as text:\n",
        "            for line in tqdm(text):\n",
        "                seq = []\n",
        "                tokens = list(line)[:-1]\n",
        "                # append a start token\n",
        "                seq.append(tgt_sos_idx)\n",
        "                for token in tokens:\n",
        "                    seq.append(tgt_vocab.get_idx(\n",
        "                        token, extend_vocab=extend_vocab))\n",
        "                # append an end token\n",
        "                seq.append(tgt_eos_idx)\n",
        "\n",
        "                var_len = len(seq)\n",
        "                var_seq = torch.tensor(seq, device=device, dtype=torch.int64)\n",
        "\n",
        "                # padding\n",
        "                new_seq = var_seq.data.new(tgt_max).fill_(tgt_pad_idx)\n",
        "                new_seq[:var_len] = var_seq\n",
        "                tgt_list.append(new_seq)\n",
        "\n",
        "        # src_file and tgt_file are assumed to be aligned.\n",
        "        assert len(src_list) == len(tgt_list)\n",
        "        for i in range(len(src_list)):\n",
        "            data_list.append((src_list[i], tgt_list[i]))\n",
        "\n",
        "        print(\"Done.\")\n",
        "            \n",
        "        return data_list, src_vocab, tgt_vocab, src_max, tgt_max\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "nudmVC3D3pAT"
      },
      "outputs": [],
      "source": [
        "# `DATASET_DIR` should be modified to the directory where you downloaded\n",
        "# the dataset. On Colab, use any method you like to access the data\n",
        "# e.g. upload directly or access from Drive, ...\n",
        "def load_data():\n",
        "  device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "  DATASET_DIR = \"/content/drive/MyDrive/data\"\n",
        "\n",
        "  TRAIN_FILE_NAME = \"train\"\n",
        "  VALID_FILE_NAME = \"interpolate\"\n",
        "\n",
        "  INPUTS_FILE_ENDING = \".x\"\n",
        "  TARGETS_FILE_ENDING = \".y\"\n",
        "\n",
        "  TASK = \"numbers__place_value\"\n",
        "  # TASK = \"comparison__sort\"\n",
        "  # TASK = \"algebra__linear_1d\"\n",
        "\n",
        "  # Adapt the paths!\n",
        "\n",
        "  src_file_path = f\"{DATASET_DIR}/{TASK}/{TRAIN_FILE_NAME}{INPUTS_FILE_ENDING}\"\n",
        "  tgt_file_path = f\"{DATASET_DIR}/{TASK}/{TRAIN_FILE_NAME}{TARGETS_FILE_ENDING}\"\n",
        "\n",
        "  train_set = ParallelTextDataset(src_file_path, tgt_file_path, extend_vocab=True, device = device)\n",
        "\n",
        "  # get the vocab\n",
        "  src_vocab = train_set.src_vocab\n",
        "  tgt_vocab = train_set.tgt_vocab\n",
        "\n",
        "  src_file_path = f\"{DATASET_DIR}/{TASK}/{VALID_FILE_NAME}{INPUTS_FILE_ENDING}\"\n",
        "  tgt_file_path = f\"{DATASET_DIR}/{TASK}/{VALID_FILE_NAME}{TARGETS_FILE_ENDING}\"\n",
        "\n",
        "  valid_set = ParallelTextDataset(\n",
        "      src_file_path, tgt_file_path, src_vocab=src_vocab, tgt_vocab=tgt_vocab,\n",
        "      extend_vocab=False, device = device)\n",
        "  return src_vocab, tgt_vocab, train_set, valid_set"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "4T7kjDck85sh"
      },
      "outputs": [],
      "source": [
        "def statistics(train_set, valid_set):\n",
        "  num_sent_train = len(train_set)\n",
        "  num_sent_val = len(valid_set)\n",
        "  print(f'Number of sentences in the trainin set: {num_sent_train}')\n",
        "  print(f'Number of sentences in the validation set: {num_sent_val}')\n",
        "  tot_q_char = 0\n",
        "  tot_a_char = 0\n",
        "  for i in range(0,num_sent_train):\n",
        "    tot_q_char += len(train_set[i][0])\n",
        "    tot_a_char += len(train_set[i][1])\n",
        "\n",
        "  mean_q = tot_q_char / num_sent_train\n",
        "  mean_a = tot_a_char / num_sent_train\n",
        "  print(f'Training: Number of characters: {tot_q_char + tot_a_char}')\n",
        "  print(f'Training: Average Question length: {mean_q}')\n",
        "  print(f'Training: Average Answer length:{mean_a}')\n",
        "\n",
        "  tot_q_char = 0\n",
        "  tot_a_char = 0\n",
        "\n",
        "  for i in range(0,num_sent_val):\n",
        "    tot_q_char += len(valid_set[i][0])\n",
        "    tot_a_char += len(valid_set[i][1])\n",
        "\n",
        "  mean_q = tot_q_char / num_sent_val\n",
        "  mean_a = tot_a_char / num_sent_val\n",
        "\n",
        "  print(f'Validation: Number of characters: {tot_q_char + tot_a_char}')\n",
        "  print(f'Validation: Average Question length: {mean_q}')\n",
        "  print(f'Validation: Average Answer length:{mean_a}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "ioP5a4y9Z5HE"
      },
      "outputs": [],
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "\n",
        "    def __init__(self, d_model, dropout=0.0, max_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "        self.max_len = max_len\n",
        "\n",
        "        pe = torch.zeros(max_len, d_model)\n",
        "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2).float()\n",
        "                             * (-math.log(10000.0) / d_model))\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0).transpose(0, 1)  # shape (max_len, 1, dim)\n",
        "        self.register_buffer('pe', pe)  # Will not be trained.\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"Inputs of forward function\n",
        "        Args:\n",
        "            x: the sequence fed to the positional encoder model (required).\n",
        "        Shape:\n",
        "            x: [sequence length, batch size, embed dim]\n",
        "            output: [sequence length, batch size, embed dim]\n",
        "        \"\"\"\n",
        "        assert x.size(0) < self.max_len, (\n",
        "            \"Too long sequence length: increase `max_len` of pos encoding\")\n",
        "        # shape of x (len, B, dim)\n",
        "        x = x + self.pe[:x.size(0), :]\n",
        "        return self.dropout(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "XwpnnV-xXi_W"
      },
      "outputs": [],
      "source": [
        "class TransformerModel(nn.Module):\n",
        "\n",
        "    def __init__(self, src_vocab_len: int, tgt_vocab_len: int, d_model: int, n_head: int, dim_feedforward: int, \n",
        "                 n_enc_layers: int, n_dec_layers: int, tgt_len:int = 3,  pad_id: int = 0, device = 'cuda:0'):\n",
        "        super(TransformerModel, self).__init__()\n",
        "\n",
        "        self.embedding_src = nn.Embedding(src_vocab_len, d_model)\n",
        "        self.embedding_tgt = nn.Embedding(tgt_vocab_len, d_model)\n",
        "\n",
        "\n",
        "        self.pos_encoder = PositionalEncoding(d_model)\n",
        "\n",
        "        encoder_layer = nn.TransformerEncoderLayer(d_model, n_head, dim_feedforward, batch_first=True)\n",
        "        self.encoder = nn.TransformerEncoder(encoder_layer, n_enc_layers)\n",
        "\n",
        "        decoder_layer = nn.TransformerDecoderLayer(d_model, n_head, dim_feedforward, batch_first=True)\n",
        "        self.decoder = nn.TransformerDecoder(decoder_layer, n_dec_layers)\n",
        "\n",
        "        self.linear = nn.Linear(d_model, tgt_vocab_len)\n",
        "\n",
        "        self.pad_id = pad_id\n",
        "        self.device = device\n",
        "        \n",
        "        \n",
        "    def gen_attention_mask(self, inp_len:int) -> Tensor:\n",
        "        # returns implemented mask for target \n",
        "        #taken from https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html\n",
        "        return torch.triu(torch.full((inp_len, inp_len), float('-inf')), diagonal=1)\n",
        "    \n",
        "    def gen_key_pad(self, inp) -> Tensor:\n",
        "        #expected shape: N,src_len\n",
        "        #return boolean mask, which is true whenever the pad_id is present\n",
        "        mask = torch.zeros_like(inp, dtype = torch.bool)\n",
        "        mask[inp == self.pad_id] = True\n",
        "        return mask\n",
        "    \n",
        "    def encoding(self, src):\n",
        "         N, src_len = src.shape\n",
        "         #generate padding mask\n",
        "         src_key_padding_mask = self.gen_key_pad(src)\n",
        "         src_key_padding_mask = src_key_padding_mask.to(self.device)\n",
        "         #do forward:\n",
        "         src = self.embedding_src(src)\n",
        "         #bec of batch-first\n",
        "         src = src.permute(1,0,2)\n",
        "         src = self.pos_encoder(src)\n",
        "          #bec of batch-first\n",
        "         src = src.permute(1,0,2)\n",
        "          # actual encoding\n",
        "         memory = self.encoder(src,src_key_padding_mask=src_key_padding_mask)\n",
        "         #save src_key_padding_mask for memory_key_padding mask\n",
        "         self.src_key_padding_mask = src_key_padding_mask\n",
        "         \n",
        "         return memory\n",
        "    \n",
        "    def decoding(self, tgt, memory):\n",
        "        N, tgt_len = tgt.shape\n",
        "        #generate attention mask\n",
        "        tgt_mask = self.gen_attention_mask(tgt_len)\n",
        "        tgt_mask = tgt_mask.to(self.device)\n",
        "        #generate padding mask\n",
        "        tgt_key_padding_mask = self.gen_key_pad(tgt).to(self.device)\n",
        "        memory_key_padding_mask = self.src_key_padding_mask.to(self.device)\n",
        "        #start forwarding\n",
        "        tgt = self.embedding_tgt(tgt)\n",
        "        #because of batch-first option\n",
        "        tgt = tgt.permute(1,0,2)\n",
        "        tgt = self.pos_encoder(tgt)\n",
        "        #bec. of batch-first\n",
        "        tgt = tgt.permute(1,0,2)\n",
        "        #actual decoding\n",
        "        decoded = self.decoder(tgt, memory, tgt_mask = tgt_mask, \n",
        "                                                    tgt_key_padding_mask=tgt_key_padding_mask,\n",
        "                                                    memory_key_padding_mask= memory_key_padding_mask)\n",
        "        #classification layer\n",
        "        output = self.linear(decoded)\n",
        "        return output\n",
        "        \n",
        "        \n",
        "    def forward(self,src:Tensor=None, tgt:Tensor=None) -> Tensor:\n",
        "      memory = self.encoding(src)\n",
        "      #decoding includes classification layer\n",
        "      output = self.decoding(tgt,memory)\n",
        "      return output\n",
        "\n",
        "    def greedy_decoding(self, src, tgt_len, eos_id, sos_id, batch_size):\n",
        "        #do the encdoding and save the hidden state\n",
        "        memory = self.encoding(src)\n",
        "        #fill first column start start of sequence id sos\n",
        "        tgt = torch.ones(batch_size,1,dtype=torch.int32) * sos_id\n",
        "        tgt = tgt.to(self.device)\n",
        "\n",
        "        #initialize boolean values and mask for stopping criteria of the search\n",
        "        mask = torch.zeros(batch_size,dtype = torch.bool)\n",
        "        mask = mask.to(self.device)\n",
        "        eos = False\n",
        "        completions = 1\n",
        "\n",
        "        while (not eos) and (completions < tgt_len):\n",
        "            output = self.decoding(tgt, memory)\n",
        "            #only extract the last \"row\" of the sequence to complete\n",
        "            output = output[:,-1,:]\n",
        "\n",
        "            soft = F.softmax(output, dim = -1)\n",
        "            most_prob_char = torch.topk(soft,1,dim=1).indices\n",
        "\n",
        "            tgt = torch.cat((tgt,most_prob_char), dim = 1)\n",
        "\n",
        "            #check if last sequence was eos_symbol\n",
        "            lines_eos = tgt[:,-2] == eos_id\n",
        "            # or the eos symbol was already in the sequence\n",
        "            mask = torch.logical_or(lines_eos, mask)\n",
        "            #fill these entries with padding\n",
        "\n",
        "            tgt[mask,-1] = self.pad_id\n",
        "            #increase completions for stopping criteria\n",
        "            completions += 1\n",
        "            #if all the last entries had eos symbol\n",
        "            eos = torch.sum(mask) == batch_size\n",
        "        return tgt\n",
        "    \n",
        "    def accuracy(self, prediction, tgt):\n",
        "        batch_size ,tgt_len = tgt.shape\n",
        "        result = torch.sum(prediction == tgt,1) == tgt_len\n",
        "        n_correct = torch.sum(result).item()\n",
        "        accuracy = n_correct/batch_size\n",
        "        return accuracy\n",
        "        "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "iagbQ-5OnWhp"
      },
      "outputs": [],
      "source": [
        "class Trainer():\n",
        "    def __init__(self, model:TransformerModel, optimizer, vocab, eos_id, sos_id, pad_id, device):\n",
        "        self.model = model\n",
        "        self.loss_fn = nn.CrossEntropyLoss(ignore_index = pad_id)\n",
        "        self.optimizer = optimizer\n",
        "        self.vocab = vocab\n",
        "        self.eos_id = eos_id\n",
        "        self.sos_id = sos_id\n",
        "        self.device = device\n",
        "        #reporting arrays\n",
        "        self.val_loss = []\n",
        "        self.val_acc = []\n",
        "        self.train_loss = []\n",
        "        self.train_acc = []\n",
        "    def train(self, train_loader, valid_loader, lr, epochs, n_accumulate, n_reporting = 500):\n",
        "\n",
        "        for epoch in range(0,epochs):\n",
        "            self.optimizer.zero_grad()\n",
        "            running_loss = 0\n",
        "            accumulated_loss = 0\n",
        "            \n",
        "            for it, batch in enumerate(train_loader):\n",
        "                if it > 21000:\n",
        "                  break\n",
        "                self.model.train()\n",
        "                #load source and target\n",
        "                src = batch[0]\n",
        "                tgt = batch[1]\n",
        "                #put to device\n",
        "                src = src.to(self.device)\n",
        "                tgt = tgt.to(self.device)\n",
        "                batch_size, tgt_len = tgt.shape\n",
        "                \n",
        "                #define input and reference for loss\n",
        "                tgt_in = tgt[:,:-1]\n",
        "                tgt_ref = tgt[:,1:]\n",
        "\n",
        "                output = self.model(src = src, tgt = tgt_in)\n",
        "\n",
        "                #in CrossEntropy the second entry needs to be the class\n",
        "                output = torch.squeeze(torch.permute(output,(0,2,1)))\n",
        "\n",
        "                loss = self.loss_fn(output, tgt_ref)\n",
        "\n",
        "                loss.backward()\n",
        "                running_loss += loss\n",
        "\n",
        "                #only update parameters every n-accumulate steps --> increase effective batchsize\n",
        "                if (it%n_accumulate) == 0:\n",
        "                    torch.nn.utils.clip_grad_norm(self.model.parameters(), 0.1)\n",
        "                    self.optimizer.step()\n",
        "                    self.optimizer.zero_grad()\n",
        "                    self.model.train()\n",
        "                    \n",
        "                if (it%n_reporting) == 0 and it >0 :\n",
        "                    self.model.eval()\n",
        "                    print(f'Epoch: {epoch} Iteration {it}:')\n",
        "                    print(f'Training loss: {running_loss/n_reporting}')\n",
        "                    self.train_loss.append(running_loss/n_reporting)\n",
        "\n",
        "                    train_pred = self.model.greedy_decoding(src, tgt_len, self.eos_id, self.sos_id, batch_size)\n",
        "                    train_pred = train_pred.to(self.device)\n",
        "                    train_acc = self.model.accuracy(train_pred, tgt)\n",
        "                    self.train_acc.append(train_acc)\n",
        "                    print(f'Training accuracy: {train_acc}')\n",
        "                    running_loss = 0                   \n",
        "\n",
        "                    val_loss, val_acc = self.validation(valid_loader)\n",
        "                    print(f'Validation loss: {val_loss}')\n",
        "                    print(f'Validation accuracy: {val_acc}')\n",
        "\n",
        "    def validation(self, valid_loader):\n",
        "      #same procedure as for test data set\n",
        "        with torch.no_grad():\n",
        "          running_loss = 0\n",
        "          running_acc = 0\n",
        "          for it, batch in enumerate(valid_loader):\n",
        "            src = batch[0].to(self.device)\n",
        "            tgt = batch[1].to(self.device)\n",
        "            batch_size, tgt_len = tgt.shape\n",
        "\n",
        "            tgt_in = tgt[:,:-1]\n",
        "            tgt_ref = tgt[:,1:]\n",
        "\n",
        "            output = self.model(src = src, tgt = tgt_in)\n",
        "            output = torch.squeeze(torch.permute(output,(0,2,1)))\n",
        "            loss = self.loss_fn(output, tgt_ref)\n",
        "            running_loss += loss\n",
        "            val_pred = self.model.greedy_decoding(src,tgt_len, self.eos_id, self.sos_id, batch_size)\n",
        "            val_acc = self.model.accuracy(val_pred, tgt)\n",
        "            running_acc += val_acc\n",
        "          val_loss = running_loss/it\n",
        "          val_acc = running_acc/it\n",
        "          self.val_loss.append(val_loss)\n",
        "          self.val_acc.append(val_acc)\n",
        "          return val_loss, val_acc\n",
        "                    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "zww9rTAde2ng"
      },
      "outputs": [],
      "source": [
        "def create_Datasets():\n",
        "    src_vocab, tgt_vocab, train_set, valid_set = load_data()\n",
        "    batch_size = 64\n",
        "    train_data_loader = DataLoader(\n",
        "        dataset=train_set, batch_size=batch_size, shuffle=True)\n",
        "    \n",
        "    valid_data_loader = DataLoader(\n",
        "        dataset=valid_set, batch_size=batch_size, shuffle=False)\n",
        "    return src_vocab, tgt_vocab, train_data_loader, valid_data_loader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "koQzB_bOe52Q"
      },
      "outputs": [],
      "source": [
        "def print_example_data(train_data_loader, src_vocab, tgt_vocab):\n",
        "    batch = next(iter(train_data_loader))\n",
        "    source = batch[0]\n",
        "    print(source.shape)\n",
        "    target=batch[1]\n",
        "    print(target.shape)\n",
        "    \n",
        "    example_source_sequence = []\n",
        "    \n",
        "    for i in source[0]:\n",
        "        example_source_sequence.append(src_vocab.id_to_string[i.item()])\n",
        "    \n",
        "    print(example_source_sequence)\n",
        "    \n",
        "    print(''.join(example_source_sequence))\n",
        "    \n",
        "    example_target_sequence = []\n",
        "    for i in target[0]:\n",
        "        example_target_sequence.append(tgt_vocab.id_to_string[i.item()])\n",
        "    print(example_target_sequence)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def training_curves(train_loss, train_acc, val_loss, val_acc,  title):\n",
        "  train_loss = np.array(trainer.train_loss)\n",
        "  train_acc = np.array(trainer.train_acc)\n",
        "  val_acc = np.array(trainer.val_acc)\n",
        "  val_loss = np.array(trainer.val_loss)\n",
        "\n",
        "  n = len(train_loss)\n",
        "  x = np.arange(0,n,1) * 500\n",
        "  fig, (ax1,ax2) = plt.subplots(2,1)\n",
        "  ax1.plot(x, train_loss, label ='training')\n",
        "  ax1.plot(x, val_loss, label = 'validation')\n",
        "  #ax1.set_xlabel('iteration')\n",
        "  ax1.set_ylabel('loss', fontsize = 12)\n",
        "  ax1.grid(True)\n",
        "  ax1.set_title(title, fontsize = 12)\n",
        "  ax1.legend(fontsize=10)\n",
        "\n",
        "  ax2.plot(x,train_acc, label = 'training')\n",
        "  ax2.plot(x,val_acc, label ='validation')\n",
        "  ax2.set_xlabel('iteration', fontsize = 12)\n",
        "  ax2.set_ylabel('accuracy', fontsize = 12)\n",
        "  ax2.grid(True)\n",
        "\n",
        "  fig_to = \"/content/drive/MyDrive/data/figures/\" + title + '.png'\n",
        "  plt.savefig(fig_to)\n",
        "  plt.show()"
      ],
      "metadata": {
        "id": "TG8FJpu1s2eF"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def example_questions(valid_data_loader, model):\n",
        "  #check the model on three validation questions\n",
        "  batch = next(iter(valid_data_loader))\n",
        "  source = batch[0].to(device)\n",
        "\n",
        "  prediction = model.greedy_decoding(source, 3, eos_id, sos_id, batch_size)\n",
        "  for index in [3, 12, 60]:\n",
        "    question = []\n",
        "    for i in source[index]:\n",
        "        question.append(src_vocab.id_to_string[i.item()])\n",
        "    print_q = ''.join(question)\n",
        "    print(f'Validation Questions: {print_q}')\n",
        "\n",
        "    answer = []\n",
        "    for i in prediction[index]:\n",
        "      answer.append(tgt_vocab.id_to_string[i.item()])\n",
        "    print_a = ''.join(answer)\n",
        "    print(f'Answer: {print_a}')\n"
      ],
      "metadata": {
        "id": "hOtfUDTTyHiy"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y7ljK4cVe6-6"
      },
      "outputs": [],
      "source": [
        "src_vocab, tgt_vocab, train_data_loader, valid_data_loader = create_Datasets()\n",
        "print_example_data(train_data_loader, src_vocab, tgt_vocab)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "src_vocab_len = 33\n",
        "tgt_vocab_len = 14\n",
        "\n",
        "pad_id = 0\n",
        "unk_id = 1\n",
        "eos_id = 2\n",
        "sos_id = 3\n",
        "batch_size = 64\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ],
      "metadata": {
        "id": "SF1O3jNW8kAU"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "Yhl2u7BfniMA"
      },
      "outputs": [],
      "source": [
        "#default parameters\n",
        "d_model = 256\n",
        "n_head = 8\n",
        "dim_feedforward = 1024\n",
        "n_enc_layers = 3\n",
        "n_dec_layers = 2\n",
        "lr = 0.0001\n",
        "\n",
        "model = TransformerModel(src_vocab_len, tgt_vocab_len, d_model, n_head, dim_feedforward, n_enc_layers, n_dec_layers, device = device)\n",
        "model = model.to(device)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = lr)\n",
        "n_reporting = 500\n",
        "trainer = Trainer(model,optimizer,src_vocab,eos_id,sos_id,pad_id, device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oFwwWINtJbrZ"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#reduce number of hidden layer d_model to 128\n",
        "d_model = 128\n",
        "n_head = 8\n",
        "dim_feedforward = 1024\n",
        "n_enc_layers = 3\n",
        "n_dec_layers = 2\n",
        "lr = 0.0001\n",
        "\n",
        "model = TransformerModel(src_vocab_len, tgt_vocab_len, d_model, n_head, dim_feedforward, n_enc_layers, n_dec_layers, device = device)\n",
        "model = model.to(device)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = lr)\n",
        "n_reporting = 500\n",
        "trainer = Trainer(model,optimizer,src_vocab,eos_id,sos_id,pad_id, device)"
      ],
      "metadata": {
        "id": "H4NhB5IoR-Uo"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#reduce number of hidden layer d_model to 128 and feed-forward to 512\n",
        "d_model = 128\n",
        "n_head = 8\n",
        "dim_feedforward = 512\n",
        "n_enc_layers = 3\n",
        "n_dec_layers = 2\n",
        "lr = 0.0001\n",
        "\n",
        "model = TransformerModel(src_vocab_len, tgt_vocab_len, d_model, n_head, dim_feedforward, n_enc_layers, n_dec_layers, device = device)\n",
        "model = model.to(device)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = lr)\n",
        "n_reporting = 500\n",
        "trainer = Trainer(model,optimizer,src_vocab,eos_id,sos_id,pad_id, device)"
      ],
      "metadata": {
        "id": "2KXZho7-HxMx"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#reduce number of hidden layer d_model to 128 and increase lr\n",
        "d_model = 128\n",
        "n_head = 8\n",
        "dim_feedforward = 1024\n",
        "n_enc_layers = 3\n",
        "n_dec_layers = 2\n",
        "lr = 0.0005\n",
        "\n",
        "model = TransformerModel(src_vocab_len, tgt_vocab_len, d_model, n_head, dim_feedforward, n_enc_layers, n_dec_layers, device = device)\n",
        "model = model.to(device)\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = lr)\n",
        "n_reporting = 500\n",
        "trainer = Trainer(model,optimizer,src_vocab,eos_id,sos_id,pad_id, device)"
      ],
      "metadata": {
        "id": "-1vnllq7SpQP"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YCe9W3K99u6j"
      },
      "outputs": [],
      "source": [
        "epochs = 1\n",
        "n_accumulate = 10\n",
        "trainer.train(train_data_loader,valid_data_loader, lr, epochs, n_accumulate, n_reporting = n_reporting)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training_curves(trainer.train_loss, trainer.train_acc, trainer.val_loss, trainer.val_acc, 'training lr = 0.0005')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "id": "ATZO0TZevv0K",
        "outputId": "27e6daa2-8c76-4aa6-ecc7-422757e4f7a7"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEZCAYAAAB4hzlwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3gU1frA8e/ZzWbTewMSSCChJdSE3ntRUCmCHRWx/hCvFa9X9F57xYJiwYYIUiyoCEiJFAEpUkMLECCUNCC95/z+mEkIIYFsSHazcD7PM89OOWf2nbDMO/UcIaVEURRFUQAMtg5AURRFqT9UUlAURVHKqKSgKIqilFFJQVEURSmjkoKiKIpSRiUFRVEUpYxKCordEULMFEL8p7bLWhhDqBBCCiEcanvdimJLKikoViWESBBCDLySdUgpH5BS/q+2y9ZXegJaLYTIEULsu9TfTwhhFkJ8IYTIEEKcFkL8q8LyAfo6cvR1NqlO3XJJMKvcUOvJVrE9dZSj1CtCCAcpZZGt47gSQggBCCllSS2tci6wARiuDwuFEBFSypRKyr4ARABNgCBgtRAiTkq5VAjhB/wATAR+Af4HfA90vVzdcuv3svd/H+XS1JmCYjVCiNlAY+AX/UjzqXJHoPcKIY4Bq/SyC/Sj1XQhxBohRGS59XwlhHhJH+8rhEgUQjwuhEgWQpwSQtxdw7K+Qohf9CPlzUKIl4QQ66q5bbFCiJeFEOuBHKBpLfzJEEI0BzoC06SUuVLKRcAuYHQVVe4C/ielPCul3At8BkzQl40C9kgpF0gp89CSQDshRMtq1FWuESopKFYjpbwDOAaMkFK6SSnfKLe4D9AKGKJP/4521BoAbAPmXGLVQYAn0Ai4F5ghhPCuQdkZQLZe5i59sMQdwCTAHThacaEQ4lchxLkqhl+rWGckcFhKmVlu3g59fsX1ewMN9OWVlY0sv0xKmQ0cAiKrUbfUUT2xfqmfeShXGZUUlPriBSlltpQyF0BK+YWUMlNKmc/5I1rPKuoWAv+VUhZKKZcAWUALS8oKIYxoR9/TpJQ5Uso44GsLt+ErKeUeKWWRlLKw4kIp5fVSSq8qhuurWKcbkF5hXjpa4qmsbOnyyspeal2Xq5sKdEK7tBStz79UolbslEoKSn1xvHRECGEUQrwmhDgkhMgAEvRFVR2ZplW4zp3D+Z1cdcv6o91jO15uWfnx6rC0fHVkAR4V5nkAmVWULV1eWdlLreuSdaWUWVLKLXrCSwIeAQYLISpLToodU0lBsbaqmuUtP/9W4AZgINqlnlB9vqi7sEgBioDgcvNCLFzHJZscFkL8XuHpnfLD71VU2wM0rbDzbafPv/DLpTwLnNKXV1Z2T/llQghXoBnafYbL1b3o6/RPtQ+5yqh/UMXakrj8TVh3IB9IA1yAV+o6KCllMdqTOS8IIVz0m6931vJ3DNPvpVQ2DKuizgFgOzBNCOEkhLgJaAssquJrvgGeE0J469twH/CVvuxHIEoIMVoI4QQ8D+yUUu67XF0hRBchRAshhEEI4Qu8D8RKKStejlLsnEoKirW9irbjOSeEeKKKMt+g3ag9AcQBG60U2yNoZyangdloj4LmW+m7L2U8EAOcBV4DxpQ+jiqEuE0IUf5ofhrazeOjwJ/Am6WPlOp1RgMv6+vqoq/7snXREvlStMtJu9H+LrfU+pYqNidUJzuKUjkhxOtAkJTS0qeQFMVuqTMFRdEJIVoKIdoKTWe0R1Z/tHVcimJN6o1mRTnPHe2SUUO0ex9vAz/bNCJFsTJ1+UhRFEUpoy4fKYqiKGXs+vKRn5+fDA0NrVHd7OxsXF1dazcgK1Gx24aK3frsNW6o37Fv3bo1VUrpX9kyu04KoaGhbNmypUZ1Y2Nj6du3b+0GZCUqdttQsVufvcYN9Tt2IcRFbXOVUpePFEVRlDLXbFLIL1I32BVFUSq6JpPC7I1HeXZdLqlZ9eFlVUVRlPrDru8p1FSHEC8yCiQPz9nGtxO7YDJek7lRUeqdwsJCEhMTycvLA8DT05O9e/faOKqaqQ+xOzk5ERwcjMlkqnadazIpRDXy5O4oM5/uPMMrS/YybcRF/ZUoimIDiYmJuLu7ExoaihCCzMxM3N3ts3VuW8cupSQtLY3ExETCwsKqXc8qh8h6Z+DJQojdVSwXQoj3hRDxQoidQoiOdR1T94YO3NMjjC/XJ/DDtsS6/jpFUaohLy8PX19ftG6ulSshhMDX17fsrKu6rHXd5Ctg6CWWD0PrejECrTvDj60QE1OHt6RrUx+m/rCLXYmqBWBFqQ9UQqg9NflbWiUpSCnXAGcuUeQG4Bup2Qh4CSEa1HVcJqOBD2/tiK+rIw98u5U0deNZUZRrnNXaPhJChAK/SimjKln2K/CalHKdPr0SeFpKedGbaUKISWhnEwQGBkbPmzevRvFkZWXh5qb12HgkvZiXN+UR4WXgiRgnjIb6faRSPnZ7o2K3DXuJ3dPTk/Dw8LLp4uJijEaj1b7/3LlzLFiwgPvuu8+ieqNHj2bWrFl4eXmVzasY+0svvUSPHj3o169frcVbHfHx8aSnX3glpF+/flullDGVVpBSWmVA61JxdxXLfgV6lpteCcRcbp3R0dGyplavXn3B9IItx2WTp3+V//tlT43XaS0VY7cnKnbbsJfY4+LiLpjOyMiw6vcfOXJERkZGXjS/sLDQ4nVZO/aqVPybSiklsEVWsV+tL89inuDC/nCD9XlWMyY6mAndQ/l83RF+3m7Vr1YUpZ545plnOHToEO3bt6dTp0706tWLkSNH0rp1awBuvPFGoqOjiYyM5NNPPy2rFxoaSmpqKgkJCbRq1Yr77ruPzp07M3jwYHJzcwGYMGECCxcuLCs/bdo0OnbsSJs2bdi3T+sRNSUlhUGDBhEZGcnEiRNp0qQJqampVv0b1JdHUhcDjwgh5qF1EZgupTxl7SD+fV0r4k5m8PSinYQHuBHZ0NPaISiKonvxlz3sOn62Vi8ftW7occlH0F977TV2797N9u3biY2N5brrrmP37t1lj3R+8cUX+Pj4kJubS6dOnRg9ejS+vr4XrOPgwYPMnTuXd955h3vvvZdFixZx++23X/Rdfn5+bNu2jY8++oi33nqLzz//nBdffJH+/fszdepUli5dyqxZs2pt26vLWo+kzgU2AC2EEIlCiHuFEA8IIR7QiywBDgPxwGfAQ9aIqyKT0cCM2zri5ezI/bO3cja7wBZhKIpST3Tu3PmCZ/zff/992rVrR9euXTl+/DgHDx68qE5YWBjt27cHIDo6moSEhErXPWrUqIvKrFu3jvHjtW6zhw4dire3dy1uTfVY5UxBSnnJDr71a1wPWyOWy/F3NzPzjmhunrmB/5v7D1/d3QkH9cazoljdtBGRNn8BrHzT17GxsaxYsYINGzbg4uJC3759K30HwGw2l40bjcayy0dVlTMajRQVFdVy5DWn9naVaB/ixUs3RrEuPpU3l+23dTiKoliJu7s7mZmZlS5LT0/H29sbFxcX9u3bx8aNG2v9+3v06MH8+fMBWL58OWfPnq3177ic+nJPod65uVMIO0+c45M1h4lq5MmIdg1tHZKiKHXM19eXHj16EBUVhbOzM4GBgWXLhg4dysyZM2nVqhUtWrSga9eutf7906ZN45ZbbmH27Nl069aNoKAgq58pqaRwCc9fH8neU5k8tVC78dyqgYetQ1IUpY599913lc43m838/vvvlS4rvSfg5+fH7t3nW/N54oknysa/+uqri8oDxMTEEBsbC2jvaSxbtgwHBwc2bNjA5s2bL7gcZQ3q8tElODoY+Pi2jng4O3D/7K2cy1E3nhVFqTvHjh2jU6dOtGvXjsmTJ/PZZ59ZPQZ1pnAZAR5OfHx7NOM+2cDkedv5ckKnev/Gs6Io9ikiIoJ//vnHpjGoM4Vq6NjYm//eEMWaAym8vVzdeFYU5eqlkkI13dK5Mbd0bsxHsYdYssvq79UpiqJYhUoKFnhhZGs6NPbiiQU72H+68sfWFEVR7JlKChYwOxiZeXs0rmYHJs3eQnpOoa1DUhRFqVXXZlLY+ytRu16GDTPg1A4oKa521UAPJz6+rSMnzuby6Pf/UFxinabHFUWpf0qbIz958iRjxoyptEzfvn3ZsuWiXgAuMH36dHJycsqmhw8fzrlz52ovUAtcm0mhIAuXnOOw7Fn4pDe8EQZzb4G/PoST2y+bJGJCfZg2MpLY/SlMX3HASkErilJfNWzYsKwF1JqomBSWLFlyQd8M1nRtJoV24/m7y0x4LA5GfQatb4SU/bD83/BpH3g9DL4bD399ACf/qTRJ3N6lMTfHBPPBqniW7j5tg41QFKW2PfPMM8yYMaNs+oUXXuCll15iwIABZc1c//zzzxfVS0hIICpK6z8sNzeX8ePHExMTw0033XRB20cPPvggMTExREZGMm3aNEBrZO/kyZP069evrAOe0qa4Ad555x2ioqKIiopi+vTpZd9X2kR3ZGTkBU10X6lr+z0Fz0bQ9mZtAMg4CQnr4eg6SFgHB/S3F80e0KQ7NOkBoT0hqC3C6MB/b4hif1IWj8/fTnhAD8IDbNdwl6JcdX5/BucT/4CxFndTQW1g2GtVLh43bhxTpkzh4Ye19jnnz5/PsmXLmDx5Mh4eHqSmptK1a1dGjhxZZf/HH3/8MS4uLmzZsoUjR47QsWPHsmUvv/wyPj4+FBcXM2DAAHbu3MnkyZN55513WL16NX5+fhesa+vWrXz55Zds2rQJKSVdunShT58+eHt7lzXR/dlnn3HzzTdX2US3pa7tpFCRR0NoO1YbADJOwdH1WoJIWAcHlmrzzR7QuBtOoT2YNbAT182HSd9s5adHeuDhZLJd/IqiXJEOHTqQnJzMyZMnSUlJwdvbm6CgIB577DHWrFmDwWDgxIkTJCUlERQUVOk61qxZw+TJkwFo27Ytbdu2LVs2f/58Pv30U4qKijh16hRxcXEXLK9o3bp13HTTTWWttY4aNYq1a9cycuTIajfRbSmVFC7FowG0GaMNAJmnteRQmigOLsMP+MvkxpqMcJZ90onRo8ZjaNi+do9uFOVaNOw1cm3QdPbYsWNZuHAhp0+fZty4ccyZM4eUlBS2bt2KyWQiNDS00iazL+fIkSO89dZbbN68GW9vbyZMmFCj9ZSqbhPdlro27ynUlHuQliCufxce2QyPH4AxX2BsN44O7hmMPfsZhlkD4PVQ+O1xSE+0dcSKolho3LhxzJs3j4ULFzJ27FjS09MJCAjAZDKxevVqjh49esn6vXv3LmtUb/fu3ezcuROAjIwMXF1d8fT0JCkp6YLG9apqsrtXr1789NNP5OTkkJ2dzY8//kivXr1qcWsvpg5nr4R7IESNhqjReF4nmTY3lpTdq/h3aCKNtn4N276BDrdDz3+BV8jl16cois1FRmqd+zRq1IgGDRpw2223MWLECNq0aUNMTAwtW7a8ZP0HH3yQu+++u+yGcnR0NADt2rWjQ4cOtGzZkpCQEHr06FFWZ9KkSQwdOpSGDRuyevXqsvkdO3ZkwoQJdO7cGYCJEyfSoUOHWrtUVBmhdXpmn2JiYuTlnv+tSmxsLH379q3VePIKixk7cwNHUrN5bYAXfZK/xS1uLgK05NDrX+DV+Iq/py5itxYVu23YS+x79+6lVatWZdO27nntStSX2Cv+TQGEEFullDGVlVeXj2qRk8nIzDui8XQ28ciSVNpsGcoNxg9Y4z6M4m3fIt/viFw8Gc5e+vRTURTFVtTlo1rWyMuZtU/1Y39SJpsTzvD3kTM8meCPyB3Egw6LuWXbHAzbvuVg0AgKezxGi1ZtMDsYbR22oigKoJJCnTAYBK0aeNCqgQd3dgtFSkni2Vz+PtKftw7up+WhWVx36jeMCxfzs+xNbOCdNAmPolOYDx0be+GuHmtVrmFSyirfAVAsU5PbAyopWIEQghAfF0J8XCA6GBjAmdNHyVjxJiMPfc+NKWv48XQvno+9geME0bqhB51CfcoGf3frdsenKLbi5OREWloavr6+KjFcISklaWlpODk5WVRPJQUb8Qlqgs/tH0Lmc7D+PUZv+YLRDuuI8xvCZ2I0323K4sv1CQCE+bnSKdSbTqE+dA7zqVH2VxR7EBwcTGJiIikpKQDk5eVZvFOrL+pD7E5OTgQHB1tURyUFW3MPgqGvInpMgb/eJ3LzLKYXL+Xt9mPY1+J+1p3xZnPCGZbtSWL+Fu29hwAXwSSHw4yNCcHTWV1qUq4eJpOJsLCwsunY2Fg6dOhgw4hqzl5jV0mhvnAPhCEvQ49H4a/3MW6eReSehURGjeb+4U9R4hvDweQs/k44w+w/43jpt728vfwAozo2YkL3UCICbf/om6Io9k8lhfrGLQAGvwTdteTA5s9h10IMUaNo0fspWnRtSUjeEfwiOvD1Xwks2JrInE3H6BHuy13dQhnQKhCjQV2LVRSlZqr9noIQop8QIkwfbyCE+FoI8aUQovJWoZQr4+YPg/8HU3ZpZw/7l8JHXWHBBFyyjxHVyJM3x7ZjwzP9eXJICw6nZDNp9lb6vLmaT9ccUr3CKYpSI5a8vPYRUNqxwNuACSgBPq3toJRyXP1g0Itacuj5GBz8g06bH4Wlz0JeBr5uZh7uF87ap/rx0W0daejlzCtL9tHl1RVM/WGX6ktaURSLWHL5qJGU8pgQwgEYAjQBCoCTdRKZciFXXxg4Dbo9wqlvH6Thxo9g9yLtPkTUaByMBoa3acDwNg2IO5nB138l8MO2ROb+fYyuTX2Y0D2Mga0CcDCql9gVRamaJXuIDCFEINAHiJNSZunz1eMv1uTqy4EWD8HEldrN6UX3wjcjIeV8t6CtG3rw+pi2bJw6gKeHtuT4mVwe+HYrfd6MZeafhzibXWDDDVAUpT6zJCl8AGwG5gCl/dX1APbVdlBKNQRHw32rYfhbcGoHfNwdVrwABdllRbxdHXmwbzP+fLIvM2+PprGPC6/9vo+ur67k6YU7iTuZYbv4FUWpl6p9+UhK+boQ4kegWEp5SJ99AphYJ5Epl2cwQuf7tD6mV0yDde/CzgUw9FVoNQL0N0IdjAaGRgUxNCqIfacz+Pqvo/z4TyLfbzlO5zAf7u4eyqDWgerSkqIolrWSKqU8UJoQhBD9gAZSyl11EplSfW7+cONHcPdScPaC+XfAnDGQduiioi2DPHh1VBs2Th3As8NbcvJcLg/O2UbvN1bzUWw8Z9SlJUW5plnySOqfQoge+vjTwDzgOyHEs9WsP1QIsV8IES+EeKaS5ROEEClCiO36oM5ALNWkG0z6E4a8Csc2wUfdYPUrUHhxN31eLo5M6t2MP5/sx2d3xhDm78obS/fT7dWVzFp3RDWloSjXKEvOFKKAjfr4fUA/oCvwwOUqCiGMaPchhgGtgVuEEK0rKfq9lLK9PnxuQWxKKaMDdHtI6y609Uj483WY0QUOLKu8uEEwqHUgcyZ25Y/HetMrwo///RrHvV9vIS0r38rBK4pia5YkBQMghRDN0Hpsi5NSHge8q1G3MxAvpTwspSxAO8u4wfJwlWrzaACjP4e7fgEHJ/juZph76yU7+IkIdOezO2N4YURr1h1MZdh7a/nrUKoVg1YUxdaq3R2nEOIX4DjQADgkpXxCTxArpJRhl6k7BhgqpZyoT98BdJFSPlKuzATgVSAFOAA8piediuuaBEwCCAwMjJ43b1614q8oKysLNze3GtW1NUtjFyWFBCf+QmjCPEBytMlYjofchDRU/TTxsYxiPtqRT1K25PqmJm4MN9VK8xnX0t+9PrHX2O01bqjfsffr16/K7jiRUlZrAHyBV4AXATd93nXAlGrUHQN8Xm76DuDDStZv1sfvB1Zdbr3R0dGyplavXl3jurZW49jPHZdy3u1STvOQ8v2OUsavvGTx7PxC+eSC7bLJ07/Km2ask8fSsmv2veVck3/3esBeY7fXuKWs37EDW2QV+9VqXz6SUqZJKZ+VUk6T+otrUsrfpJTTq1H9BBBSbjpYn1dx/aUXsT8Hoqsbm1JNnsEwbjbcvgikhNk3wfy7IP1EpcVdHB14Y0w73hvfngNJWQx/fy1Ldp2yctCKoliTJU8fmYQQLwohDgsh8vTPF4UQjtWovhmIEEKE6eXHA4srrL9BucmRwN7qxqZYKHwgPLQB+j0HB5bCh51g/XtQXHkjeje0b8SSyb1o6u/GQ3O2MfWHXeQWFFdaVlEU+2bJjeY3gIFoTxu10z/7A69frqKUsgh4BFiGtrOfL6XcI4T4rxBipF5sshBijxBiBzAZmGBBbIqlHMzQ50l4eBOE9YY/noeZPeHI2kqLN/Z1YeED3bi/T1Pm/n2MG2asU43tKcpVyJKkMBYYKaVcLqXcL6VcDtwE3FydylLKJVLK5lLKZlLKl/V5z0spF+vjU6WUkVLKdlLKflJK1XyGNXiHwq3z4JZ5UJgDX18Pnw+Ebd9AftYFRU1GA1OHteKbezpzJruQkR+u49uNR9U7DYpyFbEkKVT16Inq0eVq0GIYPPw3DHkF8jJg8f/B2y3g50fg+GbtHoSud3N/fn+0F12a+vLcT7t5aM421X+DolwlLEkKC4BfhBBDhBCthBBDgZ/0+crVwOQM3R7WLind+wdE3qg1zz1roPZ29IaPIDsNAH93M19N6MSzw1vyR1wSw95bw+aEMzbeAEVRrpQlSeEpYAXam8lb0VpNXQ08WQdxKbYkBIR0hhtmwOP7YcR74OgKy6bCOy1hwQQ4tAoDkkm9m7Howe44GA2M+2QDH6w8SHGJupykKPbqkq2kCiH6V5gVqw8CKP2f3xNYVduBKfWEkwdET9CGpD2wbTbsnAd7fgTPxtDhdtp1uI3fJvfkuZ928/YfB1h/KJXp4zoQ5Olk6+gVRbHQ5ZrOnlXF/NKEUJocmtZaREr9FRgJw16DgS/A/t+0m9Gxr0Dsq7iHD2R6xzvo1bQN//nlAMPeW8ObY9oxsHWgraNWFMUCl0wK8jLNVyjXKJMTRI3WhrMJ8M+38M8cxPw7GePix8DoMTx1uB0Tv9nChO6hTB3eErOD0dZRK4pSDZb00awoF/MOhf7PQd+pEL8Stn2N187P+bSkiKO+7fhgU1fGHx7EW7d1o5l/LbQDU1ICBVmQl35+yM/QPo2O0Gqk1lKsoig1ov73KLXDYITmg7UhKxl2zKXJtm94y/QJWee+4bcPe3Ck10QcDCbIOQN55/SdesbFO/eyofyycvO4xI3soDYw8gNo2MFqm64oVxOVFJTa5xYAPR6F7pPh2EaMm77kxr0/YV63Qlu+5jL1zZ7g5Knd5HbyBK8QMEfq88rNLx3M+vTpnfD70/BZf+3R2r7PgqNLnW+uolxNVFJQ6o4Q0KQbzk26UZzzBmt+/pz9B/ZyosCFDOmCl48/rcOC6dgilLBGDRBOnmB21846asK3GTTtp/VX/dcHELcYRkyHZhUfolMUpSoqKShWYXTxovctT1ASG8uAyBj+iEtieVwSX245i9ycRrB3DoNbBzGodSCdQr1xMFrUffh5zl7aexVtxsIvj2otwba7RXtT28WndjdKUa5CKikoVtfU3437+7hxf59mpGTms3JvEn/EJfHtpqN8sf4IXi4m+rcMYHDrIHo398PFsQY/09Ce8MB6WPMmrJ8OB5fD0NehzRjtDOYyikskx87ksP90BvtOZ3I4JRuPgkK6FRWrJ6mUq5pKCopN+bubGd+5MeM7NyY7v4g1B1L4Iy6JlXuT+WHbCcwOBnpF+DG4dRD9WwXg52au/spNTjDgPxA1SmvL6YeJ2ot3178LXo3LiqVk5rP/dCb7Tmew/3Qm+5MyOZCUSV5hCaDlEH83M8mZBax+608mDwhnVMdgTDU9m1GUekwlBaXecDU7MKxNA4a1aUBhcQmbj5xheZx2FrFibzJCQEwTbwa1DmRw6yBC/Vyrt+LASLj3D/L/molD7EuUfNCZlQ0mMbtkCPuSckjLLigr6udmpmWQO7d1aUKLQHdaBLkTEeiGs8nIR4tWsTzJzNOLdvFx7CGmDGzOiHYNa6WbUkWpL1RSUOolk9FA93A/uof7MW1Ea/aczCi7D/HKkn28smQfEQFuDI7UEkSbRp4Y9J1zUXEJCWnZ7DudqZ8BaJ/HzoTSiFd5yfQFQxPfo6nD7/wWNhXP0A60DNISgO8lzkQi/Yw8NLo7K/cm89by/Uz5fjsfxcbzr0HNGRIZhKjGZSlFqe9UUlDqPSEEUY08iWrkyWODmnP8TI6eIE4z88/DzFh9iEAPMx0be3M0LYf4lCwKirRLPwYBYX6utGnkyZjoYFoERdM0cAwlJ3+n+dJnaH74PmjwKDR5CkyXvzQlhGBg60D6twxgye5TvPPHAR74dhtRjTx4fHAL+jb3V8lBsWsqKSh2J8THhXt6hnFPzzDOZhewal8yf8QlsftkOmF+bvSM8Cu79BMe4IaTqZIbw35jIXwALPs3rH0b9vwEI9/XblBXg8EguL5tQ4ZGBvHT9pNMX3GAu7/cTHQTbx4f3JzuzfxqeastdzQtmxV7k9l+/BzNHYroa+uAFLugkoJi17xdHRkdHczo6GDLK7v4wE0fQ9ux8MsU+Oo66HgXDPqv9mhrNTgYDYyJDmZku4bM33KcD1Yd5NbPNtEj3JfHB7egY2Nvy+OqoZISyfbEc6yIS2LF3iQOJGk957mbHfglv4hk026mDm9Zs6e5lGuG+nUoSrP+8NAGiH0VNsyAA0th+JtaO0rVvBTk6GDg9q5NGBMdzJxNx/hodTyjPvqL/i0D+Neg5kQ18qyT0HMLilkXn8qKuCRW7ksiNasAo0HQOdSH/1zfmIGtAgj0cOLRWSv4dtNR1h5M4e2b2xPdxHrJSrEvKikoCmidCA1+SWv5dfFkmH8ntLgOrnsLPBpWezVOJiP39gxjfKcQvvorgU/+PMT1H6xjeJsg/jWoOeEB7lccanJmHqv2JrNibxJrD6aSX1SCu9mBPi38GdQ6kL7NA/B0MV1Q55aWZiYMiuGJBTsYO/MvHuzbjEcHNMfRQT1Wq1xIJQVFKa9hB7hvNWycAatfgRldYOA0iL7HotW4mh14uF84t3dtwqy1h5m17ghLd5/mxvaNeHRgBE18q/k4LSCl5KBfb5UAACAASURBVEBSFiv0l/y2Hz8HQCMvZ27p3Fh/C9znsjv4bs18WTqlF//9JY4Zqw+xel8K745rT4ugK09UytVDJQVFqcjooDXo12qEdq/ht8dh5wLcAsaBhbdrPZ1N/GtwCyb0COOTPw/x9YYEFu84ydiYEP6vfzgNvZwrrVf6nsYfe7X7A8fP5ALQLtiTxwc1Z2DrQFoGuVv8pJO7k4k3x7ZjUOtApv6wixEfrOOJIc25t2dT9b6FAqikoChV82kKd/4M27+D5f8m5vhjkLMK+jwDQVGWrcrVkanDW3FvzzBmrI7nu7+PsWhrIrd1bcxDfcPxdzeTnlvInwdSWBGXxOr9yWTmFWF2MNAz3I8H+4QzQL8/UBsGRwbRsYk3z/6wi1eW7GNFXDJv39yOEB/Vquy1TiUFRbkUIaDDbdByOAnznib08O+w9xftJnTfZ7S3pS0Q4OHEizdEcV/vpnywMp5vNhxl3t/HiWrkwT/HzlFUIvF1dWRYVBADWwXSM6KGbT9Vg5+bmU/uiGbRthO8uHgPQ6ev4T/Xt2ZcpxD1rsU1TCUFRakOZ28Swm4ldNxrsPEj2DgT9i6G1jdqySGglUWrC/Z24fUxbXmgbzPeX3mQA0mZTOzVlEGtA2kf4mW1SzlCCMZEB9O1qQ9PLtjJMz/s4o+4JF4d3YYA99o5K1Hsi0oKimIJFx+t+9GuD2mPr26aCXE/Q+RN0OdpCGhp0erC/Fx5d1z7Ogq2+oK9XZgzsQtf/ZXA60v3MeTdNbxyUxuGtWlg69AUK1PPoylKTbj4aC2wTtkFPR+DA8vgo66w8B5I2W/r6GrEYBDc0zOM3yb3JMTHhQfnbOOx77eTnlto69AUK1JJQVGuhIuP9sjqlF3aE0v7l2qPsS6aCKkHbR1djYQHuLPowe5MGRjB4h0nGTp9DesOpto6LMVKVFJQlNrg6guDXoQpO6HHZNj3G8zoDD9MgtR4W0dnMZPRwJSBzfnxoe64OBq5fdYmXli8h9yCYluHptQxlRQUpTa5+mltJz26E7o9rPUTPaMT/PgApB2ydXQWaxvsxW+Te3FPjzC++iuB695fW/bynHJ1UklBUeqCm7/WbMaUndpN6T0/wYed4McH4cxhW0dnESeTkedHtOa7iV3IKyxm9Md/8c7y/RQWl9TZd+YVFpOckUdSdglnswsoKZF19l3KhdTTR4pSl9wCYMjL0H0yrH8PtsyCnd9Du1ug9xPgE2brCKute7gfSx/rzYuL43h/VTyr9ifz7s3tiQi8uJkMKSXZBcWk5xaSnlNIRl6hNp5bSIY+pFcYMvKKysZL+8MAeHrtHxgEeLs44uPqiLerI76u2njFwdvFEV83bVz1pV0zKikoijW4B8LQV7T7Deumw5YvtP6iS5ODd6itI6wWDycTb9+sNZPx7I+7uO6DdfRvEUB2QdEFO/qMvCKKL3F0L4S2Lk9nEx7ODng6mwjydNKmnUx4OGvLjh46QIMm4ZzJLuBMTgFnsrTPg8lZnM0u4GxOAVV9jaujER83R3wuSiZmfFxN+LiacTIZMAihD2A0CIQQGA3adOmysmmDPi0EQi9fWveCZQbILZKk5xZSUiIpkZJiKSkpQRsvkUgJxWXjFy8vkaUD2nSJPq7Pbx7oTqMqmkm5EiopKIo1uQfBsNe0J5XWT4ctX8KOuRA+EMzuYDCBsXRw1D4NpeMO2qehwvILypRbVlrOwYyp4ByUFIOhdo6eh0YFEd3Emxd/2UPcyQw8nE14uTjSxNf1gh196VB+R+/hbMLd7FDWfSpSQs4ZSD8OGcchPVEbUhM5mxKPt6kpuPqDTwCE+Gnjrv7gGkyxix8ZxU6k5RRyNqeAtCwtUZzJPj+kZReQmlXAgaQszmQXkFtoxZvlK5bX2apfujGK27s2qfX1Wi0pCCGGAu8BRuBzKeVrFZabgW+AaCANGCelTLBWfIpiVR4NYNjrWnJYNx2OrIHiAigu1D5LCvVxfVpe2Y6sB8AGA7j4aZe0XP3BLVC79+EaUG5egDbt6nfZBOLvbubDWzte/ssLciDjhLbTTzpxfqefoX+mn4Ci3AvrGM3g2QhDiQmS9kB2CuRdfIPbCHg7OOHt6q/FXJYw/MHHH0L89W3UBxdfcosNZWcd+UXFZUfiZUfrkvNH9/rReelRe9kRfiVH8rJC+YPxh4gIb4bRIM6fgZQ7qzACJpmHuSgLc3EmjkWZOBZm4ViUgWNhJo5FGZgKMzEVZOBQmIlDYQamgkyMBRkYCzPIlS8Ad1XvB2ABqyQFIYQRmAEMAhKBzUKIxVLKuHLF7gXOSinDhRDjgdeBcdaIT1FsxqMhDH/j8uVKSvREUXBhsigugJKi8+PFRRWSSgEU5XNwx0YiGnhCdjJkpWifaYe0z6K8Sr5Q6DvZgKoTh5ueWEqKtB17+nF955944ZB75uJ1uwWCZzAERkHzodq4RyPt0zNE+24h+Cc2lr59+2rVigogJ02LOTsFslMhq9x4djJkJZ1PIsUFlf4pnZ19aOTqTyNXfzA5a8lPGLVrWqXjZfMMYDBcOM+gzxeGS5Y/bIinaa4v5KVXPZRc5sVAB2dw8jw/eAeCU3Nw8sTUMOLyv5sasNaZQmcgXkp5GEAIMQ+4ASifFG4AXtDHFwIfCiGElFI9dqAoBgMYzOBgrlH1E2l+RJTuXMuTEvIzzieK0p1sVvKFCeTMJm1exSP6ypg9wVPfwQfH6Dv7kPPz3BuCg6PlG+HgqJ1heVSj6Q0ptZ1udqqeNMonj9LtS4WcVO2ymizRhpJi7ays9FPKSuaVaEm6/LySYuDCXVVTgONOF+7UXXy11nfLz7tg8Co37lHjf+8rIayxzxVCjAGGSikn6tN3AF2klI+UK7NbL5OoTx/Sy6RWWNckYBJAYGBg9Lx582oUU1ZWFm5ubjWqa2sqdtu45mOXEmNxLo4F6ZgKz+FYoA1SCPLN/uQ5+ZNv9qPYofaa37arv7mUQAlCakNWdg4uHvWz29N+/fptlVLGVLbM7m40Syk/BT4FiImJkX0rO/qphtjyp6V2RsVuGyp267PXuMF+Y7fWy2sngJBy08H6vErLCCEcAE+0G86KoiiKlVgrKWwGIoQQYUIIR2A8sLhCmcWcv5U+Blil7icoiqJYl1XuKQAIIYYD09GeIvtCSvmyEOK/wBYp5WIhhBMwG+gAnAHGl96YvsQ6U4CjNQzJD7DXph9V7LahYrc+e40b6nfsTaSU/pUtsFpSqG+EEFuqutFS36nYbUPFbn32GjfYb+yqQTxFURSljEoKiqIoSplrOSl8ausAroCK3TZU7NZnr3GDncZ+zd5TUBRFUS52LZ8pKIqiKBWopKAoiqKUuSaTghBiqBBivxAiXgjxjK3jARBCfCGESNbbgCqd5yOE+EMIcVD/9NbnCyHE+3r8O4UQHcvVuUsvf1AIUfvt6l4cd4gQYrUQIk4IsUcI8agdxe4khPhbCLFDj/1FfX6YEGKTHuP3+guXCCHM+nS8vjy03Lqm6vP3CyGG1HXs5b7XKIT4Rwjxqz3FLoRIEELsEkJsF0Js0efZw2/GSwixUAixTwixVwjRzR7itoiU8poa0F6eO4TWiKEjsANoXQ/i6g10BHaXm/cG8Iw+/gzwuj4+HPgdEEBXYJM+3wc4rH966+PedRx3A6CjPu4OHABa20nsAnDTx03AJj2m+WgvTwLMBB7Uxx8CZurj44Hv9fHW+u/IDITpvy+jlX43/wK+A37Vp+0idiAB8Kswzx5+M18DE/VxR8DLHuK2aBttHYDVNxi6AcvKTU8Fpto6Lj2WUC5MCvuBBvp4A2C/Pv4JcEvFcsAtwCfl5l9Qzkrb8DNavxl2FTvgAmwDuqC9hepQ8fcCLAO66eMOejlR8TdUvlwdxxwMrAT6A7/qsdhL7AlcnBTq9W8GrT22I+gP6NhL3JYO1+Llo0bA8XLTifq8+ihQSnlKHz8NBOrjVW2DTbdNvyTRAe2I2y5i1y+/bAeSgT/QjpTPSSmLKomjLEZ9eTrga6vY0ZqNeQoo7eXeF/uJXQLLhRBbhdYcPtT/30wYkAJ8qV+y+1wI4WoHcVvkWkwKdklqhxT19vlhIYQbsAiYIqXMKL+sPscupSyWUrZHO+ruDLS0cUjVIoS4HkiWUm61dSw11FNK2REYBjwshOhdfmE9/c04oF3i/VhK2QHIRrtcVKaexm2RazEpVKcZ7/oiSQjRAED/TNbnV7UNNtk2IYQJLSHMkVL+oM+2i9hLSSnPAavRLrl4Ca359opxVNW8uy1i7wGMFEIkAPPQLiG9ZyexI6U8oX8mAz+iJeT6/ptJBBKllJv06YVoSaK+x22RazEpVKcZ7/qifHPid6Fdry+df6f+dENXIF0/fV0GDBZCeOtPQAzW59UZIYQAZgF7pZTv2Fns/kIIL33cGe1eyF605DCmitgra959MTBef8InDIgA/q7L2KWUU6WUwVLKULTf8Cop5W32ELsQwlUI4V46jvZvvZt6/puRUp4GjgshWuizBqB1KVyv47aYrW9q2GJAeyrgANr143/bOh49prnAKaAQ7YjkXrRrviuBg8AKwEcvK4AZevy7gJhy67kHiNeHu60Qd0+00+WdwHZ9GG4nsbcF/tFj3w08r89virZjjAcWAGZ9vpM+Ha8vb1puXf/Wt2k/MMzKv52+nH/6qN7Hrse4Qx/2lP4ftJPfTHtgi/6b+Qnt6aF6H7clg2rmQlEURSlzLV4+UhRFUaqgkoKiKIpSRiUFRVEUpYzD5YvUX35+fjI0NLRGdbOzs3F1da3dgKxExW4bKnbrs9e4oX7HvnXr1lRZRR/Ndp0UQkND2bJlS43qxsbG0rdv39oNyEpU7LahYrc+e40b6nfsQoijVS2zyuUjUUkLoBWWV9maoKIoimI91rqn8BUw9BLLh6G9NBMBTAI+tkJMiqLUVGEenDkCCevh+GbIz7J1REotscrlIynlGlGu/fZK3AB8I7WXJjbqbZY3kOcbmVKUq5KUksU7ThK7PwVvF0d83RzxdzPj5+6Ir6sZP3czvq6OOJmM1guqIAcyT0HGCcg4SfG5RPLPHKfobCIi8ySm7NM4FZy5qFqyqSEnHZtywqwPjs1INTVAipofeyYn5zH/RO007yRkMd5FKQQWHCew8DgBBYk4yryKpcrG5AXjF62t0nLl5+fmF7HywO9I9yAcPBvg5N0IN/8QvHz88XN3su6/qQWs9vKanhR+lVJGVbLsV+A1KeU6fXol8LSU8qIbBnqLipMAAgMDo+fNm1ejeLKysnBzc6tRXVtTsdtGbceeXSj5ZncenVIXMcphPbk4kimdyZZOZON0/hMnCoQTJQ7O4OAMJmcMJmeMjs6YzM44mp0xO7ng7OSMh7MJJyNorY9cHLuhOA9zfiqGnFRkdirG3FRM+Wk45afiWpiGZ1EabvLio/6z0o3T0odT0ofT0oeT0pdk4UOWgw9uhgLCOUYzeYxweZwQTmHQd6M5mDlMCIdEY+JFY+2TxmSJ6t2ALSkpwWCwLKm4yyyacJIQeYrG8hSN0T6DOY2ZwvN/E5zJwqVsWpTb9V+4m6+4j6xeOTMFuJF7UXz50kSy9CIVL84avMkwepPl4E2eow8Fjj6UOHsjnX0xO7vjYTbg4Shwdrj43/RK9OvXb6uUMqayZXZ3o1lK+SnwKUBMTIys6Y2c+nwT6HJU7LZRm7FvPJzG8/M2MyV3BqNMa5FNeiAcXSnOy6I4L4OS/HOIgiwMhdmYinO0ShKtEZTCqtebL03k4ESuwZkCgzNFDq5IoxnHvFR85dlKd/ip0oPT0oej0pc0QwuyzAHkOgdS6NoQ4dkQk1cwnp6e+Ls54udmJszNjJ+bI25mh8p3VAU5kLIXkuJwSdpDVNJuopL+gdxV58t4hkBgZLkhCnyagfHCXVKVf/OifO3yVdpBSIuH1Pjz4zlp58sZHMA7DHyjwO9G8A0H3wjwi8DN1R+3WtzRVhQbG0vf7p3JP3eS9ORj5KSdJP/sCUoyTmHISsIzN4mg/CQ8Cvfgmp8N+RfWz5cOpOBFkvQmFW8yHHzJc/KnwDmQErdAOnbqSXRUq1qPu74khXrdaqCi1JaCohLeXXGAb//cxRcuH9LJuB36Povo8xQIgRGta8ALlJRAYQ4UZEFBtvaZn0VRXibZmefIykwnNyud/JwMCnMyKM7LoiQ/E1GQjbEwG2NBDqcIZJ9LB/Kdgyhya4jBS9vZu/qH4OPpga+bmfDaukzl6AKNorWhlJSQeRqS9kDSbv1zD8SvgBK9+wejGQJaaglCTxZOuafg8J/aDj81Xtvppx2Ec8dAlpxfv1ugtrNveT34RWjjvuHg3QSMpivfpppydMEcEE5AQPilyxXkQNZpitJPkZ2WSG7aSQrTTyIzThOUfZrGeSm45sfhkpMFOUAabPf8D1zFSWEx8IgQYh5az1fp6n6CcrWJT85iyvf/kHziKMu83qVB/hG4YQZ0uP3SFQ0GMLtpQzml7V97VuO7Y2NjGWLLMzQhwKOBNkQMPD+/qABSD5RLFru1RLF9DqD1YUlpQ9UmV/BtBg07Qttx+lG/Pjh5WHuLapejC/g0xcGnKZ5hl/g3LczVkmvmadp7N6mTUKySFIQQc9FacvQTQiQC09D6xEVKORNYgtayZjxaHrzbGnEpijVIKZmz6Rgv/RZHpMNJ/vR5C+fCdLhtPoQPvPwKrmYOjhAUpQ2MOz8/OxWS9rBv0x+07DJI2/F7NNSSy7XM5Aw+YdpQR6z19NEtl1kugYetEYuiWFNqVj7PLNrJir3JTAw5ybMZ/8MgzHD3EmjY3tbh1V+uftC0D6ePSVo27WPraK4p9eXykaJcdVbvT+bJBTvJyCvki07H6Rf3H4RXE7h9kXatW1HqIZUUFKWW5RUW8+qSvXy94SgtAt35rdMOAjf8F0K6wi1zwcXH1iEqSpVUUlCUWrTnZDpT5m3nYHIW93ZvzFTjbBw2fAKtRsKoT7VrwopSj6mkoCi1oKREMmvdEd5cth9PFxOz72pHr51TYe9i6PIgDHkZDPXzDVZFKU8lBUW5QqfT83h8wXbWx6cxuHUgrw8PwXvxXXBsAwx5BbqpZygU+6GSgqJcgSW7TjH1h10UFJXw2qg2jIsoQcy5Hs4mwJgvIWqUrUNUFIuopKAoNZCVX8SLi/ewYGsi7YI9mT6+A2GF8TBrLBTlwR0/QWgPW4epKBZTSUFRLLTt2Fke+347x8/k8Ei/cB4dGIHpyCqYfxc4e8Odi7XmGhTFDqmkoCjVVFwieW/FQd5fdZAgDye+v78bnUJ94J9vYfFkCGgNty3QmnJQFDulkoKiVMOxtBxe/TuP+HMHuKlDI168IRIPswPEvg6xr0DTfnDzN/bfBo9yzVNJQVEuQUrJom0neGHxHoqLS3hvfHtuaN8Iiovgl8mw7RtodwuMeF9rx0dR7JxKCopShfScQp79cRe/7TpF5zAfbm6cqyWE/CxYeDccXA69n4R+/1YNtSlXDZUUFKUSfx1K5fH5O0jJzOepoS24v3cz1q75E7KSYc5YOL0Trn8XYu6xdaiKUqtUUlCUcvKLinln+QE+XXuYMF9XfnyoB22CtdbtnXMS4fPJkJ0C4+dCi6E2jlZRal+1k4IQ4kfga+A3KeUlOgRUlAtJKYndn8L7qw6Sl5WLa+gZ7amdeiY+OZPJc7cTdyqDW7s05rnrWuHiqP8XObaJjtueAUcz3PUrBEdfemWKYqcsOVNYCzwPzBJCzAdmSyn/qpuwlKvFloQzvLF0P38nnKGxjwvp2ZKxMzcwoGUATwxpQasGtn9aR0rJtxuP8tJve3E1O/DZnTEMah14vkDcz/DDJApN3pgmLgGfprYLVlHqWLWTgpTyHeAdIUQkcDswVwhRAMwG5kgpD9VRjIod2nc6g7eW7WfF3mT83c3878YoxsWEsDr2Tw45hPBx7CGGv7+WG9s34rGBzWns62KTOFMy83l60U5W7UumT3N/3hzblgB3J22hlLBhBix/DoI78U/jR+ihEoJylbP4noKUcg8wVQixBPgQrWvNx4UQm4HHpZQ7KqsnhBgKvIfWL/nnUsrXKixvjHZ5yksv84yUcoml8Sm2dfxMDu/8cYCftp/AzezAk0NacHeP0LLLMGYHwUN9w7m1c2Nm/nmYL9cf4dedJ7m1c2Me6R+Bv7vZarGu2pfEkwt2kplfxAsjWnNX91BE6VNEJcWw9Bn4+9OyZq8L12+69AoV5SpgUVIQQrRAO0u4FSg9S7geSAEeAn4CLuo8VAhhBGYAg4BEYLMQYrGUMq5cseeA+VLKj4UQrdH6bQ61dIMU20jJzOfDVQf57u9jGITg/t7NeKBPU7xcKn9238vFkWeGtWRC91DeX3WQbzcdY8HWRO7tGcZ9vZvi4WSqs1hzC4p5ZcleZm88Sssgd+ZO6krzQPfzBQqyYdFE2L8Euj0Cg/4HBkOdxaMo9YklN5q3oO2kvwdulVJWPGx6Rwjxf1VU7wzESykP6+uaB9wAlE8KEii9wOwJnKxubIrtZOQV8tmaw8xad4T8ohLGdQphcv8IgjydqlU/yNOJV25qw329mvL28v18sCqe2RuP8nDfcO7o1gQnU+32QbD7RDqPzvuHQynZ3NcrjCeGtMDsUO47MpNg7jg4tQOGvwWd76vV71eU+s6SM4XXgMVSyoKqCkgpLzpL0DUCjpebTgS6VCjzArBcTyyuwEALYlOsLK+wmNkbjjIjNp5zOYVc37YBjw9uQZifa43WF+bnyoe3duSBPum8sWw/Ly/ZyxfrjzBlYASjOwbjYLyyI/WSEsmnaw/z9vL9+Lg68u29XegZ4XdhoZT9MGcMZKfC+O+gxbAr+k5FsUdCSlm9gkIMBhKklAfKzWsBNJZS/nGZumOAoVLKifr0HUAXKeUj5cr8S4/nbSFEN2AWECWlLKmwrknAJIDAwMDoefPmVSv+irKysnBzc6tRXVuzZezFJZJ1J4v4Ob6QM3mSKD8jYyJMhHpW74i+urHvTStmwYECDqeXEOQqGB3hSEyg8fw1fwuk5Zbw2a589p0pISbQyIRIM26OF67H89xuona/ghQmdrV5jkyPiBrHXh/Za+z2GjfU79j79eu3VUoZU+lCKWW1BuAg0KDCvIbAgWrU7QYsKzc9FZhaocweIKTc9GEg4FLrjY6OljW1evXqGte1NVvEXlJSIn/beVL2e2u1bPL0r/LGGevkX/GpFq/HkthLSkrk0t2n5IC3Y2WTp3+VIz5YK9ceSLHo+xZvPyHbTFsqW/3nd/n95mOypKTk4kI7vpfyRV8pP+gk5ZmEWom9vrHX2O01binrd+zAFlnFftWSy0cBUspTFeadAoKqUXczECGECANOAOPRblaXdwwYAHwlhGgFOKHdwFZsbN3BVN5Yto+dielEBLjx6R3RDGodWKOjdksIIRgSGcTAVoH8sC2R6SsOcvusTfQI9+WpIS1pF+JVZd3MvEKmLd7DD9tO0D7Ei+nj2hNa8dKWlLD2LVj1EjTpCeO/1fpDUJRrmCVJ4bAQor+UclW5eX2BI5erKKUsEkI8AixDe9z0CynlHiHEf9Ey1mLgceAzIcRjaDedJ+gZTbGRHcfP8cayfayPT6ORlzNvjW3HTR0aYTRYt/E3o0EwNiaEEe0aMmfTMWasjueGGesZFhXE44NbEB5w4Sn61qNnmPL9dk6czWXygAj+r384por3JIoL4bd/aa2ctrkZbvgQHKz3OKyi1FeWJIUXgB+EELOAQ0Az4G59uCypvXOwpMK858uNxwGq/8J6ID45i7eX7+f33afxcXXk+etbc1vXxhc+pWMDTiYj9/YM4+aYYD5fe4TP1x5m2Z7TjI0O4dGB2jsOH6w8yIer42nk7cyCB7oR3aSS5jTyMmDBBDi0UrVyqigVWPJG88/6zeZ7gOvQniYaIqXcXFfBKdaVV1jMi7/s4fvNx3E2GZkyMIKJvZriZq5f7Sa6O5l4bFBz7uzWhBmrD/HtxqP8uP0EjX1ciE/OYlTHRrw4MhL3yt51SD8B390MyXth5AfQ8U7rb4Ci1GMW/W+XUv4N/F1HsSg29sX6I8z9+zgTuofyf/3D8XWr35dTfN3MPD+iNff0DGX6ioOsO5jKh7d24Pq2DSuvcHq31ux1fqbWbWb4AOsGrCh2wNI3mtsDvQA/oOx8u/xlIMU+nc0u4OPYQwxsFcALIyNtHY5Fgr1deGtsu0sXil8J8+8Cszvc8zsEtbFOcIpiZ6r9RpD+fsB6oD/wNNAG7eZweN2EpljTR7HxZOcX8eSQlrYOpfZtm62dIXg3gYkrVEJQlEuw5DXRp9BeQLsJyNU/xwCqbwU7d+JcLl//dZTRHYNpEeR++Qr2QkpY+T9Y/Ag07QN3/w6ejWwdlaLUa5a+p7BWHy8RQhiklL8LIebURWCK9bz7xwEQ8Nig5rYOpfYU5cPPj8Cu+dDhDq3rTGPdNbKnKFcLS5JCohAiVEqZABwAbhBCpKK1lqrYqX2nM1i0LZH7ejWloZezrcOpHbln4fs7IGEt9H8Oej2hHjlVlGqyJCm8AbQCEoD/AgsBR2By7YelWMubS/fjZnbgob7N6vaLSkrg+Cb8UjbAEQM4eYGzl/Zpdq+9nfbZo9r9gzOHYdRn0Pbm2lmvolwjqpUUhNaewRq0pijQLxt5A45Syqw6jE+pQ38fOcPKfck8PbRllf0eXLGU/bDze9i5ANKPEQVaK1flCeP5BFHZp7N31csc3c4nlBPb4LtxUJwPd/wIYb3qZpsU5Sr2/+3de3hU1bnH8e+bK5IAAQKR+10QEOVSEFELVhB8vMtRvKG11qe1Pq2np3q8nFptq1bsKWrVqrVWRT1RsVaKVooVjgeoIlflTsAACdcEAgmXhCTv+WOtDMM0STMhmQvzfp5nntl7zZ6Z38Yx78zea6/VoKKgqioiXwGtgtoqsENHcUtVO+KkDAAAFBpJREFU+dVf13Jq6xZ8e0zPpn3xst2w6l1YmQs7VoAkQZ8L4Fs/ZcmWMkYM7guHS+BISdD9vuOX9319bP34gXKPl5RyrEDsL4TMDnDLbOjQv2n3yZgEEc7ho+XAacC6ZspiIuhva3axbGsJj199RtNMZFNxyM1UtjIXNn0CWgWdzoSLHoXBk6FVDgBle+dDr/Mb/rrV1VBR+i+KiL/vOhIufCjwXsaY8IVTFOYDH4nIK7ghLgKD1anqy00byzSnyqpqpn20jj4dMrh6WNfGv1B1lTuZu/ItWDsLKsqgdVcY80MYMgU6NsE1D0lJ0KKNu9HjxF/PGFOvcIrCGNyIqN8MaVfAikIcmbm0gE17DvLCTcMbN6PZrtXuF8FXM6F0O6S3hkFXuELQY4zNZ2xMHAtnQLxxzRnERMbhiiqmf7yBYd2zmDAwjMMsB7a7IvDlW7BrlTuW33c8XPSIm7Yy9STpzmpMgmtwURCROr/+qdZ3JtDEklcW5bPrQDm/vW7Yv54kp7wU1s6GL3Nh8/8CCl1GwKQnYPBVkJFd//ONMXEnnMNHlQSdRwgR3YH2TYOUHKrgufl5fGtAR0b2qmWeAYCqStg83xWCdR/A0UOQ1cPNOzDkWsi2oa6MOZmFUxR6hax3Au4F/tJ0cUxzem7+JsrKK7lnYh0ngAuXQu4NULrDdfMcci2cOQW6jbIrgo1JEOGcU9gS0rRFRG7Gzb/8hyZNZZpcYclhXlmUX/egdztXwYyrXC+fa1+HfhNsekpjEtCJdhNpDXRoyIYiMlFE1otInojcW8c214jIGhFZLSJvnmA2E+TJuRuAOga9K8qDGVdAaku4eRacfqkVBGMSVDgnmmdw/DmFlsD5wOsNeG4y8CwwHigAvhCRWX5e5ppt+gH3AWNUdZ+IdGxoNlO/9TtLeXdZAd85txddQge9K9kKr13uhpme+j607RmVjMaY2BDOOYW8kPWDwPOq+nEDnjsSyFPVzQAikgtcDqwJ2ua7wLOqug9AVXeHkc3U44k568hIT+GOsSEniUt3wquXuSuGb/kAOpxEQ2cbYxpFVOvqUNSEbyIyGTdBz21+/SZglKreGbTNn3FDco/B9WZ6SFU/quW1bgduB8jJyRmem5vbqExlZWVkZmY26rnRFk729XureGzxESaflsolvY8NepdacYCzVtxPevkevhzycw60icxYQYny7x5r4jV7vOaG2M4+bty4pao6otYHVbVBN+Bp4JyQtnOAJxvw3MnAS0HrNwHPhGwzG3gPSMX1dNoGZNX3usOHD9fGmjdvXqOfG20NzV5dXa1XPrtARz4yVw+VVx574HCJ6vPnqf6io+rmT5snZB0S4d89FsVr9njNrRrb2YElWsff1XBONF8HLAlpWwpc34DnFgLdgta7+rZgBcAsVT2qql/jfjX0CyOfCTHXD3r37xeexilp/lKSioPwxjWwaw1cM8OGlzbGHCecoqC1bJ/cwNf4AugnIr1EJA2YAswK2ebPwFgAEcnGjci6OYx8JkhlVTXT5qynT4cMJg/3g94dPQK510PBYrj6JThtQnRDGmNiTjhF4f+AX9YMd+HvH/Lt9VLVSuBOYA6wFnhbVVeLyM9F5DK/2RygWETWAPOAu1W1OIx8Jsi7ywrI213G3RcNcIPeVR2Fmd92Vytf/qwbwM4YY0KE0/voR7jj/jtEZAvQHdgBXNqQJ6vqh8CHIW0PBi0r8GN/MyfgyNEqps/dyNDuWVw0KMcNcf3e99x8Bxf/Gs5qyBE/Y0wiCueK5gIRGYbrXtoNdyJ4sdpgeDHnlUX57DxwhKemnIUAzL4LVs2ECx+Gkd+NdjxjTAwL5+K1s4BiVf0M+My3dRORdqq6srkCmvCUHKrguXl5XDCgI6N6tYM598Oy19yAdufeFe14xpgYF845hddx3UWDpQEzmi6OOVG/m7+J0vJK7pnYH+Y9Cp89B2ffAeMeiHY0Y0wcCKcodFd/RXINVd0E9GzSRKbRtpcc5o+L8rlqaFcG5L0Mn06DYVPdPMk2yqkxpgHCKQo15xQC/Pr2po1kGmu6H/TupzkL4eOfweDJcMmTVhCMMQ0WTu+j6cD7IjIN2AT0AX4CPNIcwUx4Nuxyg95N77+WrHm/gP4Xw5XPQ5LNf2SMabhweh/9XkRKgO/geh9tBf5DVWc2VzjTcNM+Ws/laUu4bMuT0HssTP4jJIeeAjLGmPqF80sB4FOgHKiZnLe1iNyqqi83bSwTji/y91K5/iN+nf400nUkTHkTUltEO5YxJg6F0yX1ClxPozxgELAaGAwsAKwoRImq8pf33+b5tKeQnEFww9uQlhHtWMaYOBXOieZfAreq6lDgoL+/HTconomSxQvmcs/en1Ge2Y2km95z02kaY0wjhdsl9Z2QtleBqU2Yx4ShsnAlA/9+C/uTssi4bTZktI92JGNMnAunKOwWkRy/nC8io3E9kKx7SzQUbaTy1Ss4oC3YOPENUrI6RzuRMeYkEE5R+D1wrl+ejhvJdCXwXFOHMvVrcXgX+uqlHKyo5pF2j/HNkcOjHckYc5IIp0vq40HLr4nIfCBDVdc2RzBTh/2FnLnyQcorD3F9+f08fOmFiF2cZoxpIuF2SQ1Q1a1NGeRkcrC8kpRkIT2lCY+slWyFRc/AstdIUZha9VM6nzacs3vbeQRjTNNpdFEwxxw5WsUX+XtZkFfEwrwiVm8/gCq0apFCh8x0sjPTaZ+ZRnbIcodWabTPSCe7VToZacm1f+PftRoWPgVfzXTDVQy5lkcKR/JZQQc+nDgg8jtrjDmpWVFohKpqZVXh/kARWLJlHxWV1aQmC0O7t+WHF/QjNVkoKqtgT1k5xWXlbNxdxj82F1Ny6Gitr9kiNSlQILIz0hgh65mw7036lCykMrkluwfczJER3+NoRmfeePpTrhzahdM7tY7wnhtjTnYRKwoiMhF4Ctdb6SVV/VUd210NzAS+oapLIpWvPqpKfvEhVwQ2FrFoUxEHjlQCMODUVkw9uwdj+mUzsmc7MtLr/yc9WlXN3oMV7Cktp6isnOKyCorK3PLe0iN0KfqUi7blMqhqLcXail9X/hszjoxn//JMWJ4H5JEi8OPxp0Vgz40xiSYiRUFEkoFngfFAAfCFiMxS1TUh27XCTfv5eSRy1WdPaTmLNrlfAgvziiksOQxAl6xTmDS4E2P6ZXNOn/ZkZ6aH9bqpyUnktG5BTuugYSiqjrrDQwufhKJ10KY7nPMEbc+6gVsrU7m8rJw9ZeUUlVVQXFZO6fZNdG3bsil31xhjgMj9UhgJ5NXMxyAiucDlwJqQ7X4BPA7cHaFcAQfLK1mcv5eFG4tYkFfEup2lALQ5JZVz+rTn+2P7cG7fbHq0b9l0vX0qDrpZ0RY9AwcKoOMguOr3MOhKSE4lCWiXDu0y0uiX0yrwtPnztzTN+xtjTIhIFYUuuDmdaxQAo4I38HMzdFPVD0Sk2YtCVbWydMs+Fua5IrB86z6OVilpKUl8o2db7pnYn3P7ZjOocxuSk5q4y+fBYlj8Iix+AQ7vgx5j4JLp0G+8zX1gjIkqUdXmfxORycBEVb3Nr98EjFLVO/16EvAJcIuq5vtrIH5S2zkFEbkdN+YSOTk5w3Nzc8PO88nWo7y9vpwjVYIAPVonMbB9MoPaJ9OvbRJpyc3zhzn9yG66bXufTjvmklxdTlH7UWztfhUH2oTXi6isrIzMzMxmydjcLHt0xGv2eM0NsZ193LhxS1V1RG2PReqXQiFuDoYaXX1bjVa4EVfn+0MzpwKzROSy0MKgqi8CLwKMGDFCx44dG3aY5I172Fq6jGvOH8Lo3u1pm5EW9muEZdca3630nUC3Us75IdkdBwTGIA/H/Pnzacx+xwLLHh3xmj1ec0P8Zo9UUfgC6CcivXDFYApwfc2DqrqfY3M0UN8vhaZwXr8OVBWmM/aMTs3x8sds+Yc7ebzhI0jNgFHfg9F3QJuuzfu+xhjTSBEpCqpaKSJ3AnNwXVJfVtXVIvJzYImqzopEjogo2wNbFsJnv4Ntn0HL9jDuAfjGbdCyXbTTGWNMvSJ2nYKqfgh8GNL2YB3bjo1EphNWXgrbV0DhUti+DAqXw34/+keb7jDpCRh6I6RZ91FjTHywK5obqrIcdq7yf/yXuUJQtAHwJ+qzekDX4TDqdug8DLqNtDmSjTFxx4pCbaqrYM/6YwVg+zJXEKr9EBUZHaHLMBh8NXQZDp2H2gQ3xpiTghUFVSjZcuzb//bl7pDQ0YPu8bRW0GUojP6BKwSdh7kTxXY9gTHmJJSYRWH7Cnp+/SYUPON+BRwqdu3J6XDqGTD0Bv8LYBi07wtJ4cxFZIwx8Ssxi8KWRfTY8g50PB36T3J//LsMh44DIaWZr1kwxpgYlphFYeiNLDjYi/MunBTtJMYYE1MS87hIi9ZUpZwS7RTGGBNzErMoGGOMqZUVBWOMMQERGSW1uYjIHqCxkwtkA0VNGCeSLHt0WPbIi9fcENvZe6hqh9oeiOuicCJEZEldQ8fGOsseHZY98uI1N8Rvdjt8ZIwxJsCKgjHGmIBELgovRjvACbDs0WHZIy9ec0OcZk/YcwrGGGP+WSL/UjDGGBPCioIxxpiAhCwKIjJRRNaLSJ6I3BvtPAAi8rKI7BaRVUFt7URkrohs9PdtfbuIyNM+/5ciMizoOTf77TeKyM0RyN1NROaJyBoRWS0iP4qj7C1EZLGIrPTZH/btvUTkc5/xLRFJ8+3pfj3PP94z6LXu8+3rReSi5s4e9L7JIrJcRGbHU3YRyReRr0RkhYgs8W3x8JnJEpGZIrJORNaKyOh4yB0WVU2oG26O6E1AbyANWAkMjIFc5wPDgFVBbdOAe/3yvcDjfvli4K+AAGcDn/v2dsBmf9/WL7dt5tydgGF+uRWwARgYJ9kFyPTLqcDnPtPbwBTf/jzwfb98B/C8X54CvOWXB/rPUTrQy3++kiP0ufkx8CYw26/HRXYgH8gOaYuHz8yrwG1+OQ3IiofcYe1jtANEfIdhNDAnaP0+4L5o5/JZenJ8UVgPdPLLnYD1fvkF4LrQ7YDrgBeC2o/bLkL78D4wPt6yAy2BZcAo3FWoKaGfF2AOMNovp/jtJPQzFLxdM2fuCvwduACY7bPES/Z8/rkoxPRnBmgDfI3voBMvucO9JeLhoy7AtqD1At8Wi3JUdYdf3gnk+OW69iGq++YPSQzFfeOOi+z+8MsKYDcwF/dNuURVK2vJEcjoH98PtI9WduBJ4B6g2q+3J36yK/A3EVkqIrf7tlj/zPQC9gB/9IfsXhKRjDjIHZZELApxSd1XipjtPywimcC7wF2qeiD4sVjOrqpVqnoW7lv3SGBAlCM1iIhcAuxW1aXRztJI56rqMGAS8AMROT/4wRj9zKTgDvH+TlWHAgdxh4sCYjR3WBKxKBQC3YLWu/q2WLRLRDoB+Pvdvr2ufYjKvolIKq4gvKGqf/LNcZG9hqqWAPNwh1yyRKRmAqrgHIGM/vE2QDHRyT4GuExE8oFc3CGkp+IkO6pa6O93A+/hCnKsf2YKgAJV/dyvz8QViVjPHZZELApfAP18L4003Em3WVHOVJdZQE3PhJtxx+tr2qf63g1nA/v9z9c5wAQRaet7QEzwbc1GRAT4A7BWVX8TZ9k7iEiWXz4Fdy5kLa44TK4je80+TQY+8d8MZwFTfA+fXkA/YHFzZlfV+1S1q6r2xH2GP1HVG+Ihu4hkiEirmmXcf+tVxPhnRlV3AttEpL9v+hawJtZzhy3aJzWiccP1CtiAO378QLTz+Ez/A+wAjuK+kXwHd8z378BG4GOgnd9WgGd9/q+AEUGvcyuQ52/fjkDuc3E/l78EVvjbxXGSfQiw3GdfBTzo23vj/jDmAe8A6b69hV/P84/3DnqtB/w+rQcmRfizM5ZjvY9iPrvPuNLfVtf8Pxgnn5mzgCX+M/NnXO+hmM8dzs2GuTDGGBOQiIePjDHG1MGKgjHGmAArCsYYYwKsKBhjjAmwomCMMSbAioJJSOJGRR0bpffuLiJlIpIcjfc3pj7WJdUkNBF5COirqjc243vk40bW/Li53sOYpmK/FIw5AUFDShhzUrCiYBKSn+TlEuB+4Fp/OGelf6yNiPxBRHaISKGI/LLmUI+I3CIiC0VkuogUAw+JSB8R+UREikWkSETeCBo+YwbQHfiLf497RKSniGhNQRGRziIyS0T2+glZvhuU8yEReVtEXhORUn/Ya0SE/7lMArGiYBLZEeBR3IQzmap6pm9/BagE+uKGAp8A3Bb0vFG4iVFygEdwwxk8BnQGTscNdvYQgKreBGwFLvXvMa2WHLm4oU0648YlelRELgh6/DK/TRZuPJ1nTmSnjamPFQVjgohIDm7sprtU9aC6UTyn4wadq7FdVX+rqpWqelhV81R1rqqWq+oe4DfANxv4ft1wI57+p6oeUdUVwEvA1KDNFqjqh6paBcwAzqzlpYxpEnY81Jjj9cBNzbnDDQALuC9PwZOiBC/XFJKngPNwU5ImAfsa+H6dgb2qWhrUtgUIPkS0M2j5ENBCRFL02GQ6xjQZ+6VgEl1o97ttQDluqsgsf2utqoPqec6jvu0MVW0N3Ig7pFTX9sG2A+1qhpL2uhND4+ubxGJFwSS6XUBPEUkCUDfe/d+A/xaR1iKS5E8k13c4qBVQBuwXkS7A3bW8R+/anqiq24BFwGMi0kJEhuCGTX/9hPbKmEayomAS3Tv+vlhElvnlqUAabgKVfbgZtjrV8xoP42bg2g98APwp5PHHgP8SkRIR+Uktz78O6In71fAe8DO7psFEi128ZowxJsB+KRhjjAmwomCMMSbAioIxxpgAKwrGGGMCrCgYY4wJsKJgjDEmwIqCMcaYACsKxhhjAv4fra2Pc3rVlOcAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example_questions(valid_data_loader, trainer.model)"
      ],
      "metadata": {
        "id": "hRF9XC7cJH9D"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "DLL_Ass4.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}