From 91dd3ea4b0be65cae088d13e4e9b29e2ee228413 Mon Sep 17 00:00:00 2001 From: Karina Date: Tue, 5 Nov 2024 17:58:47 +0300 Subject: [PATCH] Deleted .ipynb checkpoint files --- .../.ipynb_checkpoints/README-checkpoint.md | 0 .../transformer-checkpoint.ipynb | 1852 ----------------- 2 files changed, 1852 deletions(-) delete mode 100644 week05_transformer/.ipynb_checkpoints/README-checkpoint.md delete mode 100644 week05_transformer/.ipynb_checkpoints/transformer-checkpoint.ipynb diff --git a/week05_transformer/.ipynb_checkpoints/README-checkpoint.md b/week05_transformer/.ipynb_checkpoints/README-checkpoint.md deleted file mode 100644 index e69de29..0000000 diff --git a/week05_transformer/.ipynb_checkpoints/transformer-checkpoint.ipynb b/week05_transformer/.ipynb_checkpoints/transformer-checkpoint.ipynb deleted file mode 100644 index c598ef6..0000000 --- a/week05_transformer/.ipynb_checkpoints/transformer-checkpoint.ipynb +++ /dev/null @@ -1,1852 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "source": [ - "# **Seminar - Attention и Transformer**" - ], - "metadata": { - "id": "jcYtDZ6yYlk7" - } - }, - { - "cell_type": "markdown", - "source": [ - "## 1. Let's build Transformer from scratch in Pytorch\n", - "\n", - "" - ], - "metadata": { - "id": "GFS003OQYv2w" - } - }, - { - "cell_type": "code", - "source": [ - "from collections import OrderedDict\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import math\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline" - ], - "metadata": { - "id": "gEmHY574YpNu" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### 1.1 Multi-head Attention\n", - "\n", - "#### Main class - MultiHeadAttention\n", - "\n", - "**Initialization:**\n", - "* _in_size_ ~ size of the input embeddings\n", - "* _head_size_ ~ size of the embeddings for Q, K, V matrices after transformation\n", - "* _num_heads_ ~ number of heads\n", - "* _out_size_ ~ size of the output embeddings\n", - "* _query_in_size_ ~ size of the input embeddings\n", - "\n", - "**Forward:**\n", - "* query, key, value ~ 3 tensors (one for each Q, K, and V transformation - these are not yet the tensors of shape $\\text{batch_size} \\times seq \\times d_k$, but tensors of shape $\\text{batch_size} \\times seq \\times \\text{in_size}$)\n", - "* mask ~ boolean mask for Masked Multi-head Attention (in the decoder)\n", - "\n", - "$$ Attention(Q, K, V) = softmax\\Bigg(\\frac{QK^T}{\\sqrt{d_k}}\\Bigg) \\cdot V $$\n", - "$$ MultiHead(Q, K, V) = Concat(head_1, ..., head_H) \\cdot W^O \\quad ; \\quad head_i = Attention(Q W_i^Q, K W_i^K, V W_i^V)$$" - ], - "metadata": { - "id": "ygtUy2dGZAC8" - } - }, - { - "cell_type": "code", - "source": [ - "class MultiHeadAttention(nn.Module):\n", - " \"\"\"\n", - " Class to calculate Multi-head attention (or Masked Multi-head attention for the decoder) operation\n", - " \"\"\"\n", - " def __init__(self, in_size, head_size, num_heads, out_size, query_in_size=None):\n", - " \"\"\"\n", - " Args:\n", - " in_size: embedding size of input\n", - " head_size: hidden size of Q, K, V matrices\n", - " num_heads: number of heads\n", - " out_size: output embedding size\n", - " query_in_size: embedding size of input for query (if not provided - same as in_size)\n", - " \"\"\"\n", - " super(MultiHeadAttention, self).__init__()\n", - "\n", - " # Store all passed layer hyperparameters\n", - " self.in_size = in_size\n", - " self.head_size = head_size\n", - " self.num_heads = num_heads\n", - " self.out_size = out_size\n", - " self.query_in_size = self.in_size if query_in_size is None else query_in_size\n", - "\n", - " # Linear transformations for Q, K, V matrices (get all Q, K, V matrices directly)\n", - " self.query_matrix = nn.Linear(self.query_in_size, self.num_heads * self.head_size, bias=False)\n", - " self.key_matrix = nn.Linear(self.in_size, self.num_heads * self.head_size, bias=False)\n", - " self.value_matrix = nn.Linear(self.in_size, self.num_heads * self.head_size, bias=False)\n", - " # Linear transformation for concatenating heads\n", - " self.out = nn.Linear(self.head_size * self.num_heads, self.out_size)\n", - "\n", - " def forward(self, query, key, value, mask=None):\n", - " \"\"\"\n", - " Args:\n", - " query : tensor for query\n", - " key : tensor for key\n", - " value : tensor for value\n", - " mask: mask for the decoder\n", - "\n", - " Returns:\n", - " output vector from multihead attention\n", - " \"\"\"\n", - " # Tensors come with the shape batch_size x seq_len x in_size\n", - " batch_size = key.size(0)\n", - " seq_len = key.size(1)\n", - "\n", - " # The number of tokens in the query will differ for the decoder\n", - " query_seq_len = query.size(1)\n", - "\n", - " # Apply linear transformations to the input\n", - " q = self.query_matrix(query) # (batch_size, query_seq_len, head_size * num_heads)\n", - " k = self.key_matrix(key) # (batch_size, seq_len, head_size * num_heads)\n", - " v = self.value_matrix(value) # (batch_size, seq_len, head_size * num_heads)\n", - "\n", - " q = q.view(batch_size, query_seq_len, self.num_heads, self.head_size).transpose(1,2) # (batch_size, num_heads, query_seq_len, head_size)\n", - " k = k.view(batch_size, seq_len, self.num_heads, self.head_size).transpose(1,2) # (batch_size, num_heads, seq_len, head_size)\n", - " v = v.view(batch_size, seq_len, self.num_heads, self.head_size).transpose(1,2) # (batch_size, num_heads, seq_len, head_size)\n", - "\n", - " # Считаем релевантность\n", - " relevance = q @ k.transpose(2, 3) / math.sqrt(self.head_size) # (batch_size, num_heads, query_seq_len, seq_len)\n", - "\n", - " # Если есть маска (для декодера), то заполняем значения по маске как минус бесконечность (чтобы exp(r) = 0 в softmax)\n", - " if mask is not None:\n", - " relevance = relevance.masked_fill(mask, -torch.inf)\n", - "\n", - " # Получаем вероятности\n", - " relevance = F.softmax(relevance, dim=-1)\n", - "\n", - " # Считаем выходы из каждой головы\n", - " head_i = torch.matmul(relevance, v) # (batch_size, num_heads, query_seq_len, head_size)\n", - "\n", - " # Конкатенируем выходы\n", - " concat = head_i.transpose(1,2).reshape(batch_size, query_seq_len, self.head_size * self.num_heads) # (batch_size, query_seq_len, num_heads * head_size)\n", - "\n", - " return self.out(concat) # (batch_size, query_seq_len, out_size)" - ], - "metadata": { - "id": "U2vT-vwEY6_S" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing MultiHeadAttention for the encoder" - ], - "metadata": { - "id": "xyTV7cqXay9b" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = MultiHeadAttention(\n", - " in_size=10,\n", - " head_size=4,\n", - " num_heads=3,\n", - " out_size=15,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "8A6Af8DEazNa", - "outputId": "80961b75-44ba-4ef2-839d-31f34e4e5fb9" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (out): Linear(in_features=12, out_features=15, bias=True)\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 3 - } - ] - }, - { - "cell_type": "code", - "source": [ - "# Check in normal forward pass from the encoder\n", - "tmp_input = torch.rand(2, 5, 10)\n", - "\n", - "print(\"Encoder-like input, no mask\")\n", - "print(f'Input shape: {tmp_input.shape}')\n", - "tmp_output = tmp_layer(tmp_input, tmp_input, tmp_input)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input, tmp_output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "3ilYxygIa33f", - "outputId": "6412f347-246a-47c8-d7bc-3368ee9743c2" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Encoder-like input, no mask\n", - "Input shape: torch.Size([2, 5, 10])\n", - "Output shape: torch.Size([2, 5, 15])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing MultiHeadAttention for a mixture of encoder and decoder" - ], - "metadata": { - "id": "if4DQzHFbAml" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = MultiHeadAttention(\n", - " in_size=10,\n", - " head_size=4,\n", - " num_heads=3,\n", - " out_size=15,\n", - " query_in_size=12,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ATH6d79ibCn2", - "outputId": "295cfa64-4d6b-4ad6-839d-e8879728c6fc" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=12, out_features=12, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (out): Linear(in_features=12, out_features=15, bias=True)\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 5 - } - ] - }, - { - "cell_type": "code", - "source": [ - "# Check forward pass in the decoder, where we mix information from the encoder and decoder\n", - "tmp_input_q = torch.rand(2, 5, 12)\n", - "tmp_input_kv = torch.rand(2, 7, 10)\n", - "\n", - "print(\"Encoder+Decoder-like input, no mask\")\n", - "print(f'Input Q shape: {tmp_input_q.shape}')\n", - "print(f'Input KV shape: {tmp_input_kv.shape}')\n", - "\n", - "tmp_output = tmp_layer(tmp_input_q, tmp_input_kv, tmp_input_kv)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input_q, tmp_input_kv, tmp_output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "FDNk8KctbEZj", - "outputId": "e7ecebaf-e8aa-466e-b9d8-b3652ef25a91" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Encoder+Decoder-like input, no mask\n", - "Input Q shape: torch.Size([2, 5, 12])\n", - "Input KV shape: torch.Size([2, 7, 10])\n", - "Output shape: torch.Size([2, 5, 15])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#### Triangular Mask in the decoder" - ], - "metadata": { - "id": "4_a1I_XnbKSJ" - } - }, - { - "cell_type": "code", - "source": [ - "def make_decoder_mask(decoder_embed):\n", - " \"\"\"\n", - " Make mask for decoder Masked Multi-head Attention based on input sequence\n", - " Args:\n", - " decoder_embed: decoder sequence after embed\n", - " Returns:\n", - " mask: mask for Masked Multi-head Attention\n", - " \"\"\"\n", - " batch_size, decoder_seq_len, _ = decoder_embed.shape\n", - " mask = torch.tril(torch.ones((decoder_seq_len, decoder_seq_len))).expand(\n", - " batch_size, 1, decoder_seq_len, decoder_seq_len\n", - " ).bool()\n", - " return mask" - ], - "metadata": { - "id": "sEZC_D24bMSe" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing MultiHeadAttention for the decoder with a mask" - ], - "metadata": { - "id": "0-KrJQ0ObP3c" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_input = torch.rand(1, 10, 256)\n", - "tmp_mask = make_decoder_mask(tmp_input)\n", - "print(f\"Mask shape: {tmp_mask.shape}\")\n", - "\n", - "# Visualize the mask\n", - "fig, ax = plt.subplots(figsize=(10, 10))\n", - "plt.imshow(tmp_mask[0, 0, :, :])\n", - "\n", - "# Add text labels\n", - "for i in range(tmp_mask.shape[-2]):\n", - " for j in range(tmp_mask.shape[-1]):\n", - " text = plt.text(j, i, tmp_mask[0, 0, i, j].item(), ha=\"center\", va=\"center\", color=\"red\")\n", - "plt.show()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 848 - }, - "id": "qQjcaCqrbR-J", - "outputId": "8c8e8714-31b9-4caa-8cab-9854d8c2cf90" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Mask shape: torch.Size([1, 1, 10, 10])\n" - ] - }, - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAygAAAMtCAYAAAB0OVVCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABPOklEQVR4nO39f5CedX0v/j+vbMgSkmxCgEASfkiU2JoIgUYoMFPqyJCeKZ3j5+NYZTxTLB2/0x7iD6IM4Bz1OCKUUhmNcPw1ih2pR+gAYvXT02+/+AEEa8mBBEGNqDSEEn4ILLshkM2PfX//gJNkIRHeyGbf2X08ZjLket/Xdd+v+5lld5973de9XSmlBAAAoAGTxnoAAACA/0NBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQjMl7+wGHh4ezYcOGzJgxI13X7e2HBwAAxkApJRs3bsy8efMyadKez5Ps9YKyYcOGHHHEEXv7YQEAgAY89NBDOfzww/d4+14vKDNmzEiSPHj369I33SvMXqn/a+Gbx3oEAAB41bZla27P/7OjD+zJXi8o/+dlXX3TJ6VvhoLySk3u9hvrEQAA4NUrz//n5S7z0BAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQeFVOaOsy43lprEeY58is3oyqyezejKrJ7N6Mqsns3rjJbPJYz3Aa23S3F/+xtvLhw9M+chBe2ma9p1fVuWMPPiS9bPzR9nQTR+Didons3oyqyezejKrJ7N6Mqsns3oTPbNXVVCuuuqqXH755Xn00Udz3HHH5fOf/3xOPPHE13q2V2X4ntft3LjpmXSXP5Vy+5E716btctKolGR7ksnd3hqvSXfm0Pxt3jJibSC9YzTNvkFm9WRWT2b1ZFZPZvVkVk9m9SZyZtUF5dprr82KFSvyxS9+MSeddFI++9nPZtmyZfn5z3+eOXPmjMaMdebs8pT6JiXdLms/fDaT3rEhw9fMTXfZU8naoZRvzUt37cZkYDjl63N3HNp97NfJT4ZSbjj8+YXhklzZn+6aweTX25MF+6WcNzs5c99vsVvTk/5u/xFr7yj3Z1lZl8OyKRszJT/K3Hwlx2Zzt/sPmQXl6fxV7snC9KckeTjT87mckPu72UmSReWJ/EXuzcL0ZyC9uSPz87Us3uP9tU5m9WRWT2b1ZFZPZvVkVk9m9SZyZtWPfsUVV+R973tf/vzP/zxJ8sUvfjHf+9738rWvfS0XXnjhaz7gaOgueTLl4wcnR+2XzJyUZOPLH7SyP931G1MuOyRZMCX50XPplj+WMrsnOWXqqM+8tw2ny1VZkkczLXOzKe/P6rwvP87nc8Ju978wd+ZXmZWVOT7D6fL6DGTbC5c4zS3P5NL8IFdncT6TpZmZoSzPmizP6pf8ZGBfJrN6Mqsns3oyqyezejKrJ7N6EyWzqoKyZcuW3HXXXbnooot2rE2aNCmnn356/vVf/3W3xwwNDWVoaGjH9uDg4Ksc9bVTzp+dnHbAKz9gqKRb2Z9y3bxk6Qtl5Kj9kjs3p7tmIGUfLyi/n0fynXLjju1VOSyf6k7esf1YpuXrZVE+mLv3+D/AnDybf8jCPNT1JUkezowdt52Vtbk5R+bG7pgdt11VluQzuSWfKydka9czGk9rVMmsnszqyayezOrJrJ7M6sms3kTOrKqgPPHEE9m+fXsOPfTQEeuHHnpo1q5du9tjLr300nzyk5989ROOhuP2f/l9drVuS7rnSvKuDSPXt5Zk8b7/WsA1OSQrd/nA3pyeHF8ey1lZmyOyMQdkW3oynN4Mp7dsy9BuTvtdn2OyInfl9LI+d2dObsvheeSFi7gWZCBHZyBvK+tHHNOTZG42ZX36RvX5jQaZ1ZNZPZnVk1k9mdWTWT2Z1ZvImY36C8wuuuiirFixYsf24OBgjjjiiNF+2N/sgBddFL+7N1vetsvfN5UkSfnGvGTui9rklH3/AvvNmTziHSEOLZtyce7IP2ZBrs7iDGZKFueJfCR3ZXKGM7Sb+/hGtyjfL0fmpDySE/No/iw/zSXlpNzRzc/UbMv3siDfzhtectzjqTiT1RCZ1ZNZPZnVk1k9mdWTWT2Z1ZvImVUVlIMPPjg9PT157LHHRqw/9thjOeyww3Z7TG9vb3p7Gz/LcFBPsnbLyLWfDO1MZ+GUlN4ueXjruLze5MWOSX+6lHwpx6V0zxew08p/vOxxD3czckNm5IYszEfLv2VZ1uWOzM8vMitHZXBcvy2ezOrJrJ7M6smsnszqyayezOpNpMyqflHjlClT8nu/93u5+eabd6wNDw/n5ptvzsknn/wbjmxbOXVqcs9Qct1g8sCWdJc/ObKwTJ+U/OWsdJ944vl91m1Nfrw5+erTz2+PMxsyPful5O35ZQ4rz+T08mDOzAN73H9K2Z7lZXWOLY9nTtmUReWJLMxTWf/C6xyvzRvzpjyZ5WV1Xl+ezvyyMSeXDVleVu+tpzTqZFZPZvVkVk9m9WRWT2b1ZFZvImVW/RKvFStW5Oyzz87SpUtz4okn5rOf/Ww2bdq041299klvnZacd2C6i59Mhkry7hnJO2ckP9t5sqxcMDs5qCfdyv5k/eNJX0/y5t6UDxw4hoOPjge6WflCOTbvys9zTu7LvTk4X8viXJBVu91/OF36siUXZFVmZSiDmZLbMz9/l0VJkn/vZuXD5bSck/tyRW5Jl5INmZ5bc/jefFqjSmb1ZFZPZvVkVk9m9WRWT2b1JlJmXSml1B505ZVX7vhFjUuWLMnKlStz0kknvaJjBwcHM3PmzPTfvyB9M6pO4Exoy+YtGesRAADgVdtWtuaW3JSBgYH09e35IvxXVVB+GwrKq6OgAACwL3ulBUVDAAAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABoxuSxHoBX5p83rBnrEfY5y+YtGesRAACo5AwKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgrsJWeUdbmx3DTWY+xTZFZPZvVkVk9m9WRWT2b1xktmk8d6gNfapLm//I23lw8fmPKRg/bSNPsGmdU5v6zKGXnwJetn54+yoZs+BhO1T2b1ZFZPZvVkVk9m9WRWb6JnNu4KyvA9r9u5cdMz6S5/KuX2I3euTdvlpFEpyfYkk7u9NV6TZFbvzhyav81bRqwNpHeMptk3yKyezOrJrJ7M6smsnszqTeTMxl1ByZxdnlLfpKTbZe2Hz2bSOzZk+Jq56S57Klk7lPKteemu3ZgMDKd8fe6OQ7uP/Tr5yVDKDYc/vzBckiv7010zmPx6e7Jgv5TzZidnjoMWK7NqW9OT/m7/EWvvKPdnWVmXw7IpGzMlP8rcfCXHZnO3+//NFpSn81e5JwvTn5Lk4UzP53JC7u9mJ0kWlSfyF7k3C9OfgfTmjszP17J4j/fXOpnVk1k9mdWTWT2Z1ZNZvYmc2b75L/Zb6i55MuXjBydH7ZfMnJRk48sftLI/3fUbUy47JFkwJfnRc+mWP5Yyuyc5ZeqozzzWZPbyhtPlqizJo5mWudmU92d13pcf5/M5Ybf7X5g786vMysocn+F0eX0Gsu2Fy8LmlmdyaX6Qq7M4n8nSzMxQlmdNlmf1S36asi+TWT2Z1ZNZPZnVk1k9mdWbKJlNyIJSzp+dnHbAKz9gqKRb2Z9y3bxk6QvfWB+1X3Ln5nTXDKSMw2+2X0xmI/1+Hsl3yo07tlflsHyqO3nH9mOZlq+XRflg7t7jJ405eTb/kIV5qOtLkjycGTtuOytrc3OOzI3dMTtuu6osyWdySz5XTsjWrmc0ntaoklk9mdWTWT2Z1ZNZPZnVm8iZTciCkuP2f/l9drVuS7rnSvKuDSPXt5Zk8cR4LaDMRlqTQ7Jyl08Gm9OT48tjOStrc0Q25oBsS0+G05vh9JZtGdrNqdLrc0xW5K6cXtbn7szJbTk8j7xw4duCDOToDORtZf2IY3qSzM2mrE/fqD6/0SCzejKrJ7N6Mqsns3oyqzeRM5uYBeWAF13gvbs3W962y983lSRJ+ca8ZO6L2uSUCXKxuMxG2JzJI95F49CyKRfnjvxjFuTqLM5gpmRxnshHclcmZzhDu7mPb3SL8v1yZE7KIzkxj+bP8tNcUk7KHd38TM22fC8L8u284SXHPZ6KM1kNkVk9mdWTWT2Z1ZNZPZnVm8iZTcyC8mIH9SRrt4xc+8nQznQWTknp7ZKHt47LaydeFZmNcEz606XkSzkupXu+gJ1W/uNlj3u4m5EbMiM3ZGE+Wv4ty7Iud2R+fpFZOSqD4/qtBGVWT2b1ZFZPZvVkVk9m9SZSZn5RY5Jy6tTknqHkusHkgS3pLn9y5Dff0yclfzkr3SeeeH6fdVuTH29Ovvr089sTkMxG2pDp2S8lb88vc1h5JqeXB3NmHtjj/lPK9iwvq3NseTxzyqYsKk9kYZ7K+hdeG3pt3pg35cksL6vz+vJ05peNOblsyPKyem89pVEns3oyqyezejKrJ7N6Mqs3kTJzBiVJ3jotOe/AdBc/mQyV5N0zknfOSH6282RZuWB2clBPupX9yfrHk76e5M29KR84cAwHH0MyG+GBbla+UI7Nu/LznJP7cm8OzteyOBdk1W73H06XvmzJBVmVWRnKYKbk9szP32VRkuTfu1n5cDkt5+S+XJFb0qVkQ6bn1hy+N5/WqJJZPZnVk1k9mdWTWT2Z1ZtImXWllLI3H3BwcDAzZ85M//0L0jfDCRxGz7J5S8Z6BAAAXrCtbM0tuSkDAwPp69vzRfgaAgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGZPHegAYLf+8Yc1Yj7DPWTZvyViPAABMcM6gAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAzTqjrMuN5aaxHmOfIrN6Mqsns3oyqyezeuMls8ljPcBrbdLcX/7G28uHD0z5yEF7aZp9g8zqyazO+WVVzsiDL1k/O3+UDd30MZiofTKrJ7N6Mqsns3oyqzfRM6sqKJdeemluuOGGrF27NlOnTs0pp5ySyy67LG984xtHa75qw/e8bufGTc+ku/yplNuP3Lk2bZeTRqUk25NM7vbWeE2SWT2Z1bszh+Zv85YRawPpHaNp9g0yqyezejKrJ7N6Mqs3kTOrKii33nprzj333LzlLW/Jtm3b8tGPfjRnnHFGfvrTn2batGmjNWOdObs8pb5JSbfL2g+fzaR3bMjwNXPTXfZUsnYo5Vvz0l27MRkYTvn63B2Hdh/7dfKToZQbDn9+YbgkV/anu2Yw+fX2ZMF+KefNTs4cBy1WZvVkVm1retLf7T9i7R3l/iwr63JYNmVjpuRHmZuv5Nhs7nb/qWlBeTp/lXuyMP0pSR7O9HwuJ+T+bnaSZFF5In+Re7Mw/RlIb+7I/Hwti/d4f62TWT2Z1ZNZPZnVk1m9iZxZ1aP/r//1v0Zsf/3rX8+cOXNy11135Q/+4A9e08FGU3fJkykfPzg5ar9k5qQkG1/+oJX96a7fmHLZIcmCKcmPnku3/LGU2T3JKVNHfeaxJrN6Mnt5w+lyVZbk0UzL3GzK+7M678uP8/mcsNv9L8yd+VVmZWWOz3C6vD4D2fbCpXRzyzO5ND/I1Vmcz2RpZmYoy7Mmy7P6JT+B2pfJrJ7M6smsnszqyazeRMnst6pHAwMDSZLZs2fvcZ+hoaEMDQ3t2B4cHPxtHvI1Uc6fnZx2wCs/YKikW9mfct28ZOkL3yQetV9y5+Z01wykjMNvHF9MZvVkNtLv55F8p9y4Y3tVDsunupN3bD+Wafl6WZQP5u49fqKdk2fzD1mYh7q+JMnDmbHjtrOyNjfnyNzYHbPjtqvKknwmt+Rz5YRs7XpG42mNKpnVk1k9mdWTWT2Z1ZvImb3qgjI8PJwPfehDOfXUU7N48eI97nfppZfmk5/85Kt9mNFx3P4vv8+u1m1J91xJ3rVh5PrWkiyeGK8FlNmrILMR1uSQrNzlE+jm9OT48ljOytockY05INvSk+H0Zji9ZVuGdnN6+fockxW5K6eX9bk7c3JbDs8jL1wsuCADOToDeVtZP+KYniRzsynr0zeqz280yKyezOrJrJ7M6sms3kTO7FUXlHPPPTf33Xdfbr/99t+430UXXZQVK1bs2B4cHMwRRxzxah/2tXHAiy5W3t2bLW/b5e+bSpKkfGNeMvdFbXLKBLnwWWb1ZDbC5kwe8c4jh5ZNuTh35B+zIFdncQYzJYvzRD6SuzI5wxnazX18o1uU75cjc1IeyYl5NH+Wn+aSclLu6OZnarble1mQb+cNLznu8VScyWqIzOrJrJ7M6smsnszqTeTMXlVBWb58eb773e/mtttuy+GHH/4b9+3t7U1vb+M//T2oJ1m7ZeTaT4Z2prNwSkpvlzy8dVxeB/CqyKyezEY4Jv3pUvKlHJfSPV/ATiv/8bLHPdzNyA2ZkRuyMB8t/5ZlWZc7Mj+/yKwclcFx/faLMqsns3oyqyezejKrN5Eyq/pFjaWULF++PDfeeGO+//3v5+ijjx6tufaqcurU5J6h5LrB5IEt6S5/cuQ3ktMnJX85K90nnnh+n3Vbkx9vTr769PPbE5DM6slspA2Znv1S8vb8MoeVZ3J6eTBn5oE97j+lbM/ysjrHlsczp2zKovJEFuaprH/h9bTX5o15U57M8rI6ry9PZ37ZmJPLhiwvq/fWUxp1Mqsns3oyqyezejKrN5EyqzqDcu655+ab3/xmbrrppsyYMSOPPvpokmTmzJmZOnUf/onvW6cl5x2Y7uInk6GSvHtG8s4Zyc92niwrF8xODupJt7I/Wf940teTvLk35QMHjuHgY0hm9WQ2wgPdrHyhHJt35ec5J/fl3hycr2VxLsiq3e4/nC592ZILsiqzMpTBTMntmZ+/y6Ikyb93s/LhclrOyX25IrekS8mGTM+t+c1nefclMqsns3oyqyezejKrN5Ey60op5RXv3O3+dfBXX3113vve976i+xgcHMzMmTPTf/+C9M2oOoEDjLJl85aM9QgAwDi1rWzNLbkpAwMD6evb80X4VWdQKroMAABANacwAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaMXmsBwDa8c8b1oz1CPucZfOWjPUIADCuOIMCAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIwjpxR1uXGctNYj7FPkVk9mdWTWT2Z1RsvmU0e6wFea5Pm/vI33l4+fGDKRw7aS9PsG2RWT2b1ZFbn/LIqZ+TBl6yfnT/Khm76GEzUPpnVk1k9mdWTWb2Jntm4KyjD97xu58ZNz6S7/KmU24/cuTZtl5NGpSTbk0zu9tZ4TZJZPZnVk1m9O3No/jZvGbE2kN4xmmbfILN6Mqsns3oyqzeRMxt3BSVzdnlKfZOSbpe1Hz6bSe/YkOFr5qa77Klk7VDKt+alu3ZjMjCc8vW5Ow7tPvbr5CdDKTcc/vzCcEmu7E93zWDy6+3Jgv1SzpudnDkOWqzM6smsnsyqbU1P+rv9R6y9o9yfZWVdDsumbMyU/Chz85Ucm83d7j+dLyhP569yTxamPyXJw5mez+WE3N/NTpIsKk/kL3JvFqY/A+nNHZmfr2XxHu+vdTKrJ7N6Mqsns3oTObN981/st9Rd8mTKxw9OjtovmTkpycaXP2hlf7rrN6ZcdkiyYEryo+fSLX8sZXZPcsrUUZ95rMmsnszqyezlDafLVVmSRzMtc7Mp78/qvC8/zudzwm73vzB35leZlZU5PsPp8voMZNsLlx/OLc/k0vwgV2dxPpOlmZmhLM+aLM/ql/zUbl8ms3oyqyezejKrN1Eym5AFpZw/OzntgFd+wFBJt7I/5bp5ydIXvuE5ar/kzs3prhlIGYffBL2YzOrJrJ7MRvr9PJLvlBt3bK/KYflUd/KO7ccyLV8vi/LB3L3HL05z8mz+IQvzUNeXJHk4M3bcdlbW5uYcmRu7Y3bcdlVZks/klnyunJCtXc9oPK1RJbN6Mqsns3oyqzeRM5uQBSXH7f/y++xq3ZZ0z5XkXRtGrm8tyeKJ8VpAmb0KMqsnsxHW5JCs3OWLzub05PjyWM7K2hyRjTkg29KT4fRmOL1lW4Z2c0r++hyTFbkrp5f1uTtzclsOzyMvXGC5IAM5OgN5W1k/4pieJHOzKevTN6rPbzTIrJ7M6smsnszqTeTMJmZBOeBFF97u7s2Wt+3y900lSVK+MS+Z+6I2OWWCXMQrs3oyqyezETZn8oh3azm0bMrFuSP/mAW5OoszmClZnCfykdyVyRnO0G7u4xvdony/HJmT8khOzKP5s/w0l5STckc3P1OzLd/Lgnw7b3jJcY+n4kxWQ2RWT2b1ZFZPZvUmcmYTs6C82EE9ydotI9d+MrQznYVTUnq75OGt4/I17a+KzOrJrJ7MRjgm/elS8qUcl9I9X8BOK//xssc93M3IDZmRG7IwHy3/lmVZlzsyP7/IrByVwXH9lpUyqyezejKrJ7N6Eykzv6gxSTl1anLPUHLdYPLAlnSXPznym6Lpk5K/nJXuE088v8+6rcmPNydfffr57QlIZvVkVk9mI23I9OyXkrfnlzmsPJPTy4M5Mw/scf8pZXuWl9U5tjyeOWVTFpUnsjBPZf0Lr0G+Nm/Mm/JklpfVeX15OvPLxpxcNmR5Wb23ntKok1k9mdWTWT2Z1ZtImTmDkiRvnZacd2C6i59Mhkry7hnJO2ckP9t5sqxcMDs5qCfdyv5k/eNJX0/y5t6UDxw4hoOPIZnVk1k9mY3wQDcrXyjH5l35ec7Jfbk3B+drWZwLsmq3+w+nS1+25IKsyqwMZTBTcnvm5++yKEny792sfLiclnNyX67ILelSsiHTc2sO35tPa1TJrJ7M6smsnszqTaTMulJK2ZsPODg4mJkzZ6b//gXpm+EEDrBvWzZvyViPAAD7hG1la27JTRkYGEhf354vwtcQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNmDzWAwDsy/55w5qxHmGfs2zekrEeAYCGOYMCAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggLAhHZGWZcby01jPcY+RWb1ZFZPZvXGS2aTx3qA19qkub/8jbeXDx+Y8pGD9tI0+waZ1ZNZPZnVk1md88uqnJEHX7J+dv4oG7rpYzBR+2RWT2b1ZFZvomf2WxWUv/7rv85FF12UD37wg/nsZz/7Go302xm+53U7N256Jt3lT6XcfuTOtWm7nDQqJdmeZHK3t8ZrkszqyayezOrJrN6dOTR/m7eMWBtI7xhNs2+QWT2Z1ZNZvYmc2asuKKtWrcqXvvSlHHvssa/lPL+9Obs8pb5JSbfL2g+fzaR3bMjwNXPTXfZUsnYo5Vvz0l27MRkYTvn63B2Hdh/7dfKToZQbDn9+YbgkV/anu2Yw+fX2ZMF+KefNTs4cBy1WZvVkVk9m9WRWbWt60t/tP2LtHeX+LCvrclg2ZWOm5EeZm6/k2Gzudv8lcEF5On+Ve7Iw/SlJHs70fC4n5P5udpJkUXkif5F7szD9GUhv7sj8fC2L93h/rZNZPZnVk1m9iZzZq3r0Z555Ju95z3vyla98JRdffPFrPdOo6y55MuXjBydH7ZfMnJRk48sftLI/3fUbUy47JFkwJfnRc+mWP5Yyuyc5ZeqozzzWZFZPZvVkVk9mL284Xa7KkjyaaZmbTXl/Vud9+XE+nxN2u/+FuTO/yqyszPEZTpfXZyDbXrhkc255JpfmB7k6i/OZLM3MDGV51mR5Vr/kJ537MpnVk1k9mdWbKJm9qoJy7rnn5o//+I9z+umnv2xBGRoaytDQ0I7twcHBV/OQr6ly/uzktANe+QFDJd3K/pTr5iVLX/jifdR+yZ2b010zkDIOv6C/mMzqyayezOrJbKTfzyP5Trlxx/aqHJZPdSfv2H4s0/L1sigfzN17/II+J8/mH7IwD3V9SZKHM2PHbWdlbW7OkbmxO2bHbVeVJflMbsnnygnZ2vWMxtMaVTKrJ7N6Mqs3kTOrLijf+ta3cvfdd2fVqlWvaP9LL700n/zkJ6sHG1XH7f/y++xq3ZZ0z5XkXRtGrm8tyeKJ8VpAmb0KMqsns3oyG2FNDsnKXb5Qb05Pji+P5ayszRHZmAOyLT0ZTm+G01u2ZWg3L2O4PsdkRe7K6WV97s6c3JbD88gLF6UuyECOzkDeVtaPOKYnydxsyvr0jerzGw0yqyezejKrN5EzqyooDz30UD74wQ/mX/7lX7L//q/si+JFF12UFStW7NgeHBzMEUccUTfla+2AF11Eurs3W962y983lSRJ+ca8ZO6L2uSUCXJBqszqyayezOrJbITNmTziHW4OLZtyce7IP2ZBrs7iDGZKFueJfCR3ZXKGM7Sb+/hGtyjfL0fmpDySE/No/iw/zSXlpNzRzc/UbMv3siDfzhtectzjqTiT1RCZ1ZNZPZnVm8iZVRWUu+66K48//nhOOGFnm9u+fXtuu+22XHnllRkaGkpPz8gveL29ventbfyncgf1JGu3jFz7ydDOdBZOSentkoe3jsvXZ78qMqsns3oyqyezEY5Jf7qUfCnHpXTPF7DTyn+87HEPdzNyQ2bkhizMR8u/ZVnW5Y7Mzy8yK0dlcFy/zafM6smsnszqTaTMqn5R49ve9rbce++9WbNmzY4/S5cuzXve856sWbPmJeVkX1FOnZrcM5RcN5g8sCXd5U+O/AI/fVLyl7PSfeKJ5/dZtzX58ebkq08/vz0ByayezOrJrJ7MRtqQ6dkvJW/PL3NYeSanlwdzZh7Y4/5TyvYsL6tzbHk8c8qmLCpPZGGeyvoXXrd9bd6YN+XJLC+r8/rydOaXjTm5bMjysnpvPaVRJ7N6Mqsns3oTKbOqMygzZszI4sWLR6xNmzYtBx100EvW9ylvnZacd2C6i59Mhkry7hnJO2ckP9t5sqxcMDs5qCfdyv5k/eNJX0/y5t6UDxw4hoOPIZnVk1k9mdWT2QgPdLPyhXJs3pWf55zcl3tzcL6Wxbkgu7+Ocjhd+rIlF2RVZmUog5mS2zM/f5dFSZJ/72blw+W0nJP7ckVuSZeSDZmeW3P43nxao0pm9WRWT2b1JlJmXSml/DZ38Id/+IdZsmTJK/5FjYODg5k5c2b671+QvhlVJ3AAGAeWzVsy1iMAMAa2la25JTdlYGAgfX17vgj/t/4tLLfccstvexcAAABJKq9BAQAAGE0KCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZkwe6wEAmFj+ecOasR5hn7Ns3pKxHgFgr3EGBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAKhyRlmXG8tNYz3GPkVm9WRWb7xkNnmsB3itTZr7y994e/nwgSkfOWgvTbNvkFk9mdWTWT2Z1ZNZnfPLqpyRB1+yfnb+KBu66WMwUftkVk9m9SZ6ZuOuoAzf87qdGzc9k+7yp1JuP3Ln2rRdThqVkmxPMrnbW+M1SWb1ZFZPZvVkVk9m9e7MofnbvGXE2kB6x2iafYPM6sms3kTObNwVlMzZ5Sn1TUq6XdZ++GwmvWNDhq+Zm+6yp5K1Qynfmpfu2o3JwHDK1+fuOLT72K+Tnwyl3HD48wvDJbmyP901g8mvtycL9ks5b3Zy5jhosTKrJ7N6Mqsns3oyq7Y1Penv9h+x9o5yf5aVdTksm7IxU/KjzM1Xcmw2d7v/tmFBeTp/lXuyMP0pSR7O9HwuJ+T+bnaSZFF5In+Re7Mw/RlIb+7I/Hwti/d4f62TWT2Z1ZvIme2b/2K/pe6SJ1M+fnBy1H7JzElJNr78QSv7012/MeWyQ5IFU5IfPZdu+WMps3uSU6aO+sxjTWb1ZFZPZvVkVk9mL284Xa7KkjyaaZmbTXl/Vud9+XE+nxN2u/+FuTO/yqyszPEZTpfXZyDbXrjMdW55JpfmB7k6i/OZLM3MDGV51mR5Vr/kp8P7MpnVk1m9iZLZhCwo5fzZyWkHvPIDhkq6lf0p181Llr7wheio/ZI7N6e7ZiBlHH5xejGZ1ZNZPZnVk1k9mY30+3kk3yk37thelcPyqe7kHduPZVq+Xhblg7l7j98Ezcmz+YcszENdX5Lk4czYcdtZWZubc2Ru7I7ZcdtVZUk+k1vyuXJCtnY9o/G0RpXM6sms3kTObEIWlBy3/8vvs6t1W9I9V5J3bRi5vrUkiyfGawFl9irIrJ7M6smsnsxGWJNDsnKXb242pyfHl8dyVtbmiGzMAdmWngynN8PpLdsytJuXflyfY7Iid+X0sj53Z05uy+F55IULeRdkIEdnIG8r60cc05NkbjZlffpG9fmNBpnVk1m9iZzZxCwoB7zogsjdvdnytl3+vqkkSco35iVzX9Qmp0yQiytlVk9m9WRWT2b1ZDbC5kwe8a5Ah5ZNuTh35B+zIFdncQYzJYvzRD6SuzI5wxnazX18o1uU75cjc1IeyYl5NH+Wn+aSclLu6OZnarble1mQb+cNLznu8VScyWqIzOrJrN5EzmxiFpQXO6gnWbtl5NpPhnams3BKSm+XPLx1XL7W+FWRWT2Z1ZNZPZnVk9kIx6Q/XUq+lONSuucL2GnlP172uIe7GbkhM3JDFuaj5d+yLOtyR+bnF5mVozI4rt8aVWb1ZFZvImXmFzUmKadOTe4ZSq4bTB7Yku7yJ0d+sZo+KfnLWek+8cTz+6zbmvx4c/LVp5/fnoBkVk9m9WRWT2b1ZDbShkzPfil5e36Zw8ozOb08mDPzwB73n1K2Z3lZnWPL45lTNmVReSIL81TWv/Ba92vzxrwpT2Z5WZ3Xl6czv2zMyWVDlpfVe+spjTqZ1ZNZvYmUmTMoSfLWacl5B6a7+MlkqCTvnpG8c0bys50ny8oFs5ODetKt7E/WP5709SRv7k35wIFjOPgYklk9mdWTWT2Z1ZPZCA90s/KFcmzelZ/nnNyXe3NwvpbFuSCrdrv/cLr0ZUsuyKrMylAGMyW3Z37+LouSJP/ezcqHy2k5J/flitySLiUbMj235vC9+bRGlczqyazeRMqsK6WUvfmAg4ODmTlzZvrvX5C+GU7gAMDLWTZvyViPAPBb21a25pbclIGBgfT17fkifA0BAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANCMyWM9AADwm/3zhjVjPcI+Z9m8JWM9AvAqOYMCAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAMMrOKOtyY7lprMfYp8is3njJbPJYD/BamzT3l7/x9vLhA1M+ctBemmbfILN6Mqsns3oyqyezejKrc35ZlTPy4EvWz84fZUM3fQwmap/M6k30zKoLysMPP5wLLrgg//RP/5Rnn302b3jDG3L11Vdn6dKlozFfteF7Xrdz46Zn0l3+VMrtR+5cm7bLSaNSku1JJnd7a7wmyayezOrJrJ7M6smsnszq3ZlD87d5y4i1gfSO0TT7BpnVm8iZVRWU/v7+nHrqqXnrW9+af/qnf8ohhxySX/ziFznwwANHa756c3Z5Sn2Tkm6XtR8+m0nv2JDha+amu+ypZO1Qyrfmpbt2YzIwnPL1uTsO7T726+QnQyk3HP78wnBJruxPd81g8uvtyYL9Us6bnZw5DlqszOrJrJ7M6smsnszqyaza1vSkv9t/xNo7yv1ZVtblsGzKxkzJjzI3X8mx2dzt/lutBeXp/FXuycL0pyR5ONPzuZyQ+7vZSZJF5Yn8Re7NwvRnIL25I/PztSze4/21Tmb1JnJmVY9+2WWX5YgjjsjVV1+9Y+3oo49+zYcabd0lT6Z8/ODkqP2SmZOSbHz5g1b2p7t+Y8plhyQLpiQ/ei7d8sdSZvckp0wd9ZnHmszqyayezOrJrJ7M6sns5Q2ny1VZkkczLXOzKe/P6rwvP87nc8Ju978wd+ZXmZWVOT7D6fL6DGTbC5cGzy3P5NL8IFdncT6TpZmZoSzPmizP6pf8RH1fJrN6EyWzqoLyne98J8uWLcs73/nO3HrrrZk/f37+63/9r3nf+963x2OGhoYyNDS0Y3twcPDVT/saKefPTk474JUfMFTSrexPuW5esvSFT6pH7ZfcuTndNQMp4/AT7YvJrJ7M6smsnszqyayezEb6/TyS75Qbd2yvymH5VHfyju3HMi1fL4vywdy9x28c5+TZ/EMW5qGuL0nycGbsuO2srM3NOTI3dsfsuO2qsiSfyS35XDkhW7ue0Xhao0pm9SZyZlUF5YEHHsgXvvCFrFixIh/96EezatWqfOADH8iUKVNy9tln7/aYSy+9NJ/85Cdfk2FfM8ft//L77GrdlnTPleRdG0auby3J4onxWkCZvQoyqyezejKrJ7N6MhthTQ7Jyl2+IdycnhxfHstZWZsjsjEHZFt6MpzeDKe3bMvQbl4uc32OyYrcldPL+tydObkth+eRFy5+XpCBHJ2BvK2sH3FMT5K52ZT16RvV5zcaZFZvImdWVVCGh4ezdOnSXHLJJUmS448/Pvfdd1+++MUv7rGgXHTRRVmxYsWO7cHBwRxxxBG/xcivgQNedHHf7t5sedsuf99UkiTlG/OSuS9qk1MmyIWCMqsns3oyqyezejKrJ7MRNmfyiHdSOrRsysW5I/+YBbk6izOYKVmcJ/KR3JXJGc7Qbu7jG92ifL8cmZPySE7Mo/mz/DSXlJNyRzc/U7Mt38uCfDtveMlxj6fiTFZDZFZvImdWVVDmzp2bN73pTSPWfvd3fzfXX3/9Ho/p7e1Nb2/jPy05qCdZu2Xk2k+GdqazcEpKb5c8vHVcvm72VZFZPZnVk1k9mdWTWT2ZjXBM+tOl5Es5LqV7voCdVv7jZY97uJuRGzIjN2RhPlr+LcuyLndkfn6RWTkqg+P67WRlVm8iZVb1ixpPPfXU/PznPx+xdv/99+eoo456TYfa28qpU5N7hpLrBpMHtqS7/MmRn3inT0r+cla6Tzzx/D7rtiY/3px89enntycgmdWTWT2Z1ZNZPZnVk9lIGzI9+6Xk7fllDivP5PTyYM7MA3vcf0rZnuVldY4tj2dO2ZRF5YkszFNZ/8L1AdfmjXlTnszysjqvL09nftmYk8uGLC+r99ZTGnUyqzeRMqs6g3LeeefllFNOySWXXJI//dM/zZ133pkvf/nL+fKXvzxa8+0db52WnHdguoufTIZK8u4ZyTtnJD/bebKsXDA7Oagn3cr+ZP3jSV9P8ubelA809BbLe5PM6smsnszqyayezOrJbIQHuln5Qjk278rPc07uy705OF/L4lyQVbvdfzhd+rIlF2RVZmUog5mS2zM/f5dFSZJ/72blw+W0nJP7ckVuSZeSDZmeW3P43nxao0pm9SZSZl0ppdQc8N3vfjcXXXRRfvGLX+Too4/OihUrfuO7eL3Y4OBgZs6cmf77F6RvRtUJHACAV2TZvCVjPQLwItvK1tySmzIwMJC+vj1fhF/9W1jOPPPMnHnmmb/VcAAAALvjFAYAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMmj/UAAACvtX/esGasR9jnLJu3ZKxHgCTOoAAAAA1RUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAADQnDPKutxYbhrrMfYp4yWzyWM9wGtt0txf/sbby4cPTPnIQXtpmn2DzOrJrJ7M6smsnszqyayezOqcX1bljDz4kvWz80fZ0E0fg4naN9EzG3cFZfie1+3cuOmZdJc/lXL7kTvXpu1y0qiUZHuSyd3eGq9JMqsns3oyqyezejKrJ7N6Mqt3Zw7N3+YtI9YG0jtG0+wbJnJm466gZM4uT6lvUtLtsvbDZzPpHRsyfM3cdJc9lawdSvnWvHTXbkwGhlO+PnfHod3Hfp38ZCjlhsOfXxguyZX96a4ZTH69PVmwX8p5s5Mzx0GLlVk9mdWTWT2Z1ZNZPZnVk1m1relJf7f/iLV3lPuzrKzLYdmUjZmSH2VuvpJjs7nb/benC8rT+avck4XpT0nycKbnczkh93ezkySLyhP5i9ybhenPQHpzR+bna1m8x/tr3UTObN/8F/stdZc8mfLxg5Oj9ktmTkqy8eUPWtmf7vqNKZcdkiyYkvzouXTLH0uZ3ZOcMnXUZx5rMqsns3oyqyezejKrJ7N6Mnt5w+lyVZbk0UzL3GzK+7M678uP8/mcsNv9L8yd+VVmZWWOz3C6vD4D2fbC5dRzyzO5ND/I1Vmcz2RpZmYoy7Mmy7P6JWch9mUTJbMJWVDK+bOT0w545QcMlXQr+1Oum5csfeETxFH7JXduTnfNQMo4/KTxYjKrJ7N6Mqsns3oyqyezejIb6ffzSL5TbtyxvSqH5VPdyTu2H8u0fL0sygdz9x6/2Z6TZ/MPWZiHur4kycOZseO2s7I2N+fI3Ngds+O2q8qSfCa35HPlhGztekbjaY2qiZzZhCwoOW7/l99nV+u2pHuuJO/aMHJ9a0kWT4zXAsrsVZBZPZnVk1k9mdWTWT2ZjbAmh2TlLt9Eb05Pji+P5ayszRHZmAOyLT0ZTm+G01u2ZWg3LzG6PsdkRe7K6WV97s6c3JbD88gLF4wvyECOzkDeVtaPOKYnydxsyvr0jerzGw0TObOJWVAOeNGFart7s+Vtu/x9U0mSlG/MS+a+qE1OmSAXvcmsnszqyayezOrJrJ7M6slshM2ZPOLdpw4tm3Jx7sg/ZkGuzuIMZkoW54l8JHdlcoYztJv7+Ea3KN8vR+akPJIT82j+LD/NJeWk3NHNz9Rsy/eyIN/OG15y3OOpOJPVkImc2cQsKC92UE+ydsvItZ8M7Uxn4ZSU3i55eOu4fA3oqyKzejKrJ7N6Mqsns3oyqyezEY5Jf7qUfCnHpXTPF7DTyn+87HEPdzNyQ2bkhizMR8u/ZVnW5Y7Mzy8yK0dlcFy/Be9EyswvakxSTp2a3DOUXDeYPLAl3eVPjvwkMn1S8pez0n3iief3Wbc1+fHm5KtPP789AcmsnszqyayezOrJrJ7M6slspA2Znv1S8vb8MoeVZ3J6eTBn5oE97j+lbM/ysjrHlsczp2zKovJEFuaprH/hmopr88a8KU9meVmd15enM79szMllQ5aX1XvrKY26iZSZMyhJ8tZpyXkHprv4yWSoJO+ekbxzRvKznSfLygWzk4N60q3sT9Y/nvT1JG/uTfnAgWM4+BiSWT2Z1ZNZPZnVk1k9mdWT2QgPdLPyhXJs3pWf55zcl3tzcL6Wxbkgq3a7/3C69GVLLsiqzMpQBjMlt2d+/i6LkiT/3s3Kh8tpOSf35Yrcki4lGzI9t+bwvfm0RtVEyqwrpZS9+YCDg4OZOXNm+u9fkL4ZTuAAALRg2bwlYz0C49y2sjW35KYMDAykr2/PF+FrCAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZkwe6wEAABh7/7xhzViPsM9ZNm/JWI8wLjmDAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAADjwBllXW4sN431GL+1yWM9wGtt0txf/sbby4cPTPnIQXtpmn2DzOrJrJ7M6smsnszqyayezOrJrM75ZVXOyIMvWT87f5QN3fQxmGjvqioo27dvz3//7/8911xzTR599NHMmzcv733ve/Pf/tt/S9d1ozVjleF7Xrdz46Zn0l3+VMrtR+5cm7bLSaNSku1JJrcx+1iRWT2Z1ZNZPZnVk1k9mdWTWT2Z1bszh+Zv85YRawPpHaNp9q6ql3hddtll+cIXvpArr7wyP/vZz3LZZZflb/7mb/L5z39+tOarN2fyzj99k5Jul7VfbsmkNzyQ3Lwp3RkPpTvqV8mdz6X74GPp3vvIiLvpPvbrdP/3f+xcGC7JyqfSnbgu3dG/Sve29cl3n9m7z220yKyezOrJrJ7M6smsnszqyayezKptTU/6u/1H/Pm/8ot8ufx/851yY/6+fC/vL3dn/7Jtj/exoDydy8utual8O98u385V5f+XheWpHbcvKk/kivL/5rvlhvx9+V7+a1nzG+9vb6k6g/LDH/4w//k//+f88R//cZLkda97Xf7n//yfufPOO0dluNHSXfJkyscPTo7aL5k5KcnGlz9oZX+66zemXHZIsmBK8qPn0i1/LGV2T3LK1FGfeazJrJ7M6smsnszqyayezOrJrJ7MXt5wulyVJXk00zI3m/L+rM778uN8Pifsdv8Lc2d+lVlZmeMznC6vz0C2vXB+Ym55JpfmB7k6i/OZLM3MDGV51mR5Vr/kzM3eVlVQTjnllHz5y1/O/fffn4ULF+aee+7J7bffniuuuGKPxwwNDWVoaGjH9uDg4Kuf9jVSzp+dnHbAKz9gqKRb2Z9y3bxk6Qsf7Eftl9y5Od01Aynj8H+AF5NZPZnVk1k9mdWTWT2Z1ZNZPZmN9Pt5JN8pN+7YXpXD8qnu5B3bj2Vavl4W5YO5e48FZU6ezT9kYR7q+pIkD2fGjtvOytrcnCNzY3fMjtuuKkvymdySz5UTsrXrGY2n9YpUFZQLL7wwg4OD+Z3f+Z309PRk+/bt+fSnP533vOc9ezzm0ksvzSc/+cnfetDX1HH71+2/bku650ryrg0j17eWZPHEeC2gzF4FmdWTWT2Z1ZNZPZnVk1k9mY2wJodk5S7FY3N6cnx5LGdlbY7IxhyQbenJcHoznN6yLUPdS7+tvz7HZEXuyullfe7OnNyWw/PICxfZL8hAjs5A3lbWjzimJ8ncbMr69I3q8/tNqgrKddddl7//+7/PN7/5zSxatChr1qzJhz70ocybNy9nn332bo+56KKLsmLFih3bg4ODOeKII367qX9bB7zooqvdXYmz68vvNpUkSfnGvGTui9rklAlyAZfM6smsnszqyayezOrJrJ7M6slshM2ZPOIduw4tm3Jx7sg/ZkGuzuIMZkoW54l8JHdlcoYztJv7+Ea3KN8vR+akPJIT82j+LD/NJeWk3NHNz9Rsy/eyIN/OG15y3OOpOJM1CqoKyvnnn58LL7ww7373u5Mkb37zm/Pggw/m0ksv3WNB6e3tTW9v4y32oJ5k7ZaRaz8Z2pnOwikpvV3y8NZx+XrGV0Vm9WRWT2b1ZFZPZvVkVk9m9WQ2wjHpT5eSL+W4lBfePfe08h8vc1TycDcjN2RGbsjCfLT8W5ZlXe7I/Pwis3JUBpt82+KqgvLss89m0qSRdbanpyfDw8Ov6VB7Wzl1arr/8XRy3WCydP901298/n+IxVOe32H6pOQvZ6X7xBMpJcmJU5PB7cmqzcmMScmfjt0psLEis3oyqyezejKrJ7N6Mqsns3oyG2lDpme/lLw9v8y/lrlZnCdzZh7Y4/5Tyvb8f/Lj3Jb5eTTTckiey8I8ldszP0lybd6Ylfl/s7yszj/l6GxOT47MxvxeHsuV3fF762ntVlVB+ZM/+ZN8+tOfzpFHHplFixZl9erVueKKK3LOOeeM1nx7x1unJecdmO7iJ5Ohkrx7RvLOGcnPdp4sKxfMTg7qSbeyP1n/eNLXk7y5N+UDB47h4GNIZvVkVk9m9WRWT2b1ZFZPZvVkNsID3ax8oRybd+XnOSf35d4cnK9lcS7Iqt3uP5wufdmSC7IqszKUwUzJ7Zmfv8uiJMm/d7Py4XJazsl9uSK3pEvJhkzPrTl8bz6t3epKKeWV7rxx48Z87GMfy4033pjHH3888+bNy1lnnZWPf/zjmTJlyiu6j8HBwcycOTP99y9I34yqX8MCAADNWDZvyViPsE/ZVrbmltyUgYGB9PXt+QxXVUF5LSgoAACMBwpKnVdaUDQEAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmTB7rAQAAYF/0zxvWjPUI+5TBjcM5cOHL7+cMCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQjMljPcBrbdLcX/7G28uHD0z5yEF7aZp9g8zqyayezOrJrJ7M6smsnszqyazeRM6sK6WUvfmAg4ODmTlzZvrvX5C+GaNwAufxbTv/ftMz6S5/KuX2I3euTZv0/J8kKSXZnmRy99rPsS+RWT2Z1ZNZPZnVk1k9mdWTWT2Z1RuHmQ1uHM6BCx/IwMBA+vr69rjf+HuJ15zJO//0TUq6XdZ+uSWT3vBAcvOmdGc8lO6oXyV3Ppfug4+le+8jI+6m+9iv0/3f/7FzYbgkK59Kd+K6dEf/Kt3b1ifffWbvPrfRIrN6Mqsns3oyqyezejKrJ7N6Mqs3gTMbdy/xeiW6S55M+fjByVH7JTMnJdn48get7E93/caUyw5JFkxJfvRcuuWPpczuSU6ZOuozjzWZ1ZNZPZnVk1k9mdWTWT2Z1ZNZvfGa2YQsKOX82clpB7zyA4ZKupX9KdfNS5a+8A931H7JnZvTXTOQ0sg/5miSWT2Z1ZNZPZnVk1k9mdWTWT2Z1RuvmU3IgpLj9q/bf92WdM+V5F0bRq5vLcni3tdurpbJrJ7M6smsnszqyayezOrJrJ7M6o3TzCZmQTngRRcQ7e5KnF2uS8qm599HoHxjXjK3Z+R+U9q+GOk1I7N6Mqsns3oyqyezejKrJ7N6Mqs3TjObmAXlxQ7qSdZuGbn2k6Gd6SycktLbJQ9vbea1eWNOZvVkVk9m9WRWT2b1ZFZPZvVkVm+cZKagJCmnTk33P55OrhtMlu6f7vqNz//jLp7y/A7TJyV/OSvdJ55IKUlOnJoMbk9WbU5mTEr+dM9vkzZeyayezOrJrJ7M6smsnszqyayezOqNl8wUlCR567TkvAPTXfxkMlSSd89I3jkj+dnQjl3KBbOTg3rSrexP1j+e9PUkb+5N+cCBYzj4GJJZPZnVk1k9mdWTWT2Z1ZNZPZnVGyeZjb9f1AgAADRn4v6iRgAAYJ+loAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA0Q0EBAACaoaAAAADNUFAAAIBmKCgAAEAzFBQAAKAZCgoAANAMBQUAAGiGggIAADRDQQEAAJqhoAAAAM1QUAAAgGYoKAAAQDMUFAAAoBkKCgAA0AwFBQAAaMbkvf2ApZQkyeAzw3v7oQEAgDHyf77//z99YE/2ekHZuHFjkuSoE9bt7YcGAADG2MaNGzNz5sw93t6Vl6swr7Hh4eFs2LAhM2bMSNd1e/Ohf6PBwcEcccQReeihh9LX1zfW4zBO+Thjb/Bxxt7g44y9wcfZ+FJKycaNGzNv3rxMmrTnK032+hmUSZMm5fDDD9/bD/uK9fX1+R+AUefjjL3Bxxl7g48z9gYfZ+PHbzpz8n+4SB4AAGiGggIAADRDQXlBb29vPvGJT6S3t3esR2Ec83HG3uDjjL3Bxxl7g4+ziWmvXyQPAACwJ86gAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAXlBVdddVVe97rXZf/9989JJ52UO++8c6xHYhy59NJL85a3vCUzZszInDlz8va3vz0///nPx3osxrG//uu/Ttd1+dCHPjTWozAOPfzww/kv/+W/5KCDDsrUqVPz5je/Of/7f//vsR6LcWT79u352Mc+lqOPPjpTp07N61//+nzqU5+KN5+dGBSUJNdee21WrFiRT3ziE7n77rtz3HHHZdmyZXn88cfHejTGiVtvvTXnnntufvSjH+Vf/uVfsnXr1pxxxhnZtGnTWI/GOLRq1ap86UtfyrHHHjvWozAO9ff359RTT81+++2Xf/qnf8pPf/rTfOYzn8mBBx441qMxjlx22WX5whe+kCuvvDI/+9nPctlll+Vv/uZv8vnPf36sR2Mv8HtQkpx00kl5y1vekiuvvDJJMjw8nCOOOCLvf//7c+GFF47xdIxHv/71rzNnzpzceuut+YM/+IOxHodx5JlnnskJJ5yQ//E//kcuvvjiLFmyJJ/97GfHeizGkQsvvDB33HFHfvCDH4z1KIxjZ555Zg499NB89atf3bH2jne8I1OnTs0111wzhpOxN0z4MyhbtmzJXXfdldNPP33H2qRJk3L66afnX//1X8dwMsazgYGBJMns2bPHeBLGm3PPPTd//Md/POJzGryWvvOd72Tp0qV55zvfmTlz5uT444/PV77ylbEei3HmlFNOyc0335z7778/SXLPPffk9ttvz3/6T/9pjCdjb5g81gOMtSeeeCLbt2/PoYceOmL90EMPzdq1a8doKsaz4eHhfOhDH8qpp56axYsXj/U4jCPf+ta3cvfdd2fVqlVjPQrj2AMPPJAvfOELWbFiRT760Y9m1apV+cAHPpApU6bk7LPPHuvxGCcuvPDCDA4O5nd+53fS09OT7du359Of/nTe8573jPVo7AUTvqDA3nbuuefmvvvuy+233z7WozCOPPTQQ/ngBz+Yf/mXf8n+++8/1uMwjg0PD2fp0qW55JJLkiTHH3987rvvvnzxi19UUHjNXHfddfn7v//7fPOb38yiRYuyZs2afOhDH8q8efN8nE0AE76gHHzwwenp6cljjz02Yv2xxx7LYYcdNkZTMV4tX7483/3ud3Pbbbfl8MMPH+txGEfuuuuuPP744znhhBN2rG3fvj233XZbrrzyygwNDaWnp2cMJ2S8mDt3bt70pjeNWPvd3/3dXH/99WM0EePR+eefnwsvvDDvfve7kyRvfvOb8+CDD+bSSy9VUCaACX8NypQpU/J7v/d7ufnmm3esDQ8P5+abb87JJ588hpMxnpRSsnz58tx44435/ve/n6OPPnqsR2Kcedvb3pZ77703a9as2fFn6dKlec973pM1a9YoJ7xmTj311Je8Tfr999+fo446aowmYjx69tlnM2nSyG9Te3p6Mjw8PEYTsTdN+DMoSbJixYqcffbZWbp0aU488cR89rOfzaZNm/Lnf/7nYz0a48S5556bb37zm7npppsyY8aMPProo0mSmTNnZurUqWM8HePBjBkzXnJN07Rp03LQQQe51onX1HnnnZdTTjkll1xySf70T/80d955Z7785S/ny1/+8liPxjjyJ3/yJ/n0pz+dI488MosWLcrq1atzxRVX5Jxzzhnr0dgLvM3wC6688spcfvnlefTRR7NkyZKsXLkyJ5100liPxTjRdd1u16+++uq8973v3bvDMGH84R/+obcZZlR897vfzUUXXZRf/OIXOfroo7NixYq8733vG+uxGEc2btyYj33sY7nxxhvz+OOPZ968eTnrrLPy8Y9/PFOmTBnr8RhlCgoAANCMCX8NCgAA0A4FBQAAaIaCAgAANENBAQAAmqGgAAAAzVBQAACAZigoAABAMxQUAACgGQoKAADQDAUFAABohoICAAA04/8Pp2TqsxxSD9QAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ] - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = MultiHeadAttention(\n", - " in_size=10,\n", - " head_size=4,\n", - " num_heads=3,\n", - " out_size=15,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cu4gFFYBbWMa", - "outputId": "58aba7a8-a103-4f17-b658-ff445ab6ac8d" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=12, bias=False)\n", - " (out): Linear(in_features=12, out_features=15, bias=True)\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 9 - } - ] - }, - { - "cell_type": "code", - "source": [ - "tmp_input = torch.rand(2, 5, 10)\n", - "tmp_mask = make_decoder_mask(tmp_input)\n", - "\n", - "print(\"Decoder-like input, with mask\")\n", - "print(f'Input shape: {tmp_input.shape}')\n", - "print(f'Mask shape: {tmp_mask.shape}')\n", - "\n", - "tmp_output = tmp_layer(tmp_input, tmp_input, tmp_input, tmp_mask)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input, tmp_mask, tmp_output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "gPA5Ba6UbYFP", - "outputId": "7790ce5c-c428-42e0-8d84-da9994d01c1e" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Decoder-like input, with mask\n", - "Input shape: torch.Size([2, 5, 10])\n", - "Mask shape: torch.Size([2, 1, 5, 5])\n", - "Output shape: torch.Size([2, 5, 15])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "### 1.2 Positional Encoding\n", - "\n", - "#### Main class - PositionalEncoding\n", - "\n", - "**Initialization:**\n", - "* max_seq_len ~ the maximum token length of the sequence\n", - "* emb_size ~ the embedding size of the input\n", - "\n", - "**Forward:**\n", - "* _decoder_emb_ ~ embeddings of tokens from the decoder input\n", - "\n", - "$$\\text{PE}_{(\\text{pos}, 2i)} = sin\\Bigg( \\frac{\\text{pos}}{10000^{\\frac{2i}{\\text{emb_size}}}} \\Bigg) \\quad ; \\quad \\text{PE}_{(\\text{pos}, 2i + 1)} = cos\\Bigg( \\frac{\\text{pos}}{10000^{\\frac{2i}{\\text{emb_size}}}} \\Bigg)$$" - ], - "metadata": { - "id": "nqpfRJPlbVpf" - } - }, - { - "cell_type": "code", - "source": [ - "class PositionalEncoding(nn.Module):\n", - " \"\"\"\n", - " Class to calculate Positional Encodings, suggested in `Attention is all you need [Vaswaniet al., 2017]`\n", - " \"\"\"\n", - " def __init__(self, max_seq_len, emb_size):\n", - " \"\"\"\n", - " Args:\n", - " max_seq_len: max length of input sequence\n", - " emb_size: demension of embedding\n", - " \"\"\"\n", - " super(PositionalEncoding, self).__init__()\n", - "\n", - " # Запишем все переданые гиперпараметры слоя\n", - " self.max_seq_len = max_seq_len\n", - " self.emb_size = emb_size\n", - "\n", - " # Посчитаем позиционные эмбеддинги в тензорном виде\n", - " pos = torch.arange(max_seq_len)[:, None]\n", - " inds = torch.arange(emb_size)[None, ::2]\n", - "\n", - " pe = torch.zeros(max_seq_len, self.emb_size)\n", - " pe[:, ::2] = torch.sin(pos / (10000 ** ((2 * inds) / self.emb_size)))\n", - " pe[:, 1::2] = torch.cos(pos / (10000 ** ((2 * inds) / self.emb_size)))\n", - " pe = pe.unsqueeze(0)\n", - "\n", - " # Добавляем полученный тензор как параметр, который будет сохранятся вместе с моделью, но не будет обучаться\n", - " self.register_buffer('pe', pe)\n", - "\n", - "\n", - " def forward(self, decoder_emb):\n", - " \"\"\"\n", - " Args:\n", - " decoder_emb: decoder sequence after embed\n", - " Returns:\n", - " output: input with positional encodings\n", - " \"\"\"\n", - " # Тензоры приходят размера batch_size x seq_len x emb_size\n", - " seq_len = decoder_emb.size(1)\n", - "\n", - " # Прибавляем позиционные эмбеддинги\n", - " return decoder_emb + self.pe[:, :seq_len]" - ], - "metadata": { - "id": "oo22_W7gbql2" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing PositionalEncoding" - ], - "metadata": { - "id": "CUlw9fUJbvW0" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = PositionalEncoding(\n", - " max_seq_len=5,\n", - " emb_size=10,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Qq1zwI4dbysA", - "outputId": "d4f63ad9-9541-4cfb-c9f3-e80528fba9d8" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "PositionalEncoding()" - ] - }, - "metadata": {}, - "execution_count": 15 - } - ] - }, - { - "cell_type": "code", - "source": [ - "tmp_input = torch.rand(2, 5, 10)\n", - "\n", - "print(f'Input shape: {tmp_input.shape}')\n", - "tmp_output = tmp_layer(tmp_input)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input, tmp_output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "xUDvRmxLb1RM", - "outputId": "5eae2649-e3de-4500-b55e-0d3471d8b2d5" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Input shape: torch.Size([2, 5, 10])\n", - "Output shape: torch.Size([2, 5, 10])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#### Let’s examine the positional encodings." - ], - "metadata": { - "id": "b1SMX9aWb4-a" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer.pe.shape" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zLpl1jWzb9As", - "outputId": "68c293f3-800c-45eb-bec3-86db51edbe24" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "torch.Size([1, 5, 10])" - ] - }, - "metadata": {}, - "execution_count": 17 - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "the article substantiates [Attention is all you need [Vaswaniet al., 2017]](https://www.semanticscholar.org/reader/204e3073870fae3d05bcbc2f6a8e263d9b72e776):\n", - "\n", - "We chose this function because we hypothesized it would allow the model to easily learn to attend by\n", - "relative positions, since for any fixed offset k, $PE_{pos+k}$ can be represented as a linear function of $PE_{pos}$." - ], - "metadata": { - "id": "iMmyRsQ1b_rF" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = PositionalEncoding(\n", - " max_seq_len=200,\n", - " emb_size=100,\n", - ")\n", - "\n", - "fig, ax = plt.subplots(figsize=(10, 10))\n", - "plt.imshow(tmp_layer.pe[0, :, :], aspect=\"auto\")\n", - "plt.xlabel(\"emb_size\")\n", - "plt.ylabel(\"max_seq_len\")\n", - "plt.show()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 853 - }, - "id": "G5gnKgeGcQPR", - "outputId": "ffa750f7-0827-4793-a10a-1ced840035db" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAANECAYAAAC6qR6BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAChSElEQVR4nOzdeZgdZZn//7uqztLb6e50Z+lOJ2QjC0sCCUuIgIKgkCiIoCKoLCKLsiiZGZ38xnGEcQyCIioRhvmi6FcQVBYRFAWURQhhCSGsgSxk7yzd6fR+lqr6/cE3sTOc535SXel0d/J+XVeuS3LnrlN9uuOV59TzeW4nDMNQAAAAAAC7ze3vGwAAAACAwYaFFAAAAABExEIKAAAAACJiIQUAAAAAEbGQAgAAAICIWEgBAAAAQEQspAAAAAAgIhZSAAAAABARCykAAAAAiIiFFAAAAABEtM8spBYsWCBjx46VkpISmTlzpjz//PP9fUsAAAAA9lFOGIZhf99EXPfcc4+cd955cuutt8rMmTPlpptukt/+9reybNkyGT58uLU/CALZsGGDZDIZcRxnL9wxAAAAgIEoDENpa2uTkSNHiuuanzvtEwupmTNnylFHHSU333yziLy3MBo9erRceeWV8q//+q/W/nXr1sno0aP7+jYBAAAADBJr166VUaNGGeuJvXgvfSKXy8lLL70k8+bN2/l7ruvKySefLAsXLizak81mJZvN7vzvHWvJUd/+prglJUV7XjnzZ8Z7OOy+L6r3qPXG7ee1eW1em9fmtXltXpvX5rV5bV57z/W2tgcyZsa7kslk1GsM+oXU1q1bxfd9GTFixC6/P2LECHnrrbeK9syfP1+uueaa9/2+W1JiXEhVZsyP9Uw9u9Mbt5/X5rV5bV6b1+a1eW1em9fmtXntPfvaImKN/Owzh01EMW/ePNm+ffvOX2vXru3vWwIAAAAwiAz6J1JDhw4Vz/Nk06ZNu/z+pk2bpK6urmhPOp2WdDr9vt+/42O3SoVhdXrF+uON93DJyY+r9/hwp74inn7kcrW+It9urNVOalJ7twddaj3Z0KHWs2HeWAtrc2qvHwZqPcgU1LraW6pf2yZM9T4aGCbixQpDL8Zrx/3oI85ZKpzDAgAAsNOgfyKVSqXkiCOOkMcf/8diJggCefzxx2XWrFn9eGcAAAAA9lWD/omUiMjcuXPl/PPPlyOPPFKOPvpouemmm6Sjo0MuvPDC/r41AAAAAPugfWIhdfbZZ8uWLVvkW9/6ljQ2Nsrhhx8ujzzyyPsOoAAAAACAPWGfWEiJiFxxxRVyxRVXxLrGcC8nGa/4bscXb5pu7Pvh9QvU6x70xJfU+i9m3a7Wb20y57M+Nup1tXdRd6VanzJis1rfUMgaa0OGmLNbIiLtoblXRCRZoWestHyWU6rnq2z5rDDV+4xVmIw5es2L0evGzGfF6A/jZqTIZwEAgH3IoM9IAQAAAMDexkIKAAAAACJiIQUAAAAAEe0zGak9YfZTXxa3tPjMp4l3LTL2rfput3rduvtSav2YE/T7+vwrM4y1351wi9r7863mfJWIyKyalWr9jfxQY21cdbPau9X31Xp1Rp9x1R2ac1DJEj0jVRD9tZ20XlczVjHmQInEmyMV+6OPWDmlmF93f+acyFgBAIA9jCdSAAAAABARCykAAAAAiIitfT1MuqldEl7xI7cLs6YZ+7701lj1uplHXlPrrYG+NbD2WfPWwEM/qu9Z+uvqiWr9J9N/rdafap9irE2q0I9OX+9XqPXh5frx6dsD8/a78lL9aPV8qG/dSyT1eiDmbWxOUj863Xb0epwjzOMcX/7ea8fo7cftcf26LRAAAKAInkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakeglVrJXCSRWub7hlv7PMeqFOvW+5vVOv3tOk5pmGLthlracP97uAvy6j1yUdvV+s3NI011s4daT4SXkTknaz+vtSXtqr1lsD841lZomekOi0ZqVRaPz5dy1i5loyUlq8SEZGEXlczVjEzUmGcI8zj5pRiHp/eb8hnAQCAIngiBQAAAAARsZACAAAAgIhYSAEAAABARGSketh0weHipUuK1p458kZj39mXn6let+OkqWp9wTJz/kpEpOHN5cbaxoI+i6n6LbUsI7xStb68cZixNmGsPkfqwe0z1PqoEnP2S0Rki19urFWnu9TejkDP46STxeeF7ZAXJSPl2WZQ6Rkqx7PMmdLE/eijP+dIxemPma/q1zlUZKwAANgn8UQKAAAAACJiIQUAAAAAEbGQAgAAAICIyEj18LmL/iIlFcXfkse6hhr7Chv0OVGr/3OkWq9YWK3WQ9+cyXmya7TaW71Mz1AlHU+tO+vMGaqRXqfau6x1hFqfM/xVtd5YqDbWhqT11+4M9a+rNGmbI2XOMSWTekbKD/U8j+PFyPvEnCMVR6wZVCJkhQAAwD6FJ1IAAAAAEBELKQAAAACIiIUUAAAAAERERqqHS6velcpM8bXl1FuvMPYdcGSbet35x96r1m//2SfUujdxnLH2wNYqvXdVo1rvDHJqvWy9OdgyzNN/fNZsr1brDQ3Nav01Jf9Vk+xQe9uClFqvSGXVereSc0pY5kDZ5ki5lpxTIErd0usr2a7d6VcxiwkAAGAnnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakePv3OqZIoTxetjf3J68a+5f96sHrdz2a2qfVfLl2p1pvOOMRYe2tFpdo7cevLan2dn1frmfXmmUmljp5DammuUOvDPD1btj5bbazVpVr11w7K1HpFUs9IZZUoUdoyg8rXMk4i4sbIWDlx50jFyRrFzSn1a8aq9+9bSDYMAAAUwRMpAAAAAIiIhRQAAAAARMTWvh66f1wviWRJ0Vp5xQZj36dOeUa97lPd+uv6rfo2tc0fMG+vK3ur+P3urjdyI9R62XrzzXuOvg53m5JqvdbVt9dt6DIf7X5o+Xq1t8nXtxVmLFv7ukPPWEt55u+HiEjecgS5Z9na5ytHrztezK19cbcGxhDG2F7HFjcAADDQ8EQKAAAAACJiIQUAAAAAEbGQAgAAAICIyEj1kP7LYkk4xXM9b/3wGGPffcMfUK97+LNfVOvjG5rU+kdnvGqsvf7oNLXXGzpUrS9s13MryY3mo9t9SxYo3aSv06tcPfjS2GE+2r16RIfauylfrdbLvZxa1zJS6YR+/HleyTiJ2DNSGtvx54Hl6HVb1kj9nsbJOO3GawMAAAwmPJECAAAAgIhYSAEAAABARCykAAAAACAiMlI9ZE+ZIb5hjtS9Z/zI2LfJz6vXrb6vXK23HVmq1i8ffp+x9o1lY9Ref1ydWn9+a5laL2lqNtbaQ30WU0mTnqkpM+TRdtjWYX5far12tffNrga1XpnoUusdYcpYK03o3299ypRI0jKHKhBzTsmNmVOyZaz05lgvHU/c1+7Hew/JhgEAsE/iiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkeqh7KoNkihPF60N9cy5mEtWfEa97pA/vanWl//rwWp9Wqp4bktExFmzQe1tOeMQtb5lo55Tmti+xljb5OvzkEqbLHOmHP3Hr6vN/HVn3G61tymv59LqUq1qvTMo/nMgIlKi/CyIiOQtMaSEZY6Ur8yCss+RssyoipOxGsQ5pVjizs/qT4P1PQcAYBDgiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkerhngP/LJWZ4mvLA//yVWNf5RJznkZEpL7jRbU+7qi1an1dwTwzyW/Vsz7bpqhlSazX711Ccz5kQyGjtpY06Vkiz7Gs49vMP54ZR792c07PSE0s3azWWwNzPqvEK6i93aH+dVnnSCnvuRtnDpTEnSPVf1mhcDDnlAAAwD6JJ1IAAAAAEBELKQAAAACIiIUUAAAAAERERqqHW1rGS0mh+Fty0PXmLJLTlVWv2/WhqWr962N+pdbvaDnS/NrJlNqbnKxnqFJ/rVLrTtqcoXo7V6e/dnOXWrdJtJnX+RlXn5fUnC1T6xlPvzdtjlSpbY6U5fOJpOXetTlSrq1XyVeJiHWuUKC8dvw5Uv2Yc2KeEgAA2MN4IgUAAAAAEbGQAgAAAICIWEgBAAAAQERkpHq4+2cni5cqPj9oxMqXjH1hVs9IrfpXPUt0UmmnWr/i+WOMtcl1W/Vrj3lbrS9ZP12te0OqjbVlnXpWyG0xz78SEcmH+jylZJs52FLi6KGX7VnzHCgRkYzbrdY35IcYa6VeTu3Nx5wjlVdyTp6nZ6Rs4s2R0st+GO/e4rw2AADA3sYTKQAAAACIiIUUAAAAAETE1r4ehv/sZUk4yaK1xsvMR5DXP7VNve68Dz2k1rOhvkWuYqH5KO/sgcPV3k8M+ZNaf3v9FLUeDDdvcXunXd9vFbbqW/tsX3eqzVwrcTy1t63bfHy5iEi5q2/HbPPNWwNtx593h/pfq6Srb+3Tqp7lCPFA9O11cXbIxdoWGPfF+9NgvW8AANCneCIFAAAAABGxkAIAAACAiFhIAQAAAEBEZKR6cCccIK5XPFtzwWV/NPbdOnSOet1Lqjao9TvbGtR63cJWY23zkRm198i0nlPyGvV8V+ch9cbahm16eKS+c6VabwsKaj3Vas7kpA1Zth26u1JqvcySkWr3zRmrEteWkdLvLeHqOSat6lp6fdFzTI6lX2/ux4xU3JxSf+acYrxvYb/edz++NgAAgwBPpAAAAAAgIhZSAAAAABARCykAAAAAiIiMVA9vX10ubmnx+UF/qDbnfZbPeUm97kvZnFr/0fIPq/Xa15Yba9s/f7jaW+WWqvWgWc9ItTWMNtZam8vV3rqc/nW3BPo6PtVuzvO4lgBHvkvPKZU7+r21Fszv2+iSZrXXlpFKuXo2LK9EaryYs5wcS+5FnUNFZgYAAGAnnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGake/nT8rZLJFF9bfmzZ2ca+3026V73u8S9doNazL9ao9SFZc0Zq2KStau82v1OtB516vWOUORjjNelZIAn1PE9zUDyPtkOy1Zwl8hz9MwCny1PrZY6eU2otmOdIpW1zpAJ9hlXK9dW6r4SRPNsMKst77sbIWDmx50jF7I8h7MfXBgAA+yaeSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR42+ynp9IuvLbtvGGns67xNz7wk7x+i1mvWZtV6YuwBxtrHR72m9r6QrVLrthxTd4M5D1S2ypKRcvWcUmOhWq0n2/UskvrSXfrQoxJHzxq1580ZqTJXn0FlmyOV9mxzpMyfb3iWrI8vet2Wc/K1n4e4c6QG6xyqwXrfAACgT/FECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9XDhHy4Tt6T4bKMJf3rO2HfF6tPV6w596G39hS05pW0nTzLWZmd+q/bevvWDat3VRznJsIYWYy14Yajl2uackYjIhryeHfPa9OyY2mvNSOn9HUpGqsSxZKQCPSOVcnufkUrY5kip1bhzpPR6YMln2bJGfqjcPXOgAADAAMMTKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1cPEm9dKwi2ejen42FHGvjf+oGdiGpoX6S8c6HOoNs0y1w5N6cGTv707Ua2Pr21S69OHrTPWXm+uUXud8nK1vi6nZ6Scji5jTc3TiEiiU39fkpbAT0c+ZayVuPp8q+15/etOWmZY+UqYyLNkpNQ5UGKfI6U3D+KcErOgAADAHsYTKQAAAACIiIUUAAAAAETE1r4ewq5uCQ3brir+xbzFreySMvW6+eOnqfXkG+Zri4gcdZT5+PS0o28rDJZVqHV/uL5VbFbli8baiqaD1F6pzqjl9V36Frmwq9tYK4i+HTJhbhURkaSjf4bQlTf/1Shx9PvuDvW/VmnL1sB86BlrCcu2QBvXsj0vUA5Qj707Lta2wrgv3o8G870DAAAjnkgBAAAAQEQDfiE1f/58OeqooySTycjw4cPljDPOkGXLlu3yZ0444QRxHGeXX5dddlk/3TEAAACAfd2AX0g9+eSTcvnll8tzzz0njz76qOTzefnoRz8qHR0du/y5iy++WDZu3Ljz1/XXX99PdwwAAABgXzfgM1KPPPLILv99xx13yPDhw+Wll16SD37wgzt/v6ysTOrq6mK91ruXThavpKRo7bWJNxv75rw9Q73uisuPUetjSsao9W/V/7extq5gPiJcRGTIW3oupatBP6r78PRaY+2e5k61N6jWr72xs6DW010txlp3qPd6XfrXnRRzDklEpDtnzp7ZMlKdfvEj9HdIu/q955R7sx5/rlbtGSlNrKPTRcRy4vzANVjvW0TCQXzvAAAMdAP+idT/tn37dhERqanZdYbRnXfeKUOHDpVDDz1U5s2bJ52d5n/kZ7NZaW1t3eUXAAAAAOyuAf9EqqcgCORrX/uaHHvssXLooYfu/P1zzz1XxowZIyNHjpSlS5fKN77xDVm2bJncd999Ra8zf/58ueaaa/bWbQMAAADYxwyqhdTll18ur732mvz973/f5fcvueSSnf976tSpUl9fLyeddJKsWLFCJkyY8L7rzJs3T+bOnbvzv1tbW2X06NF9d+MAAAAA9imDZiF1xRVXyEMPPSRPPfWUjBo1Sv2zM2fOFBGR5cuXF11IpdNpSaffn2O59uw7pSxTPJ9y47aJxtfzDjLXRESuOOkvav1/tp2q1o8pMWdm7m7TF4DVyzrU+pYZ+pypcUlzJsfd1q72dh5Sr9ab2/WsUH12k/nagZ4GSurxLUk6ekYqn1PmSFnmQGUD/a9V0tXvXZ0jZclI2aZMudaMlZKDipmR6tc5UoM1KxT3Pe9Pg/U9BwBgNw34hVQYhnLllVfK/fffL0888YSMGzfO2rNkyRIREamv1/8hDwAAAAC9MeAXUpdffrncdddd8vvf/14ymYw0NjaKiEhVVZWUlpbKihUr5K677pI5c+ZIbW2tLF26VK6++mr54Ac/KNOmTevnuwcAAACwLxrwC6lbbrlFRN4butvTz3/+c7ngggsklUrJY489JjfddJN0dHTI6NGj5ayzzpJvfvOb/XC3AAAAAPYHA34hFYZ6RmD06NHy5JNP7pHX+mDpNqksLX4i/H/cdIGxL/dJ/bpfHbJcrd97TKNa3+qbc073bTlJ7XVXbVDr7adPUutVbqmxFra1qb1dtXqWraOj+MyundfPmzNUnZboSKJb/wOuJcBRyJlzStY5UkFKrQ/39PctH5r/WiYcPV/lW94XN0ZuxTYHKrAmtAAAAPYdg26OFAAAAAD0NxZSAAAAABARCykAAAAAiGjAZ6T2ppMXf0G8svfPlxIRabjtRWPf8Kf0rM9b+axa/+r4x9X6r1sPNtZeXDFG7Z3Y9LJazzXoeR8/NOdego4utbe7Vl+n+21JtS7KrKi2QO9NdOp5Hc/R7y3MmjNSKUsWqMvX7y2Z1udn5fpwjpRn61cyiU7sOVJxegfxPCUAALBP4okUAAAAAETEQgoAAAAAImIhBQAAAAARkZHqoe6mhCQSxfMt7qRxxr4bR/1Mve4nXv+CWn/00LvV+lGLPm6slb6l57PEkgVqGNms1rcF5hxUmM+pvd21alkSreYskE1rWDzLtvPaXfq8JRsnb37f0pZZTraMVImjZ6T0OVJ6xsm3BJHi5Jxsc6Ts/YMz5xQO0vsGAAB9iydSAAAAABARCykAAAAAiIitfT04L7wujlN8W9Zbd0w39nmW7VTd941Q695Uvd9dWGWsVa/Ut5l5w/T9dUcOW6PWVxZSal2Tq9XvrWSTZWufspesxS9TW73umFv7subXTlm213Xbjj+3bu3Tjj/Xv658qH824lm2qfnSh8efxxFzW2G/Gsz3DgAAjHgiBQAAAAARsZACAAAAgIhYSAEAAABARGSketh+9lHipYofJ/7MiTcY+76+4RT1unV/WKXW//wv5gyUiEj9wk5jLdFiPp5cRMQfV6fWj6l4Qa0v7hprLrp6xsmryar15HI95+QYjqIXEWn2K9RetzOv1m28nDnYkrRkXuwZKT3nlFMyUsmYx5+7sY4/t+SrQsu1Le9boOSzAAAABhqeSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR6O/PLLkqoonm9Z66eNfc88dJh63dGNC9X6D9/9iFovWbrCXFRmLYmIbPvkIWp9anqDWv/eRnP+yy3Vsz4jalrVendbqVp3UkpGqmDJSGXjZaTcnLmmJ6BEugr6nyhx9XvrCMw/a7Y5UkGo/zx4rp6x0qqWHzWrWP2WXj/Uvy7pzxlYAABgn8QTKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1cP19YulMlN8bXngr6809k26r0m9rj9zqlrfsKhcrY9tW63WNS1T9PqYhB4+eXXzSGOtvqJZ7R1ftVWtr2gbrtad0uIzvUREmgv6e+Z0KyEnsWdq3Kw2R0p/z7IF/a9V0imo9Xxonq81kOdI2S8wSHNKMbNhAABg38QTKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1cNX1x8lqYpU0drkG9cY+wrr9VlMK783S603PKVnZhIN5pySv1nPIaUm67OcKlxzDklEpGVDpbFWX6XPQ5pSrme71rRNVutOqXnOVHNez0hJVs9IBaLndTyl3bOEZnK+p9ZTjj4LKh+a+21zpOJmpPzQXLdlpAJ1ClVMgzVfBQAA9lk8kQIAAACAiFhIAQAAAEBELKQAAAAAICIyUj28fstU8ZLFM0NDOpcZ+7wDx6nXPffUp9T6iz8cpdbbjhljrGVe0tfCJ48x37eIfZ5S6Trzj4hfo+eUDixpVOtPtOoZq7DMnN/amtVfO8xm1Xo+1LNGrpKRSjr6e57Nx5sjlQ2SSq9+30Go35tnnUNlFnecUr+OYxqss6AG630DALAf4IkUAAAAAETEQgoAAAAAImIhBQAAAAARkZHqIfPbFyXhFM+nrPqWeRZUokO/7v1Df6PWT280Z2JERDYeM95YS20frvaeUf1Htd4UdKn1inXm+T3ZmrTaOyG5Ra177XqOKciYM1Lbs+YZUyIiXk6fn5VX00AiXtb8dbuWzx/yljlSSctra3OkbHOgcqK/tq2/r3pF7HOo9N5YL92/Bum9h4P0vgEA2Ft4IgUAAAAAEbGQAgAAAICI2NrXQzjzUAkTxbeTfe/zdxj77th4rHrdDQV9C5tXXaXWJ85cbaw1rjYfjS4iMj2t7zt8LadvkatYbz4HvLNO35I4wlPOEBcRp6NbredHDTHW2rPmbX8iIkMKzfq1Lce+a7fuWfaZ5fP69rqU5Qjz7j48/jzh6l+3VrVtzfPFsnUv5tbAWOJsU2OLGwAAKIInUgAAAAAQEQspAAAAAIiIhRQAAAAARERGqoetX8uKV1a89uFSc+bmqLH3q9e94J3PqnX/cP0I86+P/r/G2mVTLlJ7q1w9A/VC11i1XrKhzVhrOnSo2lvjptS606EfvZ7PDDPW2rr0o9er8wW13h3qeR0v1/vjz31frycd/d6ygfmvZYWn5+18S6DHteSYfKUc5/jy3RFoCa3+zFcNZoP1fSOXBgAYBHgiBQAAAAARsZACAAAAgIhYSAEAAABARGSkevjL4b+WykzxteXMF75o7Ft01B3qdTc9cIBaz8/S7+vEUvO8pRGTtqi92/xOtf5cy3j9xbdsM5ayNbVqa5klIxV26RmpnOF7ISKS7bZc29fnLWUt0RFtjpRrCXAEeVtGSr+3fGieQ2Xv1f9Ku44+R0rv7XWriIhYxm8BAAAMKjyRAgAAAICIWEgBAAAAQEQspAAAAAAgIjJSPTzaWStlXvF8ysjvmd+qf19wjHrdhgfXqvUNPy5X69kwb6yd1vCa2rsoO0Stv7apXq2PannHfF9D9byOTditz0TKZcyhGr/T8qMb6PfWreSQRES8nDlL5Dn65w9hQa+ntHlJIpJT5kh5loyTbY5UwtX7taptjlRgmc0Vaw5V3HzVYJ2nBAAABiyeSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR6+8+uzxUuXFK2NXrTQ2PfQn/SM1NjV5l4RkSsmNar1+9pHGWtzMkvV3tu2fkitd66vUOth3jxQKVljnm8lIpIP9ZxSkDNnv0REcpXmYIzTHe8zAFtGyrUNmtJY5ki5lrxO1jf/tYw/R0p/bS1jZesFAADYn/BECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9TDmv9+QhJMqWmv6vDkHNe7eVv3C0w9Ry5/K6Bmqk5ecb6z9ffqdau9fV01U62Xr9KyQuOZ63ZA2tXV7oGeobLOe8hlzJsfrjPcZQEdY/Pu88/rKHCkbp6APPUpa5kjlQ/PXZstI+UqviIgrvc85xZoDJSJOjFlQcV+7P4WD+N4BAIAZT6QAAAAAICIWUgAAAAAQEQspAAAAAIiIjFQPTmVGHDddtDbray8Y+948oqBe993/nKXWK5zir7lDx8Khxlp6RlLtDZfpc6Iq1ut5Ha/S3D+haqvau8mPt07XMlKp7fGu3Rno77mb1bNIGltGyrNkZnKB+a+l69jyVXrmLeHaMla9nyPlW/JX/ZpzipHP6leD9b4BANgP8EQKAAAAACJiIQUAAAAAEbG1r4dlVzaIW1pStPZw/UPGvo+NOU297uw55m2BIiJPd+vfhvqFWWNt3aXtau+Qt/TtVBUbcmpdhtYYS1PKX1Nb3y0M0a9tEWTMWya9Tfrx5dqx7SIirUHx7/PO9lyMrX152/Hnlq19vvnercefWz4bsW0rDJS9ZHF3mdm29vmhUmeLGwAAGGB4IgUAAAAAEbGQAgAAAICIWEgBAAAAQERkpHr41ZxbpCJTfG156boTjH3rTx+tXveeEb9V6+euOEOtp19eaaw92TVG7a1epmeo3LZutZ6vqzLWJpdsVHvfydapdSeh//ilMub8VqJTz0g5np6Rsh5/HiMj5eqn4Ytnyftox5+nHP3ituPPXUs+y1fCSLbjz/tTYPm6AAAA9jSeSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR6GeHnJeMXXlkt/eJixb9iX1qrXbQv0XMsbz49T6xO2LTTW7t88Xe1139VzTKLN7hGRjulDjbWxiSa19y8th6p1x/LTV1XRZawFXeX6tZP6xdv8Ur0/H2OOlCUjlbT055U5Uq4Eem+of90JV/+6grD3GSn9zkScGLOg4vS+dwG97IfK3Q/gbBgAAOg/PJECAAAAgIhYSAEAAABARCykAAAAACAiMlI9zPnbV8QtLSlam/Tr54x9t173d/W6V64+Q63XPaenS7xhw4y1xSsOUHsnNi1W67Z5S22jzGvtEV5e7X2n1XzfIiLJUn3G1fByc31rV63a66T0JNJ2S0ZK8uagk5qnERE3rwdyPEvgJxeYvycpR8842eZjeTHyPk7MrFDcfgAAgIGEJ1IAAAAAEBELKQAAAACIiIUUAAAAAERERqqHyTdtl4TXXbSWP+5wY9/IxAvqdZf9fpJaP+D5NWq948ixxlrpMstUIkdfK4cFfehRZ4M5D1Tj6Xmc9duq1Pq4Ej1jVV/aaqy1dFnyNsmUWm7zi2fhdnCUjJSNa2m1fXqhzpFy4s2RckV/33xl4JJtjpRvmUkWy2DOV8WdgQUAAAYknkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakegjXrJXCKZ46abhhr7Ltu62HqdUc/sFGtF9auU+uNF4821oa9rM8V8obp85aCpma1nmzoMNbShvdqh64mfVZTWK7XR5WYs2PvdOlZISetZ6Q6Cnq+S5sjFVhyRraMlGcJzRQC8+cbtjlSWsZJRCTh9r7flpGysc2RCkT/nsZ78UGcsQIAAAMST6QAAAAAIKIBv5D69re/LY7j7PJrypQpO+vd3d1y+eWXS21trVRUVMhZZ50lmzZt6sc7BgAAALCvGxRb+w455BB57LHHdv53IvGP27766qvl4Ycflt/+9rdSVVUlV1xxhZx55pnyzDPPRH6dxi8eLl66+LHYC2fcZOw77K6vqtcdv3yhWnfLy9X68JmNxlrF3fr2usL4erWesBx/PmXEZrWuSTbpP15hhWVrX8q87TDRpW9RC0v0rX2tlq19Yd58NLttC5pjO/7c0bff5X3z5xuu5bXzofnodBERz9IfhNprx9za10+9sQ3m48sH870DADDADYqFVCKRkLq6uvf9/vbt2+X222+Xu+66Sz784Q+LiMjPf/5zOeigg+S5556TY445puj1stmsZLPZnf/d2mqeVwQAAAAA/9uA39onIvLOO+/IyJEjZfz48fK5z31O1qx57xCCl156SfL5vJx88sk7/+yUKVPkgAMOkIULzU+B5s+fL1VVVTt/jR5tPswBAAAAAP63Ab+Qmjlzptxxxx3yyCOPyC233CKrVq2S448/Xtra2qSxsVFSqZRUV1fv0jNixAhpbDRvh5s3b55s375956+1a9f28VcBAAAAYF8y4Lf2zZ49e+f/njZtmsycOVPGjBkjv/nNb6S0VM/YmKTTaUmn35+ROf/CP0tJRfG35KHOYcbrjbu/U30975DJar1QWTyXtcPc8fcaa/+9drra23L0oWp9aHONWp9V84qxlg3NOSIRkZImyzHfFXqOqS7ZYqx5XZYgkuX48/a85fhzJTuWD/V8lqu/Lfbjz31zzsl2/Hk+0P9K244w144/tx1frt9Z/OPTAQAABpIB/0Tqf6uurpZJkybJ8uXLpa6uTnK5nLS0tOzyZzZt2lQ0UwUAAAAAe8KgW0i1t7fLihUrpL6+Xo444ghJJpPy+OOP76wvW7ZM1qxZI7NmzerHuwQAAACwLxvwW/v++Z//WU477TQZM2aMbNiwQf7jP/5DPM+Tc845R6qqquSiiy6SuXPnSk1NjVRWVsqVV14ps2bNMp7YBwAAAABxDfiF1Lp16+Scc86RpqYmGTZsmBx33HHy3HPPybBh72WWfvjDH4rrunLWWWdJNpuVU045RX7605/26rW+VLVaKjPFH9JN/ekVxr7Rz+lzot79N/3pWEKPWMnssm3G2i1tbWpvyxS1LJWrK9X6kWUrjbXtQU7tLWnSMzH5jD4Da5hn/trcLj2IFJTq1+6wZKS8vPlIfN8yT8ktWOqWB8F+0Ps5UoElf2XPSCmv3Y8ZJ1s+y96/h25kbxus9y0i4SC+dwAAdseAX0jdfffdar2kpEQWLFggCxYs2Et3BAAAAGB/N+gyUgAAAADQ31hIAQAAAEBEA35r39505ttzJFFePDsz9ievGfucuhHqdWd9fKlaf2bNOPvNmV47qc9LSk/ertbbX9MzUpOT5v5GX//xKW2yzDzKmOcliYgMc7PGmtOtZ6QKGT0D1ZXX37eMb773ILRkoGxzpCyBHd83f77hOXpGKh/q72nSMocqCHufkdLvLH7OCQAAYCDhiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkeqh8KMRIsmSorVUlTnf0Th7tHrdXzX8QK3/vOJQtf6nziHGmjdSz2d99IC31PqfR+mDi0d4pcba4mxG7U0363OmOqaYry0iknHNWSInq1/bL9N/tDtz+pwpLSOVt86RUsv2OVK++eu2ZZzygSUj5er9vjK4KO4cKVtGSp3P1Z/5qrjzkAbrPCUybQAAqHgiBQAAAAARsZACAAAAgIhYSAEAAABARGSkekg9tlgSTvHszJs/MmeJ6idvivW6n696Va1/5s3PGWvehFq197Tqh9X6/Q1HqfWkY87cLM/Wqb2J5g61nq8sU+tlhu+FiIhYMlKFUv0zgmxe/9EPffNUpJxtjlQhXrYkCJQ5UpZpTbY5UmmxBLgUriUb5vdjpCawTbEi77N/Gay5NADAoMITKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1UP37CMkYZgj9cAnbjL2eZbsyNz1p6j1nx/wtFrfuKjeWCuboocBjkx3qvWyhna13hmYs0hvdpjvS0TEaWlT6/mKYWo97Zh/PMOcJSNVon9GkMvqc6RCbY6UJW7j5i11S4AjKGgZKf3Fg1C/tufoWSI/NL+2a+m1UcaCWTlkXgAAwADDEykAAAAAiIiFFAAAAABExNa+HiqvWifJ8lTRWrVrPja6wdOP8V70h6lq/aUvPa7W6xeaX3vNqfpauMItvlVxh0NHbFTr63zzPrXlrUPV3pL2ZrWez+jb1DzH/LWFOX3/XKFU3wvm5y2fIYTmbWx5y9a8uMefh775+knHvOVQxH78ua0/UD5bcS1HiNs2/jn9eQQ5WwMBAMAexhMpAAAAAIiIhRQAAAAARMRCCgAAAAAiIiPVwz0THpPKTPG15bg/fc3Yd+XMv6rXHXv/VrX+w499VK2XvbTaWKu/qkLt3ep3qPVjqleq9TdyI4y1jdsq1d6xXevVeqFSz+uo8raMlN4eZvUskYTmPE9eOSJcxJ6R0rJfIiKiZKRsx58XLBkp2xHmvhImsmWk4gqU97x/81X9+NoAAGDA4okUAAAAAETEQgoAAAAAImIhBQAAAAARkZHq4YfN46UklyxaO+j6FmPfzV/RM04T33hOrT/70ky9f9MiY+30hlVq76JsrVo/qlTPSD24fYaxlm3Wg0hhwTz/SkTEy+g5p3xozlDZrm2bI+Xkev8ZQrcth5SPmalRMlK2nFI+0O/NlrHylfyX7bW1fNXu9AMAAAwmPJECAAAAgIhYSAEAAABARCykAAAAACAiMlI93Pd/ThQvVVK0NmL1y8a+Cfdl1Ot6E8er9REL9WyJN2SIsXZKhTk/JSJyy5YT1Pp/1j+m1p/fOsZYSzVZZjFZZCq61Ho2NGeoQl+fQWWbI+Xk9Pdck7dlpCxzpKyCOHOkLDOuLHOkAuWzFdfy2jZxZkE5vf92xX7t/hQO0vsGAGB/wBMpAAAAAIiIhRQAAAAARMRCCgAAAAAiIiPVw7BfLJGEU3yO1IavHGnsq/uJnlNa+8/6nKjR921U67nDxxlrB6X0tfATqyaq9dqGZ9T66o3mOVQVTZZZTQn9x2toRYdabwuUWVGhZaZRmV73YmSkusPiPyM7ODEzUk5BmSNlySn16xypkDlSAABg/8ETKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1YMzcaw4Xrpo7ZJL/2Dse/jOSep1Dzp9mVrffkOzWt/4mXpjLW3IdO3gvFWh1r3j9bV0Yn3x90NEpHSrnnlxy8rU+vCy7Wq9Jej9Ot8v0e8t0d77a9syUm5en9Vk4/jKHClLzijuHClflHyWpTcuX8lv9escqJgzrPrVYL53AAAGOJ5IAQAAAEBELKQAAAAAICIWUgAAAAAQERmpHpZ/rUTcspKitcuqVhv7fvGxj6vXvWPM99X6+emT1Hr6mCZjbU2hXe0dskzPtbQH3Wq9fJ05ZFHSrMx5EhEno+ez6ks2qPUtgZ6x0vhl+tedarZ8huCYv277HKm4GSlzzTYHqmDJlXmi31ugZKxs+azAEsjpy7iOb5krRlYIAADsaTyRAgAAAICIWEgBAAAAQERs7evhz8f9t2QyxdeWp751jrEvf+Y29bpljqfWw4MmqPWrJj1qrD3ROVbtrVrWptZXF/QtUZn15u176aas2htW6Vv7RqX0921LodJcVLbeiYg4pcr+OBHxcvqPvuOZv2fdgeX489hb+5QjyC29hUD/WUtq+wZFJBfq/XG4MY4w79fjzwEAAIrgiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkephbaFUygvF15aFG+qMff99y0/V6/5/jcer9U2zqtT6pyrWGGvnrTxN7XXf3ajWX82OVOtl67vM124110RE8sP0jNTIpJ6R2pAfYqxpGSYRkURJXq172bRaF8f8GYPt+HOJm5FS2j3LMd4F5fjy9/otx58rn624lqPX/UF8xnhg+doAAAD+N55IAQAAAEBELKQAAAAAICIWUgAAAAAQERmpHi5+8BJxS0qK1iY8stDYd3Raz8yc98cj1Xr+A3rWqNRJGWsvrzhA7Z3UvFitP9euz7DyGpUcU17PIWWn1Kj1ukSLWn++Y7y5aMlIlZfm1LqT0zMxjmf+jKEj0PNVTl6f1WTjmEd37cYcKf1PuKJnpPKh+f8S4syB2p1+7c4sY8OsYvVbev3QkoljBhYAAPsknkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakeJt68WhJu8TxS+2lHG/tu396oXnfcfdvV+tG/eEWtP9FtzmCVL+v9PCQRkRe26BmryuZN+vUVXTV6jqnG7Vbr67uqjTUnoeezMiVZtd6lR6jUDFY20DNxjh8zI+WbQzn6OyriW+dI6XmdbqXftc2gCvUwkUNWCAAA7EN4IgUAAAAAEbGQAgAAAICIWEgBAAAAQERkpHoIczkxRUSG/PNqY993njxdve6kJc+r9YuGLFLrX1n5aWOt5i1l6JCIJIYPVesbNuiznio6VxlrjmWWU/dQPTNT7eqZm42dlcZaOtWmX7tEn82Vy1rmSCXNfzU6A/NcLxERKegZKdvcIUdp9ywDkWxzpDzLHCkt52TLV/Ul8lUAAGCg4YkUAAAAAETEQgoAAAAAImIhBQAAAAARkZHqYdVlk8QrKSlae2viT419H/6vL6nXTYzVZzUdkKhQ68ueH2usTXy7Se3Nj69T66n1+kwkCc3ZlLCg57OytXquJePqP35b28uNtVFpfU5UdapTrTflLJmbhPneuq1zpPQcko2WkbJ98uFbMlK2WVC+8gqu6O+ZL3p+y+3PnNNgzVjpbykAAOhHPJECAAAAgIhYSAEAAABARGzt6+F7n/mllGWKH+n9vaaJxr7U35aq1133lSPV+ps5fRta3ULzdqxw7Ua1d9vMQ9V6xTq1LK5hq6OISNDdrfbma/Stf2lH3yLX3m5+bUnrR5APS21R6+/mLcefK1v7unzLdkhfP/48sGyR03bfeZa9Xr5yfPl7/fprB6bz/2V3tgXG29rnK9tI+/X488G6LRAAAPQpnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakejirZLpUlxdeW/98Pvmjsqyt/Q71u/SdWq/UFW05U65UvmINMhbY2tbdlslqWkX/X8zxubY2xFm7eqvaW1erZr6RTPI+2Q9BmziKFKT2nNCSpv7abteRekubrdwV6PksK+ntq4+rRMpXt+HMvRs6pX48vtwgk3pHzAAAAUfFECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9fCRF88XryxdtDb6/7xk7Gs5a4Z63bsP/L5a/9AjV6v1SeteMNacpJ7XKZ3Sotfv0dfS/vAhxprb1q72jqrertZtEq3mDFVYqn/dQxN6dszLWjI1yb6cI6W/tqO0u45ljpQlI+VaXjsfmt9zz9KrzaB677V7n7HSv+q+7x+8Lx7DYL1vAAD2Ep5IAQAAAEBELKQAAAAAICIWUgAAAAAQERmpHhp+6EgiUXxt6Uweb+zbdlaHet0RXvHc1Q5DF+rfBjeTMddqzRkmEZFTDnhLrb+xuV6tt08fZaxVbDHfl4jIxMotat0P9cxNss0c0ghK9JxSTULPb7k5PccUxshIhb5lVlOoZ4W0jJRnCa74gV63zZHSck5x50g5lv5407cAAAD2Lp5IAQAAAEBELKQAAAAAICIWUgAAAAAQERmpnha/KeIUz7+8/cvDjW23z/iFetn/3KLPmRr23Fa1XphqzmflSvVv4WnVD6r115r0/FZ7wxhjrXxVhdo7uew1tZ4NC2o9qcScAsvXXe12qvU4GanugmWOVJDXy5Z5TG6MsFDBMkfKs8xy8i2zoNTePhw8FDefBQAAsKfxRAoAAAAAIhrwC6mxY8eK4zjv+3X55ZeLiMgJJ5zwvtpll13Wz3cNAAAAYF824Lf2vfDCC+L7/9jr9Nprr8lHPvIR+fSnP73z9y6++GK59tprd/53WVnZXr1HAAAAAPuXAb+QGjZs2C7/fd1118mECRPkQx/60M7fKysrk7q6ut2+ZjablWw2u/O/W1tbRUSk5ZyjxEuVFO35+4duMF6vPqFnhS7+7XFqfcI7L6j1xq8ebax53WqrzEjpfyDo1uvt5jFSUlOjL1gnphvVemeoZ4lSreZcTMGWkfJiZqTSnrHW7Vvmfvn6e+pbckqOb667lofIgSUj5VryWYGSc7LllHzLvfVnzsk2w0rv3YM3srcN0nsPB+l9AwD2LwN+a19PuVxOfvWrX8kXv/hFcXr86+bOO++UoUOHyqGHHirz5s2Tzk79H9Hz58+Xqqqqnb9Gjx7d17cOAAAAYB8y4J9I9fTAAw9IS0uLXHDBBTt/79xzz5UxY8bIyJEjZenSpfKNb3xDli1bJvfdd5/xOvPmzZO5c+fu/O/W1lYWUwAAAAB226BaSN1+++0ye/ZsGTly5M7fu+SSS3b+76lTp0p9fb2cdNJJsmLFCpkwYULR66TTaUmn9WO/AQAAAMBk0CykVq9eLY899pj6pElEZObMmSIisnz5cuNCymTWZS9KuqL4jKDVhVJj39PdVep1x9+nDETaDYVZrcZa+8ZytbfCLZ752skSAMk1mHNM3UNTau+YxDa1vj3QcyupNnOep1Cq70rNOHr+ysnrGSm/1Py+5SxzpEp9Sw4ptGWkzDXP8v0KAr3uOfq9aXOkPEu+ysaWkdKuHifjBAAA0BcGTUbq5z//uQwfPlw+9rGPqX9uyZIlIiJSX1+/F+4KAAAAwP5oUDyRCoJAfv7zn8v5558vicQ/bnnFihVy1113yZw5c6S2tlaWLl0qV199tXzwgx+UadOm9eMdAwAAANiXDYqF1GOPPSZr1qyRL37xi7v8fiqVkscee0xuuukm6ejokNGjR8tZZ50l3/zmN3v1Ot+te0UqM8Uf0h145xXGPn9oTr3upBcWq3X34Elq/ZKD/m6s/a5ihtq71e/QX9syc6thZLOx1lUzQu0d5unbsVYW9JyatrWvu8Z8PLmISLlr2YaW07f+hSnz+5L19dcu9S3bBq3Hn6tllXVrn+W186H5a7NuzVO2BQIAAOxrBsVC6qMf/aiERXIlo0ePlieffLIf7ggAAADA/oyPkAEAAAAgIhZSAAAAABDRoNjat7dcvvYYSVUUP9J78o2rjH3tRxygXjcxUj9BcNMxNWr985WvG2v5UXpeZ1G2Vq27NUPU+pHD1hhrjw2tU3urLEevb/H1o9uTbeYcU1uD/qNbZjkm3MkX1LqfNH/GkM1b/toEluPP9W5xlYyUa/nsw5aRci3HnwehuT+p3ZiI+GJ77d4fYW47/tyWO5P+PD5df1v6rre/cWQ9AGAfxxMpAAAAAIiIhRQAAAAARMRCCgAAAAAiIiPVw7JbDpZEsniupzr3trGv7LGl6nU3XjBdrW87Up9pNNQzZ4lOqTDnp0REfrLpJLXu1+kZqWMqnjHWHq45Su1NOnp+a31ez4Z57VljzS/V51+VWF5bLBmpIG3+jCFvmSMV+noOyS9ylH9PceZIhYH+2YhnSWj5ymcradHfMxvXlmPqR4H2vpD12b8M5lwaAGCv4okUAAAAAETEQgoAAAAAImIhBQAAAAARkZHqIfO7FyThJIvWVl7zAWPfuOtfUa9b8clGtf7BoWvV+vNZc4Zqeqr4/e7w5KoJan3EqFK1PjW9wVgr1OrZLpv1OT2f5XR0m19bv21JG76PO4QFyxwpJSNVKFjyV6GeQ7K9a67f+0yObY6UtV+ZI+VZZlD5YbzPZbQv2yW3AgAABhieSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR6CD0yTIFF8jtRNn7vd2Pdfr1ygXve2yT9U655lTs23133cWLtr3N/0a79VodbbR6plGZMwh1MqajvV3s4gp9Y3dFerdaejy1izZaRc2zAYW0YqZe4v5C1zpAL9+2mLQGlzpGxfV2jJSHmWWU750JL/isG1ZKz6kkPGCgAA7GE8kQIAAACAiFhIAQAAAEBELKQAAAAAICIyUj1sv7pTvLLiAZVZJS3Gvo1n6VmgA5NptZ509FzKcy9ONtbWjP6D2lu9TM+lNE3TwyMVbvHMmIjI6OoWtbfZkpFa31ml1qWrw1jyS/Wsj+dYPiPI9z4j5Rcs17bMkbIlheLMkRJLRsqWU9LmSLmWfFVg+VzGtWQBNU6MXgAAgL7AEykAAAAAiIiFFAAAAABExEIKAAAAACIiI9XDH6feI5WZ4mvLI5+72Nj3o5m/Vq9707ZJav1TmVfUet1Cc+2vp4xXe6vealPrm2br+S1fyftMqtys9m7w9Wtv6dBnXA3tbjbflyUjZRP6yrAmEfFTSq8tI2WRC/X+OOOW4s6RCpR78yw35ttmd1n05ZSpWBkrZlABAIAieCIFAAAAABGxkAIAAACAiNja18MfO4dLmVv8KPIDvmfe3/Ox33er173qDx9R69tPLFPrQ55vNNYe2DRd7XXXbFTr40bq2+uagi5jbWLpJrV3ZW64Wt/ebj5aXUSkJpc31oJSfWueTVjo/fHnkrfs9Qr1bWR5y+cXjnL8ufVYd8v+OOvWPmUfm+3odJs4x5/begPLe96vOLodAIB9Ek+kAAAAACAiFlIAAAAAEBELKQAAAACIiIxUD9ff+Snx0sVzO6NeNJ9B/kinfsz3+Hv1DNVdtUep9YmrXjbW3lhxhNo7qfkltX5krfmIcRGRt/Ll5vtKmbNbIiLPd05Q6/k2/X2TwJyDciwZKe3Y9veubcncaLfmxzzmO9T73UKMTI3l3mxZo3xQPCMoYs9X+ZZj3W2vHff4dAAAgL2JJ1IAAAAAEBELKQAAAACIiIUUAAAAAERERqqH0be9LgknVbTWdN4xxr4rnj9Mve6EZ19V69WHHq3W3bQ5sFO+rPj97mSZOzSrYplaX9w11lg7pfwNtfeX7XVq3W0z53Fs0qXmGVMiIgXRM1Shb8lYKW+rk4/3+UPOOkcqxsUt+SsbW35L7R3An8s4xK8AAMAeNnD/5QMAAAAAAxQLKQAAAACIiIUUAAAAAERERqoHd0iVuG7xPNLxVy0y9i36rj4HyqutUesjntuu1sNDzPOYat4qqL2J4UPV+sGpp9T63ZvN+a3zK/WM1IrttWo92db7dXxZSVatd4f6+yKWOVOBkpFy9XiWVT7Us2GOZcaVyjI+yzYLKlBmObmOfnHrHCnLa2scywyqvtSfrx1XOIjvHQCAgY4nUgAAAAAQEQspAAAAAIiIhRQAAAAARERGqodlV9aLW1pStPZw/R+MfbP/Uq5ed8unDlXrtXctVuubLj7CWKt/fIvamx+vz3Ia5SXV+quN9cZaxVjzfCsRkS3bMmq9tE0ti5Mw/3hWWjJSWUsGSkI9O+KnzHWnEG8oUT7U/9o5hRhZokC/N1tOKR+Y81u2fJWNa8nr+DFnYAEAAOxNPJECAAAAgIhYSAEAAABARCykAAAAACAiMlI93DP7p1KRKb62vGjNR8yNeX2wUHBmk1oPf6HPPGqb1WWs1f18g9q7bdY0tV7mKgOTRKRrQ4WxlnT0eUhBk56hSrXpmRknZb63IelOtbcjziwm6ds5Urn+nCNlySkFSk7JOkdKmUEVly1fZTOYZ0EBAICBiSdSAAAAABBRr55IdXR0yHXXXSePP/64bN68WYJg10+qV65cuUduDgAAAAAGol4tpL70pS/Jk08+KV/4whekvr5eHGffOLa41PGlzLAF6M0fmo8wT53kq9e97dAFav3fJ3xWrV8wdaGx9nRH8ePad2iZrJYlH+r3XrbOvA3Ntxwxnt6qb2FLten9Ttq8NbDWsrWvzXLEuE2QNm8FS3RaHuRa/j7kxbK1r2DZn6f1+vpr244wD5TteZ5l32AQ6u+LbWtgHH7Mo9n71GD9v8fBet8AAOwlvfrX5p/+9Cd5+OGH5dhjj93T9wMAAAAAA16vMlJDhgyRmpqaPX0vAAAAADAo9Goh9Z//+Z/yrW99Szo79e1VAAAAALAv6tXWvh/84AeyYsUKGTFihIwdO1aSyeQu9cWLF++Rm9vbTvvr5eKWFs8cTbrnOWPf27cfqV73iLR+xHjz0cPV+gXVvzTW/p4+Se0tm9yi1jf55qPVRUQq1ptzLV1hTu0t0U99l1SrJSNVVmqs1aS2qb1t2vnluyFIm+/NKcQ77DJvyW/FOf48bgypEPTdQZ7Wo9eVUE6/xnXICgEAgCJ6tZA644wz9vBtAAAAAMDg0auF1H/8x3/s6fsAAAAAgEGj1/t4Wlpa5P/8n/8j8+bNk+bmZhF5b0vf+vXr99jNAQAAAMBA1KsnUkuXLpWTTz5Zqqqq5N1335WLL75Yampq5L777pM1a9bIL39pzvQMZJN/tE0SXvHZRfnjpxv75h93r3rdW1sa1PrmD+jZkVGJCmPNqx+h9p465k21vixfpdYr1plzUM1BQe0tadYDO8l2vT8sM8/IGppsU3tbgjK1bpU037ur37aIo38+kQv7bo6UBHqgx/bJSRCa+11Lxsnv/ecyfc6x3Xs4gOdQAQCAAalX//KZO3euXHDBBfLOO+9ISck//rE7Z84ceeqpp/bYzQEAAADAQNSrhdQLL7wgl1566ft+v6GhQRobG2PfFAAAAAAMZL1aSKXTaWltbX3f77/99tsybNiw2DcFAAAAAANZrzJSp59+ulx77bXym9/8RkREHMeRNWvWyDe+8Q0566yz9ugN7k3hug0SOsXnD7XcaM61fDajzzQa98Cn1foJR7yh1l/PmWc9dR2oL1xPr3pArf+9Y7JaT23YbqxtKJjnPImIlDTpYaJEW1ath+XmjFSN16H2NhXMuTIREXEsWaK0b65ZMlKOq1+7P+dIeZaZSAUlv+WJfnFfyVeJiLjS+6/Lls/qT0GMrwsAAAxevXoi9YMf/EDa29tl+PDh0tXVJR/60IfkwAMPlEwmI//1X/+1p+8RAAAAAAaUXj2RqqqqkkcffVT+/ve/y9KlS6W9vV1mzJghJ5988p6+PwAAAAAYcHq1kNrhuOOOk+OOO25P3QsAAAAADAq7vZD68Y9/vNsXveqqq3p1M/1t40WHi5cunst5fvqPjH1PdSfV646/15y3ERH56uzH1PqCLScaa9umFJ97tcNhKfMcKBGRG9aOV+vSZM5/rcgPV1vTTd1q3W3XM1L5YeacU7XXqfauzdeodcfTZzklUkpGKq+2WudI5S1zpMTvu4yUjTZHyrNcPIg5R8oXS4ALAABgANnthdQPf/jD3fpzjuMM2oUUAAAAAOyO3V5IrVq1qi/vAwAAAAAGjXh7cSwqKytl5cqVffkSAAAAALDXxTpswiYMB9d8lYsueFhKK4q/Jfe3m/NA1776MfW6Bzz5ilo/PK3nnP700jRjzZmi568qXPMsJhGRNxtHqPVx280zrt7urlN73RZ91pPTree38uOHGGu1Xrvau7RrtFq35ZhSKfOwKMc2R8rTr90d6Jk6x9e/p2qvJSNl++REy0jFFWcWlK3XFg2zjA3rs97d4YfK3Q/g+VkAAOzv+vSJFAAAAADsi1hIAQAAAEBELKQAAAAAIKI+zUg5fR0u2MPOq1wnlZnia8vDb77S2Fe1Wk9ouJmMWt/s61miYQvNc4dKzm1Ue7darl1YX6bWw4I5ELSsXc9XyXY9xxQGehYoV2n+ujOuPqOqKWeeQSUi4nj6z2ZJ0vx1h3lLbsW1zZHS/9o5MeZI2cJClglW4gfme3ctF7d9Xa5tDlUf5rP61T76ZQEAsL/r0ydSg+2wCQAAAADYHX26kPrTn/4kDQ0NffkSAAAAALDX9Wpr39y5c3f7zx533HG9eYl+ccZbp0mivPhR5Afc/Kqxz0nob+O22ZPV+i+2r1HrQ5/bYqyd+C9L1d5nu4ep9fL1+lpa+9re3V6j9g7p2KDWbedK5yrM9YybV3u35sr1107q/eUp89HsHZbjz8XTN9B1W7bASWA7zNvMCeLtIwuUfWie5Sju7rDvPpdxOAYcAAAMML1aSL388svy8ssvSz6fl8mT31skvP322+J5nsyYMWPnnxtsGSkAAAAA2B29Wkiddtppkslk5Be/+IUMGfLe0NRt27bJhRdeKMcff7z80z/90x69SQAAAAAYSHq1F+cHP/iBzJ8/f+ciSkRkyJAh8p3vfEd+8IMf7LGbAwAAAICBqFdPpFpbW2XLlvfndrZs2SJtbW2xb6q/OD8eKk6ipGjNrTVnNPz1+hHkrWfp78n/vHasWh+/4g1jbU7Fa2rvjY0fUesV6y1Ht1dVGmtbmvVj3au6utS6Y8kS5SvNW0PLLJmZlmypWnc9/ej1sqQ5I9VlyUjZtrTmA8tfO9/8PfFD/ftlOWFcPMu9FZTjzz3L8ee248ttGau+RMYKAADsab16IvXJT35SLrzwQrnvvvtk3bp1sm7dOrn33nvloosukjPPPHNP3yMAAAAADCi9eiJ16623yj//8z/LueeeK/n8e6efJRIJueiii+SGG27YozcIAAAAAANNrxZSZWVl8tOf/lRuuOEGWbFihYiITJgwQcrLLUdOAwAAAMA+oFcLqR02btwoGzdulA9+8INSWloqYRgO6iPPk4+/LAknWbT25s0zjX1jH9BnNd0y/Xa1fuWPv6LfmGPegTkpmVJbn141Qa2PWdetv/ZQ86yosKn4zK1//AE9lxL6ek4pp0SwypX3RERke7Z41m2HIQn9665IZo21lrwlb2PJfmX7co6U/pZahZacUxyu6O+br8ywcvsz40S+CgAAFNGrjFRTU5OcdNJJMmnSJJkzZ45s3LhRREQuuugijj4HAAAAsM/r1ULq6quvlmQyKWvWrJGysrKdv3/22WfLI488stvXeeqpp+S0006TkSNHiuM48sADD+xSD8NQvvWtb0l9fb2UlpbKySefLO+8884uf6a5uVk+97nPSWVlpVRXV8tFF10k7e3tvfmyAAAAAGC39Goh9Ze//EW+973vyahRo3b5/YkTJ8rq1at3+zodHR1y2GGHyYIFC4rWr7/+evnxj38st956qyxatEjKy8vllFNOke7uf2zL+tznPievv/66PProo/LQQw/JU089JZdccklvviwAAAAA2C29ykh1dHTs8iRqh+bmZkmnLbmZHmbPni2zZ88uWgvDUG666Sb55je/KZ/4xCdEROSXv/yljBgxQh544AH57Gc/K2+++aY88sgj8sILL8iRRx4pIiI/+clPZM6cOfL9739fRo4cGenr6vr4EZJIFs/W/OHjNxn7Pu59Vb3uB/W4jtQ9q8+ZciaPN9ZceU7t9d7SDwBJNm5U67nRQ4y1dJNlHW7Ly1kyVPmMuZ529B/dti79Ta9J6P2ZRIex5hYsmRnLtfOBnqFylDlSNrY5UrZPTrRZUK7l4n7vPpfZI3zLz9J+a/BGVgEAGPB69S+f448/Xn75y1/u/G/HcSQIArn++uvlxBNP3CM3tmrVKmlsbJSTTz555+9VVVXJzJkzZeHChSIisnDhQqmurt65iBIROfnkk8V1XVm0aJHx2tlsVlpbW3f5BQAAAAC7q1dPpK6//no56aST5MUXX5RcLidf//rX5fXXX5fm5mZ55pln9siNNTY2iojIiBEjdvn9ESNG7Kw1NjbK8OHDd6knEgmpqanZ+WeKmT9/vlxzzTV75D4BAAAA7H969UTq0EMPlbfffluOO+44+cQnPiEdHR1y5plnyssvvywTJujHbQ8E8+bNk+3bt+/8tXbt2v6+JQAAAACDSK/nSFVVVcm//du/7cl72UVdXZ2IiGzatEnq6+t3/v6mTZvk8MMP3/lnNm/evEtfoVCQ5ubmnf3FpNPpolmu2itWS7K8+FymMmVAz7wTHjLWRETubKtV6+5rK9T6lnMPM9bWFDrV3iHLLKGZrc1quf2YEcZaSZN+aSelz7gKs+ZZTSIifsb8nicdPWfU3V18Htg/LqDXM0nznClbRsrx9M8nskHv50gFlllMtrKNr2SkPMvFg1D/uu0Zq76bI+UwCwoAAOxhvXoi9cgjj8jf//73nf+9YMECOfzww+Xcc8+Vbdu27ZEbGzdunNTV1cnjjz++8/daW1tl0aJFMmvWLBERmTVrlrS0tMhLL72088/89a9/lSAIZOZM8wBdAAAAAIijVwupf/mXf9l5QMOrr74qc+fOlTlz5siqVatk7ty5u32d9vZ2WbJkiSxZskRE3jtgYsmSJbJmzRpxHEe+9rWvyXe+8x158MEH5dVXX5XzzjtPRo4cKWeccYaIiBx00EFy6qmnysUXXyzPP/+8PPPMM3LFFVfIZz/72cgn9gEAAADA7urV1r5Vq1bJwQcfLCIi9957r5x22mny3e9+VxYvXixz5szZ7eu8+OKLu5zyt2MRdv7558sdd9whX//616Wjo0MuueQSaWlpkeOOO04eeeQRKSn5x9HWd955p1xxxRVy0kknieu6ctZZZ8mPf/zj3nxZAAAAALBberWQSqVS0tn5Xjbnsccek/POO09ERGpqaiIdJX7CCSdIqMx/cRxHrr32Wrn22muNf6ampkbuuuuu3X5NzZ3j/yaVmeIP6cY9fLWxb9XH/ke97rg/fUmtT+p6Sa03fSBvrD3ZZZ4xJSJSuUyfUeW3tqv19lHmh5aV7+qZF7dCn2Hl5wtq3cuYv25bRsrvtPxoJ/V6hWfOb7l5S97G0+8tH+p16cM5Up5lsJA2R8qLkXEa6ALp/XtuRT4LAIB9Uq8WUscdd5zMnTtXjj32WHn++eflnnvuERGRt99+W0aNGrVHbxAAAAAABppeZaRuvvlmSSQS8rvf/U5uueUWaWhoEBGRP/3pT3Lqqafu0RsEAAAAgIGmV0+kDjjgAHnoofcf+f3DH/5wl/++7rrr5LLLLpPq6upe3RwAAAAADES9niO1O7773e/KZz7zmUGzkLq+aaKUZIvPFzroevPQpJdOzqnXHXOvnh1JHKBvhzzzsMXG2n2bZqi97rsb1LofmGc1iYh0Npjrw1/UM05OpkKvt3eo9UxFl1pXr92t55BCS0Yq45nnSDmWOVKS0F/bOkdKyQ3asjy2jJSNH/TqIbWI7M4cqd5nhdy4A7IGq8EbOwMAYJ/X+3817QbtIAkAAAAAGKz6dCEFAAAAAPuiPt3aN9g89D8fFC9VUrQ2fO3Lxr6LX/2Cet3hjy9V69tPO1ytf2Xor421k5d8Te2dtO1Fte4kU2q9rMF8PHpqm9oqQbW+tc9tblHrQyv0rX8ar1P/jMC2ta9C2drn2rb2ufpr52xb+3x9u6XGtrXPdfS9YqFy/Lkb84hwz9Jv2xoYR5wdcuyu6yXeOADAPo4nUgAAAAAQEQspAAAAAIioTxdSxx9/vJSWlvblSwAAAADAXterjNQdd9whF1xwwft+v1AoyL//+7/L/PnzRUTkj3/8Y6yb29uG/nKxJJzix5+vv+poY1/JfXpmxknpR5A3ztLDBBOS5qxRxTI94ySufhS3W12l1g8dsdFYa9tWqfZ2j61V6+mNxfNoO9SXbVfrGq/LkgVK6+9LxjUfve4W9KxPaDv+3Nf/2oW++fq+5SRMp/fxqveuH5jfN88SwPL7MRQT88sGAACIrFdPpK666ir59Kc/Ldu2/eO0gWXLlsnMmTPl1782H4wAAAAAAPuCXi2kXn75ZVm3bp1MnTpVHn30UVmwYIHMmDFDpkyZIq+88sqevkcAAAAAGFB6tbVvwoQJ8swzz8jXvvY1OfXUU8XzPPnFL34h55xzzp6+PwAAAAAYcHo9R+rhhx+Wu+++W2bNmiVvv/223H777fKhD31IRo4cuSfvb69ypowXx0sXrX3lS7839j34oYPV67aeotenHbVCra/Km2c51bxZUHsTI4ap9XCInnM6pvp5Y+0vrRPU3u6hdWq9pEw/iGRUSaOxlg/1VIwyBkpERIKU/qNfqVzAyVvmKVkyUrlAr0vY+3lNtjlSniXHpM2RsrHNgXIdPd+lZawcSy8AAMDe1qutfZdeeql8+tOflm984xvy9NNPy9KlSyWVSsnUqVPlN7/5zZ6+RwAAAAAYUHr1ROqZZ56RRYsWyWGHHSYiInV1dfLHP/5RFixYIF/84hflM5/5zB69SQAAAAAYSHq1kHrppZcknX7/FrjLL79cTj755Ng3BQAAAAADWa8WUsUWUTtMnjy51zfT31Z8rUTcsuKzjS6qWmPsu79JzwJtOrNBrX931KNq/dbm44y18reb1d78eP3eCqV6Xueo0pXG2p/b9Wt31eh5m8pyPSPVkN5mrNkyUolOtSxhSt/VWuZkjTXHNkfK09/Tgi0j5eeMpUD017ZlpGx8JSPliSXjZMlI9SdbPgsAACCqXh828bvf/U5+85vfyJo1aySX2/UffosXL459YwAAAAAwUPXqI+Qf//jHcuGFF8qIESPk5ZdflqOPPlpqa2tl5cqVMnv27D19jwAAAAAwoPRqIfXTn/5UbrvtNvnJT34iqVRKvv71r8ujjz4qV111lWzfvn1P3yMAAAAADCi92tq3Zs0a+cAHPiAiIqWlpdLW1iYiIl/4whfkmGOOkZtvvnnP3eFe9Oixt0kmU3xt+dE3zjX2JY6tUa/7k6PvUuvHpvVgywWvHGGsTV73ltrbfNw0te7qUSOZmOwy1oKsOUckItJdq2ek/Iw5ayciUpcwL8qzoWV+Vpclz5PWc0rlrpKR8m1zpPTPJ/pyjpQlQiWu5bMTbY6Ua7l4YJlRFSen1J8Zp7gzrJzej+bqX4P1vgEA2Et69USqrq5OmpvfO+TggAMOkOeee05ERFatWiVhSKgbAAAAwL6tVwupD3/4w/Lggw+KiMiFF14oV199tXzkIx+Rs88+Wz75yU/u0RsEAAAAgIGmV1v7brvtNgmC97b5XH755TJ06FB55pln5PTTT5fLLrtsj94gAAAAAAw0vVpIua4ruVxOFi9eLJs3b5bS0tKdg3gfeeQROe200/boTe4tK/KlUp4v/pDOuWGYsW/lWfrbeGqZniXa5ner9SHPpoy1oKND7d1uGeuV3K4HIWpdZdaTZRtntlbP1BQySbVel2gx1rotOaKEOdolIiJBUv+6y528sebk9WBZmNa/LtscqWRgfl99yywnR+ndHUHQ+2CMbY6UZ8lY+b17QA68jxL1AwBgj+nVQuqRRx6RL3zhC9LU1PS+muM44vuWEwwAAAAAYBDr1UfAV155pXzmM5+RjRs3ShAEu/xiEQUAAABgX9erJ1KbNm2SuXPnyogRI/b0/fSrLz9wsbglJUVr4/+y0Nh32Y1t6nUf7ChT6xvyDWp9+KIWYy1M60eIV0zeptZbNlSqdc9R1tqWc53D2pxaz1m29g3zzPvz2ix7d5Jdlm1kaf0zhLRj/kDAKViuXa5fO+/rW/sSypbJwLKd0rEcf+7ZvmfK++rZLt6HbMef2+4s7hHmAAAA/1uvnkh96lOfkieeeGIP3woAAAAADA69eiJ18803y6c//Wl5+umnZerUqZJM7vpk4aqrrtojNwcAAAAAA1GvFlK//vWv5S9/+YuUlJTIE088IU6P7UKO47CQAgAAALBP69VC6t/+7d/kmmuukX/9138V1913jiw+cMG7knCLHzXedsbRxr65Q25Vrzvpbxep9fIK/fjzkW+sMNa8UfVq7+wD3lDrf5KD1Xp7YL4315LPGlLTrtZzFbVqvVr50dpQ0H90E5aMVC6j55RKtDxQwXL8ecKSkQr0eqlyYIv9+HO1bKUdf+5ZXjsQPX9lzTlZjk8HAAAYSHr1L5dcLidnn332PrWIAgAAAIDd1auV0Pnnny/33HPPnr4XAAAAABgUerW1z/d9uf766+XPf/6zTJs27X2HTdx444175OYAAAAAYCDq1ULq1VdflenTp4uIyGuvvbZLzbHMqRnIwkJBQsN2xeH/tNLY93beknG6t3juaodtk0r1+1IyM10Th6m9H6u6X61vHFGl1lcXzLkWp1yfjzWuulmtr8gMVetljjnH1BLo75lnmyNVq//olyg/xo41I6X/HbDNkZKg90GnuBkpbY6Ujd+HGSdbviouNXvWnzOo4v7f6eD9v2MAAAa8Xi2k/va3v+3p+wAAAACAQYPTIgAAAAAgIhZSAAAAABBRr7b27atWfflAcUtKitaWTbjF2Hfc0s+r18385TW1XrJ1klr3DhxrrG2arOevDkvl1PrMSnP2S0Tk1exIY82pzKi9kypWqfU3M/rXXeqYv7aWQM9neV0Fte6n9PetxFE+Y7BkmIKk/vlEwdfrYWjO5NgiUI4e3xLX8tlJqMyRci0BrMCSr0q6+s35fRjocfoz54ToBvP3i1waAOw3eCIFAAAAABGxkAIAAACAiFhIAQAAAEBEZKR6uP5Tv5DyTPEZP9/ZOsXY13X/CPW6GX+DWk++skKtN33iEGOtdYqeO6lwi2e+dphR+q5a/1XTB4w1v7pC7Z1U2qjW85V6DsJTckpNBf21va68WvfTljyPlpGyzJEKLHOkCoHl84vA/L74Sn5KpJ/nSFk+l0mLnlvTuNqcp34W2JJrgznvAwAAjHgiBQAAAAARsZACAAAAgIhYSAEAAABARGSkejiypE0qS4qvLefdcKKxr+5BfV5S+0emqfWSPzyv1jd/wJzJGTNhs97rd6j18ZafgJe2jjbW0rV6/mp8Sr+3Qqb3gZ5mv1ytO1k9IxXoY6Qk6RTPyomIiG/JSCX1nJFvmSOl9lrqjpKv2h1aRsqzZIFsc6Rs/DDG+0IMCQAA7GU8kQIAAACAiFhIAQAAAEBELKQAAAAAICIyUj2c9PwF4pUVz/2Muf0FY1/BkplZe1aDWj/oxTq1fvKM18295RvV3me79RlXp5W1qvUNG2qMtfpa/cdntNeu1iVjmfUUmjM5W/MZtdfpzunXTqtlcZXPGMK4c6QKSv5KRET5um2pMtscKVf0e9PGVHmWWU62jJQXY8iVG3NAlhsjvuXEi34BAIB9FE+kAAAAACAiFlIAAAAAEBFb+3o44KZQEl7x7UvOIRONfWFafxuv+8Dv1PqCI89W61eO+LGxZttu9f3Gj6r1M8qfVuup9UljratW3/M0zNPfl7JMVq1nw4Kx1pzTjz+Xbv3agfnLEhHLFrgg3ta+wHL8eagcYW475jvmDjgJA/axAQAA7A6eSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR7CJW9L6BQPzyz/1VRjn7O2VL3uZyq2q/VvfkA/DntaqviR7CIi+VDP6zy9coJa3z7qL2q9Yp251jVcbZVSJ6XWa8o71XpnaD4efVtOf8/DvOVo9bQeNvIc5TMGy3H3geVvVRAjh2Q//twSorJR2m1HkPuWz2Vc2/HpfK4DAAAGEf7lAgAAAAARsZACAAAAgIhYSAEAAABARGSketh27hHiGfJIzx7/fWPfguaj1es+0aWvV8ccrQSRRGRjod1YG+6Vqb3Jt/T6qln6vVWsN+eBWg7Se9WckYjUlbeq9e1K3mdbVv+6Utltaj3Q41uq0NezQkFSz0CFviUjFZqv72vzraR/50gFYd/NoHIdW75K51j6AQAAouKJFAAAAABExEIKAAAAACJiIQUAAAAAEZGR6uGDlz0v6Yric6TeyZvnFv1/Q5eo1z3smS+q9Ztm/Eat/7zlCGPtnKqX1N4hy/T0yJLsaLVeur7DXKzVg0a+kvURERlZqs/XagrSxlprt3m2lojIUMscqSAVIzMTWDJSlr9Vod/7zy98Sw4pdhRIub5nmwMV2jJztjlUfZexiiN2vmpgflkAACAmnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakerh3+qlRmiq8tJ/7ycmPfX865Qb1uzX3lav2kYzvV+lUvHGOsjThczxlVLdNnNS1snaDWvc3meUy1NfrX1R5m1XpDWp/11FioMtY6uvV8Vm2hoNaDdIyBS755tpaIPSMl1jlS5kxOYJ0jped5bLO9tIFM1oyU5d7cGEOubHOk4gqU97xfMf8KAIABiydSAAAAABARCykAAAAAiIiFFAAAAABEREaqh0vXfkCS5cWzN5NuWmns+/LRZ6vXrX7kTbWe/76euSlbaM4i3V8/Xe11Vm9U64s363OkhjatMdbGVukZqK2WLNHIZItabyxUG2vZruLzvnYILa8tqd7ndUJLniZIWjJQtoyU1mrLSFm+bCvLnCqNbcaVvb/vPtfp64wVAADY//BECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9bBywRRJJEuK1qoK7xj7Gh/QZzHVdbyo1h/sGKH3L2wz1l6f1qD2Ttqmv/bWDQeq9ZrubvO1KzrU3rWFSrVel2hR60+1TzHWgi7Lj64lx+Sm9DCRHyoZKkv+KrTcmhMjI5W35Ihsc6SslC/bljMKLPdmm0Olsb22LTsGAACwp/FECgAAAAAiYiEFAAAAABGxta+HivtekIRT/FjtFd+ZZew78H/Wqtft/PA0tX7TigPU+pDXVxhrFcsOV3udhP4tLlmvHyMujnnL1EGlG9TWt3N1av2YUvOR8iIi67qGmG+ry1N7bRKWrX0FMddDy/a5wPKWKpe2sm5h6/2p7u+JcYR5EPP484DPdQAAwCDCv1wAAAAAIKJ+XUg99dRTctppp8nIkSPFcRx54IEHdtby+bx84xvfkKlTp0p5ebmMHDlSzjvvPNmwYdenIGPHjhXHcXb5dd111+3lrwQAAADA/qRfF1IdHR1y2GGHyYIFC95X6+zslMWLF8u///u/y+LFi+W+++6TZcuWyemnn/6+P3vttdfKxo0bd/668sor98btAwAAANhP9WtGavbs2TJ79uyitaqqKnn00Ud3+b2bb75Zjj76aFmzZo0ccMA/ckWZTEbq6vQ8zu7wjz9MnETx489/8tn/Y+z7wTcPUa/77rx6tV7y3HC1XtVlzkjVvFVQe71hQ9V6+TrLMeFlZcba2OQWtfeBliPU+pzyt9X6xi7z8emJrnh5nFRKf9987fh07Wh0EQks8a04x5/bjhiPffy50m47vjyw5Ldcp/cBLjfG0ekiIo7l+HS9N9ZLAwCAfdSgykht375dHMeR6urqXX7/uuuuk9raWpk+fbrccMMNUijo/0jOZrPS2tq6yy8AAAAA2F2D5tS+7u5u+cY3viHnnHOOVFb+40nFVVddJTNmzJCamhp59tlnZd68ebJx40a58cYbjdeaP3++XHPNNXvjtgEAAADsgwbFQiqfz8tnPvMZCcNQbrnlll1qc+fO3fm/p02bJqlUSi699FKZP3++pNPpotebN2/eLn2tra0yevTovrl5AAAAAPucAb+Q2rGIWr16tfz1r3/d5WlUMTNnzpRCoSDvvvuuTJ48ueifSafTRRdZHV9tE688V7TnqPR242u6hx+s3tM3P/SgWr/zyo+p9cQY8yKv/O1mtTc3Qc+OZdbr2yDdGvMsp9GJTrV3WdsI/bWH62GiLR0VxprXaQmuWIItpam8Ws/HGPZkmyPl9OEcKcfvu4yUjR9zjpRvyX8NVnHyWf0pHKT3DQDA3jKgF1I7FlHvvPOO/O1vf5Pa2lprz5IlS8R1XRk+XD/AAQAAAAB6q18XUu3t7bJ8+fKd/71q1SpZsmSJ1NTUSH19vXzqU5+SxYsXy0MPPSS+70tjY6OIiNTU1EgqlZKFCxfKokWL5MQTT5RMJiMLFy6Uq6++Wj7/+c/LkCHmJykAAAAAEEe/LqRefPFFOfHEE3f+947c0vnnny/f/va35cEH39sSd/jhh+/S97e//U1OOOEESafTcvfdd8u3v/1tyWazMm7cOLn66qt3yT8BAAAAwJ7WrwupE044QUJlXo9WExGZMWOGPPfcc3vsfv449XdSmSme0zjsmUvNjWeaszwiIhdVNar1e5e8q9ZbPjzRWKt8+FW1d9vx09T6sOe3qXW/zvxkr8bVf3xWb9OfCpY5KbW+va3UWEt3q60ijp63KUvqGamsNivK8nMZJPW6U+h9ligf6u95jFFN/6/ffG+2WU62GVe2OVRxxM1nAQAARLVvprsBAAAAoA+xkAIAAACAiFhIAQAAAEBEA/r4873t9x3DpdQp/paMvd6c75jy3y+o130+q+dx/K1Nar1xljkjVfGbDrW3pfgorZ1GPKRnpFpnjTXWSi0Zp9ZtZWo96ehzpPLt5uuXdamt4iT1H+2yZPF5YTvkLDkojSXGFCvHZJ0jFeO+RSTWHKm4tK/NjTnTKG4/AADA/8YTKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1cMPfnWWeOmSorWGxYuMfd8Z8Yx63WNfvFCtNwzdrNaPOOodY601nVZ7Kyc3q3V/W4tab28w55g8y6wmb6ueobJx28yvnei0zGpK6D/aFcmsWs/GiNQElr9Vnm0GlsK3zGpygphZIGWOlGfJGQWWWU5u3CFX/cTpz3wV47EAABiweCIFAAAAABGxkAIAAACAiNja18Oo/35VEoYjvbeef7Sxry14Ur1uyX3Var17Rrla/7eRtxlr3x35SbX31NFvqvWXsvpauqPBvK0pH/pqb7o53r6kZJv53pJd+jYxJ6VvK8wk9K19HbYzzBVhQt8K5vq9//wiL/qR8eLH24YWZxdbYNmH5on+PQuUbYt9vS3Q789z3/dVbEsEAOzjeCIFAAAAABGxkAIAAACAiFhIAQAAAEBEZKR6cIfViusWP078w1ctNPZdufoM9bq1f3xbra+6fLJaP0Y54bxz0jC19/Sq+9T6S86Rat0fZT6ruz3Qc0YlTXruJBvm1XqyzVxLdFsyLamkWs5oFxeR7tCSRVKEScvR7Hq0TMQxh0v6/vhzc8mz5Ihsx5/3JVs+i7gOAADY03giBQAAAAARsZACAAAAgIhYSAEAAABARGSkelh25QhxS0uK1h4e8Xtj39S7vqJed2TzIrVefcwmtd4amHNK2ybr85KmpfRAjltRodbH1jcZa1sseZzSJn32T2egZ6RSrebrJzrjzZGq8PR8V2egBNMsbHOknEKvLy1523yrmBGpOOOabBkp1zKkylc+1/HiDLjqY35ouTcCWgAA7JN4IgUAAAAAEbGQAgAAAICIWEgBAAAAQERkpHr47SkLpCJTfG153upTjX2jH2hUr5v74GFq/eoJ96r1u9rMc6a2T9EDN2WunhVyh9ao9SNrVxtrawuVam9Jk56Bagv1QE6qzZw98br17FdYon/dVYlOtd4R6v2qhCW/ZcshOebPN3zbvCQ/RshJRETJOdk+denPOVJ9yRnA+SwAANB/eCIFAAAAABGxkAIAAACAiFhIAQAAAEBEZKR6SDmBpAwxjxU3HmTsq3hHnxO18sqZav3jZVvU+tHPf9xYGztRn0G12e9Q6/m6arU+q+JxY+2dbJ3am2zWc0jNflKtp9rMeR+vU89fhWk945RxzbO5RETa/FJz0bHklJKWjJQe7xLHNV/fNkcqbpwn3hwp/XMZT/SL+2o+K94XZpthBQAAEBVPpAAAAAAgIhZSAAAAABARCykAAAAAiIiMVA+nP3a5uKUlRWuTfmvOQXkHT1Kve9VJf1brScdT685zVcbaJ85/Ru19tnuEWu9sKP717nBwypzBeqJ1itrrbtfzWZv9CrWeajPnoNxuS0aqVM9flblZtd4RpM1FZc6TiIhryUi5loyURssRiYhIEHeOlLnk2V7aMuOqL9nmaw1kQcz8FwAA6B88kQIAAACAiFhIAQAAAEBELKQAAAAAICIyUj1M/lGTJLzi2ZjcCTOMfWuP1XNGV1avVOuPdJWr9bpnzfOYTvnKG2rv/I2nqvW2UXo+a5Rnzhotbxum9oat7Wq9sWDOfomIeO05Y82xZKQKVcocKBGp9PQ5Uu/mhppfW5nzJCLiefHmSGkZLOscKT9e3ibOuKXAkt/yLEOqAj7XAQAAgwj/cgEAAACAiFhIAQAAAEBEbO3rIWzcLKGTKlpr/7F5C9ysmrfV676V14/avvHdM9R6eql5a+CBSeWYbhF5duUEtS4N+l6uMrf4+yEisqalWu0d2b5CrW/MD1Hrbpt5+52TNW/7ExEplOlbFssc/XuyvVBmLlqOP08m9b171q19ytZB3/bZRxjzKG2l3fapi21rXxxunD2Hu9GvbTp0Bu/J6vH2agIAABVPpAAAAAAgIhZSAAAAABARCykAAAAAiIiMVA8bLpomXrr4UeYvHPYjY19noB/FPee1L6j1ppeHq/VxrWuMNVf0AEfqLf0Y8PxB5qPVRUTyoTnQ09akH9se5vUc09ruGrXudCpHlBcKaq9fqn9GUO7qGal2X8meWY4/T3q2jJSeW3GUUE4+1LNfEvf4c/2EcpUtI+WqSST70e6DVayMlaXXD2N8wwAAQCw8kQIAAACAiFhIAQAAAEBELKQAAAAAIKJ9M5TQS18+/0EprSj+ltzTVm/sO63cnGESEcnfp2eg6jfqmZpEw0hj7d3C39XeIcv0DEXVhzer9U1+l/m+mpJqr82Grkq1HnaZX1sCPQuUt2WkHD3X1loonpUTEXE8PbiSTur5rYJ1jpT53m0ZKacP50hZ0lkS9ukcKf3nuC9nWAEAABTDEykAAAAAiIiFFAAAAABExEIKAAAAACIiI9XDOZmNUpkpvrac/uMrjX1PfnKpet0Rf1gZ677aZo4x1h7vnKT2Vi7brtZn1ryr1pflq4y1dJMll+LqqZpNnRm1Xtm9Sb++olBqyTE5elCpo6DMkbLkq1KWOVK+LSOlDB4KQstnH0G8uUJO0PusUWAZeuQ5en6r2/a1xeBYXhsAACAqnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakeTnvjE5IoL56NGX3zK8a+hYnD1OuO3rRQf2HL7J/GWeONtQcaD1d7ndUb1PrM8hVq/cVO82uXNOn37ZaaZzGJiGxrL1PrmWzWXPT0/JUtI1Xu6lmi9rw5I+V4em86oc+R6rZkpBzla7PNkbLN17KJEyXqy1lOtnwVAADA3sYTKQAAAACIiIUUAAAAAETEQgoAAAAAIiIj1UPyR0MkkSie63GHm3MxY+/fql63cMw0tZ548121Pv7oNcbam8sb1N5JLS+o9cnJJrV+6/oTjLXSZj0r5FSUq/WuNmVWk4iEysAlWxqnUKrXk5b+jkLK3Ovl1N60p2ekXN+S93HNX509I6V/T/zQMmdKKXvKfCsRe0bK0y6+G/39hRlUAACgGJ5IAQAAAEBELKQAAAAAICIWUgAAAAAQERmpHhJPLJGEUzw988ZPjzb2TfrK8+p1V94wS62Pe8g8q0lE5F8O+JWx9k9/vFjtdRL6t3hkQs8pvdk4wlg7oEnPCkllhVp22iw/fsp8LS0/JSLiWzJSJY7+GUJ7zpyRGuLpvWUJ/X1xLHOkRLk3P7R89mGZSWYTJw4U9mPGybek5lxyTgAAYA/jiRQAAAAARMRCCgAAAAAiYmtfD52nHymJZPHjz/8454fGvn+a+Hn1ul/46JNq/bdNJ6j1E0u7jbWat/Sjtr0Rw9V62rCVcYfC+jJjLdHUovb6Q/Tjz5NtMdbxli1shVK9nrRs7evOKe+Lpx9BXuKZv18iIq5ta5+yddB2/LkTxNzCppxQbvtu2Y4vdx3L0ezKK7gyiLfmDdZthQPzNHoAAAYMnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGake6q5YKcny4sdep5R8x9oz6tTr/n7ob9X6fcccptazYd5YK3+7Se3NjTcfX267tohI+Xolt7K9Xe3tGl2v1pNtlhCGq+SBAtvx57aMlJ410jJStiPlSz39PXV8S2ZGyW/ZMlIS6DkkmzhxHltGyt7ff5/r+DGPjQcAAPsfnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakefjH2SanMFF9bjntwrrHviNPfVq+7ptCl1i+frM+Z+n1Hg7EWrtuo9m77kD5HakMhq9Yr1pszN2Fbm9rbXTtKrSf1dnGS5h/PMKtnpIJSPSvkWj5DyOeVLJKr99oyUq4lI+Uoc6QKgeWzD0vWJ7DNY4oRFbK1ejEubp9BZZth1fvXdgbrHCgAANCneCIFAAAAABGxkAIAAACAiFhIAQAAAEBEZKR6+O7WKZLuLj4/6KDvbzH2/eSvD6jXPe+dc9T67yb/Rq1/5JXzjLUhne+ovS2T9XzHG/mhar18XbexFnTo2a+uWn2dXtKs35ubThtrfi6n9jqlBf3alkyNn1UyUgl9llPa1V/b0csijvne7HOk4uV5tDiQZ3nP4s6RsuWc9kvkswAAGLB4IgUAAAAAEbGQAgAAAICIWEgBAAAAQERkpHr4823HipcqKVobtuEVY1/G1d/GrfePVusV88xZIBGR9oXDjLXakrVqb9XkZrX+fMcEtZ5s3G6s+Xk9p5St1fMdlWv02UBSWvx7ISLidHaqralSfZaT51g+Q8iZ66Ey50lEpMy15LdsOSbPnIMKwnhzpGws45osL61nnLwYF48zB6qvBRLjTQMAAIMWT6QAAAAAICIWUgAAAAAQEQspAAAAAIiIjFQPtb9aLAmn+BypdV872tj3zU16Jmbk79eo9Wfm6uvZ+oVZY80dWaf2zh79plpf1DRWrbtb9YyVJlerZ0dSrZZ5S2Wl5mKLPk+prET/ntg4SkZKyzCJiJR55u+XiIjj2zJS5tfOx8xIWfM8StlV5luJxJ8jZc1/xeDKwM1Y9SlGcwEA0Gf69YnUU089JaeddpqMHDlSHMeRBx54YJf6BRdcII7j7PLr1FNP3eXPNDc3y+c+9zmprKyU6upqueiii6S9vX0vfhUAAAAA9jf9upDq6OiQww47TBYsWGD8M6eeeqps3Lhx569f//rXu9Q/97nPyeuvvy6PPvqoPPTQQ/LUU0/JJZdc0te3DgAAAGA/1q9b+2bPni2zZ89W/0w6nZa6uuLb195880155JFH5IUXXpAjjzxSRER+8pOfyJw5c+T73/++jBw5MtL9OAcfKI5X/Cjyr110n7Hvht9+Ur3umLUL1fpN6z6i1tNLVhlrnUfpx5d/vPJetX73G0eo9Qmtjeaiq29xS9R06/U2fZtZUG7e2uekUmpvZYm+vc7GzSp7ohL6113i6FsW3YJlm5lr/nyjEOivLb6v1y3inDJuO/7c7cNjwvtyW2Dc3XHsrusF3jQAwCAw4A+beOKJJ2T48OEyefJk+fKXvyxNTU07awsXLpTq6uqdiygRkZNPPllc15VFixYZr5nNZqW1tXWXXwAAAACwuwb0QurUU0+VX/7yl/L444/L9773PXnyySdl9uzZ4v+/T90bGxtl+PDhu/QkEgmpqamRxkbzk5T58+dLVVXVzl+jR+sDcwEAAACgpwF9at9nP/vZnf976tSpMm3aNJkwYYI88cQTctJJJ/X6uvPmzZO5c+fu/O/W1lYWUwAAAAB224BeSP1v48ePl6FDh8ry5cvlpJNOkrq6Otm8efMuf6ZQKEhzc7MxVyXyXu4qnX5/FmrlV5PilhXP3pxXud54vd/dt1297/CIQ9T60uczan1C03PGWvPkKWrvtJSemXHWKUeMi4gE5n63vFxtravRt0x67cWPmt+hUGW+t0RK761Od6l1GzdnDmmEScvx567l+PNADyKFMY4/Dy3Hn9s4MWJMli/LyleCMV4f5qsAAAB6Y0Bv7fvf1q1bJ01NTVJfXy8iIrNmzZKWlhZ56aWXdv6Zv/71rxIEgcycObO/bhMAAADAPq5fn0i1t7fL8uXLd/73qlWrZMmSJVJTUyM1NTVyzTXXyFlnnSV1dXWyYsUK+frXvy4HHnignHLKKSIictBBB8mpp54qF198sdx6662Sz+fliiuukM9+9rORT+wDAAAAgN3Vr0+kXnzxRZk+fbpMnz5dRETmzp0r06dPl29961vieZ4sXbpUTj/9dJk0aZJcdNFFcsQRR8jTTz+9y7a8O++8U6ZMmSInnXSSzJkzR4477ji57bbb+utLAgAAALAf6NcnUieccIKa6fjzn/9svUZNTY3cdddde+R+HvvA/0gmU3xteeKrXzD2lS95Q73uu/81S63XPafnP7xhw4y11in6zKIyV5+3VLZeH9jiJM39bqZC7Z1QtVWtb+4cotYLoyqNtWRSz0jVpjvUuo2nZaQS+ucPJW5erTt9OUfKElTybRkqpexZhvvY5kh5cQJYMTmWAVnxpm8BAID90aDKSAEAAADAQMBCCgAAAAAiYiEFAAAAABENqjlSfe2tfLmU54uvLdM3mPM8iTH6MN+Pn7pIf93bGtR65xFjjbXxExvV3s2+nhXKrNfTIW51lbEWVunzr6aUr1TrmzpK1HouY84DlZa8fw5YTzWpzWrdD/W8jjYKypqRciwZKdvAJWWOVMEyR0osX5dVjFlQtoyUTaB8ba414xTvtQEAAKLiiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkerhyvsvErekeG5n3OMLjX3rr/qAet3fjrhXrX9y9dFqvfFCcwbrK/WvqL1Pd9Wr9bL13WpdhpmzYYWqUrX1wPQmtf5Ed51az1WY1/lhSp8jNTTZrtYLlslB2iiowDpHKqfWHd+SY1LnSNkyUvrXFYj+2nFGPQW2OVKWAJZvy3/1E1s+CwAA7J8G5r9cAAAAAGAAYyEFAAAAABGxkAIAAACAiMhI9TB+wSpJuKmitbYzZxr76s5YrV63OdAzM255uVqvPcY8K+rk8jfV3v/aMEetJzduU+tdBw4z1oK0vg4fm9yq1sOc/r7kKs2Zm7C0+PdphyEJfX5W3pIl8vpyjpSvZ25Cz/x1F0LzbC0REQkKet0iThyoL5NE+2tOyWE8FgAAAxZPpAAAAAAgIhZSAAAAABARCykAAAAAiIiMVE9BKKakR8M/vWNsmz/qQfWyV6w6S3/ZqSPU+tXjHzDWDkym1d6FK8ep9UlNy9V6+wkNxppt5tBIT89A2TJS+QpzLSjV50jVevocqXyo37yXM2dygoQeXLFlpCToyzlSMQZBiYg2Zsq1fO4SWuZIuZYZVoH0XSCoPzNWzn6a7+rDbycAAAMCT6QAAAAAICIWUgAAAAAQEVv7elj1lfHilpQUrS0bd4uxLxvq2+tWPjhBrTvH6Pc1u8x8jLhr+Ram3ypV60G7vgWuvcG8PyfZqbZKjae/LxLqW57ylea6X6J/3dWe5fhzy2Hd2vHnQVL//CHp6EerOwV9i1uYNB9xHli29nmB/nX5lq/bsXxPNLatfX3J5zMh7CH9+GMMABhk+NcHAAAAAETEQgoAAAAAImIhBQAAAAARkZHq4Qdn/VzKM8XzKf+x5ZBeX3f0/RvU+vL5lWrdc8yb9t8t6EGlIcv0vI4tp9Q1qmCs+U3mLI+ISNrRjyi3KWTMWaJCqf7a1W6XWu+2fN3a8eeh9fhzy3vu668dps3XL4T6Zx9ujIyTiP1Ie03MlxZf+do8y9HpAAAAextPpAAAAAAgIhZSAAAAABARCykAAAAAiIiMVA+Hp9olkyq+tvzGzR809m0/UL/u+JUL1fqXDmlT63/urDLWNuSHqL2Vy7ar9TCtz3qqHtlqrLWInu2yUrJfIiKSyRtLhTI9f5Vxzb0iIt2WYTFaRipI6r1JS57HCSxzpDzz5xu+ZY6UTWALMillLasnYp8j5cUJYMXkOvrXrd2ZY+kFAAD7J55IAQAAAEBELKQAAAAAICIWUgAAAAAQERmpHk5a9EVxy0qK1sb9/AVjX+2Mg9TruodOUevnV92u1s9+61xjLe2Z5zyJiLhrNur1IdVq/fAR6421Z7tTam821HNKTkLPOZVlssZaoUR/7TJLrqU50H/0vay5v1Cqf/6QsmWBfFtGSpkjZctIWfJXNvHmSFkybxaBmPutGSfLfC0AAIA9jX99AAAAAEBELKQAAAAAICIWUgAAAAAQERmpHsb8KJCE5xetOYdOMvaFL7ymXnf1N2ep9eFeuVrfsGiksZYbrmekJrWYs10iIu60erU+s9L8ta0aUqv2bg9yat1J6RmpmvJOY62zVJ9hVe7onxGsC/TXdvPKHKmM2iqWMVP2HJOSkfJtOSTLnChfGxQlos6RsrGNqPIsF/fJOQEAgEGEf7kAAAAAQEQspAAAAAAgIhZSAAAAABARGakewlfeltApnp1Zeeehxr6Jc0eo1z3246+o9ZeyepaofqE5B9U8Rc/6OAn9W9zVoAd+ZpS+a6y9VDVW7W30PbXulBaf2bVDXXmrsfZ2qZ7tSjv6190d6u+bly2elRMRCSzvqX5lEccyRyrwzJ9v+PvwHKm+ZJtDFYc1d9aHr20V51sycL+dAAAMCDyRAgAAAICIWEgBAAAAQEQspAAAAAAgIjJSPTR//gjxUsVzO88e931j3ymn/7N63bsablTrV649Ra2Xv7RGqR6g9np1en6rvUH/ERifMOe3JpU3qr3v5mvUulNaqtZHlpqv/0aZ2ipJR89ntQZ6PsvNmcNCviUElXQs4RJLjkkbp1Tw4332YYtAaXEe1/K5i22OlGsJYAVKxirpmjNrIiJ+HwZ6nP7MOKF3+J4BAPYCnkgBAAAAQEQspAAAAAAgIrb29XDypQslXVF839ZrOfMx4aWf3KReNx/qW5r+/tJBan1i4yJjreztcrU3N2G4Wu9o0LdEDXHN2+8mpvWve3m2Tq2HFfrWvob0NmPN13fmiWvZ6tUZpPX+nHkrWWj5W5N0LJ9PWI4/DxO9P/48tOyv82377+LsiBrAx58PWmxR27/wVwgABhWeSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR7+fdgbUpkpvrac9IsvG/t+f84P1OvOXTdbrQ9fqG+M96qrjLVwvX4EefOJ+vHn3Q15/bWVvM/YRJPa+6dtU9V6UKEHnUYmW4y1QqmeHdHuW0SkNdDzWU6uYKwFScv3yxZ0sOSUQuXkdu2I8P/3B/S6heWEclVouTfPcvi6r3yukxbz92N3uLHCX30rsB5KDwAABiKeSAEAAABARCykAAAAACAiFlIAAAAAEBEZqR4uWn28JMtTRWsH3rjc2DfpPD3r89KDh6r1MYs2qvXc4ROMNe+JxWrv9sl6NmRYQ4veH3QZayMTerZjeeswte4YZnbt0JBQ5kiVxcuVtFsGUTl58xypwPK3xrV9PuGbry0iErrmrFEQxBs0o7+yiGObM6WIm0Ky5r/6kD9wI1QAAGCA4okUAAAAAETEQgoAAAAAImIhBQAAAAARkZHqYe2CAyWRLJ6dqZSVxr4bt01UrzvmgS1q3V+5Rq1v/HS9sXbAc3rWZ8jkZrU+fdg6tb4qb15rH5TSf3zWN5vnX4mIDK/U+2u8TmMtLI2XkWqzZKQkZ56vFejRLvEcPesTBvq9BwktI2X57CO0XFvvjjVHSvo149R3nwkpkTUAALAf44kUAAAAAETEQgoAAAAAImIhBQAAAAARkZHqofyBFyXhFA/ALP/uLGPfssfMc55ERA588zm17qTTaj11jDnn5Iwy56dEROaMfl2tj0vr+a0l2dHG2uHpzWpvtrlUrecq9HV8jVsw1txSc01EJB/qE5Paff09d/Lm68eeIxXoQ4tCT2mNOUfKSrk1VyzZL8ssJs8yaUqbI+XFCm+JuDH747BE5gAAwCDFEykAAAAAiIiFFAAAAABExEIKAAAAACIiI9WD/8HDxEkUny9029n/bez7rwvOV6/rTdIzVGGZnte5YtJfjbX/O/E0tfdjlb9R6ynLZKFbtpxgrF1QqWekUk1K2EdEchm1LBnX3F9SmlN7bRmp1oJljlTB3B9aM1KWUEyg31vgmftDS0YqtOSvfEuOybHU9ReP0QsAADDI8EQKAAAAACJiIQUAAAAAEbG1r4eur22XRHl30dphqXZjn/vMUvW6a/5lplpPtep7oj6VWWWs/Xhy8ePadzg0qV87b9nat3iz+fjzbMNTam+6Sd+Glq9Uy1LmpIy18hLL1j7Rt8+15fWtfWEub6zZjj/3nL48/jzeZx+2Q8DjnBJu23ZoO4LcVz7XcW1Hp/OZEAAA2Mv41wcAAAAARMRCCgAAAAAiYiEFAAAAABGRkerhj4c8IJWZ4mvLQ56+zNh3YO1G9bqHnv6WWn9+xVi1XuGYj0dvnVJQe8tcc85od2zdUGWsNU/Lqr0lTXquZfuB+msnHXNYqKqkeJZth07LEePtBf3IeSmYv7bAkjuz8uMcf265tvUP2PpjfG2h5dh3iyBmv8aNda47AADA+/FECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9fCb9hFSanhLxn/PnGvZ+jE97HPHAd9X6z8uP0atP91t/jYdOFHPZ20smOdfiYgM98rUesl685yqTb4+w6q0Sc8CNU3v/Tp+SLpTrXdaIjEdeT07lvbN1w9j/q0JLTkkbY6UbVaTjS96f6w5UoM4hqR92Q75KgAAUARPpAAAAAAgIhZSAAAAABARCykAAAAAiIiMVA8/+b9niJcuKVob+coiY1/hGj0jVabMQxIRuaRmoVq/YtWnjLUz6peovU93N6j1D5duUOvl68z5kHcLtWpvulmfMxVk9IyVZkhKz0i1Bfq1O/N6PVUwz+cKEjEzM4EeRAr6MiNlmdUUKw5kubYn+sWD0Py5jmcJb9myX4PWPvplAQCwL+CJFAAAAABExEIKAAAAACJiIQUAAAAAEZGR6qHhf16VhFN8vtCWLx5t7PufqT9Sr/uvGz+k1m9uMOevRETeeH6csfb9z9yr9n573cfV+tiRW9V6Zr05K/ROdoTam2jqUOvpTKlaz4Z5Y602qV+7NUyr9c6sPkeqyjfPwAqTMTNSMeZIiS0jFXeYk9LuOZbPXQbwuCW3H2dB9escKmZgAQDQZ3giBQAAAAAR9etC6qmnnpLTTjtNRo4cKY7jyAMPPLBL3XGcor9uuOGGnX9m7Nix76tfd911e/krAQAAALA/6deFVEdHhxx22GGyYMGCovWNGzfu8utnP/uZOI4jZ5111i5/7tprr93lz1155ZV74/YBAAAA7Kf6NSM1e/ZsmT17trFeV1e3y3///ve/lxNPPFHGjx+/y+9nMpn3/VlNNpuVbPYfM45aW1tFRMQdPlRct3i25tQr/m683hFpPW/z+YePUOtvXvA3tV6/0DxDZ8K5es5o0cqxan3xEL1esqHNWHurvV7tdVrb1Xp1hVqWzsCckRqa1K/d4pep9Vze8qPflxkpyxwpNSPlxxssFFgGEzlBjK/N0mqdI6Xcm2uZI9Wfgri5NAAAMCgNmozUpk2b5OGHH5aLLrrofbXrrrtOamtrZfr06XLDDTdIQRmmKiIyf/58qaqq2vlr9OjRfXXbAAAAAPZBg+bUvl/84heSyWTkzDPP3OX3r7rqKpkxY4bU1NTIs88+K/PmzZONGzfKjTfeaLzWvHnzZO7cuTv/u7W1lcUUAAAAgN02aBZSP/vZz+Rzn/uclJSU7PL7PRdE06ZNk1QqJZdeeqnMnz9f0uni2/TS6bSxBgAAAAA2g2Ih9fTTT8uyZcvknnvusf7ZmTNnSqFQkHfffVcmT54c6XXe+uowcUtLitYeHv6Ase+27SPV6469f5tav+Xj+pypyhfWq3VN+i09Q/XsgRPUurvZfO8rWoeqvaXtTWp9eLlalrbQnIupSegZqWZfD2Dlc/qPfqhlhbx4mZjQOkdKyTHZ5khZ+NaMVIyLhzHvLUa/H/bdLuX+nEEFAAAGrkGRkbr99tvliCOOkMMOO8z6Z5csWSKu68rw4cP3wp0BAAAA2B/16xOp9vZ2Wb58+c7/XrVqlSxZskRqamrkgAMOEJH38ku//e1v5Qc/+MH7+hcuXCiLFi2SE088UTKZjCxcuFCuvvpq+fznPy9DhgzZa18HAAAAgP1Lvy6kXnzxRTnxxBN3/veOvNP5558vd9xxh4iI3H333RKGoZxzzjnv60+n03L33XfLt7/9bclmszJu3Di5+uqrd8lNRXH/RxZIRab4Q7pzV33c2Ldw6UT1upNeeV6t/+Hlo/T+tS8YaysKXWpvzTLzMd4iIq8c1aDW67etNNY2bhuh9o7rWqdfu1Q/XbHZTxprNZ6+tW95Vj8O389ZHsYq2wqdpL7/zVd6RUTEcsR4oB1/Hvfkddv2uTjXZwccAADYj/TrQuqEE06w5kUuueQSueSSS4rWZsyYIc8991xf3BoAAAAAGA2KjBQAAAAADCQspAAAAAAgokFx/PlAsObGScbauO161icxboxaH7pQ/za45eZzwv/aYb4vEZHM29vV+vqNlWq9Lps11nLNxY+K3yEs6O/L6BI957RZOcK82utUe7fm9ePPJasFkURE2XLqJPQMVGALC1kyVKFya05fH39u2WqrsrTajhEPlCPMvZgBLNtr294XAACA/40nUgAAAAAQEQspAAAAAIiIhRQAAAAARERGqodPPnq5uKXFcz+T7l1k7HOSKfW66y4/Uq03/GWrWi9MO9BYu3+jPnjYW7NRrZetrVHr4pizI6mtlpyRRX2yRa03FqqMtWnp9WpvU96cKxMRcXK9z8R4CX02l01omSOlZaQk3kvbs0CWEVgq24wqC+uMK62Xz4QAAMBexr8+AAAAACAiFlIAAAAAEBELKQAAAACIiIxUD1N+tEUSbrporfvDRxj70i+vVK87+oxVar1w8xa13njV0cZa23I9nzWpZZ1aL1+v53XcCvM8ppImy0wiS3asIblNrS/uHGusHVv6rtq7LVem1t0YGamEZY5UPoyZoerDOVI2lnFLscSZBeU6ccJb8Tgx3xQlZggAAAYxnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakegs1bJXCK53q6f2pec3b+crJ63XvGf1+tXyInqPX8rDZjLbMwo/Y6Cf1bXLE+p9bdoeY5UyVNlnyVYSbXDsM889clIrIhW22slVlyJ9uyfZeRSlrmSAW2YUyhXg+1jzdiRoUC9eIiEsbIA8XMVwW2GVcxuHFvrp/EzWcBAIC+wxMpAAAAAIiIhRQAAAAARMRCCgAAAAAiIiPVw/qLp4qXLp7rWTz1J8a+w848X73uCK/4bKod3Enj1fqlBz9trD3w84+ovV7dCLVeskHPKeXrqo210qaC2utU6vmtGlfPZ23oqjTWylxl2JKIbO/W81le1pLHUYb/pCwZKT9mHidQ/lbGHafkW3JIjt9/GSk/7H1Gyrdlv7DXheS7AAD7OP71AQAAAAARsZACAAAAgIhYSAEAAABARGSkerjyCw9IaUXxt+TOtnpj360zfqVe95rNR6j1LcfUqvXPV75urP3lnRlqb/ZAPSOVem21Wu88+EBjrXxdl9obVlWo9WpXX8dv6jRnrJKiZ6Q6uovPA9vBEs8Sccz3lk7o2bC8ZU6UbVZTqHxpThBv1pJv+ewkTqzFsWSc4sxy8gbpHCgAALDv4okUAAAAAETEQgoAAAAAImJrXw+fqdgklZnia8sZN11p7Hv16p+q1/3Sb45V6/lZ+laxoV65sRaub1R7t31Y39o37Bn9+PO2UeZ9ZpVLO9Te/Ajz8eUiImVuUq1vay8z1pKOvrWvu0vf2ldu2drnuOZtaratfTFPKJfQM29ji3v8eWA7JjzODrqYu++s9xaDa9mzGOfodQAAsH/iiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkephzutnSKI8XbQ2asESY9/dXxqiXnf8ve1qfdItb6n157N5Yy3o7FR7Wybr2ZCheT0s1NGg5HVa9a8re7B+rHvCcoR5V1vx74WIPSMVdOk/2tbjzz3z9dOenpHKWY43t9GOP48bwPJFzwI5ce69H08ot31dcdjyVQAAYP/EEykAAAAAiIiFFAAAAABExEIKAAAAACIiI9VD6U1VkkiUFK25deZwyry/H6xed9ILL6r1Lw99Vq3/14Y5xppb0q321k5uUuvi6lkjp6HLWAva9IxUZ61+bc/R1/FOW+9/PJ0u/dpeVs+9OEpGqsQzZ9ZERPIxIzXaOCVXf2kr3zKryQn6LiPlWbJGgTLLyY07QKsfOWSsAADYJ/FECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9eA99Yp4TrJo7Y1bjzb2jblPz0AkRjWo9YNSZWr92ZcmG2tTRm1Wez826nW1vqhymFqfWG++vt+l57OytfFm+yTber/O9zotGakYc6TKEnpQqduSQ7IJE8rsrmy89zSwfXYSI4rkBHHvrS9nQfVdxsrvzwFaNn33lgIAsN/jiRQAAAAARMRCCgAAAAAiYiEFAAAAABGRkeqh44wjJZEsPkfqz7NvNPZ99WsnqdfdeOF0tb6moM9jGvGsOejQOXGo2vvxynvU+qKhZ6v1mTXLjLVng5Ta212jZ0fyoa/Wk229D3gkuvVeL6dnZpyE+a9GqSVglY/5+USojN9yfMt74uh13xaaCftwjlSMLJFnCW8FMXNpGGDIdgEABgH+9QEAAAAAEbGQAgAAAICIWEgBAAAAQERkpHoYfflySZbruZ+iksVnT+1Q+cmNav225llqvWZRo7G27hMj1d5DU3rYIF9frdZnlq8w1p6Vg9TeQm1BrWdDfR5TstVcs+WrvC61LF7OktdRM1J6pi0fd46Up8yRijkOybfcmxNnJFLMcUpBODCDMQPzrgAAQH/jiRQAAAAARMRCCgAAAAAiYmtfD7ePeVoqM8XXluN+f7Wxr+Gj+uafn002H50uIjL7qSvU+oGrlhhrrZOHq71pR9922NGQVuuTk03GmpPUt0GW13aq9bZA3/qXajPvFbNt7UvYtvZl9X1oTsJ8Bnna1e+7O4z310rb2mc5BdwqsH12EuP481jbAqV/t/YFA3UD3wC9LQAAwBMpAAAAAIiMhRQAAAAARMRCCgAAAAAiIiPVw39uOVjSXcUzRQffsNnYt+w71ep1xyVK1HrVs3rdTZtzTJMmblB7Nxb0o7rbRulr6ZEJ82u75aVq76jqFrXeHJhzSCIiqXZzICgb6jmlRKce2HFzlrBRypwtK/Nyamt3qOfSrJRvifX4c0f/ftqOP491hLmlN86nNq4lgOVbru7FDXD1IT9GLg0AAPQfnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakeHvvvWeKliueVhm1aauy7+eg/qde9sXmKWh/+XKtaDw+aYKydXv+U2vt0d4Na72zQQzfaHConk1F7J1VuUutNgZ6xSrWac1AdoX7fiS49d+JZMlJh0vxXo8zt44xUwnxvcaM+vmUwkePHGFQVcw5Uf86R6kvOAM5nAQCA3uOJFAAAAABExEIKAAAAACJiIQUAAAAAEZGR6qHmVy9JwpAJWvfPM419p5Y9o1738gdPUusT33hZrW/60hHG2kfK31J7v7XuNLWebOhQ69kwb6wFVRVq7+SyV9R6Y6FKrSfazFmkTkueJtEdc46UkpFKu+b3RESkO0jp13YsWSDPfO/WOVIWgWWOVJw4T9wokO3e+otthhUAANg/Dcx/uQAAAADAAMZCCgAAAAAiYiEFAAAAABGRkerBOWySOF66aO1fvvgbY9/DncVnT+0w4d5utR7mzfOSRERaZ3WZr53QZzE9v2KsWp82Zr1a31DIGmuFWv21J6Ya1fobWX3Glddufu22QJ/VlOjUw0RuzlfrYdIz1vp6jpSjZaT02xbH1fNXvu2zkzBGHsjS6sUYE+WJ/v30LZk513ZzAAAAEfFECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9bD6q664ZcWzMWdnNhr7DnniEvW6ExYuVeve+APU+nlTF6l1Telben5r1vSVav2N/FBjrbtWn5c0OrFdrT/aeqhadzrN2bKWQM9nJbr1MJGT03NpYcr8V6PEsWSkLPktcfTPL9w+nCNlyxL1ZUbKJpAYIaqY/H587TgCsl8AAPQbnkgBAAAAQEQspAAAAAAgIhZSAAAAABARGakeHp/5M8lkiq8tT1j6BWNf/b16VsgbVqvWtx09Qq1fNOT/GmsrCnq2Y8gyPSt0ZJmekXqqfYqx1lWjr8PrzKOYRERkfVe1Wg87zPOzWvxytdfr0jNQtoyUX27+npa4ebW3raDnt2xcVwlC2TJSlvxVYPvsJEYGK27KKFDyW54lHGb9ugAAAPYw/vUBAAAAABGxkAIAAACAiNja18OSXIWU54rvRyu/vsrYl3z+NfW6W86eptabZujblkYlKoy1BS2j1d7MMv0I8slJvX5D01hjrXuovpmrwk2r9Y2dlWq9pLvZWGvyze+JiIjbpW+/k7y+tS9Imfclljj6tTsDfaun4+rvm+uZfx6sx59bru2H+mcnTj8ef96XXGcA3xwAABiUeCIFAAAAABGxkAIAAACAiFhIAQAAAEBEZKR6+Kd7LxS3pKRobezfFpobDT07ndmklo+rbVTrb+Y6jbUHNh6u9ibXbFDrIzz9qO7ljcOMNadGz50kHf3886aOMrU+Mmd+X7YV9OPPnW49x+RYMlJ+2vwZgzUj5evZMNsR5YlEjIyUhW/77KQPM1K2T22048/7ky1fZT2RfmB+WXbkygAAUPFECgAAAAAiYiEFAAAAABGxkAIAAACAiMhI9TDupysl4RafAdT6qZnGPq9bzxL8zyE/Ueu+6CGKBVtONNaWv1Ov9k5qXavWbTkmZ505Q1Wotcxqsuho07NlYS5nrG3N63OknKy5V0RECr5aDpJKRsrVv+5sYPlrZZn15LlaRkr/WXMsgRxrDilGRipupEa7N9eSRMqH++b/lcXOV1n6/TBm6A4AgP0YT6QAAAAAICIWUgAAAAAQ0b65Hyai8P9tZyoE5u1ghXy3uT+v72lqb9O3z9i29uXazVvJgi7zfYmIFEJ9G1qr5d6CbvP1g66Y1+7s/b1n29VWKQRZ/Q8E+vHn2ve7w/J1ZZXvl4hIwbINze9U7j1ne8/0LY1d7Zav2ze/dpyfFRGRNkt/ocP82rb7tvwVlFy7/r60l5nvTbsvEfvXpX4/RX9fbX9HrN8Ty/8/qK8do3e/fm3L3wP1tWP08tq8Nq/Na/Pae663tf29WmiJPDih7U/sB9atWyejR4/u79sAAAAAMECsXbtWRo0aZayzkBKRIAhkw4YNksn8/+3de1BU5R/H8c9yWxblIqiLVBAUE94DCQVNLTAta8ZbTpMmecksTC5T3hpE84La6Jhomv1RTaaljY2XsvKSpKaCkk6mYWkNpiLTRVERL7vn98cv9+fmpd8acYDer5mdYZ/nOTvf3fkwy5dz9tlAWSwWVVZW6o477tDRo0cVFBRkdnlowMgaagtZQ20ha6gtZA3/FMMwdObMGUVERMjL68afhOLSPkleXl7X7TaDgoL4xUStIGuoLWQNtYWsobaQNfwTgoOD/3INm00AAAAAgIdopAAAAADAQzRS12G1WpWXlyer1Wp2KWjgyBpqC1lDbSFrqC1kDWZjswkAAAAA8BBnpAAAAADAQzRSAAAAAOAhGikAAAAA8BCNFAAAAAB4iEbqTxYuXKg777xT/v7+6tixo4qKiswuCfVcfn6+7rvvPgUGBqp58+bq06ePSktL3dZUV1crIyNDYWFhaty4sfr376+TJ0+aVDEaipkzZ8pisSgrK8s1RtZQU44dO6bBgwcrLCxMNptNbdu21e7du13zhmFo0qRJatGihWw2m9LS0vT999+bWDHqI4fDodzcXEVHR8tms+muu+7S1KlTdfVeaWQNZqGRusoHH3ygnJwc5eXlqaSkRO3bt1fPnj1VUVFhdmmoxwoLC5WRkaGdO3dqw4YNunTpkh566CGdO3fOtSY7O1tr167VypUrVVhYqOPHj6tfv34mVo36rri4WG+88YbatWvnNk7WUBN+//13de7cWb6+vlq/fr0OHDigOXPmqEmTJq41s2fP1vz587V48WLt2rVLjRo1Us+ePVVdXW1i5ahvZs2apUWLFmnBggU6ePCgZs2apdmzZ6ugoMC1hqzBNAZckpKSjIyMDNd9h8NhREREGPn5+SZWhYamoqLCkGQUFhYahmEYp06dMnx9fY2VK1e61hw8eNCQZOzYscOsMlGPnTlzxoiNjTU2bNhgdOvWzcjMzDQMg6yh5owbN87o0qXLDeedTqcRHh5uvPrqq66xU6dOGVar1Vi+fHltlIgGonfv3sawYcPcxvr162cMGjTIMAyyBnNxRuoPFy9e1J49e5SWluYa8/LyUlpamnbs2GFiZWhoTp8+LUkKDQ2VJO3Zs0eXLl1yy15cXJwiIyPJHm5JRkaGevfu7ZYpiayh5qxZs0aJiYl6/PHH1bx5c8XHx+vNN990zf/4448qLy93y1pwcLA6duxI1uCRlJQUbdq0SYcOHZIk7du3T9u2bdPDDz8siazBXD5mF1BX/PLLL3I4HLLb7W7jdrtd3333nUlVoaFxOp3KyspS586d1aZNG0lSeXm5/Pz8FBIS4rbWbrervLzchCpRn73//vsqKSlRcXHxNXNkDTXlyJEjWrRokXJycjRx4kQVFxdrzJgx8vPzU3p6uitP13tPJWvwxPjx41VZWam4uDh5e3vL4XBo+vTpGjRokCSRNZiKRgqoRRkZGdq/f7+2bdtmdilogI4eParMzExt2LBB/v7+ZpeDBszpdCoxMVEzZsyQJMXHx2v//v1avHix0tPTTa4ODcmKFSv03nvvadmyZWrdurX27t2rrKwsRUREkDWYjkv7/tC0aVN5e3tfs3vVyZMnFR4eblJVaEhGjx6tdevW6YsvvtDtt9/uGg8PD9fFixd16tQpt/VkD57as2ePKioqlJCQIB8fH/n4+KiwsFDz58+Xj4+P7HY7WUONaNGihVq1auU21rJlS5WVlUmSK0+8p+LveumllzR+/Hg98cQTatu2rZ566illZ2crPz9fElmDuWik/uDn56cOHTpo06ZNrjGn06lNmzYpOTnZxMpQ3xmGodGjR+ujjz7S5s2bFR0d7TbfoUMH+fr6umWvtLRUZWVlZA8eSU1N1TfffKO9e/e6bomJiRo0aJDrZ7KGmtC5c+drvsbh0KFDioqKkiRFR0crPDzcLWuVlZXatWsXWYNHqqqq5OXl/ueqt7e3nE6nJLIGc3Fp31VycnKUnp6uxMREJSUlad68eTp37pyGDh1qdmmoxzIyMrRs2TKtXr1agYGBrmu2g4ODZbPZFBwcrOHDhysnJ0ehoaEKCgrSCy+8oOTkZHXq1Mnk6lGfBAYGuj57d0WjRo0UFhbmGidrqAnZ2dlKSUnRjBkzNHDgQBUVFWnJkiVasmSJJLm+v2zatGmKjY1VdHS0cnNzFRERoT59+phbPOqVxx57TNOnT1dkZKRat26tr7/+WnPnztWwYcMkkTWYzOxtA+uagoICIzIy0vDz8zOSkpKMnTt3ml0S6jlJ17299dZbrjXnz583nn/+eaNJkyZGQECA0bdvX+PEiRPmFY0G4+rtzw2DrKHmrF271mjTpo1htVqNuLg4Y8mSJW7zTqfTyM3NNex2u2G1Wo3U1FSjtLTUpGpRX1VWVhqZmZlGZGSk4e/vb8TExBgvv/yyceHCBdcasgazWAzjqq+GBgAAAAD8JT4jBQAAAAAeopECAAAAAA/RSAEAAACAh2ikAAAAAMBDNFIAAAAA4CEaKQAAAADwEI0UAAAAAHiIRgoAAAAAPEQjBQD4V+jevbuysrLq3WMDAOomH7MLAACgvlu1apV8fX3NLgMAUItopAAA+JtCQ0PNLgEAUMu4tA8AUOc4nU7l5+crOjpaNptN7du314cffihJ2rJliywWiz777DPFx8fLZrPpwQcfVEVFhdavX6+WLVsqKChITz75pKqqqtwe9/Llyxo9erSCg4PVtGlT5ebmyjCM/6um119/XbGxsfL395fdbteAAQNcc1df2nelvj/fnn76adf61atXKyEhQf7+/oqJidGUKVN0+fLlv/eiAQBqFWekAAB1Tn5+vpYuXarFixcrNjZWX375pQYPHqxmzZq51kyePFkLFixQQECABg4cqIEDB8pqtWrZsmU6e/as+vbtq4KCAo0bN851zDvvvKPhw4erqKhIu3fv1siRIxUZGalnnnnmpvXs3r1bY8aM0bvvvquUlBT99ttv2rp163XXpqSk6MSJE677Bw8e1COPPKKuXbtKkrZu3aohQ4Zo/vz5uv/++3X48GGNHDlSkpSXl3fLrxkAoHZZjP/3X3EAANSCCxcuKDQ0VBs3blRycrJrfMSIEaqqqtLIkSP1wAMPaOPGjUpNTZUkzZw5UxMmTNDhw4cVExMjSRo1apR++uknffrpp5L+e9aooqJC3377rSwWiyRp/PjxWrNmjQ4cOHDTmlatWqWhQ4fq559/VmBg4DXz3bt317333qt58+a5jf/6669KSkpSr169tHDhQklSWlqaUlNTNWHCBNe6pUuXauzYsTp+/LiHrxYAwCyckQIA1Ck//PCDqqqq1KNHD7fxixcvKj4+3nW/Xbt2rp/tdrsCAgJcTdSVsaKiIrfH6NSpk6uJkqTk5GTNmTNHDodD3t7eN6ypR48eioqKUkxMjHr16qVevXqpb9++CggIuOExly5dUv/+/RUVFaXXXnvNNb5v3z5t375d06dPd405HA5VV1erqqrqpo8JAKg7aKQAAHXK2bNnJUkff/yxbrvtNrc5q9Wqw4cPS5LbLnkWi+WaXfMsFoucTmeN1BQYGKiSkhJt2bJFn3/+uSZNmqTJkyeruLhYISEh1z3mueee09GjR1VUVCQfn/+93Z49e1ZTpkxRv379rjnG39+/RuoFAPzzaKQAAHVKq1atZLVaVVZWpm7dul0zf6WRuhW7du1yu79z507Fxsbe9GzUFT4+PkpLS1NaWpry8vIUEhKizZs3X7chmjt3rlasWKGvvvpKYWFhbnMJCQkqLS3V3XfffcvPAwBgPhopAECdEhgYqBdffFHZ2dlyOp3q0qWLTp8+re3btysoKEhRUVG3/NhlZWXKycnRs88+q5KSEhUUFGjOnDl/edy6det05MgRde3aVU2aNNEnn3wip9Ope+6555q1Gzdu1NixY7Vw4UI1bdpU5eXlkiSbzabg4GBNmjRJjz76qCIjIzVgwAB5eXlp37592r9/v6ZNm3bLzw0AULtopAAAdc7UqVPVrFkz5efn68iRIwoJCVFCQoImTpz4ty7XGzJkiM6fP6+kpCR5e3srMzPTtWPezYSEhGjVqlWaPHmyqqurFRsbq+XLl6t169bXrN22bZscDodGjRqlUaNGucbT09P19ttvq2fPnlq3bp1eeeUVzZo1S76+voqLi9OIESNu+XkBAGofu/YBAAAAgIf4Ql4AAAAA8BCNFADgX2/r1q1q3LjxDW8AAPwZl/YBAP71zp8/r2PHjt1wnh32AAB/RiMFAAAAAB7i0j4AAAAA8BCNFAAAAAB4iEYKAAAAADxEIwUAAAAAHqKRAgAAAAAP0UgBAAAAgIdopAAAAADAQ/8Bh9wLd2HKtcoAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ] - }, - { - "cell_type": "code", - "source": [ - "fig, ax = plt.subplots(figsize=(10, 10))\n", - "plt.imshow(tmp_layer.pe[0, :, 50:], aspect=\"auto\")\n", - "plt.xlabel(\"emb_size\")\n", - "plt.ylabel(\"max_seq_len\")\n", - "plt.show()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 853 - }, - "id": "ovX65ZQUcVSJ", - "outputId": "cae6d154-c719-4272-d1de-19de4b4f43d9" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAANECAYAAAC6qR6BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4kUlEQVR4nO3de7hXdZ33/9cXkQ3aBtwopwRPmdYoKJpEoWFgAnNbJqWpJRqKzgWa7Ltb457yNM1AOpajot5d02jdpZaNWWOJoSaoIQpIHu6GFCl05NBIgmBy2vv3h5f7146DfLabfYDH47q+18V3rfVd6725Vuqz9V1rV+rr6+sDAADAduvQ2gMAAAC0N0IKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoNBOE1LTpk3L/vvvn86dO2fw4MF54oknWnskAABgJ1Wpr6+vb+0h3q0f/vCHOeuss3LLLbdk8ODBue6663LXXXdl4cKF6dmz5zt+vq6uLq+88kqqq6tTqVRaYGIAAKAtqq+vz+uvv56+ffumQ4etX3faKUJq8ODB+dCHPpQbb7wxyVth1K9fv1x44YX5yle+8o6ff/nll9OvX78dPSYAANBOvPTSS9l33323ur5jC86yQ6xfvz7z5s3L5MmTG5Z16NAhI0aMyOzZs7f4mXXr1mXdunUN799uyaEZnY7ZvXiGn/zumeLP/KVPv//wJn/WsR3bsR3bsR3bsR3bsR3bsZvv2KvX1GW/Qb9PdXX1Nrdr9yH13//939m0aVN69erVaHmvXr3yn//5n1v8zJQpU3LllVdutrxjdk/HSnlIda1+d7eaNeWYju3Yju3Yju3Yju3Yju3Yjr1jjp3kHW/52WkeNlFi8uTJWbVqVcPrpZdeau2RAACAdqTdX5Hae++9s9tuu2X58uWNli9fvjy9e/fe4meqqqpSVVXVEuMBAAA7oXZ/RapTp0456qij8uCDDzYsq6ury4MPPpghQ4a04mQAAMDOqt1fkUqS2trajB07NkcffXSOOeaYXHfddVm7dm3OOeec1h4NAADYCe0UIXXaaaflj3/8Yy677LIsW7YsRxxxRKZPn77ZAygAAACaw04RUkkyceLETJw4sbXHAAAAdgHt/h4pAACAliakAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKNTmQ2rKlCn50Ic+lOrq6vTs2TMnn3xyFi5c2GibYcOGpVKpNHpdcMEFrTQxAACws2vzITVz5sxMmDAhjz/+eGbMmJENGzbkE5/4RNauXdtou/POOy9Lly5teF199dWtNDEAALCz69jaA7yT6dOnN3p/2223pWfPnpk3b16OO+64huV77LFHevfu3dLjAQAAu6A2f0Xqr61atSpJUlNT02j5D37wg+y999457LDDMnny5Lzxxhtb3ce6deuyevXqRi8AAIDt1eavSP2lurq6XHzxxfnoRz+aww47rGH5GWeckf322y99+/bN008/nUsvvTQLFy7M3XffvcX9TJkyJVdeeWVLjQ0AAOxk2lVITZgwIc8++2weffTRRsvHjx/f8OfDDz88ffr0yfDhw7No0aIcdNBBm+1n8uTJqa2tbXi/evXq9OvXb8cNDgAA7FTaTUhNnDgx9957b2bNmpV99913m9sOHjw4SfLCCy9sMaSqqqpSVVW1Q+YEAAB2fm0+pOrr63PhhRfmJz/5SR5++OEccMAB7/iZBQsWJEn69Omzg6cDAAB2RW0+pCZMmJDbb789P/3pT1NdXZ1ly5YlSbp165YuXbpk0aJFuf322zN69Oj06NEjTz/9dCZNmpTjjjsuAwYMaOXpAQCAnVGbD6mbb745yVu/dPcv3XrrrTn77LPTqVOnPPDAA7nuuuuydu3a9OvXL2PGjMlXv/rVVpgWAADYFbT5kKqvr9/m+n79+mXmzJktNA0AAEA7/D1SAAAArU1IAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUKhjaw/QplQqb70AAAC2wRUpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACrX5kLriiitSqVQavQ499NCG9W+++WYmTJiQHj165D3veU/GjBmT5cuXt+LEAADAzq7Nh1SS/M3f/E2WLl3a8Hr00Ucb1k2aNCn/8R//kbvuuiszZ87MK6+8klNOOaUVpwUAAHZ2HVt7gO3RsWPH9O7de7Plq1atyne+853cfvvt+fjHP54kufXWW/OBD3wgjz/+eD784Q9vcX/r1q3LunXrGt6vXr16xwwOAADslNrFFannn38+ffv2zYEHHpgzzzwzS5YsSZLMmzcvGzZsyIgRIxq2PfTQQ9O/f//Mnj17q/ubMmVKunXr1vDq16/fDv8ZAACAnUebD6nBgwfntttuy/Tp03PzzTdn8eLFOfbYY/P6669n2bJl6dSpU7p3797oM7169cqyZcu2us/Jkydn1apVDa+XXnppB/8UAADAzqTNf7Vv1KhRDX8eMGBABg8enP322y8/+tGP0qVLlybts6qqKlVVVc01IgAAsItp81ek/lr37t3z/ve/Py+88EJ69+6d9evX57XXXmu0zfLly7d4TxUAAEBzaHchtWbNmixatCh9+vTJUUcdld133z0PPvhgw/qFCxdmyZIlGTJkSCtOCQAA7Mza/Ff7vvzlL+ekk07Kfvvtl1deeSWXX355dtttt5x++unp1q1bxo0bl9ra2tTU1KRr16658MILM2TIkK0+sQ8AAODdavMh9fLLL+f000/Pq6++mn322SdDhw7N448/nn322SdJ8q1vfSsdOnTImDFjsm7dupx44om56aabWnlqAABgZ9bmQ+rOO+/c5vrOnTtn2rRpmTZtWgtNBAAA7Ora3T1SAAAArU1IAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQqGNrD0CSSqW1JwAAAAq4IgUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhdp8SO2///6pVCqbvSZMmJAkGTZs2GbrLrjgglaeGgAA2Jl1bO0B3smTTz6ZTZs2Nbx/9tlnc8IJJ+Szn/1sw7LzzjsvV111VcP7PfbYo0VnBAAAdi1tPqT22WefRu+nTp2agw46KB/72Mcalu2xxx7p3bv3du9z3bp1WbduXcP71atXv/tBAQCAXUab/2rfX1q/fn2+//3v54tf/GIqlUrD8h/84AfZe++9c9hhh2Xy5Ml54403trmfKVOmpFu3bg2vfv367ejRAQCAnUibvyL1l+6555689tprOfvssxuWnXHGGdlvv/3St2/fPP3007n00kuzcOHC3H333Vvdz+TJk1NbW9vwfvXq1WIKAADYbu0qpL7zne9k1KhR6du3b8Oy8ePHN/z58MMPT58+fTJ8+PAsWrQoBx100Bb3U1VVlaqqqh0+LwAAsHNqN1/t+8Mf/pAHHngg55577ja3Gzx4cJLkhRdeaImxAACAXVC7Calbb701PXv2zN/+7d9uc7sFCxYkSfr06dMCUwEAALuidvHVvrq6utx6660ZO3ZsOnb8/0detGhRbr/99owePTo9evTI008/nUmTJuW4447LgAEDWnFiAABgZ9YuQuqBBx7IkiVL8sUvfrHR8k6dOuWBBx7Iddddl7Vr16Zfv34ZM2ZMvvrVr7bSpAAAwK6gXYTUJz7xidTX12+2vF+/fpk5c2YrTAQAAOzK2s09UgAAAG2FkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKNQuHn/eYiod3noBAABsg2oAAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQh1bewBaWaXS2hMAAEC744oUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFOjblQ2vXrs3UqVPz4IMPZsWKFamrq2u0/sUXX2yW4QAAANqiJoXUueeem5kzZ+YLX/hC+vTpk0ql0txzAQAAtFlNCqn77rsvP//5z/PRj360uecBAABo85p0j9Ree+2Vmpqa5p4FAACgXWhSSP3DP/xDLrvssrzxxhvNPQ8AAECb16Sv9l177bVZtGhRevXqlf333z+77757o/Xz589vluEAAADaoiaF1Mknn9zMYwAAALQfTQqpyy+/vLnnAAAAaDea/At5X3vttfzrv/5rJk+enJUrVyZ56yt9//Vf/9VswwEAALRFTboi9fTTT2fEiBHp1q1bfv/73+e8885LTU1N7r777ixZsiTf+973mntOAACANqNJV6Rqa2tz9tln5/nnn0/nzp0blo8ePTqzZs1qtuEAAADaoiaF1JNPPpnzzz9/s+Xvfe97s2zZsnc9FAAAQFvWpJCqqqrK6tWrN1v+u9/9Lvvss8+7HgoAAKAta1JIffKTn8xVV12VDRs2JEkqlUqWLFmSSy+9NGPGjGnWAQEAANqaJoXUtddemzVr1qRnz57585//nI997GN53/vel+rq6vzjP/5jc88IAADQpjTpqX3dunXLjBkz8uijj+bpp5/OmjVrMmjQoIwYMaK55wMAAGhzmhRSbxs6dGiGDh3aXLMAAAC0C9sdUtdff/127/Siiy5q0jAAAADtwXaH1Le+9a3t2q5SqQgpAABgp7bdIbV48eIdOQcAAEC70aSn9m2vrl275sUXX9yRhwAAAGhxOzSk6uvrd+TuAQAAWsUODSkAAICdkZACAAAoJKQAAAAK7dCQqlQqO3L3AAAArcLDJgAAAArt0JC677778t73vndHHgIAAKDFbfcv5P1LtbW1273t0KFDm3IIAACANqtJIfXUU0/lqaeeyoYNG3LIIYckSX73u99lt912y6BBgxq2c48UAACwM2pSSJ100kmprq7Od7/73ey1115Jkj/96U8555xzcuyxx+Z//s//2axDAgAAtCVNukfq2muvzZQpUxoiKkn22muvfP3rX8+1117bbMMBAAC0RU0KqdWrV+ePf/zjZsv/+Mc/5vXXX3/XQwEAALRlTQqpT3/60znnnHNy99135+WXX87LL7+cf//3f8+4ceNyyimnNPeMAAAAbUqT7pG65ZZb8uUvfzlnnHFGNmzY8NaOOnbMuHHjcs011zTrgAAAAG1Nk0Jqjz32yE033ZRrrrkmixYtSpIcdNBB2XPPPZt1OAAAgLboXf1C3qVLl2bp0qU5+OCDs+eee6a+vr655gIAAGizmhRSr776aoYPH573v//9GT16dJYuXZokGTdunEefAwAAO70mhdSkSZOy++67Z8mSJdljjz0alp922mmZPn36du9n1qxZOemkk9K3b99UKpXcc889jdbX19fnsssuS58+fdKlS5eMGDEizz//fKNtVq5cmTPPPDNdu3ZN9+7dM27cuKxZs6YpPxYAAMB2aVJI/fKXv8w3vvGN7Lvvvo2WH3zwwfnDH/6w3ftZu3ZtBg4cmGnTpm1x/dVXX53rr78+t9xyS+bMmZM999wzJ554Yt58882Gbc4888w899xzmTFjRu69997MmjUr48ePb8qPBQAAsF2a9LCJtWvXNroS9baVK1emqqpqu/czatSojBo1aovr6uvrc9111+WrX/1qPvWpTyVJvve976VXr16555578rnPfS6//e1vM3369Dz55JM5+uijkyQ33HBDRo8enX/+539O3759m/DTAQAAbFuTrkgde+yx+d73vtfwvlKppK6uLldffXWOP/74Zhls8eLFWbZsWUaMGNGwrFu3bhk8eHBmz56dJJk9e3a6d+/eEFFJMmLEiHTo0CFz5szZ6r7XrVuX1atXN3oBAABsryZdkbr66qszfPjwzJ07N+vXr88ll1yS5557LitXrsxjjz3WLIMtW7YsSdKrV69Gy3v16tWwbtmyZenZs2ej9R07dkxNTU3DNlsyZcqUXHnllc0yJwAAsOtp0hWpww47LL/73e8ydOjQfOpTn8ratWtzyimn5KmnnspBBx3U3DM2u8mTJ2fVqlUNr5deeqm1RwIAANqRJl2RSt76mt3f//3fN+csjfTu3TtJsnz58vTp06dh+fLly3PEEUc0bLNixYpGn9u4cWNWrlzZ8PktqaqqKrqXCwAA4C816YrU9OnT8+ijjza8nzZtWo444oicccYZ+dOf/tQsgx1wwAHp3bt3HnzwwYZlq1evzpw5czJkyJAkyZAhQ/Laa69l3rx5Dds89NBDqaury+DBg5tlDgAAgL/WpJD6X//rfzU8oOGZZ55JbW1tRo8encWLF6e2tna797NmzZosWLAgCxYsSPLWAyYWLFiQJUuWpFKp5OKLL87Xv/71/OxnP8szzzyTs846K3379s3JJ5+cJPnABz6QkSNH5rzzzssTTzyRxx57LBMnTsznPvc5T+wDAAB2mCZ9tW/x4sX54Ac/mCT593//95x00kn5p3/6p8yfPz+jR4/e7v3MnTu30VP+3o6wsWPH5rbbbssll1yStWvXZvz48XnttdcydOjQTJ8+PZ07d274zA9+8INMnDgxw4cPT4cOHTJmzJhcf/31TfmxAAAAtkuTQqpTp0554403kiQPPPBAzjrrrCRJTU1N0aPEhw0blvr6+q2ur1Qqueqqq3LVVVdtdZuamprcfvvt231MAACAd6tJITV06NDU1tbmox/9aJ544on88Ic/TJL87ne/y7777tusAwIAALQ1TbpH6sYbb0zHjh3z4x//ODfffHPe+973Jknuu+++jBw5slkHBAAAaGuadEWqf//+uffeezdb/q1vfavR+6lTp+aCCy5I9+7dmzQcAABAW9SkK1Lb65/+6Z+ycuXKHXkIAACAFrdDQ2pbD5IAAABor3ZoSAEAAOyMhBQAAEAhIQUAAFBISAEAABTaoSF17LHHpkuXLjvyEAAAAC2uSSF12223bXH5xo0bM3ny5Ib3v/jFL9KnT58mDQYAANBWNSmkLrroonz2s5/Nn/70p4ZlCxcuzODBg3PHHXc023AAAABtUZNC6qmnnsrLL7+cww8/PDNmzMi0adMyaNCgHHroofnNb37T3DMCAAC0KR2b8qGDDjoojz32WC6++OKMHDkyu+22W7773e/m9NNPb+75AAAA2pwmP2zi5z//ee68884MGTIk3bt3z3e+85288sorzTkbAABAm9SkkDr//PPz2c9+NpdeemkeeeSRPP300+nUqVMOP/zw/OhHP2ruGQEAANqUJn2177HHHsucOXMycODAJEnv3r3zi1/8ItOmTcsXv/jFnHrqqc06JAAAQFvSpJCaN29eqqqqNls+YcKEjBgx4l0PBQAA0JY16at9W4qotx1yyCFNHgYAAKA9aNIVqST58Y9/nB/96EdZsmRJ1q9f32jd/Pnz3/VgAAAAbVWTrkhdf/31Oeecc9KrV6889dRTOeaYY9KjR4+8+OKLGTVqVHPPCAAA0KY0KaRuuummfPvb384NN9yQTp065ZJLLsmMGTNy0UUXZdWqVc09IwAAQJvSpJBasmRJPvKRjyRJunTpktdffz1J8oUvfCF33HFH800HAADQBjUppHr37p2VK1cmSfr375/HH388SbJ48eLU19c333QAAABtUJNC6uMf/3h+9rOfJUnOOeecTJo0KSeccEJOO+20fPrTn27WAQEAANqaJj2179vf/nbq6uqSvPW7o/bee+889thj+eQnP5kLLrigWQcEAABoa5oUUh06dMj69eszf/78rFixIl26dGn4RbzTp0/PSSed1KxDAgAAtCVNCqnp06fnC1/4Ql599dXN1lUqlWzatOldDwYAANBWNekeqQsvvDCnnnpqli5dmrq6ukYvEQUAAOzsmhRSy5cvT21tbXr16tXc8wAAALR5TQqpz3zmM3n44YebeRQAAID2oUn3SN1444357Gc/m0ceeSSHH354dt9990brL7roomYZDgAAoC1qUkjdcccd+eUvf5nOnTvn4YcfTqVSaVhXqVSEFAAAsFNrUkj9/d//fa688sp85StfSYcOTfp2IAAAQLvVpApav359TjvtNBEFAADskppUQmPHjs0Pf/jD5p4FAACgXWjSV/s2bdqUq6++Ovfff38GDBiw2cMmvvnNbzbLcAAAAG1Rk0LqmWeeyZFHHpkkefbZZxut+8sHTwAAAOyMmhRSv/rVr5p7DgAAgHbD0yIAAAAKCSkAAIBCTfpqH82somcBAKA98V/wAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEChjq09QFtS6VBJpVJp7TF2Hf6uAQBop1yRAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoFCrhtSsWbNy0kknpW/fvqlUKrnnnnsa1m3YsCGXXnppDj/88Oy5557p27dvzjrrrLzyyiuN9rH//vunUqk0ek2dOrWFfxIAAGBX0qohtXbt2gwcODDTpk3bbN0bb7yR+fPn52tf+1rmz5+fu+++OwsXLswnP/nJzba96qqrsnTp0obXhRde2BLjAwAAu6iOrXnwUaNGZdSoUVtc161bt8yYMaPRshtvvDHHHHNMlixZkv79+zcsr66uTu/evXforAAAAG9rV/dIrVq1KpVKJd27d2+0fOrUqenRo0eOPPLIXHPNNdm4ceM297Nu3bqsXr260QsAAGB7teoVqRJvvvlmLr300px++unp2rVrw/KLLroogwYNSk1NTX79619n8uTJWbp0ab75zW9udV9TpkzJlVde2RJjAwAAO6F2EVIbNmzIqaeemvr6+tx8882N1tXW1jb8ecCAAenUqVPOP//8TJkyJVVVVVvc3+TJkxt9bvXq1enXr9+OGR4AANjptPmQejui/vCHP+Shhx5qdDVqSwYPHpyNGzfm97//fQ455JAtblNVVbXVyAIAAHgnbTqk3o6o559/Pr/61a/So0ePd/zMggUL0qFDh/Ts2bMFJgQAAHZFrRpSa9asyQsvvNDwfvHixVmwYEFqamrSp0+ffOYzn8n8+fNz7733ZtOmTVm2bFmSpKamJp06dcrs2bMzZ86cHH/88amurs7s2bMzadKkfP7zn89ee+3VWj8WAACwk2vVkJo7d26OP/74hvdv37c0duzYXHHFFfnZz36WJDniiCMafe5Xv/pVhg0blqqqqtx555254oorsm7duhxwwAGZNGlSo/ufAAAAmlurhtSwYcNSX1+/1fXbWpckgwYNyuOPP97cYwEAAGxTu/o9UgAAAG2BkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKBQq4bUrFmzctJJJ6Vv376pVCq55557Gq0/++yzU6lUGr1GjhzZaJuVK1fmzDPPTNeuXdO9e/eMGzcua9asacGfAgAA2NW0akitXbs2AwcOzLRp07a6zciRI7N06dKG1x133NFo/ZlnnpnnnnsuM2bMyL333ptZs2Zl/PjxO3p0AABgF9axNQ8+atSojBo1apvbVFVVpXfv3ltc99vf/jbTp0/Pk08+maOPPjpJcsMNN2T06NH553/+5/Tt27fZZwYAAGjz90g9/PDD6dmzZw455JD83d/9XV599dWGdbNnz0737t0bIipJRowYkQ4dOmTOnDlb3ee6deuyevXqRi8AAIDt1aZDauTIkfne976XBx98MN/4xjcyc+bMjBo1Kps2bUqSLFu2LD179mz0mY4dO6ampibLli3b6n6nTJmSbt26Nbz69eu3Q38OAABg59KqX+17J5/73Oca/nz44YdnwIABOeigg/Lwww9n+PDhTd7v5MmTU1tb2/B+9erVYgoAANhubfqK1F878MADs/fee+eFF15IkvTu3TsrVqxotM3GjRuzcuXKrd5Xlbx131XXrl0bvQAAALZXuwqpl19+Oa+++mr69OmTJBkyZEhee+21zJs3r2Gbhx56KHV1dRk8eHBrjQkAAOzkWvWrfWvWrGm4upQkixcvzoIFC1JTU5OamppceeWVGTNmTHr37p1Fixblkksuyfve976ceOKJSZIPfOADGTlyZM4777zccsst2bBhQyZOnJjPfe5zntgHAADsMK16RWru3Lk58sgjc+SRRyZJamtrc+SRR+ayyy7Lbrvtlqeffjqf/OQn8/73vz/jxo3LUUcdlUceeSRVVVUN+/jBD36QQw89NMOHD8/o0aMzdOjQfPvb326tHwkAANgFtOoVqWHDhqW+vn6r6++///533EdNTU1uv/325hwLAABgm9rVPVIAAABtgZACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACjUsbUHaFMqHd56AQAAbINqAAAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoJCQAgAAKCSkAAAACnVs7QFoZRUtDQAApfxXNAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFCoY2sPQFLpUGntEQAAgAKuSAEAABQSUgAAAIWEFAAAQCH3SLFrqrgvDQCApnNFCgAAoJCQAgAAKCSkAAAACgkpAACAQkIKAACgkJACAAAoJKQAAAAKCSkAAIBCQgoAAKCQkAIAACgkpAAAAAoJKQAAgEJCCgAAoFCrhtSsWbNy0kknpW/fvqlUKrnnnnsara9UKlt8XXPNNQ3b7L///putnzp1agv/JAAAwK6kVUNq7dq1GThwYKZNm7bF9UuXLm30+rd/+7dUKpWMGTOm0XZXXXVVo+0uvPDClhgfAADYRXVszYOPGjUqo0aN2ur63r17N3r/05/+NMcff3wOPPDARsurq6s323Zb1q1bl3Xr1jW8X7169XZ/FgAAoN3cI7V8+fL8/Oc/z7hx4zZbN3Xq1PTo0SNHHnlkrrnmmmzcuHGb+5oyZUq6devW8OrXr9+OGhsAANgJteoVqRLf/e53U11dnVNOOaXR8osuuiiDBg1KTU1Nfv3rX2fy5MlZunRpvvnNb251X5MnT05tbW3D+9WrV4spAABgu7WbkPq3f/u3nHnmmencuXOj5X8ZRAMGDEinTp1y/vnnZ8qUKamqqtrivqqqqra6DgAA4J20i6/2PfLII1m4cGHOPffcd9x28ODB2bhxY37/+9/v+MEAAIBdUrsIqe985zs56qijMnDgwHfcdsGCBenQoUN69uzZApMBAAC7olb9at+aNWvywgsvNLxfvHhxFixYkJqamvTv3z/JW/cv3XXXXbn22ms3+/zs2bMzZ86cHH/88amurs7s2bMzadKkfP7zn89ee+3VYj8HAACwa2nVkJo7d26OP/74hvdv3+80duzY3HbbbUmSO++8M/X19Tn99NM3+3xVVVXuvPPOXHHFFVm3bl0OOOCATJo0qdF9UwAAAM2tVUNq2LBhqa+v3+Y248ePz/jx47e4btCgQXn88cd3xGgAAABb1S7ukQIAAGhLhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAU6tjaA7QpHSpJpdLaUwAAAG2cK1IAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIWEFAAAQCEhBQAAUEhIAQAAFBJSAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFOrb2AG1BfX19kmRj/YYmfX7163Xv6vhNPa5jO7ZjO7ZjO7ZjO7ZjO7ZjN++xV69567NvN8LWVOrfaYtdwMsvv5x+/fq19hgAAEAb8dJLL2Xffffd6nohlaSuri6vvPJKqqurU6lUNlu/evXq9OvXLy+99FK6du3aChOyq3Cu0VKca7QU5xotxblGc6mvr8/rr7+evn37pkOHrd8J5at9STp06LDN2nxb165d/Q+TFuFco6U412gpzjVainON5tCtW7d33MbDJgAAAAoJKQAAgEJCajtUVVXl8ssvT1VVVWuPwk7OuUZLca7RUpxrtBTnGi3NwyYAAAAKuSIFAABQSEgBAAAUElIAAACFhBQAAEAhIfUOpk2blv333z+dO3fO4MGD88QTT7T2SLRzs2bNykknnZS+ffumUqnknnvuabS+vr4+l112Wfr06ZMuXbpkxIgRef7551tnWNq1KVOm5EMf+lCqq6vTs2fPnHzyyVm4cGGjbd58881MmDAhPXr0yHve856MGTMmy5cvb6WJaa9uvvnmDBgwoOEXoQ4ZMiT33Xdfw3rnGTvC1KlTU6lUcvHFFzcsc67RkoTUNvzwhz9MbW1tLr/88syfPz8DBw7MiSeemBUrVrT2aLRja9euzcCBAzNt2rQtrr/66qtz/fXX55ZbbsmcOXOy55575sQTT8ybb77ZwpPS3s2cOTMTJkzI448/nhkzZmTDhg35xCc+kbVr1zZsM2nSpPzHf/xH7rrrrsycOTOvvPJKTjnllFacmvZo3333zdSpUzNv3rzMnTs3H//4x/OpT30qzz33XBLnGc3vySefzP/5P/8nAwYMaLTcuUaLqmerjjnmmPoJEyY0vN+0aVN9375966dMmdKKU7EzSVL/k5/8pOF9XV1dfe/eveuvueaahmWvvfZafVVVVf0dd9zRChOyM1mxYkV9kvqZM2fW19e/dW7tvvvu9XfddVfDNr/97W/rk9TPnj27tcZkJ7HXXnvV/+u//qvzjGb3+uuv1x988MH1M2bMqP/Yxz5W/6Uvfam+vt4/02h5rkhtxfr16zNv3ryMGDGiYVmHDh0yYsSIzJ49uxUnY2e2ePHiLFu2rNF5161btwwePNh5x7u2atWqJElNTU2SZN68edmwYUOj8+3QQw9N//79nW802aZNm3LnnXdm7dq1GTJkiPOMZjdhwoT87d/+baNzKvHPNFpex9YeoK367//+72zatCm9evVqtLxXr175z//8z1aaip3dsmXLkmSL593b66Ap6urqcvHFF+ejH/1oDjvssCRvnW+dOnVK9+7dG23rfKMpnnnmmQwZMiRvvvlm3vOe9+QnP/lJPvjBD2bBggXOM5rNnXfemfnz5+fJJ5/cbJ1/ptHShBTALmDChAl59tln8+ijj7b2KOykDjnkkCxYsCCrVq3Kj3/844wdOzYzZ85s7bHYibz00kv50pe+lBkzZqRz586tPQ542MTW7L333tltt902e9LL8uXL07t371aaip3d2+eW847mNHHixNx777351a9+lX333bdhee/evbN+/fq89tprjbZ3vtEUnTp1yvve974cddRRmTJlSgYOHJh/+Zd/cZ7RbObNm5cVK1Zk0KBB6dixYzp27JiZM2fm+uuvT8eOHdOrVy/nGi1KSG1Fp06dctRRR+XBBx9sWFZXV5cHH3wwQ4YMacXJ2JkdcMAB6d27d6PzbvXq1ZkzZ47zjmL19fWZOHFifvKTn+Shhx7KAQcc0Gj9UUcdld13373R+bZw4cIsWbLE+ca7VldXl3Xr1jnPaDbDhw/PM888kwULFjS8jj766Jx55pkNf3au0ZJ8tW8bamtrM3bs2Bx99NE55phjct1112Xt2rU555xzWns02rE1a9bkhRdeaHi/ePHiLFiwIDU1Nenfv38uvvjifP3rX8/BBx+cAw44IF/72tfSt2/fnHzyya03NO3ShAkTcvvtt+enP/1pqqurG+4R6NatW7p06ZJu3bpl3Lhxqa2tTU1NTbp27ZoLL7wwQ4YMyYc//OFWnp72ZPLkyRk1alT69++f119/Pbfffnsefvjh3H///c4zmk11dXXDPZ5v23PPPdOjR4+G5c41WpKQ2obTTjstf/zjH3PZZZdl2bJlOeKIIzJ9+vTNHgQAJebOnZvjjz++4X1tbW2SZOzYsbnttttyySWXZO3atRk/fnxee+21DB06NNOnT/d9cIrdfPPNSZJhw4Y1Wn7rrbfm7LPPTpJ861vfSocOHTJmzJisW7cuJ554Ym666aYWnpT2bsWKFTnrrLOydOnSdOvWLQMGDMj999+fE044IYnzjJbjXKMlVerr6+tbewgAAID2xD1SAAAAhYQUAABAISEFAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgDsEoYNG5aLL7643e0bgLapY2sPAADt3d13353dd9+9tccAoAUJKQB4l2pqalp7BABamK/2AdDm1NXVZcqUKTnggAPSpUuXDBw4MD/+8Y+TJA8//HAqlUruv//+HHnkkenSpUs+/vGPZ8WKFbnvvvvygQ98IF27ds0ZZ5yRN954o9F+N27cmIkTJ6Zbt27Ze++987WvfS319fXbNdNNN92Ugw8+OJ07d06vXr3ymc98pmHdX3617+35/vp19tlnN2z/05/+NIMGDUrnzp1z4IEH5sorr8zGjRvf3V8aAC3KFSkA2pwpU6bk+9//fm655ZYcfPDBmTVrVj7/+c9nn332adjmiiuuyI033pg99tgjp556ak499dRUVVXl9ttvz5o1a/LpT386N9xwQy699NKGz3z3u9/NuHHj8sQTT2Tu3LkZP358+vfvn/POO2+b88ydOzcXXXRR/u///b/5yEc+kpUrV+aRRx7Z4rYf+chHsnTp0ob3v/3tbzN69Ogcd9xxSZJHHnkkZ511Vq6//voce+yxWbRoUcaPH58kufzyy5v8dwZAy6rUb+//FQcALWDdunWpqanJAw88kCFDhjQsP/fcc/PGG29k/PjxOf744/PAAw9k+PDhSZKpU6dm8uTJWbRoUQ488MAkyQUXXJDf//73mT59epK3rhqtWLEizz33XCqVSpLkK1/5Sn72s5/l//2//7fNme6+++6cc845efnll1NdXb3Z+mHDhuWII47Idddd12j5q6++mmOOOSYjR47MtGnTkiQjRozI8OHDM3ny5Ibtvv/97+eSSy7JK6+8Uvi3BUBrcUUKgDblhRdeyBtvvJETTjih0fL169fnyCOPbHg/YMCAhj/36tUre+yxR0NEvb3siSeeaLSPD3/4ww0RlSRDhgzJtddem02bNmW33Xbb6kwnnHBC9ttvvxx44IEZOXJkRo4cmU9/+tPZY489tvqZDRs2ZMyYMdlvv/3yL//yLw3Lf/Ob3+Sxxx7LP/7jPzYs27RpU95888288cYb29wnAG2HkAKgTVmzZk2S5Oc//3ne+973NlpXVVWVRYsWJUmjp+RVKpXNnppXqVRSV1fXLDNVV1dn/vz5efjhh/PLX/4yl112Wa644oo8+eST6d69+xY/83d/93d56aWX8sQTT6Rjx///X7dr1qzJlVdemVNOOWWzz3Tu3LlZ5gVgxxNSALQpH/zgB1NVVZUlS5bkYx/72Gbr3w6pppgzZ06j948//ngOPvjgbV6NelvHjh0zYsSIjBgxIpdffnm6d++ehx56aItB9M1vfjM/+tGP8utf/zo9evRotG7QoEFZuHBh3ve+9zX55wCg9QkpANqU6urqfPnLX86kSZNSV1eXoUOHZtWqVXnsscfStWvX7Lfffk3e95IlS1JbW5vzzz8/8+fPzw033JBrr732HT9377335sUXX8xxxx2XvfbaK7/4xS9SV1eXQw45ZLNtH3jggVxyySWZNm1a9t577yxbtixJ0qVLl3Tr1i2XXXZZ/sf/+B/p379/PvOZz6RDhw75zW9+k2effTZf//rXm/yzAdCyhBQAbc4//MM/ZJ999smUKVPy4osvpnv37hk0aFD+9//+3+/q63pnnXVW/vznP+eYY47Jbrvtli996UsNT8zblu7du+fuu+/OFVdckTfffDMHH3xw7rjjjvzN3/zNZts++uij2bRpUy644IJccMEFDcvHjh2b2267LSeeeGLuvffeXHXVVfnGN76R3XffPYceemjOPffcJv9cALQ8T+0DAAAo5BfyAgAAFBJSAOzyHnnkkbznPe/Z6gsA/pqv9gGwy/vzn/+c//qv/9rqek/YA+CvCSkAAIBCvtoHAABQSEgBAAAUElIAAACFhBQAAEAhIQUAAFBISAEAABQSUgAAAIX+Px+eCHVQDvPgAAAAAElFTkSuQmCC\n" - }, - "metadata": {} - } - ] - }, - { - "cell_type": "code", - "source": [ - "fig, ax = plt.subplots(figsize=(10, 10))\n", - "plt.imshow(tmp_layer.pe[0, :, 50:51], aspect=\"auto\")\n", - "plt.xlabel(\"emb_size\")\n", - "plt.ylabel(\"max_seq_len\")\n", - "plt.show()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 853 - }, - "id": "Emdh9j6acXH5", - "outputId": "516ee9f3-2db2-4fb0-a0f9-51054bd52c84" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1IAAANECAYAAAC6qR6BAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABA4klEQVR4nO3de5hVdb0/8M/aAwzYgcFRAVG8ZlrHK5YcDE2DEuhQJqapeYu89Hj5CaefyjnmrTqQl/KolE/nWNovr3XUfDQpwBAlxAuSlwoTMTQBS5LhkgPMrN8f6LRHQPd3y8yemf16nef7PO611l7z2T2rOu++6/P9Znme5wEAAEDJCpUuAAAAoLMRpAAAABIJUgAAAIkEKQAAgESCFAAAQCJBCgAAIJEgBQAAkEiQAgAASCRIAQAAJBKkAAAAEnWZIDVlypTYZZddomfPnjFkyJB47LHHKl0SAADQRWV5nueVLuL9uuOOO+Kkk06KG264IYYMGRLXXHNN/PSnP40FCxZEv3793vP7zc3N8eqrr0bv3r0jy7J2qBgAAOiI8jyPlStXxsCBA6NQ2Py8U5cIUkOGDImPfexjcf3110fEhmA0aNCgOOecc+LCCy98z++/8sorMWjQoLYuEwAA6CRefvnl2HHHHTd7vls71tIm1q5dG08++WRMnDix5VihUIgRI0bEnDlzNvmdxsbGaGxsbPn8dpYcFqOjW3Rv24IBAIAOa32si0fiF9G7d+93va7TB6m//vWv0dTUFP379291vH///vGHP/xhk9+ZNGlSXHbZZRsd7xbdo1smSAEAQNV6632992r56TKLTaSYOHFirFixomW8/PLLlS4JAADoRDr9jNS2224bNTU1sWzZslbHly1bFgMGDNjkd2pra6O2tnbjE1m2YQAAAFUqa5mVejedfkaqR48eceCBB8aMGTNajjU3N8eMGTNi6NChFawMAADoqjr9jFRExIQJE+Lkk0+Oj370o3HQQQfFNddcE6tXr45TTz210qUBAABdUJcIUscee2z85S9/iYsvvjiWLl0a+++/f0ydOnWjBSgAAAC2hC6xj9T71dDQEHV1dXFY4Sir9gEAQBVbn6+Lmc13xYoVK6JPnz6bva7T90gBAAC0N0EKAAAgkSAFAACQqEssNrGlZIXsPXcwBgAAuq4szyKa3/s6M1IAAACJBCkAAIBEXu0rVlMTkdVUugoAAKBS8uaI9e99mRkpAACARIIUAABAIkEKAAAgkR6pIlkWlj8HAIAqVmoaMCMFAACQSJACAABIJEgBAAAk0iNVrFCIyGRLAACoWnlpeUBqAAAASCRIAQAAJBKkAAAAEumRKlZTE5HVVLoKAACgUvLS8oAZKQAAgESCFAAAQCJBCgAAIJEeqSJZlkWWZZUuAwAAqJAsSssDZqQAAAASCVIAAACJBCkAAIBEeqSKFbIIPVIAAFC9cj1SAAAAbUKQAgAASOTVvmJZYcMAAACqVGl5QGoAAABIJEgBAAAkEqQAAAAS6ZEqVihsGAAAQHVq1iMFAADQJgQpAACARIIUAABAIj1SxewjBQAA1a3EPCA1AAAAJBKkAAAAEglSAAAAifRIFStkGwYAAFClSssDZqQAAAASCVIAAACJBCkAAIBEeqSKZIVCZAXZEgAAqlVW4lyT1AAAAJBIkAIAAEgkSAEAACTSI1WskEXokQIAgCpmHykAAIA2IUgBAAAkEqQAAAAS6ZEqlmUbBgAAUJ1KzANmpAAAABIJUgAAAIm82lfMq30AAFDdvNoHAADQNgQpAACARIIUAABAIj1SxWoKEQXZEgAAqlZWWh6QGgAAABIJUgAAAIkEKQAAgER6pIrZRwoAAKqbfaQAAADahiAFAACQSJACAABIpEeqmB4pAACobnqkAAAA2oYgBQAAkEiQAgAASKRHqkheKEReI1sCAEC1ykuca5IaAAAAEglSAAAAiQQpAACARHqkitlHCgAAqpt9pAAAANqGIAUAAJBIkAIAAEikR6qYHikAAKhueqQAAADahiAFAACQyKt9xQrZhgEAAFSn3Kt9AAAAbaLDB6lJkybFxz72sejdu3f069cvjjzyyFiwYEGraw477LDIsqzVOPPMMytUMQAA0NV1+CD10EMPxVlnnRWPPvpoTJs2LdatWxef/vSnY/Xq1a2uO+2002LJkiUt44orrqhQxQAAQFfX4Xukpk6d2urzTTfdFP369Ysnn3wyDj300JbjW221VQwYMOD9/bFCYcMAAACqU15aHuh0qWHFihUREVFfX9/q+C233BLbbrtt7L333jFx4sRYs2bNZu/R2NgYDQ0NrQYAAECpOvyMVLHm5uY477zz4uMf/3jsvffeLcePP/742HnnnWPgwIHx9NNPxwUXXBALFiyIu+66a5P3mTRpUlx22WXtVTYAANDFZHme55UuolRf/epX44EHHohHHnkkdtxxx81e9+CDD8bw4cPjhRdeiN13332j842NjdHY2NjyuaGhIQYNGhTDP/y16FZT2ya1AwAAHd/6psaY8furYsWKFdGnT5/NXtdpZqTOPvvsuO+++2LWrFnvGqIiIoYMGRIRsdkgVVtbG7W1GwemPMsiz+wjBQAA1arUPNDhg1Se53HOOefE3XffHTNnzoxdd931Pb8zf/78iIjYfvvt27g6AACgGnX4IHXWWWfFrbfeGj//+c+jd+/esXTp0oiIqKuri169esXChQvj1ltvjdGjR8c222wTTz/9dIwfPz4OPfTQ2HfffStcPQAA0BV1+CD1/e9/PyI2bLpb7Ec/+lGccsop0aNHj5g+fXpcc801sXr16hg0aFCMHTs2LrroogpUCwAAVIMOH6Teay2MQYMGxUMPPbRl/lghOuGC8AAAwBZT4lJ8YgMAAEAiQQoAACCRIAUAAJCow/dItatCYcMAAACqU15aHpAaAAAAEglSAAAAiQQpAACARHqkiuRZFnmWVboMAACgQkrNA2akAAAAEglSAAAAibzaV6wQoiUAAFSzvLTLxAYAAIBEghQAAEAiQQoAACCRHqliWURY/hwAAKpXiXHAjBQAAEAiQQoAACCRIAUAAJBIj1SRPMsiL+iRAgCAapU3l5YHzEgBAAAkEqQAAAASCVIAAACJ9EgVK2QbBgAAUJ1KzANmpAAAABIJUgAAAIkEKQAAgER6pIrkWRZ5pkcKAACqVal5wIwUAABAIkEKAAAgkSAFAACQSI9UMftIAQBAdbOPFAAAQNsQpAAAABIJUgAAAIn0SBXJsw0DAACoTqXmATNSAAAAiQQpAACARF7tK5ZlGwYAAFCdSswDZqQAAAASCVIAAACJBCkAAIBEeqSKFUK0BACAalZiHhAbAAAAEglSAAAAiQQpAACARHqkiuSFLPKCfaQAAKBalZoHzEgBAAAkEqQAAAASCVIAAACJ9EgVybMs8kyPFAAAVKtS84AZKQAAgESCFAAAQCJBCgAAIJEeqWLZWwMAAKhOJeYBM1IAAACJBCkAAIBEghQAAEAiPVJF8pos8hpNUgAAUK3y3D5SAAAAbUKQAgAASCRIAQAAJNIjVSTPNgwAAKA6lZoHzEgBAAAkEqQAAAASebWvlSwi824fAABUL8ufAwAAtAlBCgAAIJEgBQAAkEiPVBHLnwMAQHWz/DkAAEAbEaQAAAASCVIAAACJ9EgVK4RoCQAA1azEPCA2AAAAJBKkAAAAEglSAAAAifRIFcmzLPLMRlIAAFCtSs0DZqQAAAASCVIAAACJBCkAAIBEeqSKZW8NAACgOpWYB8xIAQAAJBKkAAAAEglSAAAAifRIFclrNgwAAKA65c2lXWdGCgAAIFGHD1KXXnppZFnWauy1114t5998880466yzYptttol/+qd/irFjx8ayZcsqWDEAANDVdYpX+/75n/85pk+f3vK5W7d/lD1+/Pi4//7746c//WnU1dXF2WefHUcddVTMnj07+e/kWRZ5Zv1zAACoVqXmgU4RpLp16xYDBgzY6PiKFSvixhtvjFtvvTU++clPRkTEj370o/jwhz8cjz76aPzLv/zLJu/X2NgYjY2NLZ8bGhrapnAAAKBL6vCv9kVE/PGPf4yBAwfGbrvtFieccEIsXrw4IiKefPLJWLduXYwYMaLl2r322it22mmnmDNnzmbvN2nSpKirq2sZgwYNavPfAAAAdB0dPkgNGTIkbrrpppg6dWp8//vfj0WLFsUhhxwSK1eujKVLl0aPHj2ib9++rb7Tv3//WLp06WbvOXHixFixYkXLePnll9v4VwAAAF1Jh3+1b9SoUS3/vO+++8aQIUNi5513jjvvvDN69epV1j1ra2ujtrZ24xPZWwMAAKhOJeaBDj8j9U59+/aND33oQ/HCCy/EgAEDYu3atfHGG2+0umbZsmWb7KkCAADYEjpdkFq1alUsXLgwtt9++zjwwAOje/fuMWPGjJbzCxYsiMWLF8fQoUMrWCUAANCVdfhX+772ta/FmDFjYuedd45XX301LrnkkqipqYnjjjsu6urqYty4cTFhwoSor6+PPn36xDnnnBNDhw7d7Ip9AAAA71eHD1KvvPJKHHfccfH666/HdtttF8OGDYtHH300tttuu4iI+O53vxuFQiHGjh0bjY2NccQRR8T3vve9sv5Wnm0YAABAdSo1D2R5nudtW0rH19DQEHV1dfGxz30junXvWelyAACAClm/7s14/OdfjxUrVkSfPn02e12n65ECAACoNEEKAAAgUYfvkWpPeWHDAAAAqlOpeUBsAAAASCRIAQAAJBKkAAAAEumRKpZlGwYAAFCdSswDZqQAAAASCVIAAACJBCkAAIBEeqSK5NmGAQAAVKdS84AZKQAAgESCFAAAQCJBCgAAIJEeqWKFiFy0BACA6lViHhAbAAAAEglSAAAAibzaVyx7awAAANXJ8ucAAABtQ5ACAABIJEgBAAAk0iNVJI+IXI8UAABUrbzE68xIAQAAJBKkAAAAEglSAAAAifRIFbOPFAAAVDf7SAEAALQNQQoAACCRIAUAAJBIj1SRvJBFXtAkBQAA1arUPGBGCgAAIJEgBQAAkEiQAgAASKRHqkiebRgAAEB1KjUPmJECAABIJEgBAAAkEqQAAAAS6ZEqlr01AACA6qRHCgAAoG0IUgAAAIkEKQAAgER6pIrkhQ0DAACoTqXmAbEBAAAgkSAFAACQyKt9RfJswwAAAKpTqXnAjBQAAEAiQQoAACCRIAUAAJBIj1Sx7K0BAABUJz1SAAAAbUOQAgAASCRIAQAAJNIjVUyPFAAAVDc9UgAAAG1DkAIAAEgkSAEAACTSI1UkL2wYAABAdSo1D4gNAAAAiQQpAACARIIUAABAIj1SRfJswwAAAKpTqXnAjBQAAEAiQQoAACCRIAUAAJBIj1Sx7K0BAABUJz1SAAAAbUOQAgAASOTVviJ5IYu84N0+AACoVqXmATNSAAAAiQQpAACARIIUAABAIj1SxSx/DgAA1c3y5wAAAG1DkAIAAEgkSAEAACTSI1Ukj4hcjxQAAFStvMTrzEgBAAAkEqQAAAASCVIAAACJ9EgVs48UAABUN/tIAQAAtA1BCgAAIJEgBQAAkEiPVJG8sGEAAADVqdQ8IDYAAAAk6vBBapdddoksyzYaZ511VkREHHbYYRudO/PMMytcNQAA0JV1+Ff7Hn/88Whqamr5/Oyzz8anPvWp+MIXvtBy7LTTTovLL7+85fNWW23VrjUCAADVpcMHqe22267V58mTJ8fuu+8en/jEJ1qObbXVVjFgwICS79nY2BiNjY0tnxsaGjb8g32kAACgunXFfaTWrl0bP/nJT+LLX/5yZNk/fuEtt9wS2267bey9994xceLEWLNmzbveZ9KkSVFXV9cyBg0a1NalAwAAXUiHn5Eqds8998Qbb7wRp5xySsux448/PnbeeecYOHBgPP3003HBBRfEggUL4q677trsfSZOnBgTJkxo+dzQ0CBMAQAAJetUQerGG2+MUaNGxcCBA1uOnX766S3/vM8++8T2228fw4cPj4ULF8buu+++yfvU1tZGbW1tm9cLAAB0TZ0mSP3pT3+K6dOnv+tMU0TEkCFDIiLihRde2GyQ2pw82zAAAIDqVGoe6DQ9Uj/60Y+iX79+8ZnPfOZdr5s/f35ERGy//fbtUBUAAFCNOsWMVHNzc/zoRz+Kk08+Obp1+0fJCxcujFtvvTVGjx4d22yzTTz99NMxfvz4OPTQQ2PfffetYMUAAEBX1imC1PTp02Px4sXx5S9/udXxHj16xPTp0+Oaa66J1atXx6BBg2Ls2LFx0UUXlfV38sKGAQAAVKdS80CnCFKf/vSnI8/zjY4PGjQoHnrooQpUBAAAVDPzLwAAAIkEKQAAgESd4tW+dpO9NQAAgOrU1ZY/BwAA6CgEKQAAgESCFAAAQCI9UsX0SAEAQHXTIwUAANA2BCkAAIBEghQAAEAiPVJF8mzDAAAAqlOpecCMFAAAQCJBCgAAIJEgBQAAkEiPVJG8sGEAAADVqdQ8IDYAAAAkEqQAAAASCVIAAACJBCkAAIBEghQAAEAiQQoAACCRIAUAAJDIPlLFsrcGAABQnUrMA2akAAAAEglSAAAAibzaVyQvbBgAAEB1KjUPiA0AAACJBCkAAIBEghQAAEAiPVLFLH8OAADVzfLnAAAAbUOQAgAASCRIAQAAJNIjVSTPNgwAAKA6lZoHzEgBAAAkEqQAAAASCVIAAACJ9Ei9kx4pAADgPZiRAgAASCRIAQAAJBKkAAAAEumRKpIXNgwAAKA6lZoHxAYAAIBEghQAAEAiQQoAACCRHqliWb5hAAAA1anEPGBGCgAAIFFZM1KrV6+OyZMnx4wZM+K1116L5ubmVudffPHFLVIcAABAR1RWkPrKV74SDz30UJx44omx/fbbR5ZlW7quysgi8i7yUwAAgDKUmAfKClIPPPBA3H///fHxj3+8nK8DAAB0amX1SG299dZRX1+/pWsBAADoFMoKUt/4xjfi4osvjjVr1mzpegAAADq8sl7tu/rqq2PhwoXRv3//2GWXXaJ79+6tzs+bN2+LFNfuCmEdQwAAqGYl5oGygtSRRx5ZztcAAAC6hLKC1CWXXLKl6wAAAOg0yn6R7Y033oj/+Z//iYkTJ8by5csjYsMrfX/+85+3WHEAAAAdUVkzUk8//XSMGDEi6urq4qWXXorTTjst6uvr46677orFixfHj3/84y1dZ7vI7SMFAABVrdQ8UNaM1IQJE+KUU06JP/7xj9GzZ8+W46NHj45Zs2aVc0sAAIBOo6wg9fjjj8cZZ5yx0fEddtghli5d+r6LAgAA6MjKClK1tbXR0NCw0fHnn38+tttuu/ddFAAAQEdWVo/UZz/72bj88svjzjvvjIiILMti8eLFccEFF8TYsWO3aIHtKntrAAAA1akte6SuvvrqWLVqVfTr1y/+/ve/xyc+8Yn44Ac/GL17945vfetb5dwSAACg0yhrRqquri6mTZsWjzzySDz99NOxatWqGDx4cIwYMWJL1wcAANDhlBWk3jZs2LAYNmzYlqoFAACgUyg5SF177bUl3/Tcc88tq5iK0yMFAADVrcQ8UHKQ+u53v1va382yzhukAAAASlBykFq0aFFb1gEAANBplLVqX6n69OkTL774Ylv+CQAAgHb3vhabeC95nrfl7be4PNswAACA6lRqHmjTGSkAAICuSJACAABIJEgBAAAkatMeqSzrZA1H9pECAIDq1hF6pDrbYhMAAAClaNMg9cADD8QOO+zQln8CAACg3ZX1at+ECRNKvnbYsGHl/ImKsPw5AABUt1LzQFlB6qmnnoqnnnoq1q1bF3vuuWdERDz//PNRU1MTgwcPbrmu0/VIAQAAlKCsIDVmzJjo3bt33HzzzbH11ltHRMTf/va3OPXUU+OQQw6Jf/u3f9uiRQIAAHQkZfVIXX311TFp0qSWEBURsfXWW8c3v/nNuPrqq7dYcQAAAB1RWTNSDQ0N8Ze//GWj43/5y19i5cqV77uoiinkGwYAAFCdSswDZc1Iff7zn49TTz017rrrrnjllVfilVdeif/93/+NcePGxVFHHVXOLQEAADqNsmakbrjhhvja174Wxx9/fKxbt27Djbp1i3HjxsWVV165RQsEAADoaMoKUltttVV873vfiyuvvDIWLlwYERG77757fOADH9iixQEAAHREZQWpty1ZsiSWLFkShx56aPTq1SvyPO/US57bRwoAAKpbqXmgrB6p119/PYYPHx4f+tCHYvTo0bFkyZKIiBg3bpylzwEAgC6vrCA1fvz46N69eyxevDi22mqrluPHHntsTJ06teT7zJo1K8aMGRMDBw6MLMvinnvuaXU+z/O4+OKLY/vtt49evXrFiBEj4o9//GOra5YvXx4nnHBC9OnTJ/r27Rvjxo2LVatWlfOzAAAASlJWkPrVr34V3/72t2PHHXdsdXyPPfaIP/3pTyXfZ/Xq1bHffvvFlClTNnn+iiuuiGuvvTZuuOGGmDt3bnzgAx+II444It58882Wa0444YR47rnnYtq0aXHffffFrFmz4vTTTy/nZwEAAJSkrB6p1atXt5qJetvy5cujtra25PuMGjUqRo0atclzeZ7HNddcExdddFF87nOfi4iIH//4x9G/f/+455574otf/GL8/ve/j6lTp8bjjz8eH/3oRyMi4rrrrovRo0fHVVddFQMHDkz7YdlbAwAAqE5t2SN1yCGHxI9//ON//K0si+bm5rjiiivi8MMPL+eWG1m0aFEsXbo0RowY0XKsrq4uhgwZEnPmzImIiDlz5kTfvn1bQlRExIgRI6JQKMTcuXM3e+/GxsZoaGhoNQAAAEpV1ozUFVdcEcOHD48nnngi1q5dG+eff34899xzsXz58pg9e/YWKWzp0qUREdG/f/9Wx/v3799ybunSpdGvX79W57t16xb19fUt12zKpEmT4rLLLtsidQIAANWnrBmpvffeO55//vkYNmxYfO5zn4vVq1fHUUcdFU899VTsvvvuW7rGLW7ixImxYsWKlvHyyy9XuiQAAKATKXsfqbq6uviP//iPLVlLKwMGDIiIiGXLlsX222/fcnzZsmWx//77t1zz2muvtfre+vXrY/ny5S3f35Ta2tpN93LpkQIAgOrWlj1SU6dOjUceeaTl85QpU2L//feP448/Pv72t7+Vc8uN7LrrrjFgwICYMWNGy7GGhoaYO3duDB06NCIihg4dGm+88UY8+eSTLdc8+OCD0dzcHEOGDNkidQAAALxTWUHq//7f/9uyQMMzzzwTEyZMiNGjR8eiRYtiwoQJJd9n1apVMX/+/Jg/f35EbFhgYv78+bF48eLIsizOO++8+OY3vxn33ntvPPPMM3HSSSfFwIED48gjj4yIiA9/+MMxcuTIOO200+Kxxx6L2bNnx9lnnx1f/OIX01fsAwAAKFFZr/YtWrQoPvKRj0RExP/+7//GmDFj4j//8z9j3rx5MXr06JLv88QTT7Ra5e/tEHbyySfHTTfdFOeff36sXr06Tj/99HjjjTdi2LBhMXXq1OjZs2fLd2655ZY4++yzY/jw4VEoFGLs2LFx7bXXlvOzAAAASlJWkOrRo0esWbMmIiKmT58eJ510UkRE1NfXJy0lfthhh0We55s9n2VZXH755XH55Zdv9pr6+vq49dZbS/6b7ybP8sgLm68HAADo2vKstDxQVpAaNmxYTJgwIT7+8Y/HY489FnfccUdERDz//POx4447lnNLAACATqOsHqnrr78+unXrFj/72c/i+9//fuywww4REfHAAw/EyJEjt2iBAAAAHU1ZM1I77bRT3HfffRsd/+53v9vq8+TJk+PMM8+Mvn37llUcAABAR1T2PlKl+M///M845phjOk+Qso8UAABUt7bcR6pU77aQBAAAQGfVpkEKAACgK2rTV/s6Ha/2AQBAdesIr/YBAAB0RYIUAABAojYNUoccckj06tWrLf8EAABAuyurR+qmm26KU045ZaPj69evj69//esxadKkiIj4xS9+8b6Ka295IY+8YKVBAACoVqXmgbJmpM4999z4whe+EH/7299aji1YsCCGDBkSt912Wzm3BAAA6DTKClJPPfVUvPLKK7HPPvvEtGnTYsqUKTF48ODYa6+94re//e2WrhEAAKBDKevVvt133z1mz54d5513XowcOTJqamri5ptvjuOOO25L1wcAANDhlL2P1P333x+33357DB06NJ5//vm48cYb4xOf+EQMHDhwS9bXvuwjBQAA1a0t95E644wz4gtf+EJccMEF8fDDD8fTTz8dPXr0iH322SfuvPPOcm4JAADQaZQ1IzV79uyYO3du7LfffhERMWDAgPjFL34RU6ZMiS9/+ctxzDHHbNEiAQAAOpKygtSTTz4ZtbW1Gx0/66yzYsSIEe+7KAAAgI6srCC1qRD1tj333LPsYipOjxQAAFS3EvNA2YtN/OxnP4s777wzFi9eHGvXrm11bt68eeXeFgAAoMMra7GJa6+9Nk499dTo379/PPXUU3HQQQfFNttsEy+++GKMGjVqS9cIAADQoZQVpL73ve/FD37wg7juuuuiR48ecf7558e0adPi3HPPjRUrVmzpGgEAADqUsoLU4sWL4+CDD46IiF69esXKlSsjIuLEE0+M2267bctVBwAA0AGVFaQGDBgQy5cvj4iInXbaKR599NGIiFi0aFHkeb7lqgMAAOiAygpSn/zkJ+Pee++NiIhTTz01xo8fH5/61Kfi2GOPjc9//vNbtEAAAICOpqxV+37wgx9Ec3NzRGzYO2rbbbeN2bNnx2c/+9k488wzt2iBAAAAHU1ZQapQKMTatWtj3rx58dprr0WvXr1aNuKdOnVqjBkzZosW2W4K+YYBAABUpxLzQFlBaurUqXHiiSfG66+/vtG5LMuiqampnNsCAAB0CmX1SJ1zzjlxzDHHxJIlS6K5ubnVEKIAAICurqwZqWXLlsWECROif//+W7qeysreGgAAQHUqMQ+UNSN19NFHx8yZM8v5KgAAQKdX1ozU9ddfH1/4whfi4Ycfjn322Se6d+/e6vy55567RYoDAADoiMoKUrfddlv86le/ip49e8bMmTMjy/4x/5VlmSAFAAB0aWUFqf/4j/+Iyy67LC688MIoFMp6O7BjyvINAwAAqE4l5oGyUtDatWvj2GOP7VohCgAAoERlJaGTTz457rjjji1dCwAAQKdQ1qt9TU1NccUVV8Qvf/nL2HfffTdabOI73/nOFikOAACgIyorSD3zzDNxwAEHRETEs88+2+pc8cITnU4hypyjAwAAuoQS80BZQerXv/51OV8DAADoEsy/AAAAJBKkAAAAEpX1al+XZR8pAACobm25jxQAAEA1E6QAAAASCVIAAACJ9EgVybINAwAAqE6l5gEzUgAAAIkEKQAAgESCFAAAQCI9UsXsIwUAANXNPlIAAABtQ5ACAABIJEgBAAAk0iNVJMvyyAp6pAAAoFpleqQAAADahiAFAACQyKt9xbK3BgAAUJ1KzANmpAAAABIJUgAAAIkEKQAAgER6pIpk2YYBAABUp1LzgBkpAACARIIUAABAIkEKAAAgkR6pIlmWR5bllS4DAACokFLzgBkpAACARIIUAABAIkEKAAAgkR6pInqkAACguumRAgAAaCOCFAAAQCJBCgAAIJEeqWJZvmEAAADVSY8UAABA2xCkAAAAEglSAAAAifRIFcmyDQMAAKhOpeYBM1IAAACJBCkAAIBEghQAAEAiPVJFskIehYJ9pAAAoFrlJeYBM1IAAACJBCkAAIBEXu0rkmV5ZJlX+wAAoFqVmgfMSAEAACSqaJCaNWtWjBkzJgYOHBhZlsU999zTcm7dunVxwQUXxD777BMf+MAHYuDAgXHSSSfFq6++2uoeu+yyS2RZ1mpMnjy5nX8JAABQTSoapFavXh377bdfTJkyZaNza9asiXnz5sXXv/71mDdvXtx1112xYMGC+OxnP7vRtZdffnksWbKkZZxzzjntUT4AAFClKtojNWrUqBg1atQmz9XV1cW0adNaHbv++uvjoIMOisWLF8dOO+3Ucrx3794xYMCA911PlpX+TiQAAND1ZFlp13WqHqkVK1ZElmXRt2/fVscnT54c22yzTRxwwAFx5ZVXxvr169/1Po2NjdHQ0NBqAAAAlKrTrNr35ptvxgUXXBDHHXdc9OnTp+X4ueeeG4MHD476+vr4zW9+ExMnTowlS5bEd77znc3ea9KkSXHZZZe1R9kAAEAXlOV53iHeZcuyLO6+++448sgjNzq3bt26GDt2bLzyyisxc+bMVkHqnX74wx/GGWecEatWrYra2tpNXtPY2BiNjY0tnxsaGmLQoEGxx08ujJqtNv0dAACg62ta0xh//NLkWLFixbvmjg4/I7Vu3bo45phj4k9/+lM8+OCD7/pjIiKGDBkS69evj5deein23HPPTV5TW1u7yZBVKORRKHSIXAkAAFRAXmIe6NBB6u0Q9cc//jF+/etfxzbbbPOe35k/f34UCoXo169fO1QIAABUo4oGqVWrVsULL7zQ8nnRokUxf/78qK+vj+233z6OPvromDdvXtx3333R1NQUS5cujYiI+vr66NGjR8yZMyfmzp0bhx9+ePTu3TvmzJkT48ePjy996Uux9dZbV+pnAQAAXVxFg9QTTzwRhx9+eMvnCRMmRETEySefHJdeemnce++9ERGx//77t/rer3/96zjssMOitrY2br/99rj00kujsbExdt111xg/fnzLfQAAANpCRYPUYYcdFu+21sV7rYMxePDgePTRR7dYPVmW20cKAACqWKl5oFPtIwUAANARCFIAAACJBCkAAIBEHXr58/aWvTUAAIDqVGoeMCMFAACQSJACAABIJEgBAAAk0iNVpJDlUbCPFAAAVK3cPlIAAABtQ5ACAABI5NW+IoVCcxQKzZUuAwAAqJC8xDxgRgoAACCRIAUAAJBIkAIAAEikR6pIIdswAACA6pSXmAfMSAEAACQSpAAAABIJUgAAAIn0SBXJsjyyLK90GQAAQIWUmgfMSAEAACQSpAAAABIJUgAAAIn0SBWpyZqjptBc6TIAAIBKyUrLA2akAAAAEglSAAAAiQQpAACARHqkimRZHgX7SAEAQNVqto8UAABA2xCkAAAAEglSAAAAifRIFcneGgAAQHUqNQ+YkQIAAEgkSAEAACQSpAAAABLpkSpSsI8UAABUtVLzgBkpAACARIIUAABAIq/2FakpNEdNobnSZQAAABWSl5gHzEgBAAAkEqQAAAASCVIAAACJ9EgVsfw5AABUN8ufAwAAtBFBCgAAIJEgBQAAkEiPVBE9UgAAUN30SAEAALQRQQoAACCRIAUAAJBIj1SRmqw5arLmSpcBAABUSF5iHjAjBQAAkEiQAgAASCRIAQAAJNIjVcQ+UgAAUN3sIwUAANBGBCkAAIBEghQAAEAiPVJFsqz0dyIBAICuJ8tKu86MFAAAQCJBCgAAIJEgBQAAkEiPVBH7SAEAQHWzjxQAAEAbEaQAAAASebWvSLesKbplTZUuAwAAqJC8xDxgRgoAACCRIAUAAJBIkAIAAEikR6qI5c8BAKC6Wf4cAACgjQhSAAAAiQQpAACARHqkiuiRAgCA6qZHCgAAoI0IUgAAAIkEKQAAgER6pIp0y5qjW9Zc6TIAAIAKyUvMA2akAAAAEglSAAAAiQQpAACARHqkithHCgAAqpt9pAAAANqIIAUAAJBIkAIAAEikR6pIIWuOgn2kAACgapWaByo6IzVr1qwYM2ZMDBw4MLIsi3vuuafV+VNOOSWyLGs1Ro4c2eqa5cuXxwknnBB9+vSJvn37xrhx42LVqlXt+CsAAIBqU9EgtXr16thvv/1iypQpm71m5MiRsWTJkpZx2223tTp/wgknxHPPPRfTpk2L++67L2bNmhWnn356W5cOAABUsYq+2jdq1KgYNWrUu15TW1sbAwYM2OS53//+9zF16tR4/PHH46Mf/WhERFx33XUxevTouOqqq2LgwIFJ9RQijxrLnwMAQNVqii6y/PnMmTOjX79+seeee8ZXv/rVeP3111vOzZkzJ/r27dsSoiIiRowYEYVCIebOnbvZezY2NkZDQ0OrAQAAUKoOHaRGjhwZP/7xj2PGjBnx7W9/Ox566KEYNWpUNDU1RUTE0qVLo1+/fq2+061bt6ivr4+lS5du9r6TJk2Kurq6ljFo0KA2/R0AAEDX0qFX7fviF7/Y8s/77LNP7LvvvrH77rvHzJkzY/jw4WXfd+LEiTFhwoSWzw0NDcIUAABQsg4dpN5pt912i2233TZeeOGFGD58eAwYMCBee+21VtesX78+li9fvtm+qogNfVe1tbUbHe+WNUc3y58DAEDVau4My5+neuWVV+L111+P7bffPiIihg4dGm+88UY8+eSTLdc8+OCD0dzcHEOGDKlUmQAAQBdX0RmpVatWxQsvvNDyedGiRTF//vyor6+P+vr6uOyyy2Ls2LExYMCAWLhwYZx//vnxwQ9+MI444oiIiPjwhz8cI0eOjNNOOy1uuOGGWLduXZx99tnxxS9+MXnFPgAAgFJVdEbqiSeeiAMOOCAOOOCAiIiYMGFCHHDAAXHxxRdHTU1NPP300/HZz342PvShD8W4cePiwAMPjIcffrjVa3m33HJL7LXXXjF8+PAYPXp0DBs2LH7wgx9U6icBAABVIMvzvOo3TmpoaIi6uro46dfHRo9/6lHpcgAAgApZu2pt/PjwO2LFihXRp0+fzV7XqXqkAAAAOgJBCgAAIJEgBQAAkKhT7SPV1mqyPGqyqm8ZAwCAqlVqHjAjBQAAkEiQAgAASCRIAQAAJNIjVaRb1hzdsqZKlwEAAFRIc9Zc0nVmpAAAABIJUgAAAIkEKQAAgER6pIoUsjwK9pECAICqVWoeMCMFAACQSJACAABIJEgBAAAk0iNVpBDNUROlrRsPAAB0PYUS84AZKQAAgESCFAAAQCKv9hWx/DkAAFQ3y58DAAC0EUEKAAAgkSAFAACQSI9Uke5ZU3TPZEsAAKhWzVlTSddJDQAAAIkEKQAAgESCFAAAQCI9UkXsIwUAANXNPlIAAABtRJACAABIJEgBAAAk0iNVpCZrjpqsudJlAAAAFVJqHjAjBQAAkEiQAgAASCRIAQAAJNIjVaRb1hTdM9kSAACqVVPWVNJ1UgMAAEAiQQoAACCRIAUAAJBIj1SRQuRRiLzSZQAAABVSah4wIwUAAJBIkAIAAEgkSAEAACTSI1WkJmuOmqy50mUAAAAVUmoeMCMFAACQSJACAABI5NW+IoUsj4JX+wAAoGoVMsufAwAAtAlBCgAAIJEgBQAAkEiPVJHuWVN0z7JKlwEAAFRIU9ZU0nVmpAAAABIJUgAAAIkEKQAAgER6pIrURB41Udq68QAAQNdTah4wIwUAAJBIkAIAAEgkSAEAACTSI1Uky5qjkDVXugwAAKBCshLzgBkpAACARIIUAABAIkEKAAAgkR6pIt2zpuieZZUuAwAAqJD1WVNJ15mRAgAASCRIAQAAJBKkAAAAEumRKlITzVET9pECAIBqVWoeMCMFAACQSJACAABI5NW+IjVZc9RkXu0DAIBqVWoeMCMFAACQSJACAABIJEgBAAAk0iNVpBDNUbD8OQAAVK1S84AZKQAAgESCFAAAQCJBCgAAIJEeqSLds+bokTVVugwAAKBC1ttHCgAAoG0IUgAAAIkEKQAAgER6pIoUojkKWVbpMgAAgAqxjxQAAEAbEaQAAAASCVIAAACJ9EgVqYk8aiKvdBkAAECFlJoHzEgBAAAkqmiQmjVrVowZMyYGDhwYWZbFPffc0+p8lmWbHFdeeWXLNbvssstG5ydPntzOvwQAAKgmFQ1Sq1evjv322y+mTJmyyfNLlixpNX74wx9GlmUxduzYVtddfvnlra4755xz2qN8AACgSlW0R2rUqFExatSozZ4fMGBAq88///nP4/DDD4/ddtut1fHevXtvdO27aWxsjMbGxpbPDQ0NERHRPVsf3TM9UgAAUK26Z00lXddpeqSWLVsW999/f4wbN26jc5MnT45tttkmDjjggLjyyitj/fr173qvSZMmRV1dXcsYNGhQW5UNAAB0QZ1m1b6bb745evfuHUcddVSr4+eee24MHjw46uvr4ze/+U1MnDgxlixZEt/5znc2e6+JEyfGhAkTWj43NDQIUwAAQMk6TZD64Q9/GCeccEL07Nmz1fHiQLTvvvtGjx494owzzohJkyZFbW3tJu9VW1u72XMAAADvpVMEqYcffjgWLFgQd9xxx3teO2TIkFi/fn289NJLseeeeyb9nZqsOWqyrNwyAQCATq4may7puk7RI3XjjTfGgQceGPvtt997Xjt//vwoFArRr1+/dqgMAACoRhWdkVq1alW88MILLZ8XLVoU8+fPj/r6+thpp50iYkP/0k9/+tO4+uqrN/r+nDlzYu7cuXH44YdH7969Y86cOTF+/Pj40pe+FFtvvXW7/Q4AAKC6VDRIPfHEE3H44Ye3fH673+nkk0+Om266KSIibr/99sjzPI477riNvl9bWxu33357XHrppdHY2Bi77rprjB8/vlXfVIqayKMmLH8OAADVqtQ8kOV5XvXJoaGhIerq6uKn8/eKrXrXVLocAACgQtasbIov7P+HWLFiRfTp02ez13WKHikAAICORJACAABI1CmWP28vWTRHISx/DgAA1SqLLrT8OQAAQEciSAEAACQSpAAAABLpkSrSI2uKHlqkAACgaq3Lmkq6zowUAABAIkEKAAAgkSAFAACQSI9UkUKWRyErbd14AACg6ylkeWnXtXEdAAAAXY4gBQAAkEiQAgAASKRHqkhNNEdNpYsAAAAqpiZKWzPBjBQAAEAiQQoAACCRIAUAAJBIj1SR7llTdC9x3XgAAKDr6V7ivrJmpAAAABIJUgAAAIkEKQAAgER6pIoUIo+a0CMFAADVqlBiHjAjBQAAkEiQAgAASOTVviI1Xu0DAICqVmoeMCMFAACQSJACAABIJEgBAAAk0iNVpJDlUcj0SAEAQLUqNQ+YkQIAAEgkSAEAACQSpAAAABLpkSrSPZqje6WLAAAAKqZ7NJd0nRkpAACARIIUAABAIkEKAAAgkR6pIjVZHjX2kQIAgKpVah4wIwUAAJBIkAIAAEgkSAEAACTSI1WkJvKoCT1SAABQrUrNA2akAAAAEglSAAAAiQQpAACARHqkinTLIrpnla4CAAColG4l5gEzUgAAAIkEKQAAgERe7StSCMkSAACqWal5QG4AAABIJEgBAAAkEqQAAAAS6ZEqUvPWAAAAqlOpecCMFAAAQCJBCgAAIJEgBQAAkEiPVJGaLIuaLKt0GQAAQIWUmgfMSAEAACQSpAAAABIJUgAAAIn0SBXpHll0Dz1SAABQrUrNA2akAAAAEglSAAAAiQQpAACARHqkimRZFgX7SAEAQNXK7CMFAADQNgQpAACARIIUAABAIj1SRWoiixr7SAEAQNUqNQ+YkQIAAEgkSAEAACQSpAAAABLpkSrSPauJ7plsCQAA1aq7faQAAADahiAFAACQyKt9RQpv/R8AAFCdSk0DUgMAAEAiQQoAACCRIAUAAJBIj1SRQmRRiNKWOwQAALqeUvOAGSkAAIBEghQAAEAiQQoAACCRHqkiNVkhajLZEgAAqlVNiUsmSA0AAACJBCkAAIBEXu2LiDzPIyKiYVVzhSsBAAAq6e1M8HZG2BxBKiJWrlwZERE7D36psoUAAAAdwsqVK6Ourm6z57P8vaJWFWhubo5XX301evfuHVlmQ95SNDQ0xKBBg+Lll1+OPn36VLocujDPGu3Fs0Z78azRXjxr5cnzPFauXBkDBw6MQmHznVBmpCKiUCjEjjvuWOkyOqU+ffr4NybtwrNGe/Gs0V48a7QXz1q6d5uJepvFJgAAABIJUgAAAIkEKcpSW1sbl1xySdTW1la6FLo4zxrtxbNGe/Gs0V48a23LYhMAAACJzEgBAAAkEqQAAAASCVIAAACJBCkAAIBEghQlW758eZxwwgnRp0+f6Nu3b4wbNy5WrVpV0nfzPI9Ro0ZFlmVxzz33tG2hdHqpz9ry5cvjnHPOiT333DN69eoVO+20U5x77rmxYsWKdqyazmDKlCmxyy67RM+ePWPIkCHx2GOPvev1P/3pT2OvvfaKnj17xj777BO/+MUv2qlSOruUZ+2///u/45BDDomtt946tt566xgxYsR7PpvwttT/XHvb7bffHlmWxZFHHtm2BXZhghQlO+GEE+K5556LadOmxX333RezZs2K008/vaTvXnPNNZFlWRtXSFeR+qy9+uqr8eqrr8ZVV10Vzz77bNx0000xderUGDduXDtWTUd3xx13xIQJE+KSSy6JefPmxX777RdHHHFEvPbaa5u8/je/+U0cd9xxMW7cuHjqqafiyCOPjCOPPDKeffbZdq6czib1WZs5c2Ycd9xx8etf/zrmzJkTgwYNik9/+tPx5z//uZ0rp7NJfdbe9tJLL8XXvva1OOSQQ9qp0i4qhxL87ne/yyMif/zxx1uOPfDAA3mWZfmf//znd/3uU089le+www75kiVL8ojI77777jauls7s/Txrxe688868R48e+bp169qiTDqhgw46KD/rrLNaPjc1NeUDBw7MJ02atMnrjznmmPwzn/lMq2NDhgzJzzjjjDatk84v9Vl7p/Xr1+e9e/fOb7755rYqkS6inGdt/fr1+cEHH5z/z//8T37yySfnn/vc59qh0q7JjBQlmTNnTvTt2zc++tGPthwbMWJEFAqFmDt37ma/t2bNmjj++ONjypQpMWDAgPYolU6u3GftnVasWBF9+vSJbt26tUWZdDJr166NJ598MkaMGNFyrFAoxIgRI2LOnDmb/M6cOXNaXR8RccQRR2z2eogo71l7pzVr1sS6deuivr6+rcqkCyj3Wbv88sujX79+3trYAvx/GJRk6dKl0a9fv1bHunXrFvX19bF06dLNfm/8+PFx8MEHx+c+97m2LpEuotxnrdhf//rX+MY3vlHyq6d0fX/961+jqakp+vfv3+p4//794w9/+MMmv7N06dJNXl/qc0h1KudZe6cLLrggBg4cuFGQh2LlPGuPPPJI3HjjjTF//vx2qLDrMyNV5S688MLIsuxdR6n/wf9O9957bzz44INxzTXXbNmi6ZTa8lkr1tDQEJ/5zGfiIx/5SFx66aXvv3CAdjR58uS4/fbb4+67746ePXtWuhy6kJUrV8aJJ54Y//3f/x3bbrttpcvpEsxIVbl/+7d/i1NOOeVdr9ltt91iwIABGzUurl+/PpYvX77ZV/YefPDBWLhwYfTt27fV8bFjx8YhhxwSM2fOfB+V09m05bP2tpUrV8bIkSOjd+/ecffdd0f37t3fb9l0Edtuu23U1NTEsmXLWh1ftmzZZp+rAQMGJF0PEeU9a2+76qqrYvLkyTF9+vTYd99927JMuoDUZ23hwoXx0ksvxZgxY1qONTc3R8SGNz8WLFgQu+++e9sW3cUIUlVuu+22i+222+49rxs6dGi88cYb8eSTT8aBBx4YERuCUnNzcwwZMmST37nwwgvjK1/5Sqtj++yzT3z3u99t9W9iqkNbPmsRG2aijjjiiKitrY17773X/5JLKz169IgDDzwwZsyY0bLUb3Nzc8yYMSPOPvvsTX5n6NChMWPGjDjvvPNajk2bNi2GDh3aDhXTWZXzrEVEXHHFFfGtb30rfvnLX7bqEYXNSX3W9tprr3jmmWdaHbvoooti5cqV8V//9V8xaNCg9ii7a6n0ahd0HiNHjswPOOCAfO7cufkjjzyS77HHHvlxxx3Xcv6VV17J99xzz3zu3LmbvUdYtY8SpD5rK1asyIcMGZLvs88++QsvvJAvWbKkZaxfv75SP4MO5vbbb89ra2vzm266Kf/d736Xn3766Xnfvn3zpUuX5nme5yeeeGJ+4YUXtlw/e/bsvFu3bvlVV12V//73v88vueSSvHv37vkzzzxTqZ9AJ5H6rE2ePDnv0aNH/rOf/azVf36tXLmyUj+BTiL1WXsnq/a9P2akKNktt9wSZ599dgwfPjwKhUKMHTs2rr322pbz69atiwULFsSaNWsqWCVdQeqzNm/evJYV/T74wQ+2uteiRYtil112abfa6biOPfbY+Mtf/hIXX3xxLF26NPbff/+YOnVqS6P24sWLo1D4R+vwwQcfHLfeemtcdNFF8e///u+xxx57xD333BN77713pX4CnUTqs/b9738/1q5dG0cffXSr+1xyySV6PXlXqc8aW1aW53le6SIAAAA6ExEVAAAgkSAFAACQSJACAABIJEgBAAAkEqQAAAASCVIAAACJBCkAAIBEghQAAEAiQQqAqnDYYYfFeeed1+nuDUDH1K3SBQBAZ3fXXXdF9+7dK10GAO1IkAKA96m+vr7SJQDQzrzaB0CH09zcHJMmTYpdd901evXqFfvtt1/87Gc/i4iImTNnRpZl8ctf/jIOOOCA6NWrV3zyk5+M1157LR544IH48Ic/HH369Injjz8+1qxZ0+q+69evj7PPPjvq6upi2223ja9//euR53lJNX3ve9+LPfbYI3r27Bn9+/ePo48+uuVc8at9b9f3znHKKae0XP/zn/88Bg8eHD179ozddtstLrvssli/fv37+xcNgHZlRgqADmfSpEnxk5/8JG644YbYY489YtasWfGlL30ptttuu5ZrLr300rj++utjq622imOOOSaOOeaYqK2tjVtvvTVWrVoVn//85+O6666LCy64oOU7N998c4wbNy4ee+yxeOKJJ+L000+PnXbaKU477bR3reeJJ56Ic889N/7f//t/cfDBB8fy5cvj4Ycf3uS1Bx98cCxZsqTl8+9///sYPXp0HHrooRER8fDDD8dJJ50U1157bRxyyCGxcOHCOP300yMi4pJLLin7XzMA2leWl/o/xQFAO2hsbIz6+vqYPn16DB06tOX4V77ylVizZk2cfvrpcfjhh8f06dNj+PDhERExefLkmDhxYixcuDB22223iIg488wz46WXXoqpU6dGxIZZo9deey2ee+65yLIsIiIuvPDCuPfee+N3v/vdu9Z01113xamnnhqvvPJK9O7de6Pzhx12WOy///5xzTXXtDr++uuvx0EHHRQjR46MKVOmRETEiBEjYvjw4TFx4sSW637yk5/E+eefH6+++mriv1oAVIoZKQA6lBdeeCHWrFkTn/rUp1odX7t2bRxwwAEtn/fdd9+Wf+7fv39stdVWLSHq7WOPPfZYq3v8y7/8S0uIiogYOnRoXH311dHU1BQ1NTWbrelTn/pU7LzzzrHbbrvFyJEjY+TIkfH5z38+ttpqq81+Z926dTF27NjYeeed47/+679ajv/2t7+N2bNnx7e+9a2WY01NTfHmm2/GmjVr3vWeAHQcghQAHcqqVasiIuL++++PHXbYodW52traWLhwYUREq1XysizbaNW8LMuiubl5i9TUu3fvmDdvXsycOTN+9atfxcUXXxyXXnppPP7449G3b99NfuerX/1qvPzyy/HYY49Ft27/+K/bVatWxWWXXRZHHXXURt/p2bPnFqkXgLYnSAHQoXzkIx+J2traWLx4cXziE5/Y6PzbQaocc+fObfX50UcfjT322ONdZ6Pe1q1btxgxYkSMGDEiLrnkkujbt288+OCDmwxE3/nOd+LOO++M3/zmN7HNNtu0Ojd48OBYsGBBfPCDHyz7dwBQeYIUAB1K796942tf+1qMHz8+mpubY9iwYbFixYqYPXt29OnTJ3beeeey77148eKYMGFCnHHGGTFv3ry47rrr4uqrr37P7913333x4osvxqGHHhpbb711/OIXv4jm5ubYc889N7p2+vTpcf7558eUKVNi2223jaVLl0ZERK9evaKuri4uvvji+Nd//dfYaaed4uijj45CoRC//e1v49lnn41vfvObZf82ANqXIAVAh/ONb3wjtttuu5g0aVK8+OKL0bdv3xg8eHD8+7//+/t6Xe+kk06Kv//973HQQQdFTU1N/J//839aVsx7N3379o277rorLr300njzzTdjjz32iNtuuy3++Z//eaNrH3nkkWhqaoozzzwzzjzzzJbjJ598ctx0001xxBFHxH333ReXX355fPvb347u3bvHXnvtFV/5ylfK/l0AtD+r9gEAACSyIS8AAEAiQQqAqvfwww/HP/3TP212AMA7ebUPgKr397//Pf785z9v9rwV9gB4J0EKAAAgkVf7AAAAEglSAAAAiQQpAACARIIUAABAIkEKAAAgkSAFAACQSJACAABI9P8BsQHM0k+0a/gAAAAASUVORK5CYII=\n" - }, - "metadata": {} - } - ] - }, - { - "cell_type": "code", - "source": [ - "fig, ax = plt.subplots(figsize=(10, 10))\n", - "plt.imshow(tmp_layer.pe[0, 50:51, :], aspect=\"auto\")\n", - "plt.xlabel(\"emb_size\")\n", - "plt.ylabel(\"max_seq_len\")\n", - "plt.show()" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 850 - }, - "id": "NDEk6GqycZRX", - "outputId": "fdf0ca04-be31-4076-818f-eedbdc176aaf" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": [ - "
" - ], - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1kAAANBCAYAAAAShHTFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA1v0lEQVR4nO3deZjWdb3/8dcMywwoi7iAC4gaJzRX5IcHl9PCpEQ/Oy6ppbkQaXYkl2lRTj/BMkVNzFTSn5462knNMjt5LEnEMPWQKKhpx6XjcnADMtJhUUDm/v3R5fyaZAYcPjDe8nhc11yX93e5v+/7vr6IT7/3/Z2aSqVSCQAAAEXUdvYAAAAA7yUiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgrp29gDvds3NzXnppZfSq1ev1NTUdPY4AABAJ6lUKlm8eHG22Wab1Na2fb1KZK3BSy+9lIEDB3b2GAAAwLvE888/n+22267N9SJrDXr16pUk+btxE9Ole/1qt7nztO+2uf9+P/lCu88/+OzZ7a6v3aRnu+uX/nDLNtdtOqFbu/u+2Xv1r+ctXf/7xXbXd7u+7ee/cacZ7e477Ppx7a7f4WdN7a7/x+/f3ea6mxsPbHff7i+82u76za9Z1O76P3267fft+nt+1e6+H76s/fPh9i9e3u76g393dJvrzhoyrd19/7BiQLvrFyzv3e76Q/vObXPd+f8zpt19r9zpp+2uP+yRE9pdf8deP2xz3QH3fbbdfWfv/2/trh92V/vHfvgj17W5bs9fjW1330dHt71vkuz2i/aP/ejHr21z3e63tn/s333iX9tdv/vP17D/P7a9/+4/a/89/92h3293/R5r2P+RdvZfl32TZI9b1rD/Ye0cex32dewqPPZP17Dv4Ws49jrs79iO7djv3mM3LWnO9sOea2mEtoisNXjrI4JdutenS93q/+O6d6+2LxXW1q8hZGraD6Hamu7t779JXdvrurT/3Om6htlq2z92t03afv723pNkLd6XLsvbXd9j07ZP3a5rel1d2n7PkqTbJmt4z9t5X9b0uts6h9Z6/55tz96zV5d2961f3v4f97pu7Z8vm7QzW3vnYZL0WofXlazhz1jPdXtPa3t0fP912XejPvYa/vyvy79THduxHduxHdux1/exk6zxa0RufAEAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFVU1kLVq0KMccc0x69+6dvn37Zty4cVmyZMla7VupVPKxj30sNTU1+fd///f1OygAALBRq5rIOuaYY/L73/8+06dPz2233Zbf/OY3Oemkk9Zq30svvTQ1NTXreUIAAICka2cPsDYef/zxTJs2LQ888ECGDx+eJLn88sszZsyYXHzxxdlmm23a3Pfhhx/OlClT8uCDD2brrbfeUCMDAAAbqaq4kjVr1qz07du3JbCSpKGhIbW1tbn//vvb3G/ZsmU5+uijM3Xq1AwYMGBDjAoAAGzkquJK1vz587PVVlu1Wta1a9f069cv8+fPb3O/M844I/vuu2/+8R//ca2PtXz58ixfvrzlcVNT0zsfGAAA2Gh16pWss846KzU1Ne3+PPHEEx167ltvvTV33XVXLr300ne03+TJk9OnT5+Wn4EDB3bo+AAAwMapU69kfelLX8oJJ5zQ7jY77rhjBgwYkIULF7Za/uabb2bRokVtfgzwrrvuytNPP52+ffu2Wn744YfngAMOyMyZM1e734QJE9LY2NjyuKmpSWgBAABrrVMja8stt8yWW265xu1GjhyZV199NXPmzMnee++d5C8R1dzcnH322We1+5x11ln53Oc+12rZbrvtlm9/+9s5+OCD2zxWXV1d6urq3sGrAAAA+P+q4jtZO++8c0aPHp0TTzwxV111VVauXJnx48fnU5/6VMudBV988cWMGjUqP/jBDzJixIgMGDBgtVe5Bg0alB122GFDvwQAAGAjURV3F0yS66+/PkOHDs2oUaMyZsyY7L///rn66qtb1q9cuTJPPvlkli1b1olTAgAAG7uquJKVJP369csNN9zQ5vrBgwenUqm0+xxrWg8AALCuquZKFgAAQDUQWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAACioaiJr0aJFOeaYY9K7d+/07ds348aNy5IlS9rd/otf/GLe//73p0ePHhk0aFBOPfXUvPbaaxtwagAAYGNTNZF1zDHH5Pe//32mT5+e2267Lb/5zW9y0kkntbn9Sy+9lJdeeikXX3xxHnvssVx77bWZNm1axo0btwGnBgAANjZdO3uAtfH4449n2rRpeeCBBzJ8+PAkyeWXX54xY8bk4osvzjbbbPO2fXbdddf89Kc/bXm800475bzzzstnPvOZvPnmm+natSpeOgAAUGWq4krWrFmz0rdv35bASpKGhobU1tbm/vvvX+vnee2119K7d2+BBQAArDdVURvz58/PVltt1WpZ165d069fv8yfP3+tnuOVV17Jueee2+5HDJNk+fLlWb58ecvjpqamdz4wAACw0erUK1lnnXVWampq2v154okn1vk4TU1N+fjHP55ddtkl55xzTrvbTp48OX369Gn5GThw4DofHwAA2Hh06pWsL33pSznhhBPa3WbHHXfMgAEDsnDhwlbL33zzzSxatCgDBgxod//Fixdn9OjR6dWrV372s5+lW7du7W4/YcKENDY2tjxuamoSWgAAwFrr1Mjacssts+WWW65xu5EjR+bVV1/NnDlzsvfeeydJ7rrrrjQ3N2efffZpc7+mpqYcdNBBqaury6233pr6+vo1Hquuri51dXVr/yIAAAD+SlXc+GLnnXfO6NGjc+KJJ2b27Nm57777Mn78+HzqU59qubPgiy++mKFDh2b27NlJ/hJYBx54YJYuXZrvfe97aWpqyvz58zN//vysWrWqM18OAADwHlYVN75Ikuuvvz7jx4/PqFGjUltbm8MPPzyXXXZZy/qVK1fmySefzLJly5Ikc+fObbnz4Pve975Wz/Xss89m8ODBG2x2AABg41E1kdWvX7/ccMMNba4fPHhwKpVKy+MPfehDrR4DAABsCFXxcUEAAIBqIbIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAV17chOS5cuzQUXXJAZM2Zk4cKFaW5ubrX+mWeeKTIcAABAtelQZH3uc5/L3XffnWOPPTZbb711ampqSs8FAABQlToUWbfffnt+8YtfZL/99is9DwAAQFXr0HeyNttss/Tr16/0LAAAAFWvQ5F17rnnZuLEiVm2bFnpeQAAAKpahz4uOGXKlDz99NPp379/Bg8enG7durVaP3fu3CLDAQAAVJsORdYhhxxSeAwAAID3hg5F1qRJk0rPAQAA8J7Q4V9G/Oqrr+Zf/uVfMmHChCxatCjJXz4m+OKLLxYbDgAAoNp06ErW7373uzQ0NKRPnz557rnncuKJJ6Zfv3655ZZbMm/evPzgBz8oPScAAEBV6NCVrMbGxpxwwgn5wx/+kPr6+pblY8aMyW9+85tiwwEAAFSbDkXWAw88kM9//vNvW77ttttm/vz56zwUAABAtepQZNXV1aWpqelty5966qlsueWW6zwUAABAtepQZH3iE5/IN77xjaxcuTJJUlNTk3nz5uXMM8/M4YcfXnRAAACAatKhyJoyZUqWLFmSrbbaKq+//no++MEP5n3ve1969eqV8847r/SMAAAAVaNDdxfs06dPpk+fnnvvvTe/+93vsmTJkgwbNiwNDQ2l5wMAAKgqHYqst+y///7Zf//9S80CAABQ9dY6si677LK1ftJTTz21Q8MAAABUu7WOrG9/+9trtV1NTY3IAgAANlprHVnPPvvs+pwDAADgPaFDdxdcW717984zzzyzPg8BAADwrrJeI6tSqazPpwcAAHjXWa+RBQAAsLERWQAAAAWJLAAAgILWa2TV1NSsz6cHAAB413HjCwAAgILWa2Tdfvvt2XbbbdfnIQAAAN5V1vqXEf+1xsbGtd52//3378ghAAAAqlKHIuuhhx7KQw89lJUrV+b9739/kuSpp55Kly5dMmzYsJbtfCcLAADY2HQosg4++OD06tUr1113XTbbbLMkyZ///OeMHTs2BxxwQL70pS8VHRIAAKBadOg7WVOmTMnkyZNbAitJNttss3zzm9/MlClTig0HAABQbToUWU1NTfnjH//4tuV//OMfs3jx4nUeCgAAoFp1KLIOPfTQjB07NrfcckteeOGFvPDCC/npT3+acePG5bDDDis9IwAAQNXo0Heyrrrqqnz5y1/O0UcfnZUrV/7libp2zbhx4/Ktb32r6IAAAADVpEOR1bNnz3z3u9/Nt771rTz99NNJkp122imbbLJJ0eEAAACqzTr9MuKXX345L7/8coYMGZJNNtkklUql1FwAAABVqUOR9ac//SmjRo3K3/3d32XMmDF5+eWXkyTjxo1b77dvnzp1agYPHpz6+vrss88+mT17drvb/+QnP8nQoUNTX1+f3XbbLb/85S/X63wAAMDGrUORdcYZZ6Rbt26ZN29eevbs2bL8qKOOyrRp04oN97duuummNDY2ZtKkSZk7d2722GOPHHTQQVm4cOFqt//P//zPfPrTn864cePy0EMP5ZBDDskhhxySxx57bL3NCAAAbNw6FFl33HFHLrzwwmy33Xatlg8ZMiT/8z//U2Sw1bnkkkty4oknZuzYsdlll11y1VVXpWfPnvn+97+/2u2/853vZPTo0fnKV76SnXfeOeeee26GDRuWK664Yr3NCAAAbNw6FFlLly5tdQXrLYsWLUpdXd06D7U6K1asyJw5c9LQ0NCyrLa2Ng0NDZk1a9Zq95k1a1ar7ZPkoIMOanP7JFm+fHmamppa/QAAAKytDkXWAQcckB/84Actj2tqatLc3JyLLrooH/7wh4sN99deeeWVrFq1Kv3792+1vH///pk/f/5q95k/f/472j5JJk+enD59+rT8DBw4cN2HBwAANhoduoX7RRddlFGjRuXBBx/MihUr8tWvfjW///3vs2jRotx3332lZ9ygJkyYkMbGxpbHTU1NQgsAAFhrHYqsXXfdNU899VSuuOKK9OrVK0uWLMlhhx2WU045JVtvvXXpGZMkW2yxRbp06ZIFCxa0Wr5gwYIMGDBgtfsMGDDgHW2fJHV1devtI48AAMB7X4ciK0n69OmTr33tayVnaVf37t2z9957Z8aMGTnkkEOSJM3NzZkxY0bGjx+/2n1GjhyZGTNm5PTTT29ZNn369IwcOXIDTAwAAGyMOvSdrGnTpuXee+9teTx16tTsueeeOfroo/PnP/+52HB/q7GxMddcc02uu+66PP744/nCF76QpUuXZuzYsUmS4447LhMmTGjZ/rTTTsu0adMyZcqUPPHEEznnnHPy4IMPthllAAAA66pDkfWVr3yl5a57jz76aBobGzNmzJg8++yzrb7PVNpRRx2Viy++OBMnTsyee+6Zhx9+ONOmTWu5ucW8efNafjFykuy777654YYbcvXVV2ePPfbIzTffnH//93/Prrvuut5mBAAANm4d+rjgs88+m1122SVJ8tOf/jQHH3xwzj///MydOzdjxowpOuDfGj9+fJtXombOnPm2ZUcccUSOOOKI9ToTAADAWzp0Jat79+5ZtmxZkuTOO+/MgQcemCTp16+f3ysFAABs1Dp0JWv//fdPY2Nj9ttvv8yePTs33XRTkuSpp57KdtttV3RAAACAatKhK1lXXHFFunbtmptvvjlXXnlltt122yTJ7bffntGjRxcdEAAAoJp06ErWoEGDctttt71t+be//e1Wjy+44IKcfPLJ6du3b4eGAwAAqDYdupK1ts4///wsWrRofR4CAADgXWW9RlalUlmfTw8AAPCus14jCwAAYGMjsgAAAAoSWQAAAAWJLAAAgILWa2QdcMAB6dGjx/o8BAAAwLtKhyLr2muvXe3yN998MxMmTGh5/Mtf/jJbb711hwYDAACoRh2KrFNPPTVHHHFE/vznP7cse/LJJ7PPPvvkxhtvLDYcAABAtelQZD300EN54YUXsttuu2X69OmZOnVqhg0blqFDh+aRRx4pPSMAAEDV6NqRnXbaaafcd999Of300zN69Oh06dIl1113XT796U+Xng8AAKCqdPjGF7/4xS/yox/9KCNHjkzfvn3zve99Ly+99FLJ2QAAAKpOhyLr85//fI444oiceeaZueeee/K73/0u3bt3z2677ZYf//jHpWcEAACoGh36uOB9992X+++/P3vssUeSZMCAAfnlL3+ZqVOn5rOf/WyOPPLIokMCAABUiw5F1pw5c1JXV/e25aecckoaGhrWeSgAAIBq1aGPC64usN7y/ve/v8PDAAAAVLsOXclKkptvvjk//vGPM2/evKxYsaLVurlz567zYAAAANWoQ1eyLrvssowdOzb9+/fPQw89lBEjRmTzzTfPM888k4997GOlZwQAAKgaHYqs7373u7n66qtz+eWXp3v37vnqV7+a6dOn59RTT81rr71WekYAAICq0aHImjdvXvbdd98kSY8ePbJ48eIkybHHHpsbb7yx3HQAAABVpkORNWDAgCxatChJMmjQoPz2t79Nkjz77LOpVCrlpgMAAKgyHYqsj3zkI7n11luTJGPHjs0ZZ5yRj370oznqqKNy6KGHFh0QAACgmnTo7oJXX311mpubk/zld2NtscUWue+++/KJT3wiJ598ctEBAQAAqkmHIqu2tjYrVqzI3Llzs3DhwvTo0aPllxBPmzYtBx98cNEhAQAAqkWHImvatGk59thj86c//elt62pqarJq1ap1HgwAAKAadeg7WV/84hdz5JFH5uWXX05zc3OrH4EFAABszDoUWQsWLEhjY2P69+9feh4AAICq1qHI+uQnP5mZM2cWHgUAAKD6deg7WVdccUWOOOKI3HPPPdltt93SrVu3VutPPfXUIsMBAABUmw5F1o033pg77rgj9fX1mTlzZmpqalrW1dTUiCwAAGCj1aHI+trXvpavf/3rOeuss1Jb26FPHAIAALwndaiQVqxYkaOOOkpgAQAA/I0OVdLxxx+fm266qfQsAAAAVa9DHxdctWpVLrroovzqV7/K7rvv/rYbX1xyySVFhgMAAKg2HYqsRx99NHvttVeS5LHHHmu17q9vggEAALCx6VBk/frXvy49BwAAwHuCO1cAAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKCgqousqVOnZvDgwamvr88+++yT2bNnt7ntNddckwMOOCCbbbZZNttsszQ0NLS7PQAAwLqqqsi66aab0tjYmEmTJmXu3LnZY489ctBBB2XhwoWr3X7mzJn59Kc/nV//+teZNWtWBg4cmAMPPDAvvvjiBp4cAADYWFRVZF1yySU58cQTM3bs2Oyyyy656qqr0rNnz3z/+99f7fbXX399/umf/il77rlnhg4dmn/5l39Jc3NzZsyYsYEnBwAANhZVE1krVqzInDlz0tDQ0LKstrY2DQ0NmTVr1lo9x7Jly7Jy5cr069dvfY0JAABs5Lp29gBr65VXXsmqVavSv3//Vsv79++fJ554Yq2e48wzz8w222zTKtT+1vLly7N8+fKWx01NTR0bGAAA2ChVzZWsdXXBBRfkRz/6UX72s5+lvr6+ze0mT56cPn36tPwMHDhwA04JAABUu6qJrC222CJdunTJggULWi1fsGBBBgwY0O6+F198cS644ILccccd2X333dvddsKECXnttddafp5//vl1nh0AANh4VE1kde/ePXvvvXerm1a8dROLkSNHtrnfRRddlHPPPTfTpk3L8OHD13icurq69O7du9UPAADA2qqa72QlSWNjY44//vgMHz48I0aMyKWXXpqlS5dm7NixSZLjjjsu2267bSZPnpwkufDCCzNx4sTccMMNGTx4cObPn58k2XTTTbPpppt22usAAADeu6oqso466qj88Y9/zMSJEzN//vzsueeemTZtWsvNMObNm5fa2v9/ce7KK6/MihUr8slPfrLV80yaNCnnnHPOhhwdAADYSFRVZCXJ+PHjM378+NWumzlzZqvHzz333PofCAAA4K9UzXeyAAAAqoHIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEFVF1lTp07N4MGDU19fn3322SezZ89eq/1+9KMfpaamJocccsj6HRAAANioVVVk3XTTTWlsbMykSZMyd+7c7LHHHjnooIOycOHCdvd77rnn8uUvfzkHHHDABpoUAADYWFVVZF1yySU58cQTM3bs2Oyyyy656qqr0rNnz3z/+99vc59Vq1blmGOOyde//vXsuOOOG3BaAABgY1Q1kbVixYrMmTMnDQ0NLctqa2vT0NCQWbNmtbnfN77xjWy11VYZN27chhgTAADYyHXt7AHW1iuvvJJVq1alf//+rZb3798/TzzxxGr3uffee/O9730vDz/88FofZ/ny5Vm+fHnL46ampg7NCwAAbJyq5krWO7V48eIce+yxueaaa7LFFlus9X6TJ09Onz59Wn4GDhy4HqcEAADea6rmStYWW2yRLl26ZMGCBa2WL1iwIAMGDHjb9k8//XSee+65HHzwwS3LmpubkyRdu3bNk08+mZ122ult+02YMCGNjY0tj5uamoQWAACw1qomsrp375699947M2bMaLkNe3Nzc2bMmJHx48e/bfuhQ4fm0UcfbbXs//yf/5PFixfnO9/5TpvhVFdXl7q6uuLzAwAAG4eqiawkaWxszPHHH5/hw4dnxIgRufTSS7N06dKMHTs2SXLcccdl2223zeTJk1NfX59dd9211f59+/ZNkrctBwAAKKWqIuuoo47KH//4x0ycODHz58/PnnvumWnTprXcDGPevHmprX3Pfs0MAACoAlUVWUkyfvz41X48MElmzpzZ7r7XXntt+YEAAAD+iss+AAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKEhkAQAAFCSyAAAAChJZAAAABYksAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAFiSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEiCwAAoCCRBQAAUJDIAgAAKKhrZw/wblepVJIkq1a80eY2TYub21zX/Ebb+yXJm5WV7a6vraxof/+ly9tet6rtuZLkzTdr2l2f5vaPXbO00ua69t6TZC3el1Vtv64keX3Jm23v+2b7z127hudeuXQN73lz2/9vYk2ve9Xy9mdb4/7L2p592eJV7e77xoq237MkWb68/XNxade2Z2vvPEySxevwupI1/Blbtm7vafPrHd9/XfbdqI+9hj//6/LvVMd2bMd2bMd27PV57KYlf1n3ViO0paaypi02ci+88EIGDhzY2WMAAADvEs8//3y22267NteLrDVobm7OSy+9lF69eqWmpiZNTU0ZOHBgnn/++fTu3buzx+M9zLnGhuJcY0NxrrGhONdYXyqVShYvXpxtttkmtbVtf7rJxwXXoLa2drWV2rt3b39o2SCca2wozjU2FOcaG4pzjfWhT58+a9zGjS8AAAAKElkAAAAFiax3qK6uLpMmTUpdXV1nj8J7nHONDcW5xobiXGNDca7R2dz4AgAAoCBXsgAAAAoSWQAAAAWJLAAAgIJEFgAAQEEi6x2YOnVqBg8enPr6+uyzzz6ZPXt2Z49ElZs8eXL+1//6X+nVq1e22mqrHHLIIXnyySdbbfPGG2/klFNOyeabb55NN900hx9+eBYsWNBJE/NeccEFF6Smpiann356yzLnGqW8+OKL+cxnPpPNN988PXr0yG677ZYHH3ywZX2lUsnEiROz9dZbp0ePHmloaMgf/vCHTpyYarRq1aqcffbZ2WGHHdKjR4/stNNOOffcc/PX93RzrtFZRNZauummm9LY2JhJkyZl7ty52WOPPXLQQQdl4cKFnT0aVezuu+/OKaeckt/+9reZPn16Vq5cmQMPPDBLly5t2eaMM87If/zHf+QnP/lJ7r777rz00ks57LDDOnFqqt0DDzyQ//t//2923333Vsuda5Tw5z//Ofvtt1+6deuW22+/Pf/1X/+VKVOmZLPNNmvZ5qKLLspll12Wq666Kvfff3822WSTHHTQQXnjjTc6cXKqzYUXXpgrr7wyV1xxRR5//PFceOGFueiii3L55Ze3bONco9NUWCsjRoyonHLKKS2PV61aVdlmm20qkydP7sSpeK9ZuHBhJUnl7rvvrlQqlcqrr75a6datW+UnP/lJyzaPP/54JUll1qxZnTUmVWzx4sWVIUOGVKZPn1754Ac/WDnttNMqlYpzjXLOPPPMyv7779/m+ubm5sqAAQMq3/rWt1qWvfrqq5W6urrKjTfeuCFG5D3i4x//eOWzn/1sq2WHHXZY5ZhjjqlUKs41OpcrWWthxYoVmTNnThoaGlqW1dbWpqGhIbNmzerEyXivee2115Ik/fr1S5LMmTMnK1eubHXuDR06NIMGDXLu0SGnnHJKPv7xj7c6pxLnGuXceuutGT58eI444ohstdVW2WuvvXLNNde0rH/22Wczf/78Vudanz59ss8++zjXeEf23XffzJgxI0899VSS5JFHHsm9996bj33sY0mca3Surp09QDV45ZVXsmrVqvTv37/V8v79++eJJ57opKl4r2lubs7pp5+e/fbbL7vuumuSZP78+enevXv69u3batv+/ftn/vz5nTAl1exHP/pR5s6dmwceeOBt65xrlPLMM8/kyiuvTGNjY/75n/85DzzwQE499dR07949xx9/fMv5tLq/U51rvBNnnXVWmpqaMnTo0HTp0iWrVq3Keeedl2OOOSZJnGt0KpEF7xKnnHJKHnvssdx7772dPQrvQc8//3xOO+20TJ8+PfX19Z09Du9hzc3NGT58eM4///wkyV577ZXHHnssV111VY4//vhOno73kh//+Me5/vrrc8MNN+QDH/hAHn744Zx++unZZpttnGt0Oh8XXAtbbLFFunTp8ra7bC1YsCADBgzopKl4Lxk/fnxuu+22/PrXv852223XsnzAgAFZsWJFXn311VbbO/d4p+bMmZOFCxdm2LBh6dq1a7p27Zq77747l112Wbp27Zr+/fs71yhi6623zi677NJq2c4775x58+YlScv55O9U1tVXvvKVnHXWWfnUpz6V3XbbLccee2zOOOOMTJ48OYlzjc4lstZC9+7ds/fee2fGjBkty5qbmzNjxoyMHDmyEyej2lUqlYwfPz4/+9nPctddd2WHHXZotX7vvfdOt27dWp17Tz75ZObNm+fc4x0ZNWpUHn300Tz88MMtP8OHD88xxxzT8s/ONUrYb7/93varKJ566qlsv/32SZIddtghAwYMaHWuNTU15f7773eu8Y4sW7YstbWt/1O2S5cuaW5uTuJco3P5uOBaamxszPHHH5/hw4dnxIgRufTSS7N06dKMHTu2s0ejip1yyim54YYb8vOf/zy9evVq+Yx4nz590qNHj/Tp0yfjxo1LY2Nj+vXrl969e+eLX/xiRo4cmb//+7/v5OmpJr169Wr5rt9bNtlkk2y++eYty51rlHDGGWdk3333zfnnn58jjzwys2fPztVXX52rr746SVp+P9s3v/nNDBkyJDvssEPOPvvsbLPNNjnkkEM6d3iqysEHH5zzzjsvgwYNygc+8IE89NBDueSSS/LZz342iXONTtbZtzesJpdffnll0KBBle7du1dGjBhR+e1vf9vZI1Hlkqz251//9V9btnn99dcr//RP/1TZbLPNKj179qwceuihlZdffrnzhuY9469v4V6pONco5z/+4z8qu+66a6Wurq4ydOjQytVXX91qfXNzc+Xss8+u9O/fv1JXV1cZNWpU5cknn+ykaalWTU1NldNOO60yaNCgSn19fWXHHXesfO1rX6ssX768ZRvnGp2lplL5q1+LDQAAwDrxnSwAAICCRBYAAEBBIgsAAKAgkQUAAFCQyAIAAChIZAEAABQksgAAAAoSWQBs9D70oQ/l9NNPr7rnBuDdqWtnDwAA72W33HJLunXr1tljALABiSwAWI/69evX2SMAsIH5uCAAVaW5uTmTJ0/ODjvskB49emSPPfbIzTffnCSZOXNmampq8qtf/Sp77bVXevTokY985CNZuHBhbr/99uy8887p3bt3jj766CxbtqzV87755psZP358+vTpky222CJnn312KpXKWs303e9+N0OGDEl9fX369++fT37yky3r/vrjgm/N97c/J5xwQsv2P//5zzNs2LDU19dnxx13zNe//vW8+eab6/amAbBBuZIFQFWZPHlyfvjDH+aqq67KkCFD8pvf/Caf+cxnsuWWW7Zsc8455+SKK65Iz549c+SRR+bII49MXV1dbrjhhixZsiSHHnpoLr/88px55pkt+1x33XUZN25cZs+enQcffDAnnXRSBg0alBNPPLHdeR588MGceuqp+bd/+7fsu+++WbRoUe65557Vbrvvvvvm5Zdfbnn8+OOPZ8yYMfmHf/iHJMk999yT4447LpdddlkOOOCAPP300znppJOSJJMmTerwewbAhlVTWdv/TQcAnWz58uXp169f7rzzzowcObJl+ec+97ksW7YsJ510Uj784Q/nzjvvzKhRo5IkF1xwQSZMmJCnn346O+64Y5Lk5JNPznPPPZdp06Yl+cvVpoULF+b3v/99ampqkiRnnXVWbr311vzXf/1XuzPdcsstGTt2bF544YX06tXrbes/9KEPZc8998yll17aavmf/vSnjBgxIqNHj87UqVOTJA0NDRk1alQmTJjQst0Pf/jDfPWrX81LL730Dt8tADqLK1kAVI3//u//zrJly/LRj3601fIVK1Zkr732anm8++67t/xz//7907Nnz5bAemvZ7NmzWz3H3//937cEVpKMHDkyU6ZMyapVq9KlS5c2Z/roRz+a7bffPjvuuGNGjx6d0aNH59BDD03Pnj3b3GflypU5/PDDs/322+c73/lOy/JHHnkk9913X84777yWZatWrcobb7yRZcuWtfucALx7iCwAqsaSJUuSJL/4xS+y7bbbtlpXV1eXp59+Okla3c2vpqbmbXf3q6mpSXNzc5GZevXqlblz52bmzJm54447MnHixJxzzjl54IEH0rdv39Xu84UvfCHPP/98Zs+ena5d//9fxUuWLMnXv/71HHbYYW/bp76+vsi8AKx/IguAqrHLLrukrq4u8+bNywc/+MG3rX8rsjri/vvvb/X4t7/9bYYMGdLuVay3dO3aNQ0NDWloaMikSZPSt2/f3HXXXauNpUsuuSQ//vGP85//+Z/ZfPPNW60bNmxYnnzyybzvfe/r8OsAoPOJLACqRq9evfLlL385Z5xxRpqbm7P//vvntddey3333ZfevXtn++237/Bzz5s3L42Njfn85z+fuXPn5vLLL8+UKVPWuN9tt92WZ555Jv/wD/+QzTbbLL/85S/T3Nyc97///W/b9s4778xXv/rVTJ06NVtssUXmz5+fJOnRo0f69OmTiRMn5n//7/+dQYMG5ZOf/GRqa2vzyCOP5LHHHss3v/nNDr82ADYskQVAVTn33HOz5ZZbZvLkyXnmmWfSt2/fDBs2LP/8z/+8Th8BPO644/L6669nxIgR6dKlS0477bSWO/u1p2/fvrnllltyzjnn5I033siQIUNy44035gMf+MDbtr333nuzatWqnHzyyTn55JNblh9//PG59tprc9BBB+W2227LN77xjVx44YXp1q1bhg4dms997nMdfl0AbHjuLggAAFCQX0YMAABQkMgCgHbcc8892XTTTdv8AYC/5eOCANCO119/PS+++GKb690JEIC/JbIAAAAK8nFBAACAgkQWAABAQSILAACgIJEFAABQkMgCAAAoSGQBAAAUJLIAAAAKElkAAAAF/T++xgIS+CWqCAAAAABJRU5ErkJggg==\n" - }, - "metadata": {} - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "### 1.3 Encoder\n", - "\n", - "#### Picture\n", - "\n", - "" - ], - "metadata": { - "id": "n9MLJLySca14" - } - }, - { - "cell_type": "markdown", - "source": [ - "#### TransformerEncoderBlock\n", - "\n", - "**Initialization:**\n", - "\n", - "* in_size ~ input embedding size\n", - "* head_size ~ size of the Q, K, V matrices embeddings after transformation\n", - "* num_heads ~ number of attention heads\n", - "* out_size ~ output embedding size for attention and the block\n", - "* ff_hidden_size ~ hidden size for feed-forward layers\n", - "* dropout_p ~ dropout probability\n", - "* query_in_size ~ input embedding size for the query (if None, defaults to in_size)\n", - "\n", - "Forward:\n", - "\n", - "*query, key, value ~ 3 tensors (one for each Q, K, and V transformation - these are not yet the tensors $\\text{batch_size} \\times seq \\times d_k$, but tensors of shape $\\text{batch_size} \\times seq \\times \\text{in_size}$)" - ], - "metadata": { - "id": "B24kUNvlckeC" - } - }, - { - "cell_type": "code", - "source": [ - "class TransformerEncoderBlock(nn.Module):\n", - " \"\"\"\n", - " Class with one full block within transformer's encoder\n", - " \"\"\"\n", - " def __init__(self, in_size, head_size, num_heads, out_size, ff_hidden_size, dropout_p=0.2, query_in_size=None):\n", - " \"\"\"\n", - " Args:\n", - " in_size: input embedding size\n", - " head_size: size of each attention head\n", - " num_heads: number of attention heads\n", - " out_size: output embedding size\n", - " ff_hidden_size: hidden size for feed forward net\n", - " dropout_p: probability for dropout\n", - " query_in_size: embedding size of input for query (if not provided - same as in_size)\n", - " \"\"\"\n", - " super(TransformerEncoderBlock, self).__init__()\n", - "\n", - " # Запишем все переданые гиперпараметры слоя\n", - " self.in_size = in_size\n", - " self.head_size = head_size\n", - " self.num_heads = num_heads\n", - " self.out_size = out_size\n", - " self.ff_hidden_size = ff_hidden_size\n", - " self.dropout_p = dropout_p\n", - " self.query_in_size = in_size if query_in_size is None else query_in_size\n", - "\n", - " self.attention = MultiHeadAttention(self.in_size, self.head_size, self.num_heads, self.out_size, self.query_in_size)\n", - " # Если выход и вход attention-а имеют разный размер, то используем линейный слой на residual connection-е\n", - " self.adapt_residual = nn.Linear(self.query_in_size, self.out_size) if self.query_in_size != self.out_size else nn.Identity()\n", - "\n", - " self.norm_1 = nn.LayerNorm(self.out_size)\n", - " self.dropout_1 = nn.Dropout(self.dropout_p)\n", - "\n", - " self.feed_forward = nn.Sequential(OrderedDict([\n", - " (\"lin_1\", nn.Linear(self.out_size, self.ff_hidden_size)),\n", - " (\"act\", nn.ReLU()),\n", - " (\"lin_2\", nn.Linear(self.ff_hidden_size, self.out_size)),\n", - " ]))\n", - "\n", - " self.norm_2 = nn.LayerNorm(self.out_size)\n", - " self.dropout_2 = nn.Dropout(self.dropout_p)\n", - "\n", - "\n", - " def forward(self, query, key, value):\n", - " \"\"\"\n", - " Args:\n", - " block_input: input to corresponding block\n", - " \"\"\"\n", - " # Получаем на вход 3 тензора batch_size x seq_len x in_size\n", - " attention_out = self.attention(query, key, value) # (batch_size, seq_len, out_size)\n", - " attention_residual_out = attention_out + self.adapt_residual(query)\n", - " norm_1_out = self.dropout_1(self.norm_1(attention_residual_out))\n", - "\n", - " # (batch_size, seq_len, out_size) -> (batch_size, seq_len, ff_hidden_size) -> (batch_size, seq_len, out_size)\n", - " ff_out = self.feed_forward(norm_1_out)\n", - " ff_residual_out = ff_out + norm_1_out\n", - " return self.dropout_2(self.norm_2(ff_residual_out))" - ], - "metadata": { - "id": "vAsmZjqEceXe" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing TransformerEncoderBlock for the encoder" - ], - "metadata": { - "id": "yTrJWG_bdA9f" - } - }, - { - "cell_type": "code", - "source": [ - "# We check the standard forward pass from the encoder\n", - "tmp_layer = TransformerEncoderBlock(\n", - " in_size=10,\n", - " head_size=7,\n", - " num_heads=2,\n", - " out_size=15,\n", - " ff_hidden_size=20,\n", - " dropout_p=0.1,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Wne9dA-BdDEI", - "outputId": "dbabfef9-790e-40fb-dc14-a372179c88d0" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "TransformerEncoderBlock(\n", - " (attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Linear(in_features=10, out_features=15, bias=True)\n", - " (norm_1): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_1): Dropout(p=0.1, inplace=False)\n", - " (feed_forward): Sequential(\n", - " (lin_1): Linear(in_features=15, out_features=20, bias=True)\n", - " (act): ReLU()\n", - " (lin_2): Linear(in_features=20, out_features=15, bias=True)\n", - " )\n", - " (norm_2): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_2): Dropout(p=0.1, inplace=False)\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 23 - } - ] - }, - { - "cell_type": "code", - "source": [ - "tmp_input = torch.rand(2, 5, 10)\n", - "\n", - "print(\"Encoder-like input\")\n", - "print(f'Input shape: {tmp_input.shape}')\n", - "tmp_output = tmp_layer(tmp_input, tmp_input, tmp_input)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input, tmp_output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "yFAF2RRGdHGu", - "outputId": "dba6ef98-a7ec-4d34-ec49-c6831f278590" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Encoder-like input\n", - "Input shape: torch.Size([2, 5, 10])\n", - "Output shape: torch.Size([2, 5, 15])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing TransformerEncoderBlock for the decoder" - ], - "metadata": { - "id": "enhFxwBtdIlQ" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = TransformerEncoderBlock(\n", - " in_size=10,\n", - " head_size=7,\n", - " num_heads=2,\n", - " out_size=15,\n", - " ff_hidden_size=20,\n", - " dropout_p=0.1,\n", - " query_in_size=12,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "NFSh5Kzbdas5", - "outputId": "bb29185c-a9a4-4426-860f-e87467c50ae3" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "TransformerEncoderBlock(\n", - " (attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Linear(in_features=12, out_features=15, bias=True)\n", - " (norm_1): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_1): Dropout(p=0.1, inplace=False)\n", - " (feed_forward): Sequential(\n", - " (lin_1): Linear(in_features=15, out_features=20, bias=True)\n", - " (act): ReLU()\n", - " (lin_2): Linear(in_features=20, out_features=15, bias=True)\n", - " )\n", - " (norm_2): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_2): Dropout(p=0.1, inplace=False)\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 25 - } - ] - }, - { - "cell_type": "code", - "source": [ - "# We check the forward pass from the decoder, where we mix information from the encoder and decoder." - ], - "metadata": { - "id": "ASr5ZUnWdbuw" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "tmp_input_q = torch.rand(2, 5, 12)\n", - "tmp_input_kv = torch.rand(2, 7, 10)\n", - "\n", - "print(\"Encoder+Decoder-like input\")\n", - "print(f'Input Q shape: {tmp_input_q.shape}')\n", - "print(f'Input KV shape: {tmp_input_kv.shape}')\n", - "\n", - "tmp_output = tmp_layer(tmp_input_q, tmp_input_kv, tmp_input_kv)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input_q, tmp_input_kv, tmp_output" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zpId_xxbdfGb", - "outputId": "45b71cd9-3c28-4b4c-ebba-ee3d2ff3d084" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Encoder+Decoder-like input\n", - "Input Q shape: torch.Size([2, 5, 12])\n", - "Input KV shape: torch.Size([2, 7, 10])\n", - "Output shape: torch.Size([2, 5, 15])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#### TransformerEncoder\n", - "\n", - "**Initialization:**\n", - "\n", - "* max_seq_len ~ maximum sequence length in tokens\n", - "* vocab_size ~ vocabulary size\n", - "* emb_size ~ input embedding size\n", - "* num_layers ~ number of TransformerEncoderBlocks\n", - "* att_out_size ~ output embedding size from attention and the block\n", - "* att_head_size ~ embedding size of Q, K, V matrices after transformation\n", - "* num_heads ~ number of attention heads\n", - "* ff_hidden_size ~ hidden size for the feed-forward layers\n", - "* dropout_p ~ dropout probability\n", - "\n", - "**Forward:**\n", - "\n", - "* encoder_input ~ tokens input to the encoder before embedding" - ], - "metadata": { - "id": "bAAU2hKcdh4z" - } - }, - { - "cell_type": "code", - "source": [ - "class TransformerEncoderBlock(nn.Module):\n", - " \"\"\"\n", - " Class with one full block within transformer's encoder\n", - " \"\"\"\n", - " def __init__(self, in_size, head_size, num_heads, out_size, ff_hidden_size, dropout_p=0.2, query_in_size=None):\n", - " \"\"\"\n", - " Args:\n", - " in_size: input embedding size\n", - " head_size: size of each attention head\n", - " num_heads: number of attention heads\n", - " out_size: output embedding size\n", - " ff_hidden_size: hidden size for feed-forward net\n", - " dropout_p: probability for dropout\n", - " query_in_size: embedding size for the query input (if not provided, use in_size)\n", - " \"\"\"\n", - " super(TransformerEncoderBlock, self).__init__()\n", - "\n", - " # Store all passed layer hyperparameters\n", - " self.in_size = in_size\n", - " self.head_size = head_size\n", - " self.num_heads = num_heads\n", - " self.out_size = out_size\n", - " self.ff_hidden_size = ff_hidden_size\n", - " self.dropout_p = dropout_p\n", - " self.query_in_size = in_size if query_in_size is None else query_in_size\n", - "\n", - " self.attention = ...\n", - " self.adapt_residual = ...\n", - "\n", - " self.norm_1 = ...\n", - " self.dropout_1 = ...\n", - "\n", - " self.feed_forward = nn.Sequential(OrderedDict([\n", - " (\"lin_1\", ...),\n", - " (\"act\", ...),\n", - " (\"lin_2\", ...),\n", - " ]))\n", - "\n", - " self.norm_2 = ...\n", - " self.dropout_2 = ...\n", - "\n", - "\n", - " def forward(self, query, key, value):\n", - " \"\"\"\n", - " Args:\n", - " block_input: input to corresponding block\n", - " \"\"\"\n", - " # Input of 3 tensors batch_size x seq_len x in_size\n", - " attention_out = ...\n", - " attention_residual_out = ...\n", - " norm_1_out = ...\n", - "\n", - " ff_out = ...\n", - " ff_residual_out = ...\n", - " norm_2_out = ...\n", - " return norm_2_out" - ], - "metadata": { - "id": "vg8IP5CqdwAb" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing TransformerEncoder" - ], - "metadata": { - "id": "1GO52uVYd-Qb" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = TransformerEncoder(\n", - " max_seq_len=20,\n", - " vocab_size=10000,\n", - " emb_size=10,\n", - " num_layers=2,\n", - " att_head_size=7,\n", - " num_heads=2,\n", - " att_out_size=15,\n", - " ff_hidden_size=20,\n", - " dropout_p=0.1,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "id": "BBlrE6acd9fP" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "tmp_input = torch.randint(10000, (2, 5))\n", - "\n", - "print(f'Input shape: {tmp_input.shape}')\n", - "tmp_output = tmp_layer(tmp_input)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input, tmp_output" - ], - "metadata": { - "id": "6HJUmeR8eFe8" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### 1.4 Decoder\n", - "\n", - "#### Picture\n", - "\n", - "\n", - "\n", - "#### TransformerDecoderBlock\n", - "\n", - "**Initialization:**\n", - "\n", - "* in_size ~ input embedding size\n", - "* head_size ~ size of Q, K, V matrix embeddings after transformation\n", - "* num_heads ~ number of attention heads\n", - "* out_size ~ output embedding size for attention and the block\n", - "* ff_hidden_size ~ hidden size for feed-forward layers\n", - "* dropout_p ~ dropout probability\n", - "* encoder_out_size ~ encoder output embedding size (if None, defaults to in_size)\n", - "\n", - "**Forward:**\n", - "\n", - "* decoder_emb ~ tensor from the previous block or embeddings with positional encodings\n", - "* encoder_output ~ output tensor from the corresponding encoder" - ], - "metadata": { - "id": "RYI4RNxyeGR-" - } - }, - { - "cell_type": "code", - "source": [ - "class TransformerDecoderBlock(nn.Module):\n", - " \"\"\"\n", - " Class with one full block within transformer's decoder\n", - " \"\"\"\n", - " def __init__(self, in_size, head_size, num_heads, out_size, ff_hidden_size, dropout_p=0.2, encoder_out_size=None):\n", - " \"\"\"\n", - " Args:\n", - " in_size: input embedding size\n", - " head_size: size of each attention head\n", - " num_heads: number of attention heads\n", - " out_size: output embedding size\n", - " ff_hidden_size: hidden size for feed forward net\n", - " dropout_p: probability for dropout\n", - " encoder_out_size: embedding size of outputs from encoder (if not provided - same as in_size)\n", - " \"\"\"\n", - " super(TransformerDecoderBlock, self).__init__()\n", - "\n", - " # Запишем все переданые гиперпараметры слоя\n", - " self.in_size = in_size\n", - " self.head_size = head_size\n", - " self.num_heads = num_heads\n", - " self.out_size = out_size\n", - " self.ff_hidden_size = ff_hidden_size\n", - " self.dropout_p = dropout_p\n", - " self.encoder_out_size = in_size if encoder_out_size is None else encoder_out_size\n", - "\n", - "\n", - " self.masked_attention = MultiHeadAttention(self.in_size, self.head_size, self.num_heads, self.out_size)\n", - " # Если выход и вход attention-а имеют разный размер, то используем линейный слой на residual connection-е\n", - " self.adapt_residual = nn.Linear(self.in_size, self.out_size) if self.in_size != self.out_size else nn.Identity()\n", - " self.norm = nn.LayerNorm(self.out_size)\n", - " self.dropout = nn.Dropout(self.dropout_p)\n", - " self.encoder_block = TransformerEncoderBlock(self.encoder_out_size, self.head_size, self.num_heads, self.out_size, self.ff_hidden_size, self.dropout_p, self.out_size)\n", - "\n", - "\n", - " def forward(self, decoder_emb, encoder_output):\n", - " \"\"\"\n", - " Args:\n", - " decoder_emb: decoder sequence after embed\n", - " encoder_output: output from encoder\n", - " \"\"\"\n", - " # Получаем на вход тензор batch_size x seq_len x in_size и тензор batch_size x encoder_seq_len x encoder_out_size\n", - " mask = make_decoder_mask(decoder_emb) # batch_size x 1 x seq_len x seq_len\n", - " attention = self.masked_attention(decoder_emb, decoder_emb, decoder_emb, mask=mask) # batch_size x seq_len x out_size\n", - " mmha_out = self.dropout(self.norm(attention + self.adapt_residual(decoder_emb)))\n", - "\n", - " return self.encoder_block(mmha_out, encoder_output, encoder_output) # batch_size x seq_len x out_size" - ], - "metadata": { - "id": "WI673PnTeOn8" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing TransformerDecoderBlock" - ], - "metadata": { - "id": "EBJy06RwfHeY" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = TransformerDecoderBlock(\n", - " in_size=10,\n", - " head_size=7,\n", - " num_heads=2,\n", - " out_size=15,\n", - " ff_hidden_size=20,\n", - " dropout_p=0.1,\n", - " encoder_out_size=12,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zsXTUhlIfAkd", - "outputId": "1c2ea4d0-e0c1-4c05-e2a0-1ec30768c896" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "TransformerDecoderBlock(\n", - " (masked_attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Linear(in_features=10, out_features=15, bias=True)\n", - " (norm): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (encoder_block): TransformerEncoderBlock(\n", - " (attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=15, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Identity()\n", - " (norm_1): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_1): Dropout(p=0.1, inplace=False)\n", - " (feed_forward): Sequential(\n", - " (lin_1): Linear(in_features=15, out_features=20, bias=True)\n", - " (act): ReLU()\n", - " (lin_2): Linear(in_features=20, out_features=15, bias=True)\n", - " )\n", - " (norm_2): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_2): Dropout(p=0.1, inplace=False)\n", - " )\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 29 - } - ] - }, - { - "cell_type": "code", - "source": [ - "# Testing the forward pass in the decoder, where we mix information from the encoder and decoder\n", - "tmp_input_decoder = torch.rand(2, 5, 10)\n", - "tmp_output_encoder = torch.rand(2, 7, 12)\n", - "\n", - "print(\"Encoder+Decoder-like input\")\n", - "print(f'Decoder input shape: {tmp_input_decoder.shape}')\n", - "print(f'Encoder output shape: {tmp_output_encoder.shape}')\n", - "\n", - "tmp_output = tmp_layer(tmp_input_decoder, tmp_output_encoder)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input_decoder, tmp_output_encoder" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "zulYoZ6GfKVP", - "outputId": "1c43ddb6-0bee-4885-8516-b023c3ec874f" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Encoder+Decoder-like input\n", - "Decoder input shape: torch.Size([2, 5, 10])\n", - "Encoder output shape: torch.Size([2, 7, 12])\n", - "Output shape: torch.Size([2, 5, 15])\n" - ] - } - ] - }, - { - "cell_type": "markdown", - "source": [ - "#### TransformerDecoder\n", - "\n", - "**Initialization:**\n", - "\n", - "* max_seq_len ~ maximum token length of the sequence\n", - "* vocab_size ~ size of the vocabulary\n", - "*\temb_size ~ input embedding size\n", - "* num_layers ~ number of TransformerEncoderBlocks\n", - "*\tatt_out_size ~ output embedding size for attention and the block\n", - "*\tatt_head_size ~ embedding size of the Q, K, V matrices after transformation\n", - "*\tnum_heads ~ number of attention heads\n", - "*\tff_hidden_size ~ hidden size for the feed-forward layers\n", - "*\tdropout_p ~ dropout probability\n", - "*\tencoder_out_size ~ encoder output embedding size (if None, defaults to in_size)\n", - "\n", - "**Forward:**\n", - "\n", - "*\tdecoder_input ~ input tokens to the decoder before embeddings\n", - "*\tencoder_output ~ output tensor from the corresponding encoder" - ], - "metadata": { - "id": "xRLiJy5FfOUq" - } - }, - { - "cell_type": "code", - "source": [ - "class TransformerDecoder(nn.Module):\n", - " \"\"\"\n", - " Class for decoder within transformer.\n", - " \"\"\"\n", - " def __init__(self, max_seq_len, vocab_size, emb_size, num_layers, att_out_size, att_head_size, num_heads, ff_hidden_size, dropout_p, encoder_out_size=None):\n", - " \"\"\"\n", - " Args:\n", - " max_seq_len : maximum length of input sequence\n", - " vocab_size: size of the vocabulary\n", - " emb_size: embeddings size\n", - " num_layers: number of encoder layers\n", - " att_out_size: output size for attention and each encoder block\n", - " att_head_size: size of each attention head\n", - " num_heads: number of heads in multihead attention\n", - " ff_hidden_size: hidden size for feed forward net\n", - " dropout_p: probability for dropout\n", - " encoder_out_size: embedding size of outputs from encoder (if not provided - same as in_size)\n", - " \"\"\"\n", - " super(TransformerDecoder, self).__init__()\n", - "\n", - " # Запишем все переданые гиперпараметры слоя\n", - " self.max_seq_len = max_seq_len\n", - " self.vocab_size = vocab_size\n", - " self.emb_size = emb_size\n", - " self.num_layers = num_layers\n", - " self.att_out_size = att_out_size\n", - " self.att_head_size = att_head_size\n", - " self.num_heads = num_heads\n", - " self.ff_hidden_size = ff_hidden_size\n", - " self.dropout_p = dropout_p\n", - " self.encoder_out_size = in_size if encoder_out_size is None else encoder_out_size\n", - "\n", - " self.embedding_layer = nn.Embedding(self.vocab_size, self.emb_size)\n", - " self.positional_encoder = PositionalEncoding(self.max_seq_len, self.emb_size)\n", - " self.dropout = nn.Dropout(self.dropout_p)\n", - "\n", - " self.decoder_blocks = nn.ModuleDict({\n", - " f\"decoder_block_{i}\": TransformerDecoderBlock(\n", - " in_size=self.emb_size if i==0 else self.att_out_size,\n", - " head_size=self.att_head_size,\n", - " num_heads=self.num_heads,\n", - " out_size=self.att_out_size,\n", - " ff_hidden_size=self.ff_hidden_size,\n", - " dropout_p=self.dropout_p,\n", - " encoder_out_size=self.encoder_out_size,\n", - " ) for i in range(self.num_layers)\n", - " })\n", - "\n", - " self.fc = nn.Linear(self.att_out_size, self.vocab_size)\n", - "\n", - " def forward(self, decoder_input, encoder_output):\n", - " \"\"\"\n", - " Args:\n", - " decoder_input:\n", - " encoder_output:\n", - " Returns:\n", - " out: output vector\n", - " \"\"\"\n", - " # Получаем на вход batch_size x seq_len и batch_size x encoder_seq_len x encoder_out_size\n", - " decoder_emb = self.embedding_layer(decoder_input) # batch_size x seq_len x emb_size\n", - " decoder_emb = self.positional_encoder(decoder_emb)\n", - "\n", - " out = self.dropout(decoder_emb)\n", - "\n", - " for block in self.decoder_blocks.values():\n", - " out = block(out, encoder_output) # batch_size x seq_len x att_out_size\n", - "\n", - " return self.fc(out)" - ], - "metadata": { - "id": "aLuT6JaQfby_" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "#### Testing TransformerDecoder" - ], - "metadata": { - "id": "3vBYyZrXzQHW" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = TransformerDecoder(\n", - " max_seq_len=20,\n", - " vocab_size=10000,\n", - " emb_size=10,\n", - " num_layers=2,\n", - " att_head_size=7,\n", - " num_heads=2,\n", - " att_out_size=15,\n", - " ff_hidden_size=20,\n", - " dropout_p=0.1,\n", - " encoder_out_size=12,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "WOnV1Y20zV9G", - "outputId": "2375326f-a3f6-4e95-dea0-aba9ca82b975" - }, - "execution_count": null, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "TransformerDecoder(\n", - " (embedding_layer): Embedding(10000, 10)\n", - " (positional_encoder): PositionalEncoding()\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (decoder_blocks): ModuleDict(\n", - " (decoder_block_0): TransformerDecoderBlock(\n", - " (masked_attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=10, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Linear(in_features=10, out_features=15, bias=True)\n", - " (norm): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (encoder_block): TransformerEncoderBlock(\n", - " (attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=15, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Identity()\n", - " (norm_1): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_1): Dropout(p=0.1, inplace=False)\n", - " (feed_forward): Sequential(\n", - " (lin_1): Linear(in_features=15, out_features=20, bias=True)\n", - " (act): ReLU()\n", - " (lin_2): Linear(in_features=20, out_features=15, bias=True)\n", - " )\n", - " (norm_2): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_2): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (decoder_block_1): TransformerDecoderBlock(\n", - " (masked_attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=15, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=15, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=15, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Identity()\n", - " (norm): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (encoder_block): TransformerEncoderBlock(\n", - " (attention): MultiHeadAttention(\n", - " (query_matrix): Linear(in_features=15, out_features=14, bias=False)\n", - " (key_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (value_matrix): Linear(in_features=12, out_features=14, bias=False)\n", - " (out): Linear(in_features=14, out_features=15, bias=True)\n", - " )\n", - " (adapt_residual): Identity()\n", - " (norm_1): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_1): Dropout(p=0.1, inplace=False)\n", - " (feed_forward): Sequential(\n", - " (lin_1): Linear(in_features=15, out_features=20, bias=True)\n", - " (act): ReLU()\n", - " (lin_2): Linear(in_features=20, out_features=15, bias=True)\n", - " )\n", - " (norm_2): LayerNorm((15,), eps=1e-05, elementwise_affine=True)\n", - " (dropout_2): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (fc): Linear(in_features=15, out_features=10000, bias=True)\n", - ")" - ] - }, - "metadata": {}, - "execution_count": 44 - } - ] - }, - { - "cell_type": "code", - "source": [ - "# We will test the Transformer model by passing through both the encoder and decoder, ensuring that information from the encoder is correctly used in the decoder for sequence generation.\n", - "tmp_input_decoder = torch.randint(10000, (2, 5))\n", - "tmp_output_encoder = torch.rand(2, 7, 12)\n", - "\n", - "print(\"Encoder+Decoder-like input\")\n", - "print(f'Decoder input shape: {tmp_input_decoder.shape}')\n", - "print(f'Encoder output shape: {tmp_output_encoder.shape}')\n", - "\n", - "tmp_output = tmp_layer(tmp_input_decoder, tmp_output_encoder)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input_decoder, tmp_output_encoder" - ], - "metadata": { - "id": "WU6z4-uczXom" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### 1.5 Transformer" - ], - "metadata": { - "id": "X7PUNDaT0OBW" - } - }, - { - "cell_type": "code", - "source": [ - "class Transformer(nn.Module):\n", - " \"\"\"\n", - " Class for full encoder-decoder transformer\n", - " \"\"\"\n", - " def __init__(\n", - " self,\n", - " max_seq_len,\n", - " vocab_size,\n", - " emb_size,\n", - "\n", - " num_encoder_layers,\n", - " enc_att_out_size,\n", - " enc_att_head_size,\n", - " enc_num_heads,\n", - " enc_ff_hidden_size,\n", - " enc_dropout_p,\n", - "\n", - " num_decoder_layers,\n", - " dec_att_out_size,\n", - " dec_att_head_size,\n", - " dec_num_heads,\n", - " dec_ff_hidden_size,\n", - " dec_dropout_p,\n", - " ):\n", - " super(Transformer, self).__init__()\n", - "\n", - " # Store all the passed hyperparameters of the model\n", - " self.max_seq_len = max_seq_len\n", - " self.vocab_size = vocab_size\n", - " self.emb_size = emb_size\n", - "\n", - " self.num_encoder_layers = num_encoder_layers\n", - " self.enc_att_out_size = enc_att_out_size\n", - " self.enc_att_head_size = enc_att_head_size\n", - " self.enc_num_heads = enc_num_heads\n", - " self.enc_ff_hidden_size = enc_ff_hidden_size\n", - " self.enc_dropout_p = enc_dropout_p\n", - "\n", - " self.num_decoder_layers = num_decoder_layers\n", - " self.dec_att_out_size = dec_att_out_size\n", - " self.dec_att_head_size = dec_att_out_size\n", - " self.dec_num_heads = dec_num_heads\n", - " self.dec_ff_hidden_size = dec_ff_hidden_size\n", - " self.dec_dropout_p = dec_dropout_p\n", - "\n", - " # Encoder\n", - " self.encoder = TransformerEncoder(\n", - " max_seq_len=self.max_seq_len,\n", - " vocab_size=self.vocab_size,\n", - " emb_size=self.emb_size,\n", - " num_layers=self.num_encoder_layers,\n", - " att_head_size=self.enc_att_head_size,\n", - " num_heads=self.enc_num_heads,\n", - " att_out_size=self.enc_att_out_size,\n", - " ff_hidden_size=self.enc_ff_hidden_size,\n", - " dropout_p=self.enc_dropout_p,\n", - " )\n", - "\n", - " # Decoder\n", - " self.decoder = TransformerDecoder(\n", - " max_seq_len=self.max_seq_len,\n", - " vocab_size=self.vocab_size,\n", - " emb_size=self.emb_size,\n", - " num_layers=self.num_decoder_layers,\n", - " att_head_size=self.dec_att_head_size,\n", - " num_heads=self.dec_num_heads,\n", - " att_out_size=self.dec_att_out_size,\n", - " ff_hidden_size=self.dec_ff_hidden_size,\n", - " dropout_p=self.dec_dropout_p,\n", - " encoder_out_size=self.enc_att_out_size,\n", - " )\n", - "\n", - " def forward(self, encoder_input, decoder_input):\n", - " \"\"\"\n", - " Args:\n", - " encoder_input: input to encoder\n", - " decoder_input: input to decoder\n", - " out:\n", - " out: final tensor with logits of each word in vocab\n", - " \"\"\"\n", - " # Input has shape batch_size x enc_seq_len and batch_size x dec_seq_len\n", - " encoder_output = self.encoder(encoder_input) # (batch_size, enc_seq_len, enc_att_out_size)\n", - "\n", - " return self.decoder(decoder_input, encoder_output) # (batch_size, dec_seq_len, vocab_size)" - ], - "metadata": { - "id": "nkvAAfu20OYC" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "### 1.6 Testing" - ], - "metadata": { - "id": "6ZRK9UIn0Yvg" - } - }, - { - "cell_type": "code", - "source": [ - "tmp_layer = Transformer(\n", - " max_seq_len=20,\n", - " vocab_size=10000,\n", - " emb_size=10,\n", - "\n", - " num_encoder_layers=3,\n", - " enc_att_head_size=7,\n", - " enc_num_heads=3,\n", - " enc_att_out_size=20,\n", - " enc_ff_hidden_size=30,\n", - " enc_dropout_p=0.2,\n", - "\n", - " num_decoder_layers=2,\n", - " dec_att_head_size=7,\n", - " dec_num_heads=2,\n", - " dec_att_out_size=15,\n", - " dec_ff_hidden_size=20,\n", - " dec_dropout_p=0.1,\n", - ")\n", - "\n", - "tmp_layer" - ], - "metadata": { - "id": "365uJEiR0bNS" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "tmp_input_encoder = torch.randint(10000, (2, 9))\n", - "tmp_input_decoder = torch.randint(10000, (2, 5))\n", - "\n", - "print(f'Encoder input shape: {tmp_input_encoder.shape}')\n", - "print(f'Decoder input shape: {tmp_input_decoder.shape}')\n", - "\n", - "tmp_output = tmp_layer(tmp_input_encoder, tmp_input_decoder)\n", - "print(f'Output shape: {tmp_output.shape}')\n", - "\n", - "del tmp_input_decoder, tmp_input_encoder" - ], - "metadata": { - "id": "7eEBPO9F0y3r" - }, - "execution_count": null, - "outputs": [] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file