lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py36/lib/python3.6/site-packages/noisereduce/noisereduce.py:5: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f75785b3330>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import librosa\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm import tqdm\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import librosa\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import noisereduce as nr\n",
    "torch.manual_seed(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>emotion</th>\n",
       "      <th>val</th>\n",
       "      <th>act</th>\n",
       "      <th>dom</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>6.2901</td>\n",
       "      <td>8.2357</td>\n",
       "      <td>Ses01F_impro01_F000</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10.0100</td>\n",
       "      <td>11.3925</td>\n",
       "      <td>Ses01F_impro01_F001</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>14.8872</td>\n",
       "      <td>18.0175</td>\n",
       "      <td>Ses01F_impro01_F002</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>27.4600</td>\n",
       "      <td>31.4900</td>\n",
       "      <td>Ses01F_impro01_F005</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>85.2700</td>\n",
       "      <td>88.0200</td>\n",
       "      <td>Ses01F_impro01_F012</td>\n",
       "      <td>ang</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  start_time  end_time             wav_file emotion  val  act  dom\n",
       "0      0      6.2901    8.2357  Ses01F_impro01_F000     neu  2.5  2.5  2.5\n",
       "1      1     10.0100   11.3925  Ses01F_impro01_F001     neu  2.5  2.5  2.5\n",
       "2      2     14.8872   18.0175  Ses01F_impro01_F002     neu  2.5  2.5  2.5\n",
       "3      3     27.4600   31.4900  Ses01F_impro01_F005     neu  2.5  3.5  2.0\n",
       "4      4     85.2700   88.0200  Ses01F_impro01_F012     ang  2.0  3.5  3.5"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "audio = pd.read_csv('audio_df_improvised.csv')\n",
    "\n",
    "audio.reset_index(inplace=True)\n",
    "audio.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# audio['duration'] = np.array(audio.end_time) - np.array(audio.start_time)\n",
    "# audio = audio[ audio['duration']>2 ]\n",
    "# audio.to_csv ('audio_morethan2_duration.csv', index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level_0</th>\n",
       "      <th>index</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>wav_file</th>\n",
       "      <th>emotion</th>\n",
       "      <th>val</th>\n",
       "      <th>act</th>\n",
       "      <th>dom</th>\n",
       "      <th>duration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>14.8872</td>\n",
       "      <td>18.0175</td>\n",
       "      <td>Ses01F_impro01_F002</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.1303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>27.4600</td>\n",
       "      <td>31.4900</td>\n",
       "      <td>Ses01F_impro01_F005</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>85.2700</td>\n",
       "      <td>88.0200</td>\n",
       "      <td>Ses01F_impro01_F012</td>\n",
       "      <td>ang</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2.7500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>97.8900</td>\n",
       "      <td>102.9600</td>\n",
       "      <td>Ses01F_impro01_F014</td>\n",
       "      <td>neu</td>\n",
       "      <td>2.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>5.0700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>87.1500</td>\n",
       "      <td>94.3900</td>\n",
       "      <td>Ses01F_impro01_M011</td>\n",
       "      <td>ang</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.2400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   level_0  index  start_time  end_time             wav_file emotion  val  \\\n",
       "0        0      2     14.8872   18.0175  Ses01F_impro01_F002     neu  2.5   \n",
       "1        1      3     27.4600   31.4900  Ses01F_impro01_F005     neu  2.5   \n",
       "2        2      4     85.2700   88.0200  Ses01F_impro01_F012     ang  2.0   \n",
       "3        3      5     97.8900  102.9600  Ses01F_impro01_F014     neu  2.5   \n",
       "4        4      6     87.1500   94.3900  Ses01F_impro01_M011     ang  2.0   \n",
       "\n",
       "   act  dom  duration  \n",
       "0  2.5  2.5    3.1303  \n",
       "1  3.5  2.0    4.0300  \n",
       "2  3.5  3.5    2.7500  \n",
       "3  3.5  3.5    5.0700  \n",
       "4  4.0  4.0    7.2400  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# audio = pd.read_csv('audio_morethan2_duration.csv')\n",
    "# audio.reset_index(inplace=True)\n",
    "# audio.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We split the dataframe into train and test segments\n",
    "from sklearn.utils import shuffle\n",
    "train = audio[:2212]\n",
    "train = shuffle(train)\n",
    "valid = audio[2212:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_melspectrogram_db(file_path, med_duration ,sr=None, n_fft=2048, hop_length=256, n_mels=128, fmin=100, fmax=15000, top_db=80):\n",
    "    med_duration = int(np.floor(med_duration))\n",
    "    wav,sr = librosa.load(file_path,sr=sr)\n",
    "    wav = nr.reduce_noise(audio_clip=wav, noise_clip=wav, verbose=False)\n",
    "    if wav.shape[0]<(med_duration*sr):\n",
    "        wav=np.pad(wav,int( np.ceil( ((med_duration*sr)-wav.shape[0])/2 ) ),mode='constant', constant_values=(0, 0))\n",
    "    else:\n",
    "        wav=wav[:(med_duration*sr)]\n",
    "    spec=librosa.feature.melspectrogram(wav, sr=sr, n_fft=n_fft,\n",
    "              hop_length=hop_length,n_mels=n_mels)\n",
    "#     librosa.decompose.nn_filter(spec, aggregate=np.median, metric='cosine')\n",
    "    spec_db=librosa.power_to_db(spec)\n",
    "    return spec_db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spec_to_image(spec, eps=1e-6):\n",
    "    mean = spec.mean()\n",
    "    std = spec.std()\n",
    "    spec_norm = (spec - mean) / (std + eps)\n",
    "    spec_min, spec_max = spec_norm.min(), spec_norm.max()\n",
    "    spec_scaled = 255 * (spec_norm - spec_min) / (spec_max - spec_min)\n",
    "\n",
    "    return spec_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data/home/advaitmb/notebooks/Saaket/speech-emotion-recognition\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 251)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAADNCAYAAACxdqe5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOy9Wahl25rn9RvNbFa7++jjnHvPbbIyS8sqqkwx0wdRBNHCEmzQkiKFgvskKKlYWT7Vi1C+2DwpF0pIQUgVhfKhQFSoBxUks9I0TfNmXm9z+ogdsdvVzXaM4cM35pxrnxOni+acOPesPwQ71lqzb8b4mv/3/1QIgR122GGHHb550F/1Aeywww477PDVYDcB7LDDDjt8Q7GbAHbYYYcdvqHYTQA77LDDDt9Q7CaAHXbYYYdvKHYTwA477LDDNxSvbAJQSv2zSqk/VUr9RCn1W69qPzvssMMOOzwf1KuoA1BKGeDHwD8DvA/8LvCvhxD++KXvbIcddthhh+fCq/IAfhX4SQjhZyGEGvgd4K+8on3tsMMOO+zwHLCvaLv3gfe2Pr8P/GOftLCZTEKyfygfAqggfzvc+BwA9Sl7fsb6/TYA5QPKhf535QN4D87LF97L9lXciTZgNKHfp0KFAD5uwLm4TlzAxOW16haX3zpPK8g+b/zenReg3NaxENfdPl+l4rofWf+ToNXWsUPQW+e2dTz9/kP8fzy+8NHth7hM99EowtYxhu7QVL+4/DeeknYB1W7f3Pg3Xs/+WLp9dNevO2bvwTmCc/FyaDnW/n5p0FvXvz/Qre17L/+6HWxfY/XR9eIxaL11PdRwrbzfOla1dX+2tnXj2oaP/97t2ge5994Nx7z9vCsF3flu7Y9P8uK3j+Wj5xT/o561andtQrixv+5ZeOY63RYVW8vH63rj3bv5bIePPt/P3Gi334/cO+cJ3qPivcFo5H72OxjOAyB4+vsd94/+yP366LvG1lde7t+N89++9v3/4/kb9bFthhv7k/exf8e69+aTru9HxgqA4un7ZyGEk09Y4zPxqiaAZ93SG6ellPoB8AOAZHbAL/1LvynftwHdgHbDWtrJdwCmDtiNI1m18nldy4XeekGV86hKNqDXBeFqgVss5HOaoCbj4aDKCl+uP/lIHdB8jjPu1vGfc/kXhEpSAPThPsyn+PlIdp9aVIiTHOAyQztNcNnWQwn978qBbjx2LdczuSxQ62K4nlrJpNY9tK1DFVV82SAkljDO8HkSPxuCVXi7dT88mHg/zLpGbSrCSI7fTVJ8avrjMZsac7aAqpb9ZSkhTfqXVjUtYVMME7BWUFX4opTfrUWNx6ju+NoWf3VNaNuXcNU/H8zBgRzLWO5JqOWBUFmKP5zhx3LuQSlM0aCfXsnnsgRrUfFcQ9PgV2tCVX1px/61w/aA+eXd4i8EcxLH54M5fpb374pPdJwEVP/ZW4VP5HObK1yq8LI4zVTRjug/o+DHf+s333mRY3tVIaD3gYdbnx8AH24vEEL4YQjhL4UQ/pIdTV7RYeywww477PBJeFUewO8C31NKfRv4APjXgL/6SQu7BNb3owVYKUwJppDfTB0wlRosxMqTLBvMSqwi/eSS9vHpje1tR4A8N+FLB2X5Qif3OiA0YiG70ydw+qT/XgEqy9Bj8XLMfEoyGRGiRa+cWPBhJV6PX65QSqHyTLbrPG65fO7jUtpg9+aoqUzqIUtuehBa4/fGtJNoBVmFCqDr6LGtKsJ606/v52PcNKMdy6PqckOwMbwA6CZgC4cpZf3OilLxxttNgznfw7/7gRzPK7SmdZ6jHtzFH0zlWDNDsHrLwlMxLCDLqzbgc4Oa3Y4bGM4LxEvTVYtZRu/mckFottxLa1F5hjuc9+vjQa/j8ss1oa7xi1X8XWHu38UdxHuTGFTr0UX0UDYl1M0QYskzgjVD+Gxd4M4u+mdvh88H9/Sp/Cf+tZ1HcOsQN0kJiQHkfoexoXtATB3EQzAfiadGs92bFyfwvJIJIITQKqX+LeB/AgzwX4YQ/t9PWl43MDodTlIFebEB7AbyK4ct5KE0lcOsKtS7jwBor65fxSl8rRGqCtcNdJeXn708vLxJ0Tvc5eUn7lePx+g0ITXy0KOUxHILmfFdVUmcNd5XNRph8xx7S3JEXciI7qXwoKsWXcogZpQiWI1q5XnRiw3u/UfPPWjpPEfNZv0ESQj9sfrVGj3K+/CTPjkmKCXHBzTzhHascdGld6nC5dCOugkBCGDjpc8vPKMnTf/CB6vYHOc0sxkA7fgEl8Y8DhK+MzWYeKuVj9fDxQkhyHe2kHcpXfn+vZL1A3bd9tcqzMcEq/twW9ByLTtrSs/G6MkIrmVCCZsNfv2R8OkOn4ntCcEcHeLfuAuA389QHnTbhW8VQQ8hHzcCl0Gw8vvH8nPPgVflARBC+HvA3/tcC2/F8XQDpgmY+FJkC0d+Vg9Jws4ayfO48i/gBKANOpW7rqYT3Nl5b9H7zQYAcyQDospz8B5/vbjx+8uGymQAVEqh9uaoJD6VaXLTSnQeqpr2gw/7c7FvPhgSb85BVeMX4mWoUQ53jtEbueHhcI65WFB+91a/73ZsaEdb0coQtix8TxIC9pFMOP5gil5VgxVbVpiH9wgXcULyAdKkP/4wn4I1+DS+ChrcKJGBD/AhcPVWTnkc47LjYQDWrXiq2WUY1s2GpLvykvh0aZwARrKuy+MLnso6KsauyyPNxS+n+DjXeBvwuUPPZXKbTEv2sppEy7k1XrOpUlYbWcFdp5iNRjfRO6oUyRo4l+0lG4VP6d814wGjcKMkno9HVU3vbXfXmjYm3KuasFjJBL/D80OLgWAO9lDjcW/M1HNLPRUPFsQQ9gbacffsBdpxwKdbBIkXPZQX38QOO+ywww5fR7wyD+ALQW1lthGXNluI1ZEsHT4d5inVBlSWiOUG6OXqa+2G6jwnOB8pamDu3iGMMkjk1vhRgjGGcCIWv3n8FHd2jv/2Pfk9sxACZrUv2/vx2/jtcI42mPlU6HwgcWNrIIssnA8eo996Y7Dgq5qQJqjIXHEne7SzlPUdWd4ninTlaaJVUhxp2vFw/4IRy0W5b/WfXRYIMeIjIQpuJGdMrfqcj91Avb9HM9tyczW4PIYpco9KPcFFK2ltsJsRLnsgvycBnMJu5PfsMm5b3ev3p1qwZcwp1QFvomWMMC/qObSzzg2PVvhUrkc+rkltzDcERVVb1mX0Jioj59U9ribSONvO+1GooAi5rJ/PK6ajitRGRpsKZLZlZIc4/7LOuNoIm8gFRdVYuiyG85qmNQQfr4VX6Fphqo4xJR5HvRc9kMyQrAOji+7datBlgy5l/6qoYFPgLiMracc+ejWINF93foFRCl1ITsZUAT1ieFcQLzO97rxrRTseWEHhJZjvr8cEwMAtNlXAVgNXWvmAbvzgVrsgbuo6xmHrL4Fz+QIwt2+BD4QYmlFpirteYG8dA+BvH4LVfQhifSdH+UA1l6cgaNC/NO1psdWvHeBTdYOTHRTYjYSI8u/voZtwI9bbTPRAq0XyK9WePD3V/gNcBk0kYvkkoJ1Cb8WVXQ7VsWzAHhecHCy5M5JJ9yRfsZ9smMZA9MyUJGrYWRks1+2Yq0aOb+1kpE21DDoj0zA1FbMY80uUowmGpZMQ36LNWbY517V8XjcS7pgksr/ctGgVaOPbsGlTrsoRV4Us3zSWJoB38ntbWVhY8idyfceRP9DRZOt9KO63JPtyPPNRjdYe77uQkMIamYyUChjtMfGzyzVtY3FruZeqMDIBdJNVpKGHGBKy1jFJa/I44GsV0Fskcx8UmWk5nMizs65TVmVGFSectjKojcUs5diSpcKW9OExl0mYKV5yiR+bIUexORlhqlySjUQ6ohpi0MnKo2tP6EISbcAULfY6ztaPnuIuLzFzyTm45fKTaxJ2eCbc2TkmjmGjqiaf5H34Ea3xqaHek/td7RnquepzSNsTxfPitZkAtqHbWBRDTFRdrG8UaqmyFi44DOySrwraQPDoqXgkymjUaEQ4kJeivDfj8vsZTczL+UQs0O2b57KAG8v5+jTAXsv+vuQ2TiZrjvI1x5kk3o6S9ccGWfeRYKDZKmRogqH0CZto4q5cRuES1q0MpJs2YdnkXBViZV5cTVCnGTrvrMZAsKFPPNnEMUsrbo8W/fFMTUWi2xv77I5p5XIumgnvb8RDuSpHjJOa/UzuX6Zbct0wjRNArhocuj+/JhgqbxnHQdJqT+kSSpfE408JQVH7mHh1htoZTBxInfY0jcG1sS6g0phK0x2u1I0MdRFBK1SraArZ/tIZQgBfxxtWDUlRRg6T+t568a0mVAbVxBe4VdEbko/KKam7KOW1WzVT1pMcm0YGlPGEoHCt7MtVRrwHv8UMahRmI9sfrRS6HixBn0B5Emhn0VvKnHgjMSdglgYVdM8z96l4dC7GlIMFAphKNqhrI0nr+GyGJEBI0LVM5nZ9TLqA/Fx+zy8d6bLBnsu91etCcj9xgPMXl68sR/V1RlejRPd3CwqYvCmM+vxkj/owp5nK89HlCl4EuxzADjvssMM3FK+NB+DjkbhcUe5Dkgwmsk/mJNdiIZqzBf7s/Gac+xXCnJygsrR3bcN0THs8xeXD8QUFbZyVm7HGG9Vb+MWJYvX9hpN7Eld9Y37JrXzVh0B8UCzbnCZasN8an/Pd/JSHiVA37pklJyYw07HyF00THNdeaI0fupQP2n2WXix4g2esK9JodjoUpU9ZegmJLN2IazfiMoZkrrT8LRqxeG3iqEe+r+TFShxb1fK5/HDCn56P+NPkbjx5blipqlXoUveV2/IlA/OklvDSOzEs0cw9bupQo1gpbD1KhT7k4iqDKgwqxrV1G63ouH1dKdLFYGW3Y9lfR40cFYG9ZSBZxbCNCyjfYGr5nFyW6Os1zT2p3l2+mVM/0vgksp58wBYwOpMdBAvFYbzXUyseXXROTRFIV4OsRpvLc9Ada7oOmMr3y6s2oLzGRIZqUPYG71u1wol2MQcm9OjB82vHhuLQyDkTrflCoWK4q5kqQupRWx6EN5Buone9gnp/K6YcaaPdtQwG2qncHzkBOXAfn4VgDbrVuJiCa0cKFSwumcR7NUY3Hr2JFF1rCI9Od7mFj6Bj2OnxGDWdEMaR4aiFIl0+FO+5OE7kmepowC8h2vZaTADB0Cf9umKZ0E0ImSHN9RASaqdordGRe+0Xy1fiVuo//yuyfaMo74x7d6vaMxTHikYiPvgkoLzqaas+kTBO/1JZz/6tJf/IsdAivzt+wp7Z0MQTbILhdnLNfSvUun0t7nMZf7/2GZvg6EaVMiiWfsaTVrjhf1Lc4w+v7/NoOeuPPU/aPihUO0NRJ1RVTCo7g28VoZDPZmkkP9DnYBSz5UBt3I4pC2TANVWME1+UhESjqyEEpDbVQPs0GnW1hJE81CGXydRP40MeAqpx6HNxf8PelHaeg44hnDygXEM7kkE3XTQx6R0LAVclIbED7TOxIlcRc0Shbgbe9TMQqfOon0tF/fx/F4qtStN+mdA6iFIS7vKSUfxeTyao6QRlh9coVFUfrnSXl6gk7WUp1N4cNcpF2gLwkxw/sn2+Swfh/ldT+b2eaqGQdiUPiVACu5i+CkJDzS9lf+nSSa1BRztNFT4x/fq6gWTjyS5j0tcFmrml6WLKWmGagI1FdfVUUy407TjmP0w3CRHvRWD81DH+UK61uVgLeaDLA/hAqGtCpCi3vwAFmJ8LkeZpv/UQtzchZFvxXh8IZivwohWbWzIBrO9oykNFO+nCkTevt8QrQ18AcPO9fD68FhMA0LMwmrkj2ICKiTO71oxONSFawOnUkp8lmPNYPRcCPM8EEAcoM5tBlvUvKXmGO5hx/X0ZUM//nKI+cpBFCzWrmYwrDnMZgBItFmuIN2VZpSzXeZ+auHd4zZ/df8zD/AKAsa5Z+pwntSQFKm8pfdInPXPdUAfLtZNhZuVyrtsRFzHu+mizx+lyyuo66sxUBr02PfdbedgUw4QkdRUw6wbsTcCWgfRKBoH0fA1GYS4kx+CfnqPSgZLlzi8+9TJ+UsX1F0HYXv/9m/Tm7gHtXiFlrej7vIT9fiJ8IJRyf9XeDPfo3b4OA8Dej4wipQhNQ5jHDPrTCzg6ACtHq9+4gx8l1HFAbydSz1BPhwG68zBAmEneDh6AS8SK74wNNwp4G4bArZf720zlC7uRZH8ntmcLGD/1jB7Lw2Av1qimhZg/ax+fkmiDjtpYKjLDuvza2LmPW+sx59VjK+nr2AHA7Mm77edj3DzFx7FFO49qt8QA9ZB4B3ke2ulWDscG8PQ5H1OKsRz0y0u073IAO+ywww7fULwWHkDQ0M5jtWHuUDoQYpzRtYpmoihOohuaKEyZYB/HGPr1gk+VxO3Qacdkmbji0eoJRhNGKfWRfG4mVkJSE1m+eVjxF7/9Lv/ovoQIvpudcmIXpNHeKUPCEzfjtJE43VhXHNkVsxjKyVXTLwdw7qa8Xx/hO4+hzfnx4hYfLsRqWF6OodbCIYeexaGLGAK50mQXcBB57PN3W1zmGb8vgVhztiBsSlGWBEJdo0c5LkormIMDkcDtmBlR98fHyupXnlv5HPfKzOeEWgLjKs/w33nQV6u6XHSFejVTL/LSZiPL+zwBHzDrwXLVy4KwEi/RnZ19bP/m+Aj33fsAbO6NqKe6j7OWR4rq8H5fvevtEHtVDsxGkV3GmoOr46h82lX6Kspj1cfog4mMoxgta/YCzZ6DWBegdCA4LUwjwKwMphqqnk2lSK+GkIBoItHnEJJlIL/ypItYV7BpsVcb8UwQyuHH4F3/DPB5JKD8zs7/LHSV0sZo0sWsD/mopoVmS7JUa8Iow6w773KM8obqILLA4jNnSnm+kpXc+y48/otTB2BCH2IJrUJVlqTjNl8r8ovA6FzegtHTmuTRgrCMIYvPGLDM/h5Y28dwAdRkjDsUv7qdpjQzKcGGoZTfxXL8JGs5ytYcW3k7ZrrA4PsBvRv8Z7GS6cQu2NebGzTGKz/mvfoIgD9aP+D/Or/Posj746nKhGYh7ndyaTGF6hNxuhFXPl3IwzB+2pAsWsxK3npzsSCUg7hbSNOPleq7LTfeXV5K7DoWmunxWGSLXZck9YS2uTFI2vv3+sKxMMqgqlFRHqB+eMT6XtbnPHQLtvR9bEh5MIXDNLL9za30hqyDLTzJyvfy3soHlreyvk6hmYh+TnUgG2wOfCykin5zZVBOETopBwWq0qSX8lKl1wrVQhaLadLlWyIvXm/x7VPVJ3Y3txXVkSfclmt2eLDmwWTFfir3d2JrdByRW29YthnnpYSALtZjVuucdiUXw1wLr9JnXdZXaJy9mNfYocctxm5RelvJ0QAEE1BOi5wDEu+3m0C67oQRA+miJVnIs9BJoesoHsfTi51sw1cId3YOz5p0PwHz04ekV7dY35Xnp5nEhG9HM06kYPFl8P87vBYTgGoUyWMZYIThMcSw00Ugv/BM3xELTl+tUZuybwjyWfCrtcSN4wCnpxPww0uStB5dJySryK01Cp9o1nfk0tSrlB9d3uEyxuBz01C6hMdrsdjPV2OypGWcDsU8LiiKOvLUy5SmtviOG14ZzEryGgDZVWDv0vdid6ptSJYNyXvxwYnx5E6PhcRCVfex2VA3qIM92I+FBs5htCKsh8IzRvmgMR8CapTj51FbaJzSzIdBuVfn7MX4HC1w9V25P9WBEiu447bHQrHytgzgdr9mOinJ4/VItCdo30u1J0GhvKZ2cl7XVcpmlcGlzLijx5p0CW3MtLpULKFmHhOdJxseHl3xYCKsquN0xdgMQm/X7Yj3N/v8/Eoqpy/OZtinKc20G1SN1DX0ej1K6k7i+WzebLj98JK39uT6H2crMt2yjhbBqk2ZxgF7P1txmK6ZWbkXVnm09qyMTO61CaADKsZsQ6MBgymihV9Y1KlFt52yKeTlMNnrRiaBgUXksWtHsow1EU8WhNOzwYIn1p2xw9cF5vgIDvYACGlCcZKwvi/PR70XcHnAJ12dRkAF1VeWq5dwo3c5gB122GGHbyheCw8gmEAbLTyz0iRLGD8eJGyTlUNFzfLw4Sm+qj53h6fQtjeWdZc1XF72rA41ykmSoeMUIYi38KvfAmD9dsoH17d5bxw1vHVANRq7kLkzP1OoZSCG5ElWgXTpOTkTD8PllmAULuuYGtFSXorVaD44E03+SDt0l5fo2Yw2WnU6z+HPvIUbR2rgXkqyamnHnVSE0GZdrNxNFw7dHJBcDvv3mcElw1yvXWB1L9YVOCgO9Q3qzU3JYAMa1hIip7rdkOxVTEZidedpQ2YcRg/MEOc1TcfjDwrnTJ/zAPGSxklUuExrmnHBZj96THdTlo0dwiCNRlUGVUca7jLjSTolidoWNoZjupBbt5+ujoDoeXXaPoQurDZYUUFvudWpx2qPZ8jRXHjDaSGssNPllDyRe3iQF2S23fJmcpbrnOY6xg9NwKTCEgPRb/de4focgvQt8NHCt0GRFRLHBwnxuEzhO3loJdRNs475Eec/sXvg54WZD/kwrNBpu/Cqe0Zl6g5fDNs04NC2BOf6d12PcpS1tEcSjl7fH1HPFcVJzCEdNdi8xXZV7U7jS4OKNUO6enH7/bWYAJRTJJeDnsnsPc/0g+hWX1foqxVEWp4P4aW09+trBz6BQjr+mSR176+mbG5nNP2AKwJi6UoGnI5i1zfVKBvU9bKXO2azEa2UyIOnbQl102vKh/EYdf8OVSz2cKlmc9vSxsV95HR3D0V97CAxPReYAHrUYJKYQ/GatjJQRGmKEIuB0sFfDKXp/cfx0Yb9SdFLDK/rlIurCfpDOYBkodjusKNKQ5tbKjvo4TivcbHYqKhSylWGPpMBPb3WfVtIkL/tKNDMu8K6FjNqsbbr8QvaOEIckUOtIIYFAfTThOLJPv+fluv1o5lDjR06idQ5D7602Et5tOcfKGwRbhTPbIsNZhcNpmy5+p7E8avDhA/NPmeLSX88TW1xCzmf5Mp0ygYsvbqR1DU1jJtBV6k6DPjKEPqEfpB81358VkzAOyX3A/CpwSeKal8+J2vZdlcDo52inqVsbsdjKebkTw9JY7jQn18SiuJzvx/6z/8Ky2/PKPe7/BeiBRTPL1kHsmtHsoxJ5bJFFQ36Sp7t9sPHu6Twp8AcHNxsPxvbnPaUc2tpHh6zeEvinZs7mmYGbhYNjOMl9+YLplH3qvWay2rMoysJ9xbL7IWP8TWZACRZBxL/NFWgOugSIRa7n2PX8aWpDzGbCi4kBvxZPPXnhfuTnwCg/wSmn8VaOTrsY+7uGUlpt1igO7XN28eEvTHtTG5edZjQjBWb2/JQrB94wlFFNoox9GhtjqMV8A8fnvFrBz/jl3IpLLtjFsx00+v/JApypRgrGUQylZCom1kjFzxVkO2uQsNTp/lpI0nqPyze4P+YvMW7Y6mMrWtDUySopTwqdq0xFzm6jKyhAtp6aJQ99jBtIFvIgJxdluja9w1b0Bp9vcHty4tR3hrRTPKhSYq9yW5QHgiDWJtP4iAeXa5soTH1llaOk+fHroq4fsBebgZN+6KSngQdrKipjg7kfsx/lrBZZ7Rd314t7Js8Mn3y86HIz9uBkQGx5qIIVIeRxz9zmFlDlsuLb41Hq0ATPYaqTCKNJ06GWYBC9TpF3kLIB82XjvGTRsPcVlITEEaxajlNCEUhXiNIJ7YQ8NF46gbrzvsNStGMVZ8f6foVdJN1M1XUc4WJRYTKQVJ4RqcyYCVNI/2Wd5W9z8bJIe3hRIrxEI/NbBpYdwnOhOJOTpsPNTy6ligIiDFlledWJhPuyDRUY8vJSDy0070Z777gIe5yADvssMMO31C8Fh4A0IcYvFGUB4PiomkgXeme951cruDJ+auPT34BWdttL0TPZqg0QY2ihvvJPm6e0sRy8HpuKA801cFQ7VkfePyJWAXfe/CEv3j4Lm9k4tbvG/Es6hgSmeiaO/aKQ92pZ3oaFFexlLoMlolqODRice/pwJi09wJc8LQ4VkF+P3eKp27ChZOQ0XU7kpDOtnxyNTBXkmXs2RyNPhv1bwYWk0gJpE8iLfWn7+DL8kasejtokMd/HcztW7Tfucv6gVy/cl/RTgb5g0H5sqPxQH4RyM9jbmVVS8vQRdz/enODCqmsxZwcE2YxThO7mXVhlnYsFbduEs1gHfC5HAPA+s3hTHSlMeWgv68ryOshOof1JGlLFr04azytG66td4qP9fULN8NlIXpAADpaiB1LaPYn16i6EX45QJpi7t65wRwLq3Uf5lRZhp5OpAsVUJyMqPY0dUcgG4UYAopV+AF8M9RtdGQrH/NJ4eQQ3bY3aMY7bEGpGz2eu/sYRrG3Rp7gsiF/52NkdvQk6m61Uz4cl4ytXPjDdNPnpgBS/eLht+eeAJRSD4H/CriDVOT/MITwnymlDoH/BvgW8Dbwr4YQPpWMvM271070Z5LIdU7WnmTdYroYe+vwIdxI2n7VUEmK3hcql5qMCKMMN5ETqg9z6pnp6wzasfB7O5pjeb9h//aSX7/3cwD+8flP+E7yhJmWm94EzRM35e1GktCPqgP+qHhAEyeEwiW8Vxzw8ysJ4azLFOd0f1ms9STWYc3wsDTOUFVRnqA2+NIOmvVOoWpFeiXHe/xOiHrxQ+GVbocBP71qsE+XqGUccFsHTd0Xnn1RuNMnJHnGJAiNM71O8Inuk7TeStK7G7CTlSc73aCXcZCrGsJySRv1ZwhBBv3b0mLSH+9Rz/M+KY9W4APFsbwK1X7A7bekM7n+t/eXfGfvjLuZnE+mW1bxYX2vOODH5ydcnkUdpkpT3NGEmG/J5hWHszXTJMZ+VaBsE1xMUDe1xdi2p4m2maFWCbqx8drClsq20HPboRCtORljVw36Op473GzP2QgBQiUx6Zhl8t50OQIf0C70YnHKidZMTy8Mcr+7gT9dekanFclpvLZXy74V6Q7PgHPoxvVq3rr1qMb18tjKmP4aC4T3333qDIuuBuVhfkGiHMs09srIc/6XFzzEF/EAWuDfDSH8vlJqBvwDpdT/DPybwP8aQvjbSqnfAn4L+BuftiHlIL5fmFIsyvQ6Wk0bh12U6AuJg4XVWhKor4WFA2oAACAASURBVMHAD3HwP9yHPRkE3N5YmDdRwbEdaYpjTdH1lJ0EghmSkpPjDb9+7+f8E/MfA/C99JSZbmjiAo/dnN/bfJs/XAgN57Ias2kSyjhIrDY5zdmI7DQKUBWQlkPHq/zC41LVN/kIRpEqmNdDtapuhxi+KQPKe/InEmfEBfSm6q0WaRIeehGe+mREMz2gHUuDG5fJ9jqtIdN4dNHiJp06HujG005sf310G4ZCsNqz2UupZzfrErrqWlMF2hF9o/UwN3g7IVlFRcXaoau93mDAeTBDwx03SXsLFoRF5bNBIz8YQMMbx2Kz/Nrxz/jl/EMOzapf53G71///Z+YIyi7DjBSpdR+V1EEkW5Nv6zV1vHeu1jhnBr1/L0qqfRGgk5xHZ+gJeync6F2A96iYXAxlRTg5pN2P+ZnMQLiD6Tp+rSpU2fQDjN20mCod1EhN1CHaerX0Vj7HlJ6QaNDxBINHHxz0Vefb9Qg7AFcLNKBNzAF0OkxdP+35iHTh6JSubAH1XPUkgq4Pg4teoguaTDc94y3TL06Gee4cQAjhUQjh9+P/l8CPgPvAXwF+Oy7228C/+KIHucMOO+yww8vHS8kBKKW+BfwF4P8EbocQHoFMEkqpW5+wzg+AHwAk0wNMEStPqwAh4PKon+ECJjH4vUhrnI0x8ynu8ROAr5yBEJwjbAp0lATW1kQLM/L+C09QhtDLQzPQAoGmMTwuZ/xRIj1tn7ZzEtWy8WLRflDt89PVCe9cRb361Qh3nZA/kf2NLmBSB5JNJ98cSFaOZBE12MsWva5QVbSIvSdMRoTITGhnGW5keqaJyzQ+0dQzud6mCRCmvUXe5kicOFqN5bGiuONxe9HiH7dkWdPHvbOkJTOut4J9UGxay6KMErjLnLBI0YV8tmuNLaS6GIQ7rxvVyyEkq4C3g4a98hKz1rFjl60CtpBrAOJB6sb1Cow+FWu/q6Po5JY7D0O3iunBpq80npqStc966Y8rN+btUrydt1dHVI1FTeRcg1fQalTMl1SbCe9cZ6h0qJEIpcEsIkNrrWM3MuK9k/Prehc0E9HB8oMyNfVMUR7J+uu7BrtJya7lXuXnLSoEXDZQlpUL6JhzcJOMsDfqw1/1nqU8UhS3Ywhqr+21uEA6nK1Kg1l3NSyWZJWQXsm9yq+OSReO5DrKkqwrkeeO4UB/eflSKNtfV7izc/SmQHVS6M6JMkFXB1DXjFdz7L3YPfAowaWDN2o3iqene/xuvH9vT47IbEvb1dj457bfe7zwBKCUmgL/PfDvhBAWajvp8SkIIfwQ+CHA6O7DUJ4MJ50spS8wQJtr2qnBbqIbumlJnr3JLwVmPoe7t/BjeQlCZgguEDrxsdbRji2rB3KT13cV1WHoub1do/CuwQrvTviD977L76ffke1ZicnqcpDDTlZg4wB4fO2xZUC7KOd83WKX9RD3dUH0+DuEgHLDANTe2qOdppz9udgS8k6g3XOQdMs4qDW2G6QupWFJX5eQSQ+ELoTlck/2cMX9A4nhHWQbtApsWjn/ok1ItCM3g1RGmxj289jTeU/doEYu6ozL5Zh6E0NOjQyoPhu0gVTME4GEREzFDe0h7UIf8lKtR5dt32c1JLqXWwbpfxs0fd9bUyqKIuXHV5JzOS1mpNpROnlVrsoRVytJ4FTrVHIn8V6atZG+vMVwLD5Jt2QnpOl9sopFjmuPbgJ2E+mZjRxLM4/5iD2Ny6Hei41Yci+TbywAsktFeq36XgnrO0a0l+pODlomwy5flrQNZlVjr7uY0pT0Svf9AwgWN9EiOY1cU13poQajkRaU3fmZymMqNzxfWstE28mUfIMH/w5+s/lYrVF/XTYbTFHSze+6mmDXad9/oZlo6qcp1Vzye+/kh/hkKFp8GbLQLzQBKKUSZPD/r0MI/0P8+lQpdTda/3eBJ5+5HQ9JDB8qFwhm4D4ntSe9bknOY6KrqOHi6kuz/HWew/e/RRPVQsuJpdrTVNvFM4DdiBWWrgLrO5rVd2Mxx71rvrN3zX46PASLesS711LIdPnhHumZYfwosmxWXbJVllWtTIbJOvLqz2uSp6u+AYpqWkn8xbisn+Q09/b6HERXPdoNej7VuExR3Ik8+u8v+N7BJYdZ1FpSnkU94r2FxLkvPtgnuTD4LFqFWZDCsqhgmeYN83FJZtq4fsAHRdXKo/VkOWW1zPFtFzdW6MQxn8socne+YD8t+nhmkSdM05rTVK7nZpPhjO3a2mIK6TbWJS7TFuZvV6g2FoIlGtV47FUs0Du7FH37tmPiGJI8H4pxALKU9q54WJtbU9zPRpw+khnvsQFdK5JlVPy8hHGcXEZWoZsw6FatPOnSYdeyr3ovodozfQ0DRK2rtvM2Asl6Kzk/N7hE9UJ41WFkiM2jd5VHBdzo7TTWgtbYVZy8KkkQdxakyxTNOGzxzAP502WfM8hbj12NKC7kXItD3XsPIN6fbthi5HnShWf8vuRD1AfSaGeXA3h+uMWi7wWsgPHBAdwV46PdHxG0opmJybu+a9ncVn3zLMznM7Y/Dc/tQygx9f8O8KMQwn+89dP/CPxG/P9vAH/3+Q9vhx122GGHV4UX8QB+HfhrwP+jlPqD+N1/APxt4L9VSv114F3gX/nMLW1RoYJROEuvgAeaZKlJYy9UtVzTXm5RDD9PL4DPglLYO7dl/7PJwKMG/CTj6V+YsvyWfG5uNeR7BdOReCBj2+K87tU/L8oEaz3fP4g9gKeXnKTL3sLd+BSjAheZeBTX04ZwZfqYuq650etTLMbBalQBmluzwe0O4HODrmKMPTM0E7tVWSsx7k5PxjRBqj/3Zfm39q/4s3uPOE4GlstZM+35xusio0qyXlOnD1/F/gS1Sji1I04z8WhUIlLQXctJe20YnylsdICUD7SjlM2hWJ0/Op6T7FXkUVsoMQ7vNXUdW1g2GlUYkqtBeylZB9IYJ8/PGtInq0FnXSnxii7l+rfPoqM+w1K10Sq+bTTVQdpb7UFLSKkLQaYXJfaRMIRCUYDzvaxI55V2lbjjt94g3R8NevA+oGvXe2/BaAlNxRCN7b0/Wb86MCIf3fXG8ELX7XpDJNcaUw7SDaaKIa0u5BRpozfo4t7TvvOeLH9wAHtvSFc9JH9kGoaam00gu2z6Knxaj16X8ORMjuc5qb47fDLc5SU2Mgr1KKHZz3tpkHouHcPceFAHfVE89wQQQvjfuCEhdgP/9Bc6iI1n72fykFUHVlzWjsdeeOzGoco4QhppYde7my84+JujQzg6GGL6VoOTPrUA7TShPFI0h1GS4aDgYFIwz8TtHdsauyWE1nqNVqEv0qi84SfrEy4rGfDPo2Z8s5EJQ60sybXqi3uypcSF+0InJNbaxLjg5ffGVIeDlo6ftjHxGsMEkdPXRKphVSRwlfQhjGY/oA9K3jyWAfK7szNupQtyNUxQa5dxVcZy/8RhDwraGMKpiwSukz5kNToNJIU0NwcZ5CQxG8McdYMtHcmj6OZuSun7uxcHucOMam9EEzVTXCb9oLuwdN7GYrPu+lw15I/X6DMZfNzZ+UspRGofn8rxPT69UZj2zGU/4/e+R8Uf/xjFR16SJEXPI6FhNAKzRasEqBvmV/J7sppTHtpBh8qaXswORJYhWfmB4htbDHYDuq48dtNir2LQ/unljf7I7vIS839XzA5k8iZNIARC1287yjx8UhHfDq8G7dsi8GCbe7jp7Z7SnV2GvmgMBqPuRfB6VAIrbnCzdRP6RFZ22WKvi/63bSG1lwF3foGum74fqgpBdH0id9ff+TOYSixZgMJN2ZgJH2xzpWcN85kc096oJDFuSGqWGYvlGH8RG75ca7RTjLqXeCU9ASancQCvHM10aNTtE9HPr/ajOuVbjuR2wZ09sdgP8oLcNDcmIU1ARzeiDbpnDYBUD05szX4iVuueLWi85drLAHxazXlvvU8ZY/hlkdJuLMQJQLUKs8VVT9eB/KwhO41x4dWGsFrjOi8t6s/cGDi2+v5+tBL486Br5P6qoGcz+LbUXTRHY5qJ7V+8oAeP0xaBdNlgYkMWc3ZN+977H9ueysS4MIcHhNmEkHfdzRJ8avpYrmpDbBoeJ+uZoZmo/t5Xx4Fm7qBjFQWFqnX/bGYXN3sb2DIwOrdMNzEBnybYu3cI0QOhKHGLxSCMuMNrhfaDD0m1xp6LQeB/ZV9yXx3h4SXsY6cFtMMOO+zwDcVr4QG0I83yYayW8+Libhe5hSzBdz19lUJX1UvVAvLL5Sf2Qx399IwTf0w7Mf2xBqWEHw8QoDjKaWYSMjlLuSGpa0o4XEuPAID8vCI9XQ5SFkoREiOWIOAzi08GhcbySFHPA24a5Z7HjhBgFXn0ZWPxXrMpxcNoKouxnjSTA8iSFmtuVqM2zvTyzQCpdb0+f8fmyW2kmWYN7TohuYo8+5XQHO2mqyQWS7XejzkAt4duQx8CMoXQBPty9xgH7+sSWof7yc9feWV3Z4XrUS5WdlQEVeMxajbBT2MXr+MJ67sJxa2OKw9tHnrqnXb0uki6BlNbTCH3fnQxY/LohORxrFofpVTHI5pZzGck4j1sx+SDkhwDgK6kIruruVjdMxR3A82RXKt8v2RvVPWyHlVjWa5GtL7TUdEEhpi/XSu81bhEeOajwxHZB9eo2B+Z8QjDTvf/dYWZz8FoVMf4c0Ih72NyL+GVeS0mABgGfF1DsvH9gGk3LXpRoKKkbVitv9QHtv35O9ifv/OpF2r8Kb8BoJTkGgAO9giTvC/E8qnBZaYvTKpnWrSCxoM8soK+LoAigdOEOuqENNHlz2PYee8yMDp3pIsoPTEay6TViahZJdGtrs/AseLyTYfalzBGNmpuaAdZ40mmNW3Hdd+YG9TAak+zeFNTH8j9cnseM2lIYn8CrT3OadpO/mBtMde2b6RuKgj6Tn//7SaQXwYm70tIzaxicVGnOdO0orEeQ3Qi2N/gux7R6/Wzb4HtegZr0B6VRAnlLCVkyVAYNzEUx5ryMIbQJqGvzQAIbssFD1HLrVNyMOASjc1kX902hwR+iC1PB12loIakv649qvW4XCbzdBmlpePvzinKxhIi4aAqE9plgi2GBD1b+nJD/UEno+GE4BDDnX1jgx1eL+j4bN86onpw0Evj11N9Q4L8ZfQGfi0mgGDo45ymomdNAASV4lMjOtqAGefYNMVHlsdnNYV/LRBCb4H6SY4bDaVsnU5+cSLflQdS/NMNKuk12FMpkANpZJJdNeii7ddXjYOzq7hBBc71jSiSPCMklhD36cYWl+jeo9ncscLaiaNFCIoAfQ6jri1tPTQ1qU4czUz3SWXdQvFGw/4dsXrvzhccZpue9dQEzVU95ryISfBkQjOy1PfjoGQd3mmaIjZSP0+on2iWD2JDljBBN5Cs7gIwOnPkZ+XQgKf1Ig53KHULdrGm/eDDj3kU/cTwCRNEJxZnDnLSpek9NL9QsbG9LJdfCddfLpYMrraINRFP1vDeo36bCsjTRCYd6PMhXSVo34GrG4i1QuUZyWm8Ft85wmUWU8pnnya0ClRcfByLymKDKFyubuj5S9N4aeoCoDcNal30zZVC2+J3Sp6vHcxbbwDQ3N9n8a2M8igaexPRB3KjjiHz4i7ALgewww477PANxWvhAegaJo9iubsR3nrHw9ZOk2xazEXkqV+v8Ov1185y6bo2Ce97WwJW0LvpTWSBdBKyTcAWkF+KFZeflajK9VQ/1XrUqujDgWo8ws9GuMg06SUQTFe5bHCppozc4uogoA5qJrPYXyBp0SpQt1Erx3h04nFdpS2a0A5dsbpj7ZhGiyqncrbXKVlWKcvViGYZWVaVJkwc2Uj2N8pqtIIq1gGs05x1lmI3ndUsEgRDNavB5SNM7EjWsXBU13HMOXSWfWHP0J1KwXretuTvHeLnEtf3iY5yEpFJc7kSjf24L1+UhEaO/WVTJMdFxfjHidA/QeoHrleE6MWoyQSVb7UFbBppOxh1j/rj66QZeDaF1RyL1AD7c8jSG+HJbZYSPmpLreTaqqKSngPxWruLq12LyJcAvy/eb3GS0mz1wlBBejKozlvv7vML4LWYAFDDydhS+u32+iiVw6zrvg6gf7FfEznozwv307cBqTsweT4kgdsWkybsXcYBJ0tAD8U4tF7yH1cSA/fLFcpa0RIHQuyN0IV8QpqAFjkEiG5/0/Zyzsol6MxQHHWCIuArw8bIQFKQ4VpN2MQ2gI0CrzBd4/KNwq5Vn1NoJwGzNKwLSTRuwhxdKpJ1p60DuYNRx9sMUqzWhbyKgwadu/5y+EajDLis0zaKxXBRCyJZC+99ewJV3g/FVd6/kAaNO7+ArQY/3SvmP/L3y0D7/gcf+04lKdIlhpdC3zTzOf7hHQCawxyXa1yUEWnGUYzODlISpoKkkAHKVAFTepJNpDCv7qCvN4RTKRTbSUN8QWiDOdyn3Jd3sc0VygVsuZUPbFVfNBpeQvzmtZgAdBOYPI4x/k0rybFudgugGtc3nvjaPlRxwnJn559v8U/77VneT5cYfyQVnno8GpZvW9S5TKiJtYTZhNFEFC3rDy1VkfZ5iaAgaen1ZZJVrMvoOoBVgWTj0NUWGVnRj4zaBXTjhdMeoVp/Q5DOjyx11DdZ3UuoDlOaSWTCJNxgsphCkS5gdBZJAaXHlL6vfDZli14WtD97+1Ou2OsLlWVDgtq5G56Lmc/hwR2aQ5ncm3mCy3Wf3NVtwK4dNg7AuvFiMHRV82UF16sbxV8fQ5YN3qQT9pbq+id7jQqatnMyVKwsjkns9LolPd+gF3Eiqhv8Yvn1fUe/Ypi9OWo06vNzxYmm2qfX4QJQrSjWvizscgA77LDDDt9QvBYeQDCKeh4pc2MdFRajxbduUEWFW++qFbfRW43GoOfzvpIZownpEDdWVYNSCn9H4rz14YgmKpqCWHPJcuhru60jA+BTsfq6fgPJRqQ50nOhxeinV4StSmpCIGyKG314ZYMxjJCmJKOcJLYpnI5z2tv71AdiZrqRSApv1xEkqwYVLX5d1nB2hY86NKGpP1Oa4XnQtVE0RwcwHg3Xsx1a+oWyJKzWL8ZEc07YQCChO2sH1tCDOxRv7FEeRvXPkQI91JikWzIQIAwvYPCO/GcHrNzTpxA9BAOYLW0tlWXo8VgkIjrE6uEOr7oq+xuFNCHMJ1v9taV3iMu3tH/coKvVMQNfBK/FBEDYbvnnsesWs+wSTbWItd0XGmAoK8Jy+fWgf0bYb79J/UDqAOo9aQTdxe90K3pH6ZXEWOzpNWG16TXVu5dNz2LLyX/oLdYP8n4Ad5kkYfs2gUrh8kGm2uXQTjw+UsdC4kFvvbZBfaS4RN3oV2DXihxFG2mQLjWYse6biuRWY06v+sSou7p6dn4mfheq6uPaPe+8R/rxNW6uHv9+GSlG+9a3qN6U+7U+SEQvv2PeVX7oTVE5zKbFXApBIZyefeHwR2jbT85ZfHhKnlp0I+E8Cf8oTC37TxY15rpAbeRdCOvNjQnpua7V1r175r3a4ZXBnT7Bpil2X+735NSinKadDjmA7bqRl4HXYgLQLpBeDR2sVNWgquGlCHkKWRRP04rwGsYYOwVIlaZSpHRHYuzVvTmL2wnVXizMGiuC2dJr2QTyS4WpovU2G0lcbizbM7eOIEtxExkiV2+MWN/T1LEtrctjMVHXP8AFSaqb4Xc/a0nGcn1t4ghB0Tax8KmwqNIQ+mokIr+9255YItvoGsMDNHsZxZ17PWurnmqaqRo6elki6yluvpG+w11TEVuGPpkIYAuHXVZDo/PVBnf55fV/gFh5nmz1U/BDMZcpPHYZGUF1iy5bYcPAS/dS3dU1/MF1/5I+62XdcW5+gaAUoa4x19G4vZ/jRlBH/X+Xi0cwvJtfYT+AHXbYYYcdvt54LTwA/NC20FuNJuklclVRo9YFIVYvuournnf9OqEPSdUNOs96qQKX6Rjikp87+uIQxw2MT6tePwatCNagYuUzZYVyHh35/MnGxyrc2PWpUKL73rF0Cmkr6KKaaJsr3CLBxzhuEy3ajlqWbenJy/FJWKpj8ZhaQlRdNampPLr26Mg0Wb45ZvFtTXkcpSDm0ldW2yHE5BtNKGOOp4gtBns3VqGcwsTf00VCdpkxfirMl/xJgbp1gI51CDQtalPiI1XTV5XEzF8m//yDU0aRdpznqTyfMczizy8G/X92FvgOLw9mNoP5tK/aryeKem+Qfg/Zy8+2vBYTQLCKzV2JGYhgltD9ALKLhvTtsi/U+apg798DIMwnhNT2Sc1gNe007fuy+lRJH+N8i7vrIVlH6tzCkWxaVN0JPHmC0RTflpaE5aGhnqkbDUmUk4EdIL/07P+0QnfrNw69adBR4Cus1pKkHG2JLLftkIS1FqwlxBDT6pePWD601JJikAYoLaQxz5euJNzTTKI42sxgmkB63TVgEQ2cbV2S0Gpc1wKy1uhCk6w66QhFMw34se/XV43qQ2IEmdA6SeR2mkrD965QIAR0McIk8dE9v4z0yTgDvoSJYLtN3w47fFkIbYteriEWgikfef+RENEVyXZWh/IvHgJ6LSYAb1TfmLqLt3ZxrmAVfm+K9TIAtx8++tKLwOzDBzQPhUXTzNMbzVq6frvlwVbXnnwYEJWL1n+3ijK4ke6Tti7T1NNB/K06gHrf40bdXQZdatKroaK3GWfYauBiZwG8Fc1w5mPpj9txuesGyoqwxcMPeUpzOxZu3TJU+1LQBXECaIZqQ13Lg9YP8EoeyrbLyQQYnwbSRRzwlUa1w4SVFBLj7xQN21xTHiqqg058LdwQuGrmsr9uQg3aCAspdssaPW1IfMAdy/HryQh1tYROG2qnbb/D1wTKWiF3dNpnIRCc75Vyp48a2nGCcvJ7O5EK/JB07+pOC2iHHXbYYYfnxAt7AEopA/we8EEI4S8rpb4N/A5wCPw+8NdCCJ8atFeBXl9feuAOevJdLPpFyvtfFP7qGhu1fHSVC1+7jtWX12vCKMM8FFqOqazQPKPFb6tAuhykLVxm+j69PQK0MWLTjoL0Q+6mZhUISaCNfUDLI0WSgboYqjdV0aCivr1IG6u+mrPzBDr5gNA4WK7Rx7O4flfdGRfTwi7ojt+nkKxBx7Z02oGpPUENIa5sAaPzSE1ctqTnJeZpVCdtW8LBvNfbD0YxfU9R70dW031LdTDonQC0YwmlddsPGqqD2B/hMCO7Tnq58HTRYscpeh7lCZ6ci5zDVshohx1eR4S2xa/W/bupsgx9sA9Rdyq5qpicGoLpeqUofEpfrKNeQkrgZYSA/m3gR8A8fv6PgP8khPA7Sqn/AvjrwH/+aRtQfmi6rduA2tJ6Uc6DUb3oldmb4xarL1V0ShrG3KSedsNKdw+SH8W/z1hfWYv+/lsANCcT2tz0XN70qsHuJ7STTi9foZweNL81EnbZKsxyGTSTOCAeJwQzJbuIDcnXlfTdjX1dQ90QqupGaETPZuiVzMnTDzOyhcZth7V86JuU2MKTXtTY89jysW5EAKyJM0bdQPBDiKlpCCH0DXxIErS16E67xxpoWpJHMoFmZ3MW35lQHkX9makM+p3YXEdJHXIEN0OGPtEif9A1mMlzzHy+a3Kyw9cC24SW4BxojY7vjrGG9DolG3XWmI7GUVz+q9YCUko9AP554D8EflMppYB/CvircZHfBv4WnzEB4Okt4mrPEMyQ+EiXGl21InIGqNEIXTdfq1hvaFvCOyLsla4PScZ5P6uH3KLHhtHTrgHOTWtYx3h6UgyFcrry/QCtGo9d1aiNTACqqPBPz3pWkkpSlNG93j3zKWGSUx9JsUm1b2jGSvj6dAyhcKMjWzNPQEmOwT5divxPN8CnqejYR0/DL5aE5XIg+ZTlpxZH6Uc5B6s3aE7Egq9nCe1I9Ulg5btCuWgVPb6Gxarff2gawnqDixNQcG6nSLnD1xMh4JdLQh17TJvb6HpCdh0LD+uAywZxvpcxAbzoJv5T4N9nMISPgKsQQjd8vA/cf9aKSqkfKKV+Tyn1e03z7CYdO+ywww47vDo8tweglPrLwJMQwj9QSv2T3dfPWPSZQdgQwg+BHwJMDx+G4rDrmyruftL1Sa29dH+6Eiuy/TRlw9cZbojJq6Lq2/KFzJBeVn0Ix+WWei/paZfBxBBIVz8Qrf9Osrc+TmjGoxvSEsnmfh8fbEeKNtvKOQTZZsc6asdyzTuWj4QXB1qmqQzJEkbncn+So0wkamMXLLus5f5EfRxVNx8Ll30afFnCH/+4t0TyT116x7vf4RcYsXVseHAbgOJ4RHmcUMZuiW6kelo43Oyb/rx4kRDQrwP/glLqn0Pe2zniEewrpWz0Ah4AH37WhryV2G8HXW81O1AS1nBR/Ovrir5QbLFE5TnEGL1ZaMgz3OFwAexmaPjS5tJAps1jw5U3LNXBQNt0uSck7sYAL7TNOO92v/WC9gq26WOtRrVbyytJApt1LMRzNxv0uNi83tTyN0s12VlAxUI1xhn6/km/eVU1qKsl7kwKt17HIr4ddviqoJIUPZd3X41GhMmol30JVkcKdZfvAreVZPxKk8AhhL8J/E2A6AH8eyGEf0Mp9d8B/zLCBPoN4O9+nu11LJRkEyQGvVX84PanmFq0ddyTs6/dIKKSFHNfmm6EyUh62HYdtpwnZLbv2BWswuX6Rq0BDPG+YMCnAZ924m5dLiDy9qOqpxtvaYjbrSclQHBaSoIBVSt0o270XzC1Ir2Sz6PTMDQ/R3SbTEMvSGa6RubjNO7XSg4n/m6vFSZNUF3hllaYwwPCTGL+GI2qGsJCvIaP9UvYUqfcYYdfNISmFtYaYO/cxs9ymqm8K81UU810b3wpB7aVJksguboXxauoA/gbSEL4J0hO4O+8gn3ssMMOO+zwgngplcAhhL8P/P34/58Bv/pF1tctjC4GK9UlMW4NmFShQo5eC2tFz6f4xeq19QI6HfmuL6ueTfEP71CcRPW4vAAAIABJREFUyPG3IwnpdB21bOGkk1MYch5GqV7pzwapheg8AFNZ8jPdW+zKizSDqTuWkCNoJfsB2szckGnoMjIdi4gA1YGmiQZ5VwncVd4qH7t8td3+A7bw2CLWQRQtITG9Vo++rtBFg1pIYj9sNgRAddIUtbB2iB5AezCmHVuCEQ9PuW8R7OAByfEMOQezbtBVKzLhIJTX9aZnGn2V9SI77PA86Hp7hKpGryts1ALyicYmoc/XeSv/2tjsr/5FkYJQbqgDCBpMEdvbAaZ0mHXD/8/eu4ValmzZYWNGxHrs93nno6q6qtp9+7alNo1NIwuDhXH7R8bQ/rCEMBjJlrl/svGX+k/+8IcMBtNgEDR+tcAg2cKgD4MxNAZ/uWV32yBLcrtbt27deuTrnLPPPvuxXhEx/TEjYu2TlVmPPFl5z81cEw6Z++x11l6PvWJGjDnmGNQE4+07PPgDsqQjY6BGQc9jNgVnvfSDrv0NSMVONJhMOt9sY5Gt6mTMzURQnYPayTmPftxKsohmH20ntLHIs49G4DYUZbUG5TnUXBq/+HAOP+6Nv9uDQgrH6z3tIu4LTNnOI7+yyXYQDKg93j01FlQ3MqgDoju0b+npGcycaJuklRigRBopCeWTTSx0KXTjXkuJCQI5NbH2kEE3Htl1oIVeKBAzVHiI/NXqTn8/XkfEAYNGI6jFHDyREYEzI7Wj8KxgvQWvrn+uvDPexeBIEHFCECEnQojEAr+aYMakLBIFVP7w9p99JxIAK6QZHzmWBzxoYuvHS/B6Axcbm+7Cw620KH4Covq557xEWoMmY7jgwFW9P0GzUDe0jkzFSeyOVeDgB+0bN1LQlUkYuxjk9jddAaDLFVzQvvkmnXy2FiDVN25drqAuAb4nhifuXgmX91o/ykkdJtv0fBtfKHRGVjZma6XzOPQd4Ooa9uLyO+P0tJIZe1bfA390gvpI9t9N6MbxxC957BQvnzXIlhVoK4YCfHkF+440fVFRQB0sgIUkcz8r0Y4ycCjAEzOo9VBtSBDTEejkECYIBfonz4ZkcBcj+oUHEUI9lQTgzwp0o95bAwj+3GEIfB1F4EELaIghhhjiHY07sQIg3nOwypWwYHLJggUArRRUmAW7qze/AlDjMehD6WfbfXSA+ljDhvZsr2XWHFlM5CFyBYGu5QrRv6HU1wCZ4Qa/A+UCrhdpliXA2qTtsw1QLh1UJ7dKz3KYcQ4dMfWqllVI1AJyXjpzY6estQAzaCzXk8scyAzsXP4+zrZTH4AWHXIXICITMH8dfWZrK7P/tWD8/mr1SiyduJKzn36GbLNFdiYrJj/OxQsi1hSaTjwAgusWX6/h1ut3khnETSOy6EEanbIc2WTUewozg9sOCKs9Zobv7NAZ/XMWUdq9WI7QThRcGSnY4d8APtwVLaDXEhEz9oZuJATWSr7Y9ZuxBCRjoO+dwR+KtJGbF9jcL7G9Fxqv5pS0OACALG5g+mwAn3EvrcAy6EcD5/yagW4v4WkxbWkX/c2Vwue+SbtGVoXr0yi4XMGUoTFrWUnBNcBQCWOPx0cEKAV3ImJ1XGh4o7D6JcGNqzOCHfU0U9WKF0D+JEhxXFnkyzrVIEAkej4BhtCjUuSmQ/GVsgw8HYNHgcustawzk42dA9VdgnDceRBvC1S452MYul4e3LU/kwnREK8v1HgMmoU+gCyTiVwnz5IPZIiUAAoZD+JkU7+GW38nEgC53mDE5QrKMbJrOTvz7Bq82Yjz05s4ljwHiEDhJpDN4A3tsW5kkEx9C1sGWaCb9p21pAg6DnidqGmOzoN65SY4nwWtG9ZSA4jcXrK4oX8/PrcwWwcVZuC+1GjnBvWRzODbPzVCOz/uPXhz8QhWrtfzVy16raVDRnfkQGO5ntp4MBNcFYqsKwNQb/qeHeUorrPEOTY7D7Pp0gqFnBdWTpixu1EGN+o9dcUPmJOqq+ocSGuo0PegvYd9/OSdnNEPMYTf7YCX6JqV3kPv7qE5lil/NxVyRJxc3gUtoCGGGGKIIX5O406sAFhRmnGm38XO2DIHjAEZAdW/bxaQ3+1uKI3qk2PM+CGKK8nCPiPAC10TALKLLdy0QPVAIJVurMB6j0a58SifVjCPlnL8my24aXpa5vEBusMR3DjQGDVBdR5mKzvQu1ZsJ+fy+ev3C+zuUfIJdSXgje9ZM4bBxgN5hIQYpOQHALT2mBUdyizsX3lYp7Er5fpWWYFqZKB3QZ55q9DNCMVVcOQCoBuV9KmV9aDWJlqo3lbQnYUPnb0vUgIdvHSHGOLloQ8ErhW6NNDO5FlsDoQhF6Xh354awB61VbTf+QYkQMaA8jeTAJ4Pd34BnF+8UOcf6Aey0XP/Ph/PtyelJPP4CRQRzCj85V5BFwAoz6DKEm4slpjKsRi6B0hKeiYI+VquV7ECqpMMjVgMi/9uwfAmSGsw4Noxdj5CUIxo8wgAWQvoHSET+X+YmqFroLiOfRkettS9HLVnKKUSbsmr60GLf4ghXiFUWYKmk74xrMxDk6S8z4pEuDFCQLfvA7sjCSAwZwDAlQQ7MtBB7VJ1Hur8qk8ISr99rAbml/sb7ABgBX0hK4iD/0OJftB+IlQaas8v4QBIPqNUFOBRAR4HHPFojHZhUB3Lt2rzvkK38PBlqE0UUnzab9Qiy+hSN6JCtvEwa5nxm6sd6GoNHz15B575EEO8Uvi6BvaeH1oWKAHkV8Lg6+YFtg8ztLOeMHPbGGoAQwwxxBDvaNyJFQBr6YAFhFcPAMr0ln/K2p8rB7Bbh9LQQSIWWoPrBn4btHW6F2zvHXxYEOjRCMgzkWQAwNstKDPwUVpCEXxGibXkxgIRsYmsHhK5h8hyWjPyLUMn7SKPbN1B7/ZWIGUBdU8koJV14LqGXwkMNGjzDDHEq4Uaj1F/fITtfVndtzOhbPuX4dGvEHciAQBidQgAWcdiAtOGxqNdBxwfINY43XL5MzrCNxNUFCKXHPRdQATV2SSm5i6vvgqB7UtTOAc4JZxiAChLcBGaqyByAbrhRBu1Ew9MLXQWICBLsMoA4Yp3U0C3SqikAEylkV8bjC6E558vW5AXQTcA6GYGtuylHMhJHSFbh6Jz44U6GrSPqLWgbQ1cCIT0tt/fIYZ4WVCWQ5+dwB9LD1I3L9HONNowWetmSBRQAG+PFpCyDBUSgM8VvNGI8JZqLEgp0DiogT5ncP62BeU5uKrAX4OpU1FA3xePX7+YwM5LtNNQJNciINc30hGqQ2HyAL2hdH0cvj3zDuWkhQ6NC21r0DY6NZlk1wRT9Z+tLEPZPV6/9WCt4EPNxhsEUxnZXreMbOuQr8IOrQdnqi9gKQXl+Y31eQwxxF0N7lpw24GasGomgnKM8ip4a9SycrfjsP3QBzDEEEMMMcSrxp1YAbAi+NAH4DPBoH0bOm+tBy6WcD9jTFlNRN5ZnR7DH0xhp71tGzFDBX18vWmAiyv4IG1AeQ4qCoDDCqeqwU2TqF76wX24swXsLLB0jGR9swm8+qstyHlwcNTqzmbYPShQHcv1aeeiFuiDMxgbwBUMP5fjKecNFpMKh4XMsAtt4Zmw6+T4t20O6xR8mJJnmYOfdmjek/21C4N8RUkuWjdx6i5LDJ+XMFsHs5Mpf3HZQF9uwZ8/kvfDau1lq1XGVymyQwzxzgb35H7VWKjOpJm+N0KTj6oBL3Rg/45xNxKA7hvBVCdy0NEAhJWSpqmNFEHxBhLAPsTC0zHswQir9wU0r49IhOAiS9Lhxuimuhmy7TGy6hfDyYlZS34dGseWNVTnkok62g5q1964Eaq2oJUQ8XmzAWYz+IUkoISxR8MZC6AGdB15/QAvAP+cWUQc4Dun0XqNbSsJYLUpYVsDk8n1VknDov9bVj32SE48h6MpBSsNWypk270v7riAOpJGhG8L10UjHcoz0X6K4nZtO8hEDPHOROw7ijH+5X8G5qE0hlWnGapjBS56747bxq12QUQHAP5LAL8KGQb/PQB/BODvAvgIwE8A/EVm/trKHnmGCo5WPiN0EwM+DjPktkBxVSJKYttPPr3NIX9jRHEmDrx6P8qwu1+gOolicDLLThh3I4yZeDOcAXze43S6IZiKoWwsY5eijZM8cyvQ8hrqcVC7rCq4Pdxfz+fg+QQurDjIMcZPLaafh7/fdaDOJS0e6hw4N2iPZISuTyZoZlMso4Adi4hUxOD1ewR74tCNoxocgI6gdkENdEdQLaWaQL5i8QuoQtF+7WC2HdRGNlDBhMRerb71NTcffwh7T77k3US0l6LWkNlZmIst+LMv5X68bfUfpUE6MLRe0ORo7t9D94viJ90cFXBF3wmqK4/yvIZ+HLrMdzv41fXAvHpLwnzwPtqzGbpZaAxTMvFM/uB3oAbw2wD+Z2b+FQC/BuCfAPgtAL/HzD8A8Hvh9RBDDDHEEHcsXnkFQERzAH8OwF8BAGZuAbRE9JsA/pWw2e9CvIL/+tfuzCNhyGwIqlO9Q5UVWmickeuDBdx3mF1+10jqfEFzXZ+eYpw/hG4Fo49euybMgIuLBuZii/aBULfahYE3BBVYMtnaoXiyAX/ymZxfZ0FapVmai7IPL4E5uG2hrrfIgtRCRgRqO/CV1ETccikIVCHHR0UBNR4hx3Hah6l06hokzyLnMI4OZBrEGjasAERJFDBb2b5YMkYXHtnapb9XHcPsQifw02vw+WWCbLycVN/O7tw3Qjj2k09hWpG60Iup2FVG/4TNDu7Rk290Pvt5igh3qflU1GdDcFXd+G6r2Qw8n6KbyzbtPFhlhsuZ5QTd5FC7YOg8G0OdHCUlW1TSj/HWrZre4tDzOehQVsPtB8fYPSyxC+hDNxdJ6Cjr8rOuAfwigGcA/hsi+jUAfwDgPwRwj5kfAQAzPyKis2/aERtCexBs7Jzw1HWwJFSNg6ptr4/jbw4majIRm7yQIEAk2wZfVL9nGP4q4Z49Az17hvLrtgGg/0j+/yItIHrvIfiXPwIAcGnAnqG3wdS8auDHJexBqDGcFWinCi6cTrRoLK7koTahCYtGMuDr0yPBzAsZJOyihJ2YVFMRfwVOfRbiv+xSDaG4EgP6ZHCT9R4Gsn1ozoum9JVHdrmDugwm7JsN/G7Xww5Er4TZ2y8E4sEXN3//GvSu7lxEqMe9xAMhhl+vofMMxUS+G7op4DOVKLjZuoV+dAn75WMAwY40M/CxfvIWJc13IfTBAnh4D92h4MebXyix/kChOQzCjyMGiKHaKPX+szWFNwD+BQB/jZl/n4h+G98B7iGiHwH4EQDk48M9z1wCyPcn2TmopgO54KE7KqH2ZpXq7ATd/QN0CxkAXRHZKvEh6WCerYGn5/L+97h6eFHQP/+ncfWDGeqDOMCSzMLDs0lOFD2bw9jswXB5EtuEaoB8rWB2oQZgMxCPk4dyOwO6BaObh4Q57zAaV8hN6ARmQt1mqNeSMNTKQDdZqgH4jIMgXPQgDhhzeKlaQrZRGD2TP5g8IbCawAT1UL2ZSKNaFIOrarinz4bC7WuKfbMcDaSGSOCrDCr27m54Zg/xSuGuVtAnR2mSm+08iiXd0PzxeY/9+/z2z9htagCfA/icmX8/vP57kITwhIgeAED49+mL/piZf4eZf52Zfz0rJrc4jCGGGGKIIV4lXnkFwMyPiegzIvohM/8RgN8A8I/Dz18G8DfDv3//m/blNVK7MwCQ12hmMtfJDwxGjzRMkApAVUu3XJQ+sA7muoZqewiCrE/69LSp4C+XSUvn+w5VlqDRCO2vfQwAWP6wQHVKsnwDwIqhG0IWFJNZA7uHHv5AZm66cAATbHDoMhcZyPVZ384J7YzhxlG9M+j4FIGHX3ZYjCtMs7A/5dE4g6tSYIT1aATbGHAbpCEaBdUSyIbXrl99ACI1HZlOgHQZ+1zB53J8VGQyi7ABomu77zz714eHoAOpoXBmAO97D+D15q2Sl9Ynx0A4VwTYjnbC+uKL5Vt1rkN8t9AHC5B1MFdSs5lUHbyZoT7eW/cRetr5a9CDvi2T9K8B+O+IKAfwYwD/LmRV8d8T0V8F8FMAf+Hb7ChCNz6X84q0Q28UyBcY23sAAKo62NMx6mOBIOqFQrugZJSM4JmZr4Kl4jMPXT9I2jO68dCbFvqZJBT7xZffasCKRU0yBmzty6l2WQbcP8H2oTzczYEM/i7U+ogBrxkclnXtguEnDqboefjOKcBHCAwwO+5N5y0BILimv/mqA7JNpIoVWM7neDaLhjEMzvbOzxJ0TanIm6/Ep1gHGq5uGbrb82KwYn6TP5KBia438NfrXpwOLzF3CXLS+uAANJ+Cx5KAWGvxBd7Vabv2w2PUx3KButBjEc83XzuUz2qonz6R83ny3IJSaZCi10p91MdHoGlYlWoNdBYc6ki3hhDtHlymxO86eSm0A3zzTkdRgE0/2DMRvO4p2KoDQPRa6J8xbpUAmPn/BvDrL3jrN77rviLvm+mm6QGY4LZ72jGFhis02kmsjBO6qcyEw+ZQXY+buVIj2yjkm17NUhcKuTmS10Rwj5++FDtVsxno/incsahzVicl2rlKNQvh1fOeZ65DtrEoL11635b9YK1bRr7xqY9ge1+jWWbweS/xl7dAkRKYQ3HeInsqAxDtain6RrXPXXWjyE1ZDjXpS9F+W33l3FLhHIC7fwg7K/qicSZG1KnRLSwHksl7JgO6jiyk5wdkouTeBoRBrW7EnB4AjJZ6TijSc10jywyoFYe0fCwsKh37JNYt9JMr2GcXeGF4t988efv4M/8cNg9HqS+CCdAdEgsq21joWq692jagy5V4Gu+Fmsm50P1T8KQEB/9nahxovQUvJYm49XqolQwhHieQZ5msBQVzKHc8gR1RUv/0GeDGHpzHAt1r+Ojb72KIIYYYYoifx7gTUhCg5yAghZ7/R7FTOEAk1zuMvzjH6DBIph5P4MYm0RrJeoFZ9irnpnYwVwI5qKdL+M02LbedZ1BZQC1k1gatBe4JSzF/OMPVD+fY3QsrjsnNTmBTAWYLdJPw+YcKujFpRZDtPIorTrh6PK44w53/xMFOep5+O1NoDgjVibyuTgx0Y1AsZVYweWyR7WyiArYHGdbvGbSLsOIJUFjS898AxZXH6FxmraN/egHaVgmSiTWTZCm8c9KpHFhX1DioTQXa9Fxytg7+ZVAI840VB3ct8E31l6sV6I/D5z+/O7xZrSByPi29gSCtoZCKMKwzZOFeGeuhZhNoFqZzXA1xLfULZR2YOcF5xAw4n1ZvpPXQtTtEknf367V0/h/JQ8wmyONEZd4NQTdaoMPXFHcmATzvc5kMj8O/UXwt6xyos6BrGVTyzZ5WMQAuMvC4AOdRm4Gh6hZ0FXjrWxnIVGicgtZAnvX6+UYLfk19AiHuB3zVAbBINM7iilFcO9jEuw++xuG4zdahPTCojuX9CAeZKhqwAO2C0AR55vbYgcYWSgdpjE4BGwP7WIbo6l4GO8pSUdkuLIrDDQ5ncl6LooZRHpWV87ncjvHkYgrzVF6rf/k+2PTGO4D0GkQtIbMD8itGcR0S2NZDtSMAR+H8GWbTwjyRBOLPL8Ft99bQD+mffIJZ9T66I+Fiu+BzoIM2lVlWoGdCy+RdBd+2X+Hbx2vxvGzJW2ZkOsT3EO76GjqMTXmeYaYJ+Vq+g7tTheaIYEN5av8ZftUYIKAhhhhiiHc07sYKYF/iFLGQK/83FUvhtgnzJ+vEIjFAEFQWiZkD4IVdv7edeS0+O8LiRGbAsRhKVZjxXq7gl0uRfAagZlNhAoXi3uWf+wCrjxW6RZjR51IA7qUWCM0Ro/tAZpEHix3GRQvnJTevqwK7dox2Ia/d3MLMOhSlXKCpdlDKg8PSadMWMMon9U8iRjZu0Z2Gk2mVNNulFQ2BWoIKk1izBUzdF+XBIscRX6vOQzU2wRxvil77poIKWT1yFiAfJY175OOKzSVF2m/VYb4v9ubcV93chhhiL8gYUICfu0WJzXs5qtOeMWhH3DeAqdsTCO5GAtijs6o2al7LyeVbhtk6qFXAoJfX4PU6LbPfBPRAWZZqDGCWOkOUczZauN3hIYfRgFJJ3XLznkJ93wGzwBzJPLwjNBO59D7TcCOGMj2VpbUGdSfvV5sC2YWB2YVOYW/gWoWtieqgBFUp5Ffy/vgpI19zYinxnFCMgGAnAGUDzBMgnuLaIVu7lGBVY6GuK/BPRZPhRY5kb6M8g/n4QwDA5Z99gN19hS4ss1kHKu5WrnexGsFUorOkYlK0kQHmYdYd6jOZDGweaHRTSvCmUGoZo8uYTCP1NiirbizMsoKKFOVHj7/PUw4HRTD3gvT5bCLf35jsmhZ8eTX0JrypUBpqMe/HEpaang3lunbBwEGLfBR0uMztn8Q7kQDIA9m2x8SV7Xnp2cZB7zpQaDTyu50MSoE6paJs8wsGqtd3gHs1ASKh9cURNQz4ic7nPZBnWP2SYMjdFFLjsKFIazXQKmSr0Oh2RfBbgq2lyLs2JchTKvyMN4TisqeZtjOCK3rwTzdAsfIoL4JY3LqD3nbQq8DTv1x+J+76S3n9P+ex7zdwQ4CtbW+IpXUTQnPI8rAB4MLLKqgOcuArheIi1kvCAB7mAl4TmIBmLvfHTkQWPAkbdoCyBBvLT0nWN3phaOgiA4L/s5pMwE3zvRWK9eEh3A/ex9WH8l1tDlQiYwAyCSuvHmAUpMrNdQ2q2r5J73o9JIfXGKLllIHLIGszNiAH2En/XVSKYYO0vLO3LwIMNYAhhhhiiHc07sQKgJ8zOfAZJbEzADC1SR1ylOegPUvFOJtLYnG3hYSIpHs1dILyfIL2cIQmdKruzrS4goWs7AMlNEI05SVj+qUTpy4AoydAfq3BccXSAdmaMXkSpBuW4ngVO5VdadAcZ30jkmLotocJvBFYoQ0KgXbmwaUDZWE56DW4yWBW8/D5DzH50iPfBPnqyxbkfJIYdoUCqz1566c74I8/feuw/RuQYVWDAqSXakdBLPDgT45hR2VqvrFEosAWEUDF6d6SD8qpAcoxtUfxtEJxIavR8dMSbqTAFJlfHsWzHdRPxC4TSveSJghy0OcXb4z26jdb6G2L/DrQDhWljnUgnM/SpiZEfvQUvm7eGsbXXQk9l2eVjg/h52O4cVQRMNg+IHSnssScHO9wOtvgsJAVa64dfnzLz74TCYC4TwAuJ8FdA65KnuCN6tfLeQa1mPe0Ta3BbQdUrw4BqbJMnbF8OIeblfCjgMGf5Fj+UKN6GPTwFzVM3oMk3mq4RqMN2jrVe4TVL6sg2SDcb+qEwwsIbJBVffFGJCEImwey7N8+0GjnSAMQOSmQ+1CU3P2CRXlS4WwmA/TRaIdc2VT0XXcl1k2B7Zl8ieoPMqxrAwpSEdlqJL6+3O/f1JKUAKCcz1Ac/wryC6HXqvMVuOtAkXvMDL/epASh753Bf3CG9kAGEVdqkZ8OCSW/2EE9XcI9kwH2TvDeX9A9HM+n+OQZpicPAQqFuKmGN0iDvtn1FN7ySuon2UogEf3FOeyjxwluGh8fgqdjgQghmLo/v4S7I8mVuxbuH/0R8n8kr/OXbPc2QoJ3KWghCcAdTdEtZNIAAKwJfu+mdJ3GdV0kgkimb39n7kYC8D2vHsRg22sBmUoMYVJ4hluu0gyO8hzc2VvNSmg2gz8TD9v2ZAI70amIWh8p7D7qcPRAcPST8Q56j2f/bD1FxTnKhQyYRWbBTGhCEbfeFKCrDKoLlfwDgiv2JF4JYngT5QIsQ9d9DUBZuQaxf4A8Qan+elQ2w9oXuNpJDWF1NQY2vRSD7JOgq9CAVhHydT+ImYpRrDzyqyCeZ714Fodw9w7gpjlsGfBGIujaIVvK+V7/YI7VRxrtQbT0BMB7K6KLAtMv5yiuxPDFbDrYaYbdvWByMiO4fM8CsgLG5xbl49Cvsa5B2wr+Wmahr+rtEFlaZIw0q4VGQPYMNSqhDg8AALtfuYdmptAGvbb61INHDgh9GegUTKrfSJNetgkD/gcTFFcPoereHMcXOq0eVOuhZxOYZY+b82QELvv7RVULRKmI85fIXwzx1oQqy76eGBfxAf1oZuGL40L90Cns6gK7OqzYXoOKyFADGGKIIYZ4R+NurAAco7ySWVNkIexT68j6JDfMu92NJfx3hRQoy28yQbQC5RkQTNV1ZQW2GQUIYCwzuM7J622XQxFj04SawKYAtwq8J/hGxPCh/Z9rjeJKJQjBZaHPIbzO1zLbjxj89HHotg0Wmea6gaotfOhsnn45RXU8x/VYIKuVjmqgcr3ef+YwerSDagLt9HwF9+Rpsmx82bQhsWTKAlSWck0AkSxWPY7NWmbrkblCTjqJKcxSSDOw5y9DjqWPwMa+Aka7MNidyd83h6JYGkPXhOYow+hApDmK6wny6w56cxiuLcGVRmbWCOJ1pqdayj30CYfX59fy3XnuvJMcgzFAkcMHaQyfK9gxoZsH5sXUQo9sYio7reGa0NU9FggvyZCwAqsM+Sp8hx2DSRzZALnvfpqjvfcAAFCdZmkFJBvIimx0eQIAmHxympzXAICbFrxef7+MtyHeaFBZgCO8rQDlPCiwe1g/J81ODK371f/rWAHciQQA6jVybBEKwMn3VCzwXteBcte+WJ/mi3Qo0ACKD94HADSL9+H+aYb6WYAIcgaYUuPa9IpAFmiDI9m2kAEwSbaeWvh/dgNlAizgFKptjvo8YPRHGqbSCZLRjVBi44BrR1qSoe/lmvMN945gE6A+ZbhSvhiXmgA7gQmQT748RHnxYdq/smFgCoOW18F2civHl1+20Ls2JUSgH2SBoE/CgA6Q1ehxjepojJ2odcOedcgnLSicb+UVrlsDF2oQelMIxBUgP90A+TWlxj/dMLKNyGsAkmBcpsCzAOFYj26R4foXAsR2IgX52BYvfsYGxVKub3kxRb72yK+CP8K2E++I0MfBLE1eai3fh/GnQDc5gM9Dgqozsdjcc19K5mkZwHvQTWxgAAAgAElEQVQwrOqC2myAgIgBV+g+QRBBVR2KdRPObQJTG7i8X4ibyqO4lGNVyw388irZofq2GxrJ3rJwVyuo0FSpH94HiJCFZ79cEtqFht4GHbIsg/eqV+h9DZ9/JxIA7+numJqBhtPZmdpDVxYIWj5vgp2iDxbgQmbAi/93jdGzEnYatHiONKozQhf09qtTBnn0A+6KROP/I8HI7x+uMc97rZhVU+LcaviRjCItFDjr+wxcCdTH1M8KWIrAcQCvj6Vz2B3IAJbPG0zLFibMDDwD3iu4UBT2XmHdadg2zJhbLeJkYcauKoVsTcjWcr6mymB2Y+TrIMdcMeyI0sqMFUG3DFdGcTyFzYeAuy+D1vxgh2nZn2/VZrCd7kWevDBN7CSwthSDXF+jyNbiT9zOI3MGyDceupbtlVXQtUe5DA+BJqiu10gnJwJ4o3PZfvSsg9m00Cu5H7StwE1fLyKt5NrHfpLxzVKoKwE3c6AyDOrE8F28lgq2UXBp9UfQrU7S2nJ8/akrB+iZSfWeZi6rjbQCAKBbQhc8gMfZGQrvYb8MrKFBOvqtjLii8z/+CfTyEEUwha9O7/WaaIAIYxKnBPA6DGGGGsAQQwwxxDsad2IFAAB2FCCNKcHnexDQViAHsxBDFrq+/or64m2DikLkHCC8f3s0QTuXGaEdK7QzheuPw/F92ODgaIOTMMvVymPXZbjaSDflZl0gn7T4+GQJADgtNyi0xboTCOO6LcCOxIg9Bu8lcwo/4W3dCiQUDW1cQXAlgUO7eNuVaFXZY/CNmLn4MsqRhh3F961o/0T1z2xDyPZYQfJ5HsWVYOR6Z1EQEubOiqCsuKoBgJkXGD8qQEGHejfOsTWcaLCmIoyXvcGNN8D2IaE5DSuAiQO0RxcsKe3EgM9lVQIIROUK1VtWetmHG/Xy1+R7yVxTMcpLxuxTYRGZR0uxmYw0VqOByoEDbZiJQJlJGL8vNFxG8FE6owWo0kAw4TAjC1XKtXGO0FUZLIWelIVCXSmUexxTVkgsIAegnZrEtW/nhG4O2GQXKhBddh26jmcFRkcfwGyFQZWvWujLbW8hudnc3qFsiDsVbC3IByl2L6y6BCcz4KxKYwP7268A7kQCIM/ItmFAIAVnsQcBMXTtkxQEGfPaE4AalQmCoaaD2nXQgfZYH2nUx4T2KBRlyw5aMbrAxW2cxq7J4cIAm406LKYVMiXb72yOy2aMZ1tpLLt8NodeGmSBFmq24g+cr6M+DKA7TkXh/Noiu9ilYuBBVYG3u68UAnXoY4AxAKmEFXNV35A6AAB9egr/oYD2zXGZ4BxAMOh82UA/Eslj5Bnc0bSHpLTg2tWJJGRbEkwFFJLv4DOgmzIwCzh2YWGNi/kH1ip0jQFvgzz1lYFq9wroYYBvD8IvvMgn7AOerIH2IDTOLVwakAHAVwZmabC7J8eXbSaBShuu59qLlk8qUnuoyibpDHNRIT/OYVexyE1QltCyHG/bqD6pWoLaaeRL2bY8B8orn+AzkPS1JBqoZZDraX7ZltBd9/CaUIK5b9pbOmQ7CxV0mvRyBzw9hx0G/bc2/HqdCBij8xab90ZpssaZBnd9AoiTvtvErRIAEf1HAP59yCH9Q4gn8AMAfwciIP+HAP4dZv5akj5ZRvlUBvXiShgnycN320JdbeCDJeD3UQNwVytg76GiokB5XwSydHUE1uOkv9NtJ1hm43TxdUVgxbBHIWvPW2zrPPUBOK9QVzl8LEqea6hGsG1AZvdm12sfEUsh1ISiLHUedl6ie09YMfWRhh0T2sARbg4Zdsbw4yDmNrIwuUs4obMartXgiPkXDuWoRZEF3j+16JxGtZMZvL3OoddTqE4GUN0QzHYvQbWSoHTTF5WrE0J9Kq+7GYMLBx1WOPE4EiuKCaQZaiqf7wsF3/aDqhlZUObS9l1jwLXuE4BhGVDjJNsR/Drr70etxDgjWnRWoqOUryRJ5JcVqNtjBXVSX7LPnqX7P/2TEvOH9+XtBweoT3K0s+jnYNLqgEJ9pgyrpfGXVS9aCKC7N5eu7lHvBcGK0E7jvYPcuz1RL9USsrVsnx8q6NokrSHdjpFtj5Ctw7k83QCfPx70eN6yiEq7ZtVAdWXyIvmKCuPtx/9XrwEQ0XsA/gMAv87Mvwohz/wlAP8pgP+cmX8AYAngr97+MIcYYoghhnjdcVsIyAAYEVEHYAzgEYB/FcC/Hd7/XQD/MYC/9fW7YXCgwtmRASuZeQKAqklgjDfYPs9NA/vpZwAA+vQzHP5BgeMAsVBZCFzUBVP2UYHdD0+wvSeXspuM4LNRytZZBxjVK0KSB8yWMb6QtD7+fAdVdfCjADEcFLBTjeYo4MrOQDceNmgDtXNhyFgpOcAXwqSJQQrIc4tRLtPGTLsbE4VMO+TKwYffdk6jtgY6zNSb3KKZZuhCN7Ffa5An8Eb+3tQsvP/oS+0YowsP3UZMXgGUJ1pn4v9HEpAh1Cc9hIPSyTRE9SsGrT1MWBEoxbDGQwWWU55b5MYiDzRTZkLVZtisg2bu0wKsCc1hf9bKcmqvb87GUF1glkFgFTJaOjIhGKy6fwa3iHrQjGzt0ixMj9VNyunOo7gIFpDLDahuwTO5OT5XwU+gPw7leo6wHRNciV6XiG7yvqWHAqlLmpz4MlCk6Drf93fcImL9CwdzYUNFO9BdDf/sfOg7+B6DjIEah4e5KKQvYCpd/Vc/nGP3AGhPwup+2sEYBw7ws/9Z1gCY+Qsi+s8A/BRABeB/AfAHAK6YOYKynwN470V/T0Q/AvAjACiKRfp95FBHCIgalzjwP4vQBwvg4T10xzIgNMcZ6oVOAzBrwacjhFNcMXTHqE7kJtXHgBtzQjBUB+QFQQcqId0fQ9cO7UJuRbNQSfYBCDWQpod87DjQTcOq3zwSvnARECxWBruzEvWxbHc9B+yYwdmewUtDMHUcsBl24sGR5+5F+riIMMQKyFeMfNsXcXdHvVRC9HDmMGB3cwfMO2QBlzfGQSlOhjXOKbhOA4GWiloDHYBwPezGwBLAecygHkRIHr3eKXRGowmwiQ7mNzq87iYeuiaYTWzOAmypEhff1B4GgAraTZwZMeEYyUOnJmO07x+hPhVIrJ0pdGMIMQE3aZ0iYUJoZ/K3U6Ng1g22Hwp8tn7PoJv1RTxlxYdh/DRg/NfSzxHPjXyAq4IsR3a+kSa2WJ9qO/C2gl9KwcW9Bl0lfXwE/7EUmaUetEdJrRnF+QH0p0/k8/ZgsiFeT5AxSQqCtMjKUyP33xWB3hwfzU6BrZI6AAB0P8MEQESHAH4TwMcArgD8DwD+/As2feHozcy/A+B3AGA+ecj6OhQNWyt644Gr7a/XcK+o//Ktggjm/ffgj2VEs4tCViFhfOoCC6g6C9pApx5ubkFREI4AeEp6/7AiBMex1VcLxYeaqPke2ESB5+6Nhm5V7yFccShUxk5oB1ZAtgs1hVAwjAknX1lkVzXUTq4X7WqMf3+bViiyUZa0cGA0OM+w+yWZ9W0fGBE8i+JzwYwn+jNkOw/dcBrEfCY1j3i8riDYEdCFhMCaQUxwLgywnuCZUg2CmaA0Q4Xr5xWDKw3q4oDMMuhHXJxJOq1DJ7bXDKsNEg3AkxRjw/XNKkK2JozOg2DbUmbvcRatG49s3d4wGLpRJH94hvYwR30YmTgUVln9JqnLucONb7crFPSGUsE52zLEP0LeV52Y9RRLecBV4+BGJjWdgRmmdjBBZ4k/f/zK2kffJtR4DP/RAzTHsvqxYwUmpPqO2VqoqgMCA8/kGfz1+ns9pnctfF0DzxM6gjqoqe8jW6vE+HMjJc+Hurmqvk3cpg/gXwPwCTM/Y+YOwP8I4F8CcEBEMbG8D+DLWx7jEEMMMcQQ30PcpgbwUwB/lojGEAjoNwD8nwD+VwD/FoQJ9JcB/P1vtbcoBz3JwfMyKVKq+QT6vPzelp/67BR+MYWbCEvHlhrdRKGdhk7XKaE5AqqHATNetBjlNkEabWPgO5XUIim3IMVIDpIe8I0GVZHHTvAZYNPMkZKCprwvtMd2Gt/WUjeoewimWSjYUXhfaZAt0oy9XDKyrUvd1cQs2kLbcPybBrStMf7DTwEAo10VumHjDXDgtgUHHFifncB+cIJuKksENgr5tUs1iXqhoZu+ZpNdG7AxcEVgIY0YvuQbfQ/OAwiQlBpZ6FkHH1cIlQFtNcjJV1N1FPjQ4fwLTnBTuoJd30lcXJDYYgYqpeoCbn6DRro3dWKfPHvTe3uQXrYF+DkI/CYE1N+b7LqFXq5R/lQ6d4u6kX1HjL9uvoKnG+zpMGkF3zTwb6jjV50eY3s2Rn0c4LdSKKtR5oQ1AB5B13LvqSyglAK6wOAaagPfS/D+/Y99QZDVNWceCN4f9BpWALepAfw+Ef09CNXTAvi/IJDO/wTg7xDRfxJ+9199q/3pCAGIdkqsb5DNoCYjqFpokK97+emePIVuGphWBLhUO4apDbJt0JppNbqpgt6EZRjnqFTe0w63Clx6mDNZti+mNQpjYUOhZlMXqJAnBlczUsItryNtkWC2BLOHQtgR0k0nJ5TR1Jhkw+AUxdlIBrkoBldedMjPd6LZA8h2jkX/BgBVjVj57UsNKw0dDHDocCHesKV84ObBGNt7Gt0kQFZZ4PrP+wGeM6F+AkA2bTGb1JjsFaE9UxLTs16htRp1G3n1GWxlgAABqUohXyqU4fDIM1xB6fNdyYLDhzE7JgQ7DVIYGcFOFMpz2WD+KSNfdamOpKwHVV3v6VwU8hOab6qHM2ze02gOemkOn3G/3L4hdCdCft0sineNMdYKOhSQ2SjYUQaEhKMaB32+hvuTT7AfvVkN3mjYn36O8WSEbCfHGyW/42Qhu9iCP/ksDfSMt9MP+q5FHOPKiw7Vcd7rcCkFR30DGP+sTeGZ+W8A+BvP/frHAP7Md9rRXipTdQdV25u+o1er79VIZL8PwLz3UFYdwZdTNwWUyzF6FgdAfUOCgzzQLjTqSh6ii8lIcPC9Jg3Ofa8nTwzqFPSu75TVDdIKwI5lgI0zVsGv91g3HaNY9Y1iunEgzzdm/L4w0FeSUfjzR/C73deaepCink1SN1BE4DBA56MMtiSoiNFrwcOzbWARTQjtAcOG87OtwcpNcBW1zVsNqjVU1ScQHrkbHcpqq1FcyP6Li1j47vsMvOH0eS4jWS0mFhKkTqD616byGJ3L+Yx+ugLt6uQoFw23k8MckQz+AbVsDsTxLXoC+5JvqLeS61k9ZPmGDpEtCW5s+mTbORjrE2uHdjX8+eXX3Ik3HMxw//j/Szjw84Ywg+zcmw8yJtXrVOPQTQl17Jqfd9C57weHQQtoiCGGGGKIV427IQVhfVJrhHluhjYaQbXd99rtaD54H+5MqKjtNIc3KuHE3VSjPiS0i8jeYdgpwxd73ZuNSrTD/EsNO2G0J0GPf2JhtBcNDwC+y2A2hPIybL+K8gDhWCqZ8Wc7yfKjC4t82YCq0LnLDLsoE01xd1rAlT1LhSyQVYxiKcyOclbCPF2BN6GPgj3YeZn5QvBGbtu+z+K5fotsdYbZ7hR21vvG6sZBr4KC4TTH1Q8m2L4XILNjfUM9UxUONLLgcP241aBKQ1+qcL4kngKxb4CFgVSsQie0Z2HXtHEFIbTJLtD02zmlawcgraYiW6o9mwKYJqYNK+kyj/IKqnVQjU3+Bu2MYMcMNw6zrkCP9eF6adfLUrAWSIrCktzuxOktwpnkGNRaUB0gnl2VnMiGGOIrQSQ+JeG7qKwX6DHWFzVDaZcAk7fHD0CRCHZBBgBYB4qNVlUFt3kNTWDhqqnRCNBaCp8A6GABe7ZAFzxt7UgFqmMYQKaE5oBQ3Q8Y84Fw3GPjlOs0PGeJqtVNGd2hg5qEoqtx8F7B13J+ZqVhtrSnRQMUa983+3iBFkwV9F9qC7VrQZerdD2y7QKqkppIOTIAEVQbBrRNDVyu4J5K0ZyZv7PJOBWF6CMBoPEIPje9haUKlM64dvQs8s6hKO1HHigcVCpUSQ9ApIHCyblHGEXXwo2PhUfdiiyGjgN040QvKXgie0OwEw1WgRYacYu9msk+Tq8aB71rwVmcWCjAczLMoboDNW1qfpp/OgKQpcJoN2HwXoIhu9ek2EqROL8OyfrcimBblJ7e7MB1DR9a+99kM+MQP4dBSiYIFIURMzSHDHUYpNZnO+TGoQsU61hXu03cjQTguS/KMYvwWzTsaNrXY4IR0qWvG6g86wtuVQ29KXuxM2/gSp0GlojxJhYLE2yrwWFGr9YGo3OVXK26Aw8UPr3f1RpqYzB+Iq9H54JvR4w723lkG4vssuoPNdNASAhqU4G/fHLTSPxqBfxE/ksIvqJh1uCq6tZTA24auCC4R5stsqpGFhJCvGb2qZi8m/ceYPxgBFECAeyFhsv7vgKfSxFV7w3QuiJke53F0kgW9t2KOU3xSTCRv1oBD85gD0LnHbT0PqyDg9wzSdixU9fsHPLzLSgwcfxmC4xGUEV0gNPCdNrJ9XbPCeWNjIbXZ1Bt7Oymr9RkVBd7CoB861FeyJep+PQS7rMv4W7hTz3EOxzB6TB2BpNjZGvCbiUP08qPYXJ3wxXstjHUAIYYYogh3tG4GysAoGcCaVEDjfLPrz28g6/39l3XwJOnieVXzGZQx4fg6BE7yjA6H6P5NMxwg1NY5H6XSwfVdVi/LzPMdiEz4ARxNNJZa5q+O3T0pEb26Eo2YAaPCvgyLjkQcOOA+VsHms+gghaRe3iM+qxEN4mdsc9ZOi5bmIteMx7MgHMyE8Z3p9GqxRx8OIef9DUA1YxhxoL5+HEJWyrUp6FT+pjhpi5xlWOn9L56p2oUbFDENBsSvZvYWZwRlDXIQk3GKAU3KWBnct1docC6l4ZwueDuKqyYfEZQbYl8Id2UejKRVvu4Kmpa+Kb9ikR2DL5YYvTlGLqRWZgda/icbmiyxx4BXTmYrYWJjKvLZaJ0DjHEK0cYC822Q3FZwJXR+0OhHbukQkD69kWAO5IAOPGw4RhUt/ArKfq+Ku8/NdcEbW2KNYayFM38aNR9MEP1/ixp8bhMBrnIM49aHHEAqE+FIhj8TwBo6Nb0ReBrMWi3gbfeLgA+pCQBUVwxVFeAvAxwqo6SEaHImWv4eQGf9YVEvbMw53Id1K6FrnpusNIE1TKyTdDeudiCVusEcfimuZV/gru4hKob6Nk0/Y7rGjbcH/PRL0C3C2SpRk/wK9NDMjWQrVmKsyF81vP44RnZjpOGfraxyJYV6LE0AtgnT0E/NSijGN9kDM6zZOEYTeppI4Ow+/IJuGu/c90jne/1NfTuBHoaEp4Rv+II+5hdl+4ZtVYsJper9LdDDHHbiLUi8+gSo8sJ7CQQGAiwTsOXsQ/g9p91NxIA79UAIp89qDMq5144W9Onp/KnD09gZ0XvWEWhmzYURXXjxOA8JhjrRWArYP71ezPUxyZptNsRicDZc/UVO45iZ17ctmIThiMQq1QzsCMCT0Q/BhDtGmUJFCbkphKhOx8atSjToDZorkAYUWQ9lO6ZAGpTJ3ZUdziCK1Tyke3G8tmbh4E19UslTHMCnRyyPLK1RXYp11BtarkWcUZsBQ93QWDsReG325cWMN3njzAbl1CdDNC7M412QegCZN8ciCBeDHLS9xAb37I1w+WE5iDUMIoM5Bj5RUzcuax+TsQhxpU5oCk1uvlMgw2BjuQDzeEM+tkV/LUkTMoz+U6F79e3GaTdH/8Y9Mfh+F50Pb5xD0MM8RqCGeR5T3mXoFtA2dsXf2MMNYAhhhhiiHc07sYKgAgcWRpEogYZWTnsQZ29ga3qe2fge0cAgPZkjObQJNcl1oFpEnHalqFbv8fccNA7m2bA3VRkDm7Y8nUQiWLIMsvnvRQDGKBGJd56tlHIV/373VTkCWKoRhgv+VW0fGS4UoEChaubZ+imGnbv83XLMHWvZYOTMvUltDMVuOpRTVQYKvF4TCVdwT7KM08UXJ7BToQ2qquJwBjB05e8Bx/PoT4Qi0jqHGi9g19KjcLvdl/LKuKuBT57hDL2CZgS5HSa4XstnbMqiqdaDpaXQc10LfcjqXVWHdTFNeznX6TPcM+eAXtaUGRM0u8xRMLsCSworhvYAYcf4i0ItlY8JMKzrRrIsx2l3X+WWkCvNZh73r/RoXAZGnGa9quFtaaBWgvGneUGrlAJE3egsGyKUgKMfdN1GRxVEuiyBcHlYswBfBVXYyU/JkgZmJ1gwnGAMzvRcI80wW5CcBmlz9ct35RycCK3m3wPHIfBXaXPI98XOcEC45gruQbFktDOs2RwEiGvFEkbKNgGPtkAz5bgoOeijg7QfHyK648FMmlnJB614fhNzSiuPYpLsUTMLytQ2ydMqlvwetM3lgGAIvGrBZCPpGegCQOyH4uBu42Nal5sD23RJyggSwnANCWy+xOYD0WbKVtWwPkyQThQBG47+I3wSPkNCacNMcSbDvfkKab/zwS6Egy1OjVoDlTSxXp7agDUN4IhM4JRRz17/1XE9YaH7ydAQYTxgWDENJ2Ay76b7ivhvRhrhCLiJFMActTJpQk3OksRxmYXOkJdCfjSJ0VKsgSzVSguQpH3kkHM6KLvayaDXkwY6lwG/9joRNc18qecHMHcJJMEFcY16Vq18LnMeJuTHM1cp05YVwpPfZ+lIjuW68lUAjhJ77MO57Q/bu4lka6lUGOQ47ETDbPtj1e1JVSZQxnZv9/uwJ2F2kpCNpsRsqLHKHUrCeaGgmbLiUWlWy/HkhKQg7luoC9lgPfnl4P+/BDvbNCeGRY5QLUME9VlX0MpYKgBDDHEEEO8o3EnVgCsFfwksH5aC2o6cGiffxGFUR8fgd87AwDYwxG6sUkYefLejR6urYeuet5/pFdG/Kw5MKiPFBpZQMCNRdogWiiyZkABHByqaOSgNIOj2uU2Ey2fcCW3H5CsFqLHrQP0rleQZAXUxxmywO3NrpVo0UT5ZsdQ3oMCBKZaB6ot9CbICRQa3UglWimYYbZINEtTC6smSjO4kuR8wvFxwONV2F518l66bvamxr1uPMyuSxAPtd2NVRkRiZ5Q8MGtT0rURzqtgFxBN2YqUfeni8dPSuoY4RZlO41yrJGXsgLRiwn0pgKFFQZbBzRN7xI3QEBDvOURZWlEZwo3Keq3jDuRAIil+AgIr5vHZeLt6yyDv1zeoIL6zRY61AAwL+ELQjvptWJor1kHW4j2S5RW6GRw7eZSdO6CsUkcQMkTDPUesLZkKerynsWhJ1DQgynWUgR2e7RPqL0EVBHyNZCvw4DaBQgkmpJvW9Cu6X1Bi6AfH4834O9uLgnSFeqGuYmdiEF8lKKQL8mefn0a8KP/gJyKjZ7nAf5JCaTp4RkgfPk8i14OAK5qwNoE0fmmAZyDCoJnpnIwde/Bq9pe4wiQmoeyoTYT9u91/76pPbJrm8Tm1K4GX6/7Rra2ez3SIG8g1Cw08EUpauvAVdWbrhc5oHWa5PjvWfZ8iJ+v0MdHqH7xGLszGRvaBcEVewngbSkCsyJwwMBhPci7mzM7fRPs4qYBh8YfvZ3CjE3Sw/dGCrJxgAEDvlBp0qobD9O5NACbRqPzSCsIV4pRug+YP2f8FR9OVh4cZtjNHGiPNbJl0Pp5Ssg2wnSRc5NjyIJ+f7G0yC8qqOvQPbregsYl3Fx26KY5WBGoiyOm9AbEPoduolAfELpZnGHLFyKtCDzD55SYSFx4eAYo79U0zY6Svr2KM/6gPppvGfnawexCH8W2g77cwD+TxqyXCppVMmCbVYOSAN2ExrpSiuI3DG5qDxM6l+N9UuF8Vd2J+N0mNLJdLuGrGqqUDKvnUyDLQaFTm8sCXJheXK9ugOUqJQwA4M6+0aRh7gujyn50D/UiBweCAjkgW3fQ695gBUqlFZU6mAMXV1/bkzHEuxOUZXCl7ieX2R4b8TXFUAMYYoghhnhH406sAMhzPwOfFEJlDDRJwwycv8ArL8zo1LZGlimo4GDFKtBA3d4KYs9BipwXp6Zd4MGfiP5/zKxs+GZ13RPIIunC67GFyfpleteKJ3D8m24qtMfoaamsyAbHXKsbDaYRTPAgxv0F2kWObhr7GETaIGHwYyNSEEuZEc8vKowPR9g+lGlBdarQznrmkmKCrvc8ejeM0SVj/ERgBlVZ+EKos/ILEvnpnVxjfblNLmIxvs3cOVpM6rIQH+JWzs/nqq+7IPQB1DbJMcOLBDhFF61dDV5vYIO8QrzP6Xgim2oSLCynE6isp21x14HbDggOZ+z5tcz+1Uz6KKgsE4UY1sKvrr8C29jHTwAAuq4xPlgkxhm8B212cBfLcKxDv8IQLw9//xjViUYVdLa6BYvX9mt0BPvGBEBE/zWAfwPAU2b+1fC7IwB/F8BHEGHiv8jMSxKXkd8G8K9DHtW/wsx/+E2fwZrgRoFWmAW99viQEb2Q0unCAKEBmF0Nk7RhlGwfjWU0yT4ij90KxdQdygDSLBTsZA9Dj8XQPd9XnyNZGBIxvFOwbdjwKsfomUqQUXMkHrURkjFbQr6SxCDnp1GsCDqIw+nKQjce9VH0EyB4rRKElF8rTBoHzsP1yTW6qREDb0AKvHuFIflQkd0HRMvGjgm7E4GYTB319vca5RqGCxmQ5iPosxPwIxnEXlSEV2UJCmJrNCrhFxNUD+UEq1OR1YgJiTgUltve4lF1WQ9BOYayDJ2M1UcwywImSIH45ZX4HZxI4x9PRuBMJ9pse1Cgm/Z9IOQY2c4jv+onDT5T4jMBSYB60ySxPK5qoKqTUYu6d4rtn76P3alcj1hMjzWS4toj24TJRyfJzIQCOX/6xc3EuU9XHmKIbxnmgfTgVCcjNAuCnQYK+sgncgqA10KA+DYrgP8WwH8B4G/v/e63APweM/9NIvqt8PqvA/jzAH4Qfv5FAH8r/Pv1wUgOTWQZYE5aPtRZePeCGVyY1bkL8VglE4uoBSjPQQEzplHZN+9Zcj0AAByPSURBVJdBWCxcN/D3RbsmaQfZyCKSIm5cEfiROFqpoLzHAIzxGI3C7G1RoX1o0KzD561NahoDgG7m0c2lXwCQhrJ6q7E7C6ynrkR1Jr66AODKkBhqSXr5lUJzUMJUsn22k67m0WXwvL18bvBnKbzGBBOx9Riqc9DrJhnMuMvljUGegW8UUmPngSA2R0oBmKTPybYe5HpDHeVYBsqu74RW3Z7hSyWrgbQC2FTwz87hQuNajJfVHsb374EP5+Cin2WrjRSOgdAXEpIGII2AnGlgFO4XACiVEtr2T93H+n2Ddt57GN/o43CqL1hXHqpTKTmrsxPgYjn0LQzxyqHGY/BUGHXVaYb2EOgOwlg4scgy91pXAN9YA2Dm/w3A807Wvwngd8P/fxfAv7n3+7/NEv87gAMienDroxxiiCGGGOK1x6vWAO4x8yMAYOZHRHQWfv8egM/2tvs8/O7R8zsgoh8B+BEAlNkcJtD+4kwtzgjRdiBjhMkBvBTPjTgsWwtst2lFoMZjkQ6OjmBtB25b6FqYGqZmdA1BhSvhSnGwipi/7FNBB9x/PtthVrTQSqaBjTVYMUFlAasuFKym1CkM4wEmUBulHsTvIFlUdrLaSMmcpWcg0kjJ93IUgEhXUE5JKoL1TTkIZRm6Q5KQlX3xnhaSgjcKOsxa9agEr7dJKoKr6iamHXxKo0UkjAGcT5CJv15D5RnoWGYtPiPYknqHLxYpbLMHORnvoaMFJss1oT2Yj92319u0j58AAXOPceOvQ20inU5RAFrDh+NP5xosJosHh+imY5CLaqNyDaNHc3npMPo8rC6+eCq1htCt6Xa7gcY5xK2CO5vgSt2x6P4Ebw2TWxjTf7u9fwM1gO8YLzqiFwJVzPw7AH4HABbjhxwxXR8SgNKRR16A1vqbC3nhAY4iYbGPAFoDpAAXHszwwCbIgeWon4dRYuWTmMBQoCAHzUyorUHTyf7XmxH8edFz80sPLl0/orcKulJQbT/g6xrIrqMYGmPyqK892PFNLaFsxygvXfIIBguu77Oe1unNXkIIySImENWK9ETsg+imBtUHObpJHLAFCos1gXzNGD+1KD8TiIjWO7FQTHLaVq5hGLCJCHAeJojL5aUCeQ0XatxMlGCg+Dm68dBBU1+tdqIvFBr/3PX1ay2OkjEAqbRPbpoXfyHj9+sf/ENM/wEwfdE2cdPXdnRDDHEzqCzAY4Ens42H2Wq0YfLorN4vZ4L97Umcr5oAnhDRgzD7fwDgafj95wA+2NvufQD/f3vnFmrbd9f3z2+MeVmXvfe5/a/nrzFR4kNEqEFCoMUKQtVQSAsW4ot/qiBCCvogmJAHfRGqUh9KSyHFUC02qdCKeVAwFcEX46UaY2Ka+M8//5r/Jed+zt7rOi/j58MYY861z9n7XPfZa529xhcOe6255lpnzDHHHGP8bt/v2w/8tdZhr4WgbuugadDaB/HcZPpwgiYuZn2Ev3ESOUb5yV71mRjD50e0RdHtyKX1efIRKj4zyN30E+aBGQU+nZ4LqJxLJ2qu1mcESVhv8imUt3p2z2yulDeXZNc9143Ml35SHca89igsEGMioSYiKqapotZQv+xLl+cvFlQ7pp9wbVDJ6grTALKeCygjMIUGn3wrmKqvgxDnq6Pt0k+BdlBgZkufXw+wFNQpEkuhJWYvxR2z4LIVMjuAFTI8aRTTuq7QzTdC+swad+T0fA+6LKDRCMlsv0AB7XteYvKuUJl8XnxRX/hZu1AGdxzlrVAHclBhFjUyD1lhVY2WBW433I/cegtqEm7wtZtdxlNCwklDRKCOvFuO4g7MXwrPplGyrF2ZCk4gu+0xv/dZ4NXw+lXg91aO/4R4fBC4E11FCQkJCQmbhYdJA/008IPAcyLyJvCLwL8HfkdEfgr4B+DfhNN/H58C+ho+DfTfPlQr2hYNEoPaOqhr3F1ZII+EVTvpuM9DllA79Lw1lU/zphn5OoDep84hm1/zUCUcs1Rr0Gol736qZDMlnx/Oesnv+B1mfuUOeutOlynyuD5j85r/uzsYgLW962tvl+byRRYv+PzFas/QDKSvczCA0HH1VHtKc45uh+zTUAXTeJMiLy3ZQYZZhKwZVa8ZEDWHF0t0WHY7eGlDn9FTQJhauzTQ6NqKfk5pg0LbI/aD7AQui70dXFl0v0dV43LbudSQYAVFg0SEhRjaUDtQlob8IOtiItI4mt2SejemJQumUbKxP78QwT6CulhCwqOg3d/HBm+AvTACLbGzwFR8p2CWrwQMT8AX+cAFQFV//JiPfuiIcxX46CO3QvDBRfAsxkWOCRPasdQDx8HY3v8PfnFpV6glRDBlidsLLoILlmq3l3xsh4H2Idb6WAWr3QQpS0N2YMgP/AnFAZS3HEXIDbcLR7YIojN4Hzc3bnXpqo863dvz55DhsBc8qWp0Ou3yze9ZKPf34c23GIb+HJUlMhwge36Fq188x+yVAZMQa7GVhEhs6L7aT9IxjTSbNth53XE1+YtoOy4gnc7AOWwQ9CmMYGrbu5Tau3z+iwaZLZH9wOc/m6NV5TmF4KFym81g4E1l8Lw6da9X4M6NOHhXyfz5u+i9V64vF+kkM10uuNxg6p6MT1rXU4kQAumN6/6/GABPSHgakCL6c/2z2MUXleCTju83Lwh8Mjim+Ouh4Fp02fZBYSOItf1OWxVX1dipnziLA0e2sD05WuMJ1eiyeBTJHRoi7up8nUAsZIrB0xiUtUtBKtf5jOVgilssD/nwH+16Hq/Y4+6sqJgNI6/DGDj37ncBUH3HJRYXi66wDJRsqdgYs7g9R27c7rOEqho3nx++jv19uOLDQPmLL5Cf2+0CWa6wPj6weh3O9TGe+eLwAv0QkKLoNgyaWa/vHL7f7BYsLhkWl8L7ceBycpEMT8LiFIPm1leeRy4i6GoYur4UPEEfgDX9A/okVmpCwjGICRFm0VDuK9XtmNEXkitOsw4gISEhIeFsYjMsAKWjyMX4HPlOE9g8RAroUeiygsJ7kUOfSdO7bMzSHuLr19x5t09omy4sEqgd7MyQTaXX6ax8qmZkz8wWrfdrR0nLusbNF49dtt3u7/sd9gkjsnvaC7vYcUYb2EZdFmoNdkM67nJIXtXe1QOHqA6ObO+Vq3DlKpL7XbKxBm2aziI5iRRKN5t1LkJpW28NRInPwXlPjz0MltnQgfRpsWp8jUJkWGwraEpDFriRpHHYeU22Hyy4VqFpezrsg8mT+/6DdWovXfQV62Fs6HzeuQoTthcSaW3yvuocQvzOrLijTyMGcDrQPgYwHqJ51gmi2LL0k8qjQAQz9EFQKXL/26uU0isuB808VXGkapAGKOm5f0IRhoZcXNcKOu/pjWNQtVtgXBBxiT5ywBQ5brFZHPZdbOWvv0z51zCI/T8cgnOHYi+P0/Iu7/4IHr8nhTZNR/csVXWIUrm4fZHiTtYR0DWNdJTcEFx3Sleo5tNWFVfG++tdQraOkp3TQ8VxLgjTPDZEyC57rhc9t0Nb9I+gWYywWfbo4z3hTEEDzYpZ1L6wM4Sc7HLF/89ZWgBkJQ+8dYjWnQh47IxjYazniC9DlooxfsKPIhzdxLzSWyuVuNIc3pl73U1DV2OROcQqFP77buBohtIJrLAnaGYwu/4L2dKST3OKnZBFc2uA2RkjBz7o2d66c6RF0+2Y93a8jzkuUIsF7f7kqfPZdzGDDeSxyV65jJ7zdQlaBPW3eDudZ3eVufebVsOMZiTUOzEm0IYgfrzfgllITx7nBFMJGkrB7a7FLpUyWERF42A+7yrR9SheqkeBKs2bb/nXbx7+aLO2CAnrQtzc2IM5xe0xxV5kERBksKKwdwKCMCkGkJCQkLCl2AwLAEFG3mXj9kaeXTGqJE12sFeMp9Y9AmY4QHZ20HH4fpmh1vZZG+B3iJFbqHWHrYG+CeEHfeVvFwMA1AnahLUyagFHH7J6JtEofYiGatxR7NohtsgwITU1y3Pc7TuH0zeN7fx+iPiq1shnf8pqVpsGKUt0WHb0z67MfIwoxoxq/E0IFp+pHXbus30ANDO4gaJm1Xam2/q4zKuWxfqEaBk2Q/972bjEDoedRbrN9yLhdLA6F6zK29q5oNJrf5wZTWCge8BkukAWtpukZTKjnRxfCyA2UD2HCSCKxMcHXFrI5k2Xly/z2gubhwm5HRrako7Pvx04dNiSDQNVQCB5q2v/+22dHza9Qq5uJ23Y+iBNpEKQwmIaB0EikbJAxqPO9NLWYc7tdnn6kdZYlqE/8hyJRGzgqSCqGjeZdO/PMnS5hNv7mMg91Hj3mCxC/8wW6LLqBYIujPtFnEC0txI4oxWklm6ByGZeDjObRwIusJXr0mClanxMZH52Uj5N0FqQ4RDalja6/c74WHpWEONv7vUppTWgFwDIZxl14AqDM7UA+EkNQFwMuvrIR3uE4hL4AikAXnye6oVdlhf9xNkMzSGRcdNCMzLYneDjXRRk0xoTCpN8oVKvqSutoK3ggqJKjfVC8MuQdbKwZFNhcM3/fjFRsrmjOIhcPw1SO0zQHDYzLz4Sc3u1rnzueyh8MqMRsrtDe84XIrhBaOc89Ic1yKJaYYDyAXMT+umRC+WeQbTXb2CjoMto5ONFIYvLB/RdV2eQXdunvD3uFdYE2rZXbJPWr99dMWUObSl9QE0UMIivE8TUA7JWiW7X9vqNh+Om2mB01meqY9hsfOB7ufVdY5bng/hT2RNKAoeKFR8XKQaQkJCQsKXYCAtAm7bLf7Y7Y7C2swDu2f2LYHd34flLADSXxtQ7GW0R6JEzOhMJQhm/E2QY8vgrQ5EJZSztF3CF1wCI73GCBj54MQ4xigQebs0dLpeO6sBZXzkcK4F1IdjadZYMmUV3hhBdO4vK57BHLptLF2gujmlGIfc3k0OZSdY5r1gVJQz3D7zJvmXmepcfHxXgYtZXlgW3WJB0tIbytmMZMidcSPONXEjiPH9Tv4s6zFWUzb1FF+s67KTCTGa4oDD2rO/+zWiECc+OBvqOmEHlbt7aCovyWYFag8uFdhBrXDhU0xJVDJ8EG7EAIJ6yATw1QFcExsqDHvP4rUWeu0j1opfwqy4U1GNDtRODd3KYzK1STN13ljgvVN6ENE2XCc2g5wJyQ582qOH8tolE/UHUPRSC5ZMgEHLHYZeuuyleW2CFysIqUrvuIdPp1F9joBMwlXdHWRP9el4U3kbR+tkSDqa0N2/7z5OQONBPxHdPyO5gQnlzyWgU7lctfkx0QRc/6XeBtaWX2cxm0YXXkk1qzH6QvNyf3COb+SxDRkN0FKiuyyCUFIn5BiWyXCZRmzXD7vm5TadLxA27lGVXgrO93vgZigGI9+0CMg4KXq6vjtTVAFzbotdvkodFQnQPKNA44cYinxXyL1v3xRT5tKW4VXUKZC7bobxtuu83Y8EV2kXaIxFczPvPJ4Z86pXEwC8w2awlux18+gczZFn3FsxsTjuZ3LNjjxOKOzhAypIsXH8siOtPdI/MlLnN0OUSO1li6hg495N/ZwGE+W6VHdVZ0/H/SKuYwkLgMjJti6kq2jOyALTXb9yjkpawWZAg1jS/vEu1K53Wh8vUMxVE1oITSEhLMYCEhISELcVmWACqHceMNI2nbYhskcdkKkiwCvLmZcxih2IUXDqFCZWiMa3PeXbOWNpfNZjJotMfGO5PccXLHTfAPBPagXZUECqKWZpOJSzfh2y6kvbZKKbqff5SN+h83nMbwWE20qMuf7k8fof5IG2DhHvQjsuO3dRlvi6jo34I8QAbJUeXIJlSjyK3h/X3dLHCYWFsp0CmVZ3ccAlPF503I7oxwtssWKyhpuWQjO1jYiMWAMks5pz3e3n+/pVCrWPI4OJD2L79LSwvIe1KGqVTzDKkeVaNXwxCYZUsqj6QClgRsllLMQl1BGMf4G2bOIGoTw2NN8HGvyFGMBBUckwsHNopsQdjzH4gTQsL1WNTLKTJ/5FgRiPq0nb3qS28Ce2CGR1jAKvFNJoJGT2td71rkVDplzcOM1/i9gMVxBmY/Dt3a5GjdZMCvxuGmHCQH9Tk05xmGD84HN+UE3ABbcQCQOs6P79kGSKCBsUmaZqe0fMI6HJJ843/3x+QwP8fPwcfOI788fhdXMe1s6yQxnVZIdlMfUZPXHwzwPWVpaYCTJ9p1EXiYwzBCK4YwaVR36RGe0GU21P0m293lo0Zj5Gdcd++qkYPDp5MEW2L4WYz7KIhW/jxk88UEFyUgzjC6en5n/zrbKHYhbfq/BfUj6l4f57xWED2ymXcBb/ZcqXXO7YTH/DWt688kO014ekjqgXm79xmvJNjaj/26pEcim/KCewNHxgDEJFPichVEfnSyrFfE5H/JyJfFJHfFZHzK599XEReE5GvisgPP3kTExISEhKeBh7GAvhvwH8Cfmvl2OeAj6tqIyK/Anwc+AUReR/wEeB7gMvA/xGR79aHkK/XFe4bberHd32o3utvb5pjd27m/DnqvZxmGF0+EnRsw885/96G93bpaQP8zhKyaUu2aNHIITO2LM/lfV1CVAoLaYflnQGDcyPs7WB2lwVuVOAyvxabusVMz2Ov+7TP9sbNxD9zBCKdAcb4MbPimjGTirbwFli9I9Q70BZ9nYe0gg3DIavBLrwKGoTK4VKQkP5rFkFx7IxAl0uvoQCos2iwmAFPOZIsgI2BHkyw84vYKmp1yGFS4xOwAB5GE/hPROTddx37w5W3nwd+LLz+MPAZVV0C3xCR14APAH/6gP9jfaa1kcDnE97WweQPc25MF4wLgjiwFRR3AlfQtMbM6y6NtN4ZU49NJ7relt7tIEGScHHeUO1adt8IhUqlZXkhpy1CWmur5JMBg/BQWtf6uEigI3bzxb0LgrFdHcW6crhj7nIk9YuSjzhFyqIjWdO6RifTTgM4Tj73bbcI9tJF/3Iw8L8Vn4S29aI7gS9K6wY3yv19JAR5ByDB5+8F4hUXuCAa9TQgQRESa33tQCT309yiRd6J0BtVL4kJz2R8pr1+AyuhRiKI0WjglTqOcDFhPdDJlPzGlLaMxGa+4DXSmJzKAvAQ+Engf4bXr+AXhIg3w7GNhbt5m8G1c2D8xGUaLxIfiy88cQwdC6itIJs7T/AGyLJF5lUnRJ/NWsp96WIDzUBwWX+z7NJbA3HHL42SzRy2qytw5PtLzPVb/vtHKESZwQDzXKjm3B3hRgUaRN7VCKZ2mGlYUN+5Rnvr1sl01jEw4zG88iIA9fkRbmD7vHoXCuHiBrx1mHmNna8EU53zfEeAu7Pf+UC739/Z8RM/eNK/1nXcUdTV4aQB12Jm9Uqhl5LNpVuAOwGflfshjWLDgmErXxVs2phpYXB7w46MzuQ5UnttApZL3HT+bAWGjUWiVkbWky4mbB7cYgF/9zXK1/3YH15+ieryeeq9wIy7wnjwuHiiBUBEPgE0wG/HQ0ecduQ6JSI/Dfw0wIDRUackJCQkJDxFPPYCICKvAv8S+CHVzhZ+E/j2ldO+DXj7qO+r6ieBTwLsycW12dLu4AD7zasM2ucByGYlxUFGFdgk29KngMYdZTFxlDeW5N/y5rK7doN2MiF7ye+Ai9xiqoJ8EOijh5ZmIN2O2LRhxz/1O1jz9jWyqs8518bXETT3cYm4xQKJWULDEmlc52IR1UMKWe5BrjXpq7Dh8dhF3XSKveqrS7O68ZXcEXWDzJdoaK9Opo+c4eQODu6xCu4HM5lhak+vbWtwC+0sgGYYUkBXhOLU9GZ1tNYaife/QJqcPNSZ5DctMvF+cq1qH696hiB51ruu6uaQ+yxhMxGfF7l5C3tpl6b107Z5cgPg8RYAEfkR4BeAf66qq1GjzwL/Q0R+HR8Efi/w50/cyqeM9uo1snGkYjA+VzxqAAdt2Vj4lU1qslsz9KZ3q3QTU/RlixcXyaZ+Ai9uLJC67QVqVDGTBe5bXve1jQVwkfOoKLwIShQoOSYA3MZy/iPK+lc8Vg+G9jUST4K7ydrWieaNf2AUFiG7PE91ricLVOGQC8i0gQwu8P/bqnfvQeBmahQ7CdxM+1NcvPfPYP68Lpc0UXNYTEoweJYQdbtXmOGf+CcfdIKIfBr4QeA5EXkT+EV81k8JfE78zvPzqvozqvplEfkd4O/wrqGPPkwG0NqhSvP6GwDY6QsU5/dwgQsGK0jjPCkbwI1b/WS3gl7n9a279WLu8YEdNTkfR252GjiLNQft174OwGB6mfL8Lu04VIqXXlNY2hgkbrCTZce2ymLpBWYirIGqpt33leNngpWp05bY/EczYQXVyVubD5MF9ONHHP6N+5z/y8AvP0mjEhISEhKePjajEniD0F65Cleu9mmNwTWkIfUvpco9W2jeehvesZhwH22R+3hJiLG46Zz2WcriSdhauPkCGtdZr3ICQYC0AByDaPIT/yY8u3DtIwWRExI2Edq2GNVj8iofD2enxDEhISFhm3ACUeC0ACQkJCRsKdICkJCQkLClEN0APhMRuQZMgevrbsuG4DlSX0Dqh1WkvuiR+sLjOWCsqs8/7g9sxAIAICJ/qarfv+52bAJSX3ikfuiR+qJH6guPk+iH5AJKSEhI2FKkBSAhISFhS7FJC8An192ADULqC4/UDz1SX/RIfeHxxP2wMTGAhISEhITTxSZZAAkJCQkJp4i1LwAi8iNBQP41EfnYuttz2hCRN0Tkb0XkCyLyl+HYRRH5nIj8ffh7Yd3tfBoQkU+JyFUR+dLKsSOvXTz+YxgnXxSR96+v5SePY/ril0TkrTA2viAiH1r57OOhL74qIj+8nlafPETk20Xkj0XkKyLyZRH52XB868bFffri5MaFqq7tH2CBrwPfCRTA3wDvW2eb1tAHbwDP3XXsV4GPhdcfA35l3e18Stf+A8D7gS896NqBDwF/gGfz/yDwZ+tu/yn0xS8BP3/Eue8Lz0oJvCc8Q3bd13BC/fAy8P7wehf4WrjerRsX9+mLExsX67YAPgC8pqqvq2oFfAYvLL/t+DDwm+H1bwL/ao1teWpQ1T8B7hZXOO7aPwz8lnp8HjgvIi+fTkufPo7pi+PwYeAzqrpU1W8Ar+GfpWceqvqOqv5VeH0AfAWvK7514+I+fXEcHnlcrHsBeAX45sr7jReRfwpQ4A9F5P8GnWSAF1X1HfCDAHhhba07fRx37ds6Vv5dcG18asUVuBV9ISLvBr4P+DO2fFzc1RdwQuNi3QvAQ4vIn2H8U1V9P/CjwEdF5AfW3aANxTaOlf8CfBfwT4B3gP8Qjp/5vhCRHeB/AT+nqvfjZN/GvjixcbHuBeChReTPKlT17fD3KvC7eJPtSjRjw9+r62vhqeO4a9+6saKqV1S1VVUH/Fd6c/5M94WI5PgJ77dV9X+Hw1s5Lo7qi5McF+teAP4CeK+IvEdECuAjeGH5rYCIjEVkN74G/gXwJXwfvBpOexX4vfW0cC047to/C/xEyPr4IHAnugTOKu7yZf9r/NgA3xcfEZFSRN4DvBf489Nu39OAeJHx3wC+oqq/vvLR1o2L4/riRMfFBkS6P4SPbn8d+MS623PK1/6d+Kj93wBfjtcPXAL+CPj78Pfiutv6lK7/03gTtsbvXn7quGvHm7f/OYyTvwW+f93tP4W++O/hWr8YHu6XV87/ROiLrwI/uu72n2A//DO82+KLwBfCvw9t47i4T1+c2LhIlcAJCQkJW4p1u4ASEhISEtaEtAAkJCQkbCnSApCQkJCwpUgLQEJCQsKWIi0ACQkJCVuKtAAkJCQkbCnSApCQkJCwpUgLQEJCQsKW4h8B/VB6WDuIdc8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(spec_to_image(get_melspectrogram_db('/data/home/advaitmb/datasets/sentences/' + audio.wav_file[56] + '.wav', 4 )))\n",
    "get_melspectrogram_db('/data/home/advaitmb/datasets/sentences/' + audio.wav_file[0] + '.wav', 4 ).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tensor([[[  1.,  10.,  11.,  15.,   9., 100.],\n",
    "         [  2.,  11.,  12.,  16.,   9., 100.]],\n",
    "\n",
    "        [[  2.,  11.,  12.,  16.,   9., 100.],\n",
    "         [  3.,  12.,  13.,  17.,   9., 100.]]])\n",
    "therefore, with dimensions (2=batch_size, 2=seq_length, 6=input_size).\n",
    "\n",
    "where input is of shape(seq_len, batch, input_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Expected object of scalar type Byte but got scalar type Float for argument #2 'mat2'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-46-0dc7921a13cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspec_to_image\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_melspectrogram_db\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/data/home/advaitmb/datasets/sentences/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0maudio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwav_file\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'.wav'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munsqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mrnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    562\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_packed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 564\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    565\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    566\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mGRU\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRNNBase\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward_tensor\u001b[0;34m(self, input, hx)\u001b[0m\n\u001b[1;32m    541\u001b[0m         \u001b[0munsorted_indices\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 543\u001b[0;31m         \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_sizes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_batch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msorted_indices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    544\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpermute_hidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munsorted_indices\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/rnn.py\u001b[0m in \u001b[0;36mforward_impl\u001b[0;34m(self, input, hx, batch_sizes, max_batch_size, sorted_indices)\u001b[0m\n\u001b[1;32m    524\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_sizes\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    525\u001b[0m             result = _VF.lstm(input, hx, self._get_flat_weights(), self.bias, self.num_layers,\n\u001b[0;32m--> 526\u001b[0;31m                               self.dropout, self.training, self.bidirectional, self.batch_first)\n\u001b[0m\u001b[1;32m    527\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    528\u001b[0m             result = _VF.lstm(input, batch_sizes, hx, self._get_flat_weights(), self.bias,\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Expected object of scalar type Byte but got scalar type Float for argument #2 'mat2'"
     ]
    }
   ],
   "source": [
    "x = (torch.from_numpy(spec_to_image(get_melspectrogram_db('/data/home/advaitmb/datasets/sentences/' + audio.wav_file[1000] + '.wav', 4 )).T)).unsqueeze(0)\n",
    "\n",
    "rnn(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 251, 128])\n",
      "torch.Size([1, 32128])\n"
     ]
    }
   ],
   "source": [
    "rnn = nn.LSTM(128, hidden_size = 64, num_layers=2, bidirectional=True, batch_first=True)\n",
    "x= torch.from_numpy(spec_to_image(get_melspectrogram_db('/data/home/advaitmb/datasets/sentences/' + audio.wav_file[1000] + '.wav', 4 )).T).unsqueeze(0)\n",
    "print(x.shape)\n",
    "out, _ = rnn(x)\n",
    "print(out.reshape(out.size(0), -1).shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "class IEMOCAP(Dataset):\n",
    "    def __init__(self, base, df, in_col, out_col):\n",
    "        self.df = df\n",
    "        self.data = []\n",
    "        self.labels = []\n",
    "        self.c2i={}\n",
    "        self.i2c={}\n",
    "        self.categories = sorted(df[out_col].unique())\n",
    "        print(self.categories)\n",
    "        for i, category in enumerate(self.categories):\n",
    "            self.c2i[category]=i\n",
    "            self.i2c[i]=category\n",
    "        for ind in tqdm(range(len(df))):\n",
    "            row = df.iloc[ind]\n",
    "            \n",
    "            #If all files are stored in a folder named sentences\n",
    "            file_path = base + '/' + row[in_col] + '.wav'\n",
    "            \n",
    "            \n",
    "            self.data.append(( get_melspectrogram_db(file_path, 4).T ))\n",
    "            self.labels.append(self.c2i[row['emotion']])\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[idx], self.labels[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 2/2212 [00:00<02:37, 14.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ang', 'hap', 'neu', 'sad']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2212/2212 [02:25<00:00, 15.25it/s]\n",
      "  0%|          | 2/731 [00:00<00:38, 19.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ang', 'hap', 'neu', 'sad']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 731/731 [00:44<00:00, 16.56it/s]\n"
     ]
    }
   ],
   "source": [
    "train_data = IEMOCAP('/data/home/advaitmb/datasets/sentences', train, 'wav_file', 'emotion')\n",
    "valid_data = IEMOCAP('/data/home/advaitmb/datasets/sentences', valid, 'wav_file', 'emotion')\n",
    "train_loader = DataLoader(train_data, batch_size=32, shuffle=True)\n",
    "valid_loader = DataLoader(valid_data, batch_size=32, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.rnn = nn.GRU(128, hidden_size = 64, num_layers=2, bidirectional=True, batch_first=True)\n",
    "        self.fc1 = nn.Linear(251*64*2, 64)\n",
    "        self.fc2 = nn.Linear(64, 4)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x,_ = self.rnn(x)\n",
    "        x = x.reshape(x.size(0), -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.softmax(self.fc2(x), dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device=torch.device('cuda:0')\n",
    "else:\n",
    "    device=torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Network()\n",
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(251, 128)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_loader.dataset.data[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def setlr(optimizer, lr):\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "    return optimizer\n",
    "\n",
    "def lr_decay(optimizer, epoch):\n",
    "    if epoch%10==0:\n",
    "        new_lr = learning_rate / (10**(epoch//10))\n",
    "        optimizer = setlr(optimizer, new_lr)\n",
    "        print(\"Changed learning rate to {}\".format(new_lr))\n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, loss_fn, train_loader, valid_loader, epochs, optimizer, train_losses, valid_losses, change_lr=None):\n",
    "    \n",
    "    for epoch in tqdm(range(1,epochs+1)):\n",
    "        \n",
    "        model.train()\n",
    "        batch_losses=[]\n",
    "        if change_lr:\n",
    "            optimizer = change_lr(optimizer, epoch)\n",
    "        for i, data in enumerate(train_loader):\n",
    "            x, y = data\n",
    "            optimizer.zero_grad()\n",
    "            x = x.to(device, dtype=torch.float32)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            loss.backward()\n",
    "            batch_losses.append(loss.item())\n",
    "            optimizer.step()\n",
    "        train_losses.append(batch_losses)\n",
    "\n",
    "\n",
    "        print(\"Epoch - {} Train-Loss : {}\".format(epoch, np.mean(train_losses[-1])))\n",
    "        model.eval()\n",
    "        batch_losses=[]\n",
    "        trace_y = []\n",
    "        trace_yhat = []\n",
    "        for i, data in enumerate(valid_loader):\n",
    "            x, y = data\n",
    "            x = x.to(device, dtype=torch.float32)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            trace_y.append(y.cpu().detach().numpy())\n",
    "            trace_yhat.append(y_hat.cpu().detach().numpy())      \n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "        valid_losses.append(batch_losses)\n",
    "        trace_y = np.concatenate(trace_y)\n",
    "        trace_yhat = np.concatenate(trace_yhat)\n",
    "        accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)\n",
    "        print(\"Epoch - {} Valid-Loss : {} Valid-Accuracy : {}\".format(epoch, np.mean(valid_losses[-1]), accuracy ))\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 2e-5\n",
    "optimizer = optim.Adam(net.parameters(), lr=learning_rate, weight_decay=0.01)\n",
    "epochs = 20\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "train_losses=[]\n",
    "valid_losses=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Train-Loss : 1.1719778333391462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  5%|▌         | 1/20 [00:07<02:21,  7.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Valid-Loss : 1.2192200992418372 Valid-Accuracy : 0.5034199726402189\n",
      "Epoch - 2 Train-Loss : 1.1390072056225369\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 2/20 [00:14<02:12,  7.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 2 Valid-Loss : 1.2018405095390652 Valid-Accuracy : 0.5430916552667578\n",
      "Epoch - 3 Train-Loss : 1.1179372072219849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 15%|█▌        | 3/20 [00:21<02:03,  7.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 3 Valid-Loss : 1.226969319841136 Valid-Accuracy : 0.5020519835841313\n",
      "Epoch - 4 Train-Loss : 1.1126976702894484\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 4/20 [00:28<01:55,  7.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 4 Valid-Loss : 1.1950005344722583 Valid-Accuracy : 0.5554035567715458\n",
      "Epoch - 5 Train-Loss : 1.1101831989628927\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 25%|██▌       | 5/20 [00:35<01:48,  7.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 5 Valid-Loss : 1.200543911560722 Valid-Accuracy : 0.5088919288645691\n",
      "Epoch - 6 Train-Loss : 1.0978545469897134\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 6/20 [00:43<01:40,  7.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 6 Valid-Loss : 1.1901778604673303 Valid-Accuracy : 0.5581395348837209\n",
      "Epoch - 7 Train-Loss : 1.0910514261041369\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 35%|███▌      | 7/20 [00:50<01:33,  7.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 7 Valid-Loss : 1.1873185064481653 Valid-Accuracy : 0.5622435020519836\n",
      "Epoch - 8 Train-Loss : 1.0805894110883985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 8/20 [00:57<01:25,  7.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 8 Valid-Loss : 1.2085576057434082 Valid-Accuracy : 0.521203830369357\n",
      "Epoch - 9 Train-Loss : 1.0792598818029677\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 45%|████▌     | 9/20 [01:04<01:18,  7.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 9 Valid-Loss : 1.2184535005818242 Valid-Accuracy : 0.5102599179206566\n",
      "Changed learning rate to 2.0000000000000003e-06\n",
      "Epoch - 10 Train-Loss : 1.069644934790475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 10/20 [01:11<01:11,  7.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 10 Valid-Loss : 1.193997455679852 Valid-Accuracy : 0.5389876880984952\n",
      "Epoch - 11 Train-Loss : 1.0668810537883213\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 55%|█████▌    | 11/20 [01:18<01:04,  7.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 11 Valid-Loss : 1.1933472933976546 Valid-Accuracy : 0.5403556771545828\n",
      "Epoch - 12 Train-Loss : 1.0677506932190486\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 12/20 [01:25<00:57,  7.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 12 Valid-Loss : 1.1935150260510652 Valid-Accuracy : 0.5403556771545828\n",
      "Epoch - 13 Train-Loss : 1.0625212626797813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 65%|██████▌   | 13/20 [01:32<00:49,  7.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 13 Valid-Loss : 1.1950822446657263 Valid-Accuracy : 0.5362517099863201\n",
      "Epoch - 14 Train-Loss : 1.0660860214914594\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 14/20 [01:40<00:42,  7.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 14 Valid-Loss : 1.194730898608332 Valid-Accuracy : 0.533515731874145\n",
      "Epoch - 15 Train-Loss : 1.0625592640468053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 75%|███████▌  | 15/20 [01:47<00:35,  7.11s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 15 Valid-Loss : 1.1926310425219329 Valid-Accuracy : 0.5430916552667578\n",
      "Epoch - 16 Train-Loss : 1.0648209248270308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 16/20 [01:54<00:28,  7.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 16 Valid-Loss : 1.1898632827012434 Valid-Accuracy : 0.5526675786593708\n",
      "Epoch - 17 Train-Loss : 1.0683996447495052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 85%|████████▌ | 17/20 [02:01<00:21,  7.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 17 Valid-Loss : 1.1932113948075667 Valid-Accuracy : 0.5417236662106704\n",
      "Epoch - 18 Train-Loss : 1.0621749937534333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 18/20 [02:08<00:14,  7.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 18 Valid-Loss : 1.1909380218257075 Valid-Accuracy : 0.5458276333789329\n",
      "Epoch - 19 Train-Loss : 1.062884521484375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 95%|█████████▌| 19/20 [02:15<00:07,  7.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 19 Valid-Loss : 1.1906652917032656 Valid-Accuracy : 0.5458276333789329\n",
      "Changed learning rate to 2.0000000000000002e-07\n",
      "Epoch - 20 Train-Loss : 1.0572853990963527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [02:22<00:00,  7.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 20 Valid-Loss : 1.1904672384262085 Valid-Accuracy : 0.5471956224350205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_model(net, loss_fn, train_loader, valid_loader, epochs, optimizer, train_losses, valid_losses, lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ResNet(\n",
      "  (conv1): Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
      "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (relu): ReLU(inplace=True)\n",
      "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
      "  (layer1): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (layer2): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (downsample): Sequential(\n",
      "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (layer3): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (downsample): Sequential(\n",
      "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (layer4): Sequential(\n",
      "    (0): BasicBlock(\n",
      "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (downsample): Sequential(\n",
      "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
      "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      )\n",
      "    )\n",
      "    (1): BasicBlock(\n",
      "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (relu): ReLU(inplace=True)\n",
      "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
      "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "    )\n",
      "  )\n",
      "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
      "  (fc): Linear(in_features=512, out_features=128, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from torchvision import models\n",
    "resnet18 = models.resnet18(pretrained=True)\n",
    "resnet18.conv1 = nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
    "# resnet18 = nn.Sequential(*list(resnet18.children())[:-2])\n",
    "# # print(resnet18)\n",
    "resnet18.fc = nn.Linear(512, 128, bias=True)\n",
    "# i = torch.randn(32, 1, 251, 128)\n",
    "# a = resnet18(i)\n",
    "# print(a.shape)\n",
    "# a = a.permute(0,2,1,3)\n",
    "# a = a.reshape( a.shape[0], a.shape[1], a.shape[2]*a.shape[3] )\n",
    "# a.shape\n",
    "print(resnet18)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 8, 128])"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rnn = nn.GRU(2048, hidden_size = 64, num_layers=1, bidirectional=True, batch_first=True)\n",
    "x = torch.randn(1,8,2048)\n",
    "x,_ = rnn(x)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Given groups=1, weight of size 32 1 5 5, expected input[32, 32, 125, 63] to have 1 channels, but got 32 channels instead",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-238-f107315ef0a5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# x = torch.randn(32, 1, 251, 128)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mla\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcnn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcnn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mla\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbn\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mla\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    545\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 343\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconv2d_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    344\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mConv3d\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ConvNd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py36/lib/python3.6/site-packages/torch/nn/modules/conv.py\u001b[0m in \u001b[0;36mconv2d_forward\u001b[0;34m(self, input, weight)\u001b[0m\n\u001b[1;32m    338\u001b[0m                             _pair(0), self.dilation, self.groups)\n\u001b[1;32m    339\u001b[0m         return F.conv2d(input, weight, self.bias, self.stride,\n\u001b[0;32m--> 340\u001b[0;31m                         self.padding, self.dilation, self.groups)\n\u001b[0m\u001b[1;32m    341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Given groups=1, weight of size 32 1 5 5, expected input[32, 32, 125, 63] to have 1 channels, but got 32 channels instead"
     ]
    }
   ],
   "source": [
    "layers = [ 1, 32, 64, 128, 64, 32 ]\n",
    "cnn = [ nn.Conv2d(layers[l-1],layers[l], 5, 2, 1 ) for l in range(1, len(layers)) ]\n",
    "bn = [ nn.BatchNorm2d(layers[l]) for l in range(1, len(layers)) ]\n",
    "# print(cnn)\n",
    "# print(bn)\n",
    "# x = torch.randn(32, 1, 251, 128)\n",
    "for la in range(len(cnn)):\n",
    "    x = cnn[la](x)\n",
    "    x = bn[la](x)\n",
    "    \n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "#         self.conv1 = nn.Conv2d(1, 32, 5, 2, 1)\n",
    "#         self.bn1 = nn.BatchNorm2d(32)\n",
    "#         self.conv2 = nn.Conv2d(32, 64, 3, 2, 1)\n",
    "#         self.bn2 = nn.BatchNorm2d(64)\n",
    "#         self.conv3 = nn.Conv2d(64, 128, 3, 2, 1)\n",
    "#         self.bn3 = nn.BatchNorm2d(128)\n",
    "#         self.conv4 = nn.Conv2d(128, 64, 3 , 2, 1)\n",
    "#         self.bn4 = nn.BatchNorm2d(64)\n",
    "#         self.conv5 = nn.Conv2d(64, 32, 3, 2, 1)\n",
    "        self.cnn = resnet18\n",
    "        self.rnn = nn.GRU(2048, hidden_size = 64, num_layers=1, bidirectional=True, batch_first=True)\n",
    "        self.fc1 = nn.Linear(8*128, 4)\n",
    "\n",
    "\n",
    "    def conv(self, x):\n",
    "        x = self.bn1(F.relu(self.conv1(x)))\n",
    "        x = self.bn2(F.relu(self.conv2(x)))\n",
    "        x = self.bn3(F.relu(self.conv3(x)))\n",
    "        x = self.bn4(F.relu(self.conv4(x)))\n",
    "        x = F.relu(self.conv5(x))\n",
    "        return x\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.cnn(x)\n",
    "        x = x.permute(0,2,1,3)\n",
    "        x = x.reshape( x.shape[0], x.shape[1], x.shape[2]*x.shape[3] )\n",
    "        x,_ = self.rnn(x)\n",
    "        x = x.reshape(x.size(0), -1)\n",
    "        x = F.softmax(self.fc1(x), dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "crnn = CRNN()\n",
    "crnn.to(device)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3114, 0.2048, 0.2416, 0.2422],\n",
       "        [0.3127, 0.2347, 0.2232, 0.2293],\n",
       "        [0.3113, 0.2175, 0.2141, 0.2571],\n",
       "        [0.2996, 0.1960, 0.2485, 0.2559],\n",
       "        [0.3498, 0.2207, 0.2096, 0.2199],\n",
       "        [0.2895, 0.2225, 0.2374, 0.2507],\n",
       "        [0.3408, 0.2192, 0.2297, 0.2104],\n",
       "        [0.3554, 0.1855, 0.2090, 0.2502],\n",
       "        [0.3370, 0.1937, 0.2343, 0.2350],\n",
       "        [0.3293, 0.2019, 0.2325, 0.2362],\n",
       "        [0.3560, 0.1969, 0.2540, 0.1931],\n",
       "        [0.3448, 0.1820, 0.2480, 0.2251],\n",
       "        [0.3342, 0.2078, 0.2279, 0.2301],\n",
       "        [0.3469, 0.2127, 0.2093, 0.2311],\n",
       "        [0.3184, 0.1812, 0.2580, 0.2423],\n",
       "        [0.3276, 0.2204, 0.2436, 0.2084],\n",
       "        [0.3643, 0.2146, 0.2365, 0.1847],\n",
       "        [0.3114, 0.2050, 0.2373, 0.2463],\n",
       "        [0.2897, 0.2127, 0.2482, 0.2494],\n",
       "        [0.3347, 0.2114, 0.2243, 0.2296],\n",
       "        [0.3235, 0.2020, 0.2357, 0.2388],\n",
       "        [0.3418, 0.1868, 0.2421, 0.2294],\n",
       "        [0.3621, 0.1958, 0.2125, 0.2296],\n",
       "        [0.3447, 0.1894, 0.2606, 0.2053],\n",
       "        [0.3021, 0.2086, 0.2532, 0.2361],\n",
       "        [0.3205, 0.1784, 0.2370, 0.2641],\n",
       "        [0.3362, 0.2063, 0.2330, 0.2245],\n",
       "        [0.3074, 0.2109, 0.2681, 0.2136],\n",
       "        [0.3413, 0.1811, 0.2423, 0.2354],\n",
       "        [0.3388, 0.1798, 0.2435, 0.2379],\n",
       "        [0.3553, 0.1912, 0.2308, 0.2227],\n",
       "        [0.3497, 0.1720, 0.2403, 0.2381]], device='cuda:0',\n",
       "       grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = torch.randn(32, 1, 251, 128).to(device)\n",
    "crnn(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "class IEMOCAP(Dataset):\n",
    "    def __init__(self, base, df, in_col, out_col):\n",
    "        self.df = df\n",
    "        self.data = []\n",
    "        self.labels = []\n",
    "        self.c2i={}\n",
    "        self.i2c={}\n",
    "        self.categories = sorted(df[out_col].unique())\n",
    "        print(self.categories)\n",
    "        for i, category in enumerate(self.categories):\n",
    "            self.c2i[category]=i\n",
    "            self.i2c[i]=category\n",
    "        for ind in tqdm(range(len(df))):\n",
    "            row = df.iloc[ind]\n",
    "            \n",
    "            #If all files are stored in a folder named sentences\n",
    "            file_path = base + '/' + row[in_col] + '.wav'\n",
    "            \n",
    "            \n",
    "            self.data.append(( spec_to_image(get_melspectrogram_db(file_path, 4)).T[np.newaxis,...] ))\n",
    "            self.labels.append(self.c2i[row['emotion']])\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "    def __getitem__(self, idx):\n",
    "        return self.data[idx], self.labels[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 2/2212 [00:00<02:37, 14.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ang', 'hap', 'neu', 'sad']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2212/2212 [02:26<00:00, 15.06it/s]\n",
      "  0%|          | 2/731 [00:00<00:40, 17.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ang', 'hap', 'neu', 'sad']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 731/731 [00:44<00:00, 16.60it/s]\n"
     ]
    }
   ],
   "source": [
    "train_data = IEMOCAP('/data/home/advaitmb/datasets/sentences', train, 'wav_file', 'emotion')\n",
    "valid_data = IEMOCAP('/data/home/advaitmb/datasets/sentences', valid, 'wav_file', 'emotion')\n",
    "train_loader = DataLoader(train_data, batch_size=32, shuffle=True)\n",
    "valid_loader = DataLoader(valid_data, batch_size=32, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 2e-7\n",
    "# optimizer = optim.Adam(crnn.parameters(), lr=learning_rate, weight_decay=0.01)\n",
    "epochs = 10\n",
    "# loss_fn = nn.CrossEntropyLoss()\n",
    "# train_losses=[]\n",
    "# valid_losses=[]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Train-Loss : 0.7657878773553031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 1/10 [00:11<01:47, 12.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Valid-Loss : 1.1560689314551975 Valid-Accuracy : 0.5991792065663475\n",
      "Epoch - 2 Train-Loss : 0.7602540671825408\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 2/10 [00:23<01:35, 11.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 2 Valid-Loss : 1.1568336538646533 Valid-Accuracy : 0.5950752393980848\n",
      "Epoch - 3 Train-Loss : 0.7608691734927041\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 3/10 [00:35<01:23, 11.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 3 Valid-Loss : 1.1625111906424812 Valid-Accuracy : 0.5909712722298222\n",
      "Epoch - 4 Train-Loss : 0.7605856316430228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 4/10 [00:47<01:11, 11.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 4 Valid-Loss : 1.156731849131377 Valid-Accuracy : 0.6032831737346102\n",
      "Epoch - 5 Train-Loss : 0.7604665628501347\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 5/10 [00:59<01:00, 12.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 5 Valid-Loss : 1.1577506998310918 Valid-Accuracy : 0.5964432284541724\n",
      "Epoch - 6 Train-Loss : 0.7618333390780858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 6/10 [01:12<00:48, 12.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 6 Valid-Loss : 1.1657061265862507 Valid-Accuracy : 0.5841313269493844\n",
      "Epoch - 7 Train-Loss : 0.7611525441919055\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 7/10 [01:24<00:36, 12.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 7 Valid-Loss : 1.1616346265958704 Valid-Accuracy : 0.5896032831737346\n",
      "Epoch - 8 Train-Loss : 0.7612968598093305\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 8/10 [01:36<00:24, 12.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 8 Valid-Loss : 1.155289738074593 Valid-Accuracy : 0.6073871409028728\n",
      "Epoch - 9 Train-Loss : 0.7605901181697845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 9/10 [01:48<00:12, 12.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 9 Valid-Loss : 1.1604523969733196 Valid-Accuracy : 0.5978112175102599\n",
      "Changed learning rate to 2e-08\n",
      "Epoch - 10 Train-Loss : 0.7622888045651572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [02:00<00:00, 12.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 10 Valid-Loss : 1.1543341978736545 Valid-Accuracy : 0.6019151846785226\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_model(crnn, loss_fn, train_loader, valid_loader, epochs, optimizer, train_losses, valid_losses, lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn.GRUCell??"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 128])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(32, 1, 251, 128)\n",
    "x = resnet18(x)\n",
    "print(x.shape)\n",
    "# rnn = nn.GRUCell(128, 32)\n",
    "# x = rnn(x)\n",
    "# x.shape\n",
    "# fc2 = nn.Linear(32, 4)\n",
    "# x = fc2(x)\n",
    "# x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WrongNetwork(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "#         self.conv1 = nn.Conv2d(1, 32, 5, 2, 1)\n",
    "#         self.bn1 = nn.BatchNorm2d(32)\n",
    "#         self.conv2 = nn.Conv2d(32, 64, 3, 2, 1)\n",
    "#         self.bn2 = nn.BatchNorm2d(64)\n",
    "#         self.conv3 = nn.Conv2d(64, 128, 3, 2, 1)\n",
    "#         self.bn3 = nn.BatchNorm2d(128)\n",
    "#         self.conv4 = nn.Conv2d(128, 64, 3 , 2, 1)\n",
    "#         self.bn4 = nn.BatchNorm2d(64)\n",
    "#         self.conv5 = nn.Conv2d(64, 32, 3, 2, 1)\n",
    "        self.cnn = resnet18\n",
    "        \n",
    "\n",
    "        self.rnn = nn.GRUCell(128, hidden_size = 64)\n",
    "        self.fc = nn.Linear(64, 4)\n",
    "\n",
    "\n",
    "    def conv(self, x):\n",
    "        x = self.bn1(F.relu(self.conv1(x)))\n",
    "        x = self.bn2(F.relu(self.conv2(x)))\n",
    "        x = self.bn3(F.relu(self.conv3(x)))\n",
    "        x = self.bn4(F.relu(self.conv4(x)))\n",
    "        x = F.relu(self.conv5(x))\n",
    "        return x\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.cnn(x)\n",
    "        x = self.rnn(x)\n",
    "        x = F.softmax(self.fc(x), dim=1)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "WrongNetwork(\n",
       "  (cnn): ResNet(\n",
       "    (conv1): Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (relu): ReLU(inplace=True)\n",
       "    (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "    (layer1): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (layer2): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (layer3): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (layer4): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace=True)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "    (fc): Linear(in_features=512, out_features=128, bias=True)\n",
       "  )\n",
       "  (rnn): GRUCell(128, 64)\n",
       "  (fc): Linear(in_features=64, out_features=4, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn = WrongNetwork()\n",
    "wn.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, loss_fn, train_loader, valid_loader, epochs, optimizer, train_losses, valid_losses, change_lr=None):\n",
    "    \n",
    "    for epoch in tqdm(range(1,epochs+1)):\n",
    "        \n",
    "        model.train()\n",
    "        batch_losses=[]\n",
    "        if change_lr:\n",
    "            optimizer = change_lr(optimizer, epoch)\n",
    "        for i, data in enumerate(train_loader):\n",
    "            x, y = data\n",
    "            optimizer.zero_grad()\n",
    "            x = x.to(device, dtype=torch.float32)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            loss.backward()\n",
    "            batch_losses.append(loss.item())\n",
    "            optimizer.step()\n",
    "        train_losses.append(batch_losses)\n",
    "\n",
    "\n",
    "        print(\"Epoch - {} Train-Loss : {}\".format(epoch, np.mean(train_losses[-1])))\n",
    "        model.eval()\n",
    "        batch_losses=[]\n",
    "        trace_y = []\n",
    "        trace_yhat = []\n",
    "        for i, data in enumerate(valid_loader):\n",
    "            x, y = data\n",
    "            x = x.to(device, dtype=torch.float32)\n",
    "            y = y.to(device, dtype=torch.long)\n",
    "            y_hat = model(x)\n",
    "            loss = loss_fn(y_hat, y)\n",
    "            trace_y.append(y.cpu().detach().numpy())\n",
    "            trace_yhat.append(y_hat.cpu().detach().numpy())      \n",
    "            batch_losses.append(loss.item())\n",
    "\n",
    "        valid_losses.append(batch_losses)\n",
    "        trace_y = np.concatenate(trace_y)\n",
    "        trace_yhat = np.concatenate(trace_yhat)\n",
    "        accuracy = np.mean(trace_yhat.argmax(axis=1)==trace_y)\n",
    "        print(\"Epoch - {} Valid-Loss : {} Valid-Accuracy : {}\".format(epoch, np.mean(valid_losses[-1]), accuracy ))\n",
    "        \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 2e-7\n",
    "optimizer = optim.Adam(wn.parameters(), lr=learning_rate, weight_decay=0.01)\n",
    "epochs = 10\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "train_losses=[]\n",
    "valid_losses=[]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/10 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Train-Loss : 1.3941045658929008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 1/10 [00:11<01:46, 11.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 1 Valid-Loss : 1.3978080438530964 Valid-Accuracy : 0.11491108071135431\n",
      "Epoch - 2 Train-Loss : 1.3938811336244856\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 2/10 [00:23<01:34, 11.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 2 Valid-Loss : 1.3975042934003084 Valid-Accuracy : 0.12859097127222982\n",
      "Epoch - 3 Train-Loss : 1.3936347688947406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 3/10 [00:35<01:22, 11.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 3 Valid-Loss : 1.3974374791850215 Valid-Accuracy : 0.12311901504787962\n",
      "Epoch - 4 Train-Loss : 1.3941247429166521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 4/10 [00:47<01:10, 11.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 4 Valid-Loss : 1.3976746849391772 Valid-Accuracy : 0.12038303693570451\n",
      "Epoch - 5 Train-Loss : 1.3942324229649135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 5/10 [00:58<00:58, 11.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 5 Valid-Loss : 1.3980843917183254 Valid-Accuracy : 0.11764705882352941\n",
      "Epoch - 6 Train-Loss : 1.3942568012646266\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 6/10 [01:10<00:47, 11.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 6 Valid-Loss : 1.3976275558057039 Valid-Accuracy : 0.12038303693570451\n",
      "Epoch - 7 Train-Loss : 1.3940265774726868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 70%|███████   | 7/10 [01:22<00:35, 11.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 7 Valid-Loss : 1.397538195485654 Valid-Accuracy : 0.12448700410396717\n",
      "Epoch - 8 Train-Loss : 1.3941379070281983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 8/10 [01:34<00:23, 11.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 8 Valid-Loss : 1.3973677624826846 Valid-Accuracy : 0.12175102599179206\n",
      "Epoch - 9 Train-Loss : 1.393904003075191\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 90%|█████████ | 9/10 [01:45<00:11, 11.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 9 Valid-Loss : 1.3978221675624019 Valid-Accuracy : 0.11080711354309165\n",
      "Changed learning rate to 2e-08\n",
      "Epoch - 10 Train-Loss : 1.3940645609583173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [01:57<00:00, 11.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch - 10 Valid-Loss : 1.3973237587058025 Valid-Accuracy : 0.12722298221614228\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_model(wn, loss_fn, train_loader, valid_loader, epochs, optimizer, train_losses, valid_losses, lr_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 - AzureML",
   "language": "python",
   "name": "python3-azureml"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}