neural_machine_translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# dependencies\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "import data_utils\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# read dataset\n",
    "X, Y, en_word2idx, en_idx2word, en_vocab, de_word2idx, de_idx2word, de_vocab = data_utils.read_dataset('data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence in English - encoded: [108, 5, 867, 93, 38, 25, 2583]\n",
      "Sentence in German - encoded: [166, 262, 8, 474, 268, 324, 67, 15, 130]\n",
      "Decoded:\n",
      "------------------------\n",
      "They walk in here and \n",
      "\n",
      "Die kommen hier herein und\n"
     ]
    }
   ],
   "source": [
    "# inspect data\n",
    "print 'Sentence in English - encoded:', X[0]\n",
    "print 'Sentence in German - encoded:', Y[0]\n",
    "print 'Decoded:\\n------------------------'\n",
    "\n",
    "for i in range(len(X[1])):\n",
    "    print en_idx2word[X[1][i]],\n",
    "    \n",
    "print '\\n'\n",
    "\n",
    "for i in range(len(Y[1])):\n",
    "    print de_idx2word[Y[1][i]],"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# data processing\n",
    "\n",
    "# data padding\n",
    "def data_padding(x, y, length = 15):\n",
    "    for i in range(len(x)):\n",
    "        x[i] = x[i] + (length - len(x[i])) * [en_word2idx['<pad>']]\n",
    "        y[i] = [de_word2idx['<go>']] + y[i] + [de_word2idx['<eos>']] + (length-len(y[i])) * [de_word2idx['<pad>']]\n",
    "\n",
    "data_padding(X, Y)\n",
    "\n",
    "# data splitting\n",
    "X_train,  X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.1)\n",
    "\n",
    "del X\n",
    "del Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# build a model\n",
    "\n",
    "input_seq_len = 15\n",
    "output_seq_len = 17\n",
    "en_vocab_size = len(en_vocab) + 2 # + <pad>, <ukn>\n",
    "de_vocab_size = len(de_vocab) + 4 # + <pad>, <ukn>, <eos>, <go>\n",
    "\n",
    "# placeholders\n",
    "encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "targets = [decoder_inputs[i+1] for i in range(output_seq_len-1)]\n",
    "# add one more target\n",
    "targets.append(tf.placeholder(dtype = tf.int32, shape = [None], name = 'last_target'))\n",
    "target_weights = [tf.placeholder(dtype = tf.float32, shape = [None], name = 'target_w{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "# output projection\n",
    "size = 512\n",
    "w_t = tf.get_variable('proj_w', [de_vocab_size, size], tf.float32)\n",
    "b = tf.get_variable('proj_b', [de_vocab_size], tf.float32)\n",
    "w = tf.transpose(w_t)\n",
    "output_projection = (w, b)\n",
    "\n",
    "outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                            encoder_inputs,\n",
    "                                            decoder_inputs,\n",
    "                                            tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                            num_encoder_symbols = en_vocab_size,\n",
    "                                            num_decoder_symbols = de_vocab_size,\n",
    "                                            embedding_size = 100,\n",
    "                                            feed_previous = False,\n",
    "                                            output_projection = output_projection,\n",
    "                                            dtype = tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# define our loss function\n",
    "\n",
    "# sampled softmax loss - returns: A batch_size 1-D tensor of per-example sampled softmax losses\n",
    "def sampled_loss(labels, logits):\n",
    "    return tf.nn.sampled_softmax_loss(\n",
    "                        weights = w_t,\n",
    "                        biases = b,\n",
    "                        labels = tf.reshape(labels, [-1, 1]),\n",
    "                        inputs = logits,\n",
    "                        num_sampled = 512,\n",
    "                        num_classes = de_vocab_size)\n",
    "\n",
    "# Weighted cross-entropy loss for a sequence of logits\n",
    "loss = tf.contrib.legacy_seq2seq.sequence_loss(outputs, targets, target_weights, softmax_loss_function = sampled_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# let's define some helper functions\n",
    "\n",
    "# simple softmax function\n",
    "def softmax(x):\n",
    "    n = np.max(x)\n",
    "    e_x = np.exp(x - n)\n",
    "    return e_x / e_x.sum()\n",
    "\n",
    "# feed data into placeholders\n",
    "def feed_dict(x, y, batch_size = 64):\n",
    "    feed = {}\n",
    "    \n",
    "    idxes = np.random.choice(len(x), size = batch_size, replace = False)\n",
    "    \n",
    "    for i in range(input_seq_len):\n",
    "        feed[encoder_inputs[i].name] = np.array([x[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    for i in range(output_seq_len):\n",
    "        feed[decoder_inputs[i].name] = np.array([y[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    feed[targets[len(targets)-1].name] = np.full(shape = [batch_size], fill_value = de_word2idx['<pad>'], dtype = np.int32)\n",
    "    \n",
    "    for i in range(output_seq_len-1):\n",
    "        batch_weights = np.ones(batch_size, dtype = np.float32)\n",
    "        target = feed[decoder_inputs[i+1].name]\n",
    "        for j in range(batch_size):\n",
    "            if target[j] == de_word2idx['<pad>']:\n",
    "                batch_weights[j] = 0.0\n",
    "        feed[target_weights[i].name] = batch_weights\n",
    "        \n",
    "    feed[target_weights[output_seq_len-1].name] = np.zeros(batch_size, dtype = np.float32)\n",
    "    \n",
    "    return feed\n",
    "\n",
    "# decode output sequence\n",
    "def decode_output(output_seq):\n",
    "    words = []\n",
    "    for i in range(output_seq_len):\n",
    "        smax = softmax(output_seq[i])\n",
    "        idx = np.argmax(smax)\n",
    "        words.append(de_idx2word[idx])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# ops and hyperparameters\n",
    "learning_rate = 5e-3\n",
    "batch_size = 64\n",
    "steps = 1000\n",
    "\n",
    "# ops for projecting outputs\n",
    "outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "# training op\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "# init op\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# forward step\n",
    "def forward_step(sess, feed):\n",
    "    output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "    return output_sequences\n",
    "\n",
    "# training step\n",
    "def backward_step(sess, feed):\n",
    "    sess.run(optimizer, feed_dict = feed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------TRAINING------------------\n",
      "step: 0, loss: 9.51545906067\n",
      "step: 4, loss: 9.85502529144\n",
      "step: 9, loss: 7.60950708389\n",
      "step: 14, loss: 7.24870491028\n",
      "step: 19, loss: 9.95916461945\n",
      "Checkpoint is saved\n",
      "step: 24, loss: 6.82820177078\n",
      "step: 29, loss: 8.01150417328\n",
      "step: 34, loss: 8.57325744629\n",
      "step: 39, loss: 7.71085071564\n",
      "Checkpoint is saved\n",
      "step: 44, loss: 6.8006734848\n",
      "step: 49, loss: 6.0620880127\n",
      "step: 54, loss: 7.5349149704\n",
      "step: 59, loss: 6.55568885803\n",
      "Checkpoint is saved\n",
      "step: 64, loss: 6.03047990799\n",
      "step: 69, loss: 6.19692897797\n",
      "step: 74, loss: 5.42797708511\n",
      "step: 79, loss: 19.0027732849\n",
      "Checkpoint is saved\n",
      "step: 84, loss: 22.5006141663\n",
      "step: 89, loss: 13.7472429276\n",
      "step: 94, loss: 6.8521771431\n",
      "step: 99, loss: 6.28392124176\n",
      "Checkpoint is saved\n",
      "step: 104, loss: 5.56026077271\n",
      "step: 109, loss: 6.5281291008\n",
      "step: 114, loss: 6.57611370087\n",
      "step: 119, loss: 7.07715034485\n",
      "Checkpoint is saved\n",
      "step: 124, loss: 7.29706668854\n",
      "step: 129, loss: 5.67437314987\n",
      "step: 134, loss: 5.85692214966\n",
      "step: 139, loss: 5.47358703613\n",
      "Checkpoint is saved\n",
      "step: 144, loss: 6.84951019287\n",
      "step: 149, loss: 5.23042392731\n",
      "step: 154, loss: 5.53386831284\n",
      "step: 159, loss: 5.66608428955\n",
      "Checkpoint is saved\n",
      "step: 164, loss: 5.82921504974\n",
      "step: 169, loss: 5.60702848434\n",
      "step: 174, loss: 6.9934220314\n",
      "step: 179, loss: 5.99299049377\n",
      "Checkpoint is saved\n",
      "step: 184, loss: 6.82448387146\n",
      "step: 189, loss: 4.78254985809\n",
      "step: 194, loss: 6.02455806732\n",
      "step: 199, loss: 5.84940433502\n",
      "Checkpoint is saved\n",
      "step: 204, loss: 5.24849748611\n",
      "step: 209, loss: 4.79305458069\n",
      "step: 214, loss: 5.34067440033\n",
      "step: 219, loss: 5.04536867142\n",
      "Checkpoint is saved\n",
      "...\n",
      "...\n",
      "...\n",
      "step: 904, loss: 2.8909842968\n",
      "step: 909, loss: 2.84566783905\n",
      "step: 914, loss: 3.24181413651\n",
      "step: 919, loss: 3.86452913284\n",
      "Checkpoint is saved\n",
      "step: 924, loss: 3.79612350464\n",
      "step: 929, loss: 5.16866064072\n",
      "step: 934, loss: 3.3992767334\n",
      "step: 939, loss: 5.7843875885\n",
      "Checkpoint is saved\n",
      "step: 944, loss: 6.06335735321\n",
      "step: 949, loss: 4.78808069229\n",
      "step: 954, loss: 5.28374910355\n",
      "step: 959, loss: 3.95825266838\n",
      "Checkpoint is saved\n",
      "step: 964, loss: 2.74541044235\n",
      "step: 969, loss: 2.67334342003\n",
      "step: 974, loss: 3.19328260422\n",
      "step: 979, loss: 5.3072104454\n",
      "Checkpoint is saved\n",
      "step: 984, loss: 2.69399261475\n",
      "step: 989, loss: 2.86632490158\n",
      "step: 994, loss: 4.03659677505\n",
      "step: 999, loss: 3.52744889259\n",
      "Checkpoint is saved\n",
      "Training time for 1000 steps: 2469.76520491s\n"
     ]
    }
   ],
   "source": [
    "# let's train the model\n",
    "\n",
    "# we will use this list to plot losses through steps\n",
    "losses = []\n",
    "\n",
    "# save a checkpoint so we can restore the model later \n",
    "saver = tf.train.Saver()\n",
    "\n",
    "print '------------------TRAINING------------------'\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    t = time.time()\n",
    "    for step in range(steps):\n",
    "        feed = feed_dict(X_train, Y_train)\n",
    "            \n",
    "        backward_step(sess, feed)\n",
    "        \n",
    "        if step % 5 == 4 or step == 0:\n",
    "            loss_value = sess.run(loss, feed_dict = feed)\n",
    "            print 'step: {}, loss: {}'.format(step, loss_value)\n",
    "            losses.append(loss_value)\n",
    "        \n",
    "        if step % 20 == 19:\n",
    "            saver.save(sess, 'checkpoints/', global_step=step)\n",
    "            print 'Checkpoint is saved'\n",
    "            \n",
    "    print 'Training time for {} steps: {}s'.format(steps, time.time() - t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbIAAAEiCAYAAACcFVdfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvWmcI3d17v/8atEu9d7TPUvP1hqP7fFuDF6wjcPqADG5\nkIQ/GOfea5JcYl8byGIM4SZ8EpYLF/Mh4U8gkISLWRJiCAn7ZozB2B6v4208mn3rfdEu1fa7L6pK\nqpKqJHW3uluaOd839qjVUkkt1annnOecwxYXFzkIgiAIoksR1vsACIIgCGIlUCAjCIIguhoKZARB\nEERXQ4GMIAiC6GookBEEQRBdDQUygiAIoqtZs0B22223IZlM4qqrrqrc9sEPfhBXXHEFrrnmGtx8\n883IZDJrdTgEQRDEGcKaBbK3ve1tuO+++1y33XDDDXj44Yfxy1/+Ejt37sQ999yzVodDEARBnCGs\nWSC78sor0dvb67rt+uuvhyCYh3D55Zfj1KlTa3U4BEEQxBlCx9TI7r33XrzqVa9a78MgCIIguoyO\nCGSf+MQnIMsy3vKWt6z3oRAEQRBdhrTeB/DVr34VP/7xj/Ef//Ef630oBEEQRBeypoGMc/d84p/8\n5Cf49Kc/je9973sIBoNreSgEQRDEGcKapRZvvfVWvOY1r8HBgwexZ88e3HvvvfizP/sz5HI53HTT\nTbj22mvx3ve+d60OpylfSeXx+edzrtueX1DxR7+YX6cjWh9SqdR6H8JZCb3v6wO9793JmimyL3zh\nC3W3vf3tb1+rp18yBgcE5r5NZObtBEEQROfQEWaPToR7BCyBAToFMoIgiI6CApkPHF6KjEH3inAE\nQRDEukGBzAeDc9TEMYikyAiCIDoOCmQ+mIrMHcqoRkYQBNF5UCDzweBATRyDQKlFgiCIjoMCmQ8c\n8EwtkiIjCILoLCiQ+cA97PcCA3RjfY6HIAiC8IYCmQ/eiozBoNQiQRBER0GBzAevhmjqIyMIgug8\nKJD5YM6FrHEtChTICIIgOg0KZD54N0RTICMIgug0KJD5YIBqZARBEN0ABTIffF2LFMcIgiA6Cgpk\nPng1RFMfGUEQROdBgcwHL/u9wBg4QOlFgiCIDoICmQ+ce785pMoIgiA6CwpkPnBwsNrcIqhORhAE\n0WlQIPPBqyEaoJ1kBEEQnQYFMh+8amQA9ZIRBEF0GhTIfOAerkWAamQEQRCdBgUyH7waogGagE8Q\nBNFpUCDzwashGrCme4AkGUEQRKdAgcwHg3MwD01GiowgCKKzoEDmA4dPjUygGhlBEEQnQYHMBw7v\nN0cA2e8JgiA6CQpkPvi6FmknGUEQREdBgcwH/4ZoCmQEQRCdBAUyH/wbomknGUEQRCdBgcwH07VY\nD81aJAiC6CwokPlguhbrQxlN9iAIgugsKJD54NcQTYqMIAiis1izQHbbbbchmUziqquuqty2uLiI\nN73pTbj88svx27/920in02t1OE3xG1FFNTKCIIjOYs0C2dve9jbcd999rtvuueceXH/99Xjsscdw\n7bXX4p577lmrw2mKn/2eFBlBEERnsWaB7Morr0Rvb6/rtu9973t461vfCgB461vfiu9+97trdThN\n8WuIJvs9QRBEZ7GuNbKZmRkMDw8DADZs2IDZ2dn1PBwX/mtcaLIHQRBEJ3FGmD20VbARGpxD8KiS\nkWuRIAiis5DW88mHh4cxPT2N4eFhTE1NYWhoqOnvpFIp179VA7j7UBwfT2bbemz5QhSnTy8gldZd\ntxcLEZw8tYBURmvr83Uyte85sTbQ+74+0Pu+tiSTyRU/xpoGMl6Tknvd616Hr371q7jzzjvxta99\nDTfeeGPTx6h90WnFwOKLsxgfH/fs+1ouoekFbN40iORQwHV7YiGNDSNBJDeG2vZcnUwqlWrLB41Y\nGvS+rw/0vncna5ZavPXWW/Ga17wGBw8exJ49e3Dvvffi3e9+N+6//35cfvnl+PnPf44777yz6eM8\nNq24/l22nBdqm3eE+a1xIdciQRBEZ7FmiuwLX/iC5+3f/va3l/Q4syV3qk+xoopicATE9iky3w3R\ntI+MIAiio+g6s0epRg6VrbjWbkXm1xAtAORaJAiC6CC6PpApljxS2pzv45xD8Jq1KDBKLRIEQXQQ\nXRjI3P8uO1KL7cR/jQvVyAiCIDqJ7gtkWk1q0Vgls4dvQzTVyAiCIDqJ7gtktalF3f5ve6OLAe83\nR6DJHgRBEB1F1wWyYp3ZY5VSi6TICIIguoKuC2Rlv0Cme917+Rjcx7VINTKCIIiOousCmV+NbFXM\nHj5Dg2kfGUEQROfQdYGsNrWorFpq0XtoMCkygiCIzqLrAplfQ3TbU4vwr5FRICMIgugcui6Q+dXI\n1FUwe3iOqGKATm4PgiCIjqHrAllxLWtkHreLjKHNLWsEQRDECui6QFbfR8YREtufWvS13wuATpGM\nIAiiYzgDAhkQk4W2KzLfhmhQHxlBEEQn0XWBTNHhsr+XDY6YzFahRsZ9FBlN9iAIgugkui6QBcSq\nUxEwzR5xWVgd16KP/Z4UGUEQROfQdYEsJDJXetEMZGxNR1SR/Z4gCKJz6M5A5nAuKjoQl4X2pxZB\nQ4MJgiC6ga4MZM7pHnaNbFVmLZIiIwiC6Hi6L5BJzNUUXdatQLaGDdFUIyMIgugcui+QedTIVst+\n778hmiIZQRBEp9CVgazoqpGtTmoRnPu4FhkpMoIgiA6iKwOZS5EZptljVRqiqUZGEATR8XRfIPOo\nkcUD9TWyyYKO/Qvqsp/Hz35PfWQEQRCdRdcFsrDDtagZHJwDEYlBrUkt/uhECf9yqLDs5/EbUSWS\n/Z4gCKKj6LpAFnT0kSkGR0BkCAj1iuxIVsNMafnTfbnP+HtarEkQBNFZSOt9AEslJFYHBys6EBTh\nGciOZrW6bdJLwa8hmqbfEwRBdBZdF8jCEsN82YwkZZ0jKDIEata4aAbHqbwODnv4r5eRvjG+DdFg\nrqHFBEEQxPrSdalFc0SV+f9lgyMoMMiCe/r9qbyOwZCIsMiQVpYXdDg4BI/coihQapEgCKKT6LpA\nFnTY78u6d43sSFbD9oSIwZCAmdLyGsz8FBm5FgmCIDqLrgtkYYnV1chkK7XIrZTf0ayObXEJg2ER\nM8XlF7S8J3uQa5EgCKKT6IhA9pnPfAZXXnklrrrqKrzzne+Eoii+93VO9ijrHAGBQWTM5SY8mtWw\nLS5hKCRgdpnORcNn1uKZqMhO5jR8/KnMeh8GQRDEslj3QDYxMYHPf/7zeOCBB/DQQw9B0zTcd999\nvvcPidWG6LJhmj0At3PRDGRmanF2manFs2kf2UzJwIG0tt6HQRAEsSzWPZABgK7rKBQK0DQNhUIB\no6OjvvcN1dTIKoHMSi+qlmNxLCZhaAWpRb+G6DOxj0wzuGtaCkEQRDex7oFsdHQUt912G/bs2YNz\nzz0XPT09uP76633vH5Kqkz0UnSNgvQJbkZ3M6dgQFhEUmaXIlh7I7Fqbl23/TKyRqQYokBEE0bWs\nex/Z4uIivve97+GZZ55BIpHAO97xDnzjG9/AW97yFs/7T544ilwphlQqhRMLMsolCanUNKDHkDp8\nFKfLAnoRQCqVQrEk4FQmglRqZknHZHCAIYFUKlX3s4mygFI54vmzbuVERkJBCfu+pjPptXYT9L6v\nD/S+ry3JZHLFj7HugeyBBx7Atm3b0NfXBwB4wxvegEcffdQ3kO3euQP6sTkkk0k8fbiAobyOZHIz\noifnMLplCOlFFZuYimRyE4YVA5mTc0t+ozSDg7044/l74ZwGcSrdlje/UzhxqgR9Kuv5mlKp1Bn1\nWrsFet/XB3rfu5N1Ty1u3rwZjz32GEqlEjjneOCBB7Br1y7f+4dEd2qx1uyxWDbQa+UbEzKDonPX\n/rJW8BtPBdj7yM6sNJxmcCiUWiQIoktZ90B22WWX4Y1vfCOuvfZaXH311eCc4/d///d97x8UAc0w\ntzSXnTUykUHVOdIKR0/ADG6MMQyGxCU7F/2aoYEz07WoGoDGafM1QRDdybqnFgHgrrvuwl133dXS\nfRljlQn4igHEZVuRAYoBpBUDW+Ny5f5DYdPwsSW2tGPym854JvaRaUbVBRqRlj6XkiAIYj1Zd0W2\nHMLWBPyyV2pRMdATqL6swZCwZAu+XzM0YLkWz7BIplovR1leyx1BEMS60pWBLGiNqXL3kZn1sHRd\nIBOXPG+Rg4P5aDKBAWfa+d6pyAiCILqNrgxk9gR8Z41MFsxaT7psoDdQDUJ2anEpNFRkZ+A+Ms16\nPeUzTGkSBHF20JWBbDAkYLKoQzFMJQZUU4um2cOdWpxdYmqx0elcPANrZPYKHHIuEgTRjXRlILt4\nIIAnZpW61GJe5SjpHDHZoch8XIuf2pdFTvUOcLxZjewMc/fZ3QnlMy1nShDEWUFXBrJLhwJ4csYK\nZELVtThbMpAICK7RUuZOsvqA9ZNTJZzKe5+5G9nvz0TXYkWRnWkvjCCIs4KOsN8vlV09EqZLBooO\nRSYLDJNF3VUfA4CBkIDFsgHN4JCsoGdwjrzKMV/2UWTwj/BnYh9ZpUZ2pr0wgiDOCrpSkUkCw0UD\nMiYKBgKieVtAZJguuh2L9n0TAQELjqCV1zg4gDkfE4jBuefAYMCc7MGt+5wpqORaJAiii+nKQAYA\nlw4GAMDRRwbMFPW6QAagbgp+zmqcmm/gZmzUFrwUw4fBeWWafqeiU42MIIgupvsDmVA1e8yVDPQG\n61/SUFjATLF6lrZNHn6pxUb2e2BpO8n+91NZfPd4qbU7rxOqwSExqpERBNGddG0g2xYXcdmgjHig\nWiPTOCpzFp2YzsV6ReaXWuRopshacy7OlXT85GTJ93k6Bc0AotaAZYIgiG6jK80egDlz8f9c1Vf5\nt51R7PVJLc7UBLLeAMOcTy6tmSJrNbX4H0eLEAUg72Pz7xRUgyMqCVQjIwiiK+laRVaL3RjtXyOr\nBq2samBrXPJ3LTY5n7fiXFR0jv84VsIbtoaRW+IambVGM4CYzODzdhAEQXQ0Z04gsyRUj2eNTHQN\nDs6pHGMxEfMlw9OIwdG8RtZMkT04Ucb2uIjz+2Tk1c4OZKphTr0nRUYQRDdyBgUy879eNbJa12JW\nNTAQEiELrFIvc2Jw/6HBQGs1sqfnVFw9EkREZshrnS11NE41MoIgupczJ5A1SC2ag4P1ivrKqRxx\nmaE/JGDOI5/WDkV2MKNivEdCTBLapsjufmQRhzNaWx7LiWbVyMi1SBBEN3LmBDLBP5BFJAECY5Va\nVU41EJMZBoKCZy9ZoxFVQPMJ+DrnOJzRsTMhISoz5NtUIzue033Haq0E1XItUh8ZQRDdyBkTyGQB\nCIus0iBdy5BjwWZW5YjLgq8iAxrb7wU0Ti2eyuvoCzLEZAFRiTVUZMUlBLmsamB+ibvVWsFUZFQj\nIwiiOzljAllIYujzMHrYOJ2LOZVXFJlXj1cz+31YaqyyDqY1JHtkALAUmXewzKsG/r+fzvk/kQPO\nObKO+ZAG57j3QL6l322Gym3XIgUygiC6j5YD2S9+8QscPXoUADA5OYk/+qM/wh//8R9jampqtY5t\nSWyJivj4lT2+Px8Ki5i2FJmZWhQwEBIw75FPa9YQPRYTcTznr4wOpjWMJ8wWvbDIoBjVLcxO0grH\ngjXQuBkFjcPgqCjIhbKBL+zPt2WljGZwRCSBzB4EQXQlLQeyP/mTP4EomhN6P/CBD0DTNDDGcMcd\nd6zawS0Fxhg2Rf37u0ciIiYLbkXW71sj4xAaFMm2xSUczfqbLlJpDeM9UuW4oj4KLmM1ShdaSC9m\nauZD2kG51Ib6m2aAUosEQXQtLU/2mJiYwJYtW6BpGn7605/imWeeQSAQwO7du1fz+NrGaETA3mkF\nQNXs0R8SfV2LjdgWF/Gjk/7zEw9mqoEMACJWnawn4L5fTjGfKa9yJGp+VktONcBQnQ85bc2OLOoc\nUbnJATdBMziZPQiC6FpaDmTxeBzT09N44YUXsHv3bsRiMSiKAk1rvx18NRiNiJgo6FANDtUwU35+\niqzRhmjAVmTeZ/25kg7d4BgKVcVuTBasOpnouq+tyHIeP6slo3CMRARHIDP/uxSziB8qNxUZ2e8J\nguhGWg5kf/AHf4AbbrgBiqLgIx/5CADg4YcfRjKZXLWDaydmatGopBUZYxjwcS0aaFwj2xQVMVPU\nzQ3VDpekonN8+2gR4z2Sa59ZVPJuvM4q5nO30mdmj9V6fEYB57wyzb8dgcxUZDRrkSCI7qTlQHbn\nnXfi9a9/PURRxPbt2wEAGzduxKc//elVO7h2MhASkFENzJV0xGRTLSVkBt0wg0RcriqoZopMEhg2\nRkUcz1XdiXMlHe96cAHbExL+5wVx1/2jMvOsg2WtANZKn1lO5egPCgiKDBmVY9pSksU2BB/VmrVI\nZg+i0zmZ0/CVVAF/fklivQ+F6CCWZL/funUrJiYm8M1vfhMAMDo6im3btq3GcbUdkTEMh0UcymiI\ny2aUYoxhe0LEkZppGc0aogFge016MZXWsDkq4qMv7cW2uPv6wOwlq1d+dmqxlen4GcVAQhYq6dDp\noo6IxFZs9uCcQ+dmSwENDSY6nZmSgUOrMN2G6G5aDmTPPfccLrvsMtxxxx24/fbbAQC/+tWvcNtt\nt63awbWb0YiAA2kNMbkapXYmJM8vRpM4hq1xCccczsW5koHBsHedKyYLnhPwcypHQEBL0/GzKkc8\nYNX1ygamiwa2xsQVKzKNAxIzF5RSapHodEo6p1ouUUfLgey9730v7r77buzduxeSZCqOq6++Gg8/\n/PCqHVy7GYmIOJjWXGnEHQmpbn6hwTmEJqFsW1zEEWcgKxsY8GnIjsre0z0yioGRiNhyjcyeRjJd\n1LFYNrApKq64RqYaHJLAIAumEm2lp40g1ouyztHhM7iJdaDlQPbCCy/gd3/3dwGgYmSIRqMoFour\nc2SrwKgVyJopMo7mqcVtcQnHHKnF+ZKBgZD32xnxGVOVVXnrgUyxBh0HBaTSGnqDAmKysOJAphnm\neC/GGIIi6GqX6GgUnUOlzyhRQ8uBbGxsDE899ZTrtscffxw7duxo+0GtFiMREXmNV8weALA9IeFI\nRofhmJDRbEQVAGyOipgq6hWDxHzZP5DFJO8xVVnFwEhYbGnNS1Y1EA+YNbL9iyqGwwLCEltxatFW\nZIC5QUChXjKigynrpjmJIJy0HMje//7343d+53fw4Q9/GKqq4pOf/CRuueUWfOADH1jN42srI1YN\ny6nI4rKARIBhwpr6UdZ50xFVgOlcHAqZwQwwXYv9vqlFwdt+r5q9Ya24FrPW6pmBkICDaQ0bwqIZ\nyNqgyCTrxVKdjOh0ygan9DdRR8uB7LWvfS2+8Y1vYG5uDldffTVOnDiBe++9FzfccMOKDyKdTuOW\nW27BFVdcgZe97GV47LHHVvyYXoxG7EDmftlmnUzHd44Vcdcji+AtuBbNxxNw2gqAcw0Umd8ql6xq\nYHSpNbKgAMUwd6yFxfYoMvvtCIo0OJjobMo6J0VG1NFyHxkAXHzxxbj44ovbfhB33XUXXvWqV+FL\nX/oSNE1DoVBo+3MAQF/QrAM5FRlg1smemVPw45MlMMaaNkTbbIyKmMybCzvnSwYGgt6uRS/7vWIV\nrQdCrS3ezCqWa1Ezn2M4LCIkrlyR6RzV1KIAUmRER6PoHFobBmUTZxYtK7K/+7u/w759+wAAe/fu\nxZ49e3DRRRfh0UcfXdEBZLNZ/PrXv8bb3/52AIAkSUgkVqfZkTGGkbBY6SOz2Z6Q8G9HirhiQxBZ\n1UBZ501rZIBZczttTQuRBYaQ5P1L5ogq95fPVFjMsuY3vsTUDG7OVJTMrdYAMByyFFkbXIuy9WKD\nVCMjOhy7RsYpmK0LUwUdzy+o630YdbQcyD772c9i69atAIC/+qu/wrve9S68973vxfve974VHcDR\no0cxMDCAd73rXbj22mtxxx13rKoT8tZzYzi31z1ld1ePhLDI8M5zoxgKCZgq6GAtaDJ7fuNc2agE\nGC+8lmuafWHNF28C1rR+iUFgDIkAg8BMReY0e8wUdfzLwaUrWc0AJOvQAyLNWyQ6Gzv13aal68QS\n+fqhAr5xaHUyZiuh5dRiJpNBT08Pstksnn32WXz729+GKIorNnvouo6nn34an/jEJ3DJJZfgrrvu\nwj333IO7777b8/6pVGpFzzcCYDIHTNbc/rGdwMKJRcQRxfOniyiWBaRSMw0fSyuKOLoQxr5Dc4gY\nId9jK+hAVkm4fn6wIELWQ5g8frjuZ7VMKQKCiFTuc14kgtLUUcyVRMxnQ0ilZvBERsJnT0UxUJjA\n1nDrRYQjBRFa2Tx2vRTBkROLiC242xFW+p4Ty4Pe93pm5sMAAtifOogG144rgt53bwwO/Ox4HBtD\nOlKp6bY9bjvm9bYcyDZt2oRHHnkE+/fvx1VXXQVRFJHJZCo7ypbLxo0bsWnTJlxyySUAgN/6rd/C\npz71Kd/7r/aQ4h25DLKqgZgEJJObGt53qGzg06fmEBrYiM16GcnkRs/76ZxDOTCDHePjEC0Xycxk\nGUOlIi44Z7TuZ7WoCyoG5rJIJkcBAH9nvQWhBRUsk0UyuREvHisiNp3DD/L9+MSFfS2/3tysglgu\nj2RyI/oW0hjYEERyY6jy81Qq1TWDoc8k6H33JphJA+kyxrbvRE+g/ZGM3nd/npxVUEYamhRoem5c\na1oOZB/60Idwyy23QJZlfPnLXwYA/PCHP8Rll122ogMYHh7Gpk2bcPDgQYyPj+OBBx5Y1x1nGyIi\nDkxoGA43/5L0BBgMDhzPab5TPQBzzmNIMgcHTxY07EhIyKrm7ESBMYStn9XW7mxqhxrbOO33acXA\n67aE8NCUgsemFVw+3GTBmYXbtQjaSUZ0NLYZiSz4a8/9p8p41eYgHrX2OnYSLQeyV7/61di/f7/r\ntptuugk33XTTig/iYx/7GN75zndCVVVs27YNn/nMZ1b8mMtlQ1jAREHHSKR5IGOMYTQi4tl5FS/b\nEGx436jEcDyr490PLeAvL+9BRqkGLnuEVbxmQea3jhTQExDAeb3TEoDLtZhWTPv/W8cj+O7xYsuB\nzKyRORqi6QRBdDD2hVY7LfgFzYDImGslE+FGMzh+MVHCx6/sxU9Oltf7cOpoOZDt378f/f39GB4e\nRi6Xw6c//WmIoojbb78dsryyFcUXXHAB7r///hU9RrsYiYgoaLwl+715fwGPzyj4za3hhveLygx/\n/3wOIYnh6TkVIRGIW6kRL8PHREHHP7yQx0BQwJt2hJHwSKM4zR4ZhWNbXMDOhIRvHWndLKMZHDKz\n7ffUEE10NvaFVjsV2f99sYDBsIA374i07THPNPYvahgIiRhPSNZy4qrbuRNoOcl86623Ip1OAwD+\n4i/+Ag899BD27t2LO++8c9UObj2wp3+04loEgI0RESUdDVOLABCTzIkc77kwjqfnFGRUpyIT6sZU\n/f1zOfzuzggCIsP9p8qeacfa1GJCFrAlJuJkXoPeoj1Z5VXXYlCkQEZ0NvbnU2mjIstrRku9nGcz\nGcXAUFgAYwzxgIBMO/8AbaBlRXb8+HEkk0lwzvGd73wHDz/8MEKhEC666KLVPL41ZygsQEBrkz2A\n6rSQRvZ7AIgFGN68M4wrNwTx0Scz6A8KlTYAcxZj9Yv0xIyC/Ysq7r40gcGQgI8/ncU1I/WpS1kw\nw61qcGQUAz0BhogkICELmC4YGI02N+Jorj6y9p4gCKLd2D2e7VRkJZ3TBVwTSjpHyEq99gQY0grH\nQKjJL60hLSuyYDCIbDaLxx9/HJs2bcLAwACCwSDK5c7Ll64ESWAYDAstNUQDqASLZorsPRfGcXMy\niqDIsKtHxhOzCuIBd40MMI0dH3sqgzsvjCMoMrxycwgJmVXuW4utytIKr6Qfx2Lm9upWUA1SZET3\nUNbNenE7a2RlHTSarQllnVdqiHG5ixXZm9/8ZrzxjW9ELpfDO9/5TgDA008/jbGxsVU7uPViJCy2\nXCMbjYiQBW8zhpNhx9LNiwdl7JtXkbDsghFrhBXnHJ98OosrNwRxpWUeCYoM77kojvGE95/KDmSm\nIjMfb2tcwvGcjpduaH78msEhOWpkit5ZH1CCcKIY5nCAdo6pKuu8ssWC8MapyBIB1r2B7CMf+Qh+\n9rOfQZIkXHvttQAAQRDw4Q9/eNUObr3YEBHQ6gXa5qiIP70oXtnR1goXDQQAFCrBz94gvXdGwaGM\nhn+4rt91/+s3+mv4kGimJXMar6i2sZjY8jp41dpHBtDQYKLzKescfUERahvbREo6p9FsTShpHPYo\n2Z6AgEyH1RSXNDT4hhtuwIkTJ/Doo49idHS00sR8prEhLGKy0NonWxIYXr2lsWOxlvP7ZMgCKgrK\ndi3+/HQZb9gaXpINOCwxTBd1xCRWaagei0m4/3RrKV+d87NyaPBMUcdQeGXN/MTaU9a5mVpsoyJT\ndE4XcE0o6RxhW5HJAtIdpsharpFNTk7ixhtvxGWXXYabb74Zl156KW688UZMTEys5vGtC5ujIgKr\n2FMSkhi+9IoB9Fp1tajMkFUNPDRZ9jR1NCIsMkwVdJc9fywu4lh26Yos5rM37UxjrqTj9++fdy1T\nJTofg5srXCISQwu7aFuGzB7NKeuoXGCbqcXOer9aDmTvec97sGfPHhw5cgQvvvgijh49igsuuADv\nfve7V/P41oVXbg7htj2xVX2OjQ5HYVQSsHdGwUBIbMlp6CQsMUwWTceizUBQgGrAddXk5/LSDA47\nZvcFBSyUO+tKazXIqhx5jVeWqRLdgaKbF10Bob2N+2X97MlELBd3jazzzB4tB7KHH34Yf/M3f4No\nNAoAiEaj+NCHPrTiNS6diCSYNva1IiYzTBYMXDPS2jQOJ2GJYbJGkTHGMBYTcSJXPVH/6cOLeGSq\nPt1oKjLzA3q2BDLbIXo4Q4GsmygbpnNOEtprvy9TjawpTteiWSNr/TxR1jmem1/d1S8tn617e3vr\nRlSlUin09PS0/aDONqKW6ePqJaYVAdPsMVXU6yZ/jMWkigWfc44Dixp+5lE304xqjcwOZLW7nv7+\nuRz2L3bCQqgCAAAgAElEQVTeDqLlUrDyUq0aYtaaH50o4Xhp7S6kugX7ZCoLbbbfG1Qja0atazFt\npRY/93wOv/a4QHbywoKKv302u6rH17LZ44477sBNN92Em2++GVu2bMGJEyfwla98Be9///tX8/jO\nCvqCAjZGBCR7luS9AWArMgN7+t01vW1xEYetE/VMyYDBgV9Pll2BCzAne9g1srC186yg8UpwBYCn\n5syet929KxtF1ikUNDOdeijdmYHsFxMlbOISfmO9D6TDUHSOgGAGsnYrMkotNsapyBKOPrJHpxX0\nBQVc2aDVp7gG72/Ll3233HIL/vEf/xFzc3P4wQ9+gLm5OXzuc5/DqVOnVvP4zgrGYhL+6RUDS7Lw\n24RFhvmyUbfS4pxeGfsXzRP1kYyG8/okbIqKeGrOraxqA1tfkNWlFxfKBlIdetJfDgWNY1evtCJF\ndjKn4Y8fnG/jUVXJKBxZrXPm2HUKZR0IiWYDf7sUmW4ZSGhYthuDc3zpxXzl37WKLKsYUHSOo1mt\nab2sqPFV36qxJAlw3XXX4brrrqv8u1wu481vfjOpsjaw3MnbYamat3ayq1fCwbQGzeA4ktWwPSGh\nPyjglxNlXD5UrcVpDtciUE0vbra8LpybgezAYnsC2Q9PFPHKzSHf3WtrQV7j2NUj44cniihoxrLq\noQtlA4cyGjjny7oAaURGNZAVKbVYi10jkxmD2qbAYysFWl/kJqdy/NOLebwtGYEkMDOQSQ6zh8px\nOKtB56Z5qhFr4Qpd8beltp5CrC12IEvUTBaJyQKGwgKOZXUcyejYHpfw8tEgHpwou04CqsEhOX61\n1vBRMgCBAfNlA7kVXgarBsfHnsxitth+Q4lmcGRbPL6CNbB5LCbhyDINHwWdo6QDi6tgQ84qHBlS\nZHWUdY6AyCCLaJv9vqwDEgNN9qjBVlkFrRro7YZoWTBX3uyztni0pMhWWfGuOJC1+2qUWBp2k2KP\nx6zH3b0S9i+qFUW2JSZhT7+Mm34wi489mQHnHBpHTWpRwILj5JzVBfQHBexMiDjoSC8ulA28/adz\nS7qQmSzoMIAlOZ5a5WenSnjj92dx24MLTR1SBY0jKjHsTCw/vWhvHWi1cb5VOOfIqAYFMg8UvarI\n2pUKLOsc8YBANbIa7H5SO5A5U4uAmV58YkbBBf2BpheQa6HImqYWH3jgAd+fqeqZ42TrVqqKzCuQ\nyXh+QcWxnIZtcfNy6q9e0lMJQn94Xqxur1CtIstoDH1BAckeGam0hosHzbTkCwsqTuZ1zJcNDIRa\n6307mTdP+tlVUDHTRQP/ZUcYEmP48ckSzu/3N6bkNQOjEQk7ElLFELNU7EA2UdBxbl/7TDAl3az/\nZHRKLdZS1jmCgmm/L7SpZFvWzdmN6TJaXn10NmBfbFYVGXeVPxKygKfmVPz+OVH87FSp4WMVNbMO\naXAOYZWET9NAdvvttzf8+ebNm9t2MMTSCYveNTLADGRfOpBHX1Bw1YH6gmbacb5sWBui4frZsWxV\nZdiBbFevhCdnqyvOX7Ds+MdzeuuBzOprWw1FtqAYGAmLGImI+MGJxotFCxpHRBawJSjgwYnlbW+w\nF5q2W5FlVQNRiZHZw4OSld6SBQaVt+czZJ+gA6u0+WE1aqhrgV33ylvf1VpF1hMQUNI1XDoo49+P\nFho+VtFRhwwv3ZjdEk0fdt++favzzERbqCgyjzUv4z0S8irH+R6KoT8oYK6kuzZE27c/NVtV2hld\nsBSZhH89VP3A7l/QMBAUcDyn45LB1o71VF6HgNVRZItlA7t7ZQyFBUw3qcHlrdTihoiA6dLyAlFR\nM2fPtXs6SEYxMBwWcCpnmM8hdd9JcLVQDKtGJrSvRmafoAMi2t4UnVUM3HL/PL7x6oEVmZvmS0bT\nfYftJmvXyHQfRRZgGIkI2BgVm36fSw5Vt1qfZ8pfdDlhiSEqMVedyyYoMmyPmym0WgZCAuZKhlUj\nq97eG/BOLW6LS5go6ChqHJxz7F9U8crNId+9Z1MFHYs1Nv6TebNWtyqKrGyYSjMkYLrY+IxU1Dgi\nEsNAUMRcqb4BvBWKGsf2hIjJQntfS1Y198rFJX5WTFlZCvbJVBLa61oMiEBQaL8i+/VUGfNlo3Ii\nXw4LZQP/9edzbTyq1rAVWUHl0AwO3TCHitskAgKSCRlRyXQ0NurrsxXZarY4UCDrcvqCAsZi/qm9\nV28Juez2NgNBEXNlw3QtNqiRZXWG3qAAWWA4p0fGo9NlnMrriEgMFw/KOJ71Dhpf2J/Dvx12pxxO\n5nWc2yetiiJbKBvoC5jqsaA1Li7nVTOQhSTTfbWclRRFnWNHQmp7ajFt7ZVLiAYFshrsGpksoI2B\nzJyOExTbO78RAB6cMFPxhRUEsqxqIK3wSk12rbAVWV7jUKy2B2eKdGdCwkuGA2CMISazhhZ8pyJb\nLSiQdTmDIRGfvbbf9+e/szOCSwY9AllIwHypvkbWX2f2MF2LAPBfdoTxr4cKeGFRw+5eueEm6hcX\nNRx0GCkUnWOuZCDZI9cpsrmSjp+fblwwbsZi2UBv0JxMMhgSMNNAlZm9Y+aXctBSpkulqHFsj0uY\nLOqeU/QPpbXKyWApZBWzNSAhcSx22GDWVvjB8eKyDTQ2Pz9d8gzipiKDNdljRU9R85is7dvRSxrH\nE7NK5cJqudi/O7fMFPhyyahm6ryomUG0tgz++q1hvHGbub4q0WRjtK3IShTIiHbTHxIqisxZI4vJ\nzDWyJ6sx9Fo5hWtGg1goG/jWkQJ290oYiYhY8Eid5FUDJ/O6y64/UdCxISyiL8DqFNkTsyo+9mR2\n2X1qOufIqLxieBkOi5hpEJzyGkfUcnkOBAXMLuMkUdQ4eoMCYhLzDIT/++kMftLEzeVFRjXM1KLY\nnanFbx0t4vmFlbmZv3yggH1zSt3tigHHrMU2KTLDNnssryn6a6k87jtcb3bYO6PgnF4Jw+EVBjJL\n6dif50emyvirx9LLfrxWyaoGRiIC8tZUjmCD2lYi0FyRMbS/BumEAtlZykDQqpHVKDLGzFSiXd/K\n6GaNDABExvCWnRE8v6Bhd58MkTFsioo4kXdfgafSGnb3Sihq1ZPxybyOzVERcY/J2ZMFHSWd43vH\nm5/4Oef4/nG3KzGjmMsW7RRpszpZwaqRAaaitQPR6bzespooWosGR6L1S1gXrUkoy+lRsxVZXOq+\n1KJmcBzOLE+J2nDOMVnQMeeryFhbR1TZZo/gMlfDHMxorgs2mwcnynj5aBARibVFkc2WqoOuH5wo\nr+g9boWswrEhIqKgGXWOxVriTda6FHWORGB1XKE2FMjOUqpmD3cfGeBOL2Y1oRLIAOC1W8K4dFDG\nOb2mgcScsu8+kb+4qOGcXhnjPVJlMO/JnI5NMdEzDTFV1PHaLSHcd7jQdBhsRuX42FNZ1xfZNnrY\nDIVFX+eionNwXi1cD4SEykniP48V8X8P5D1/rxbbUTgaETFRY/h4fEZBf1BY1lBiW5ElutDscTyn\nQzWA3IpqQuauOC+V655+356TomKbPZaZWpws6HXO1cWygYen2hvI7NTiZMEAB/Dg5PLaRlolq5rt\nLHa9udEIvYQsNGyKLmocvQFhVad7UCA7SxkICZgrmyee2lGD5nQPw1oBD8Qd46/CEsMnr+qr9KWN\nxUUcr9lGfSCt4pweCckeCam0mWY6mdcsRVafhpgs6Lh+YxBDIREPNfmC2krrtOPkURvIhsP+isxW\nY3bh2lkjO57T8Myc2pKL0Q5kI2ERkzXP9eiMgt/eEcaRrL7kJtuMYiAhs65MLdp/65WYeWx16xfI\nAkK77fdVs8fyAplRF8i+msrjhk0hDIZEq860/IO1Vw7ZqcWJgo5Xbgrh/lOrHci4mVpUeQuKrPHG\n6JJupv1JkRFtJyIJYGBIKwakmh6XXkuRLSoG4lLjhs6xmIT9ixp+crKER6bNL1dFkSWkiuHjRM5M\nLXoqsoKBkYiI146F8IsmDcozxeoX2maxbFTqeEDjGpnZDO3Yph2q1siO53QsKkZLlnq7j2w04k4t\ncs7x2LSC60aD6AsynMovrTCQVc2RSQnJ6DqzRyqtYWdCannmpReTBR1B0TuQmTUyQGrz0OCgyBAQ\nmtfI5muOqayb48TMwQLm8UwXdfzgRAk374oAML9nfopsrqQ3bRUpaBwbwkJlPulkUcebtofx/IJa\n197STjKKgQ1h0Zpc31yRNWqpsevJFMiIVWEgJFgbot2326nFhbKBhNj4w7erR0IqreHnp0v46BMZ\nPDJVxmzJwFhcxHiPOYH/hQUVx3M6zuuTERQBjqoV1+AcU0Udw2ER2+JSZYyVk/2LauX+U7Yic9xv\n3kORzfikFvOagahDgto1Ms0wazMvHQ7gmfl6o0EtRd1Udlvj7hmUR7LmiXhzTMJ4Qvasn9g8O6/i\nf+11F+6r9vuVKbJ27utqlYNpc9JDbhntDDaTRbOx3culV6qkFttXI3NO9mhUIzuS0fA/f7Xgum26\nqGMoJKI/WG3C//KBPF6/NVyZdtMotfjto0X8y6HGUzEKGsdYTMJsyXTHThV0bI1LuGI4gH9+MY99\nc0rbBx6XdQ4O0xCW15orsoSHgcvGXpMTl9mqbhigQHYWM2Cd/GubqccTEh6dVrBYNhCXGp8xtsYl\nfPM1g/jrK3rxpxcn8MHH0hjvkSAyhq2WPf3/PJ3FH5wbRVQWwBhzqbKFsmmFD0sMm6MiTub1utTe\nR5/M4PEZM7hMFw0MhgS3IlNqamQh0fdKt+gwegDVGtnpgo6hkIDLhgJ4poW17HZq8dw+GSfz1ebv\nR6cVvGTY3PS9s6fxUOKjWQ1Ha9KyVbPHygLZ/3osja8fbHySbCeccxzMaLhkMODrPl0sG/j3I42P\naaKgY0+/jNkGZg9ZYNBaTNn+/89lGwb1itmjSWpxqqhjtuT+bE4WdIxEBIxExMrn8ddTCn5za6hy\nn4jsH8gyCsd0E/Vf1DjGYiJmS+aFZdT6rrxjVxRlneNvnsjgO8caj2RbKlnVQEwWKkG43CyQNVBk\nJc383dAqjQCzoUB2FjNgjb2pVWTXbQxipmjggYkyElLrH76rRoJ4WzKKKzcErMc1V6WERIZXb6l+\nuZ11ssmCgQ0R8+q1xxqzlXZc3WkGx6m8XjGUzBR1XDwguxRZbY2sx3JIFTWOr6Xyrmn4+ZpAZqvP\nY1kdYzEJF/TLlUD2i9OlunQSYJ60i9YXVBYYLhyQ8YQ1h/Lnp0u4ZsR8/TsTUkPDx3RRrxhN7Me1\nzR4xkSOnNp6Y0IhTeR3/9GJuxT1drTJZMBASTRerlyLjnOOjT2Xw98/nG9YgJws6dvfKKKi8Tmko\njhpZK4qsoBn410PFhq0Y9mMGBdZQ2cwUDZT0ak+UeaxmSnzUCmRzJR2KzrExUm26ikjMt5k5qxpN\nR6TlNY4tMRHzZQOn8zpGrMfenpDw55ck8JotIaTbnILOKhwJmSEqCaZrUeOVFS5emDUyn0Bm7TFr\npnhXCgWysxh7flvtxZYkMLwtGcH3j5cQb5JarOUdu6J4WzJa+ffNyQj+7JK4a+q1U5FNFXWMhM1v\nCWNVVWYzWTANKSesxuvpooGLBgIuRbZQUyNjjGEoLOI/jxVx35Ei3v/oIp6wFF1BdQeygMgQlRme\nmVewJSZiR0LCdNHAN48U8MHHMnhwor4lQLEMMraSfclQAI/NKDie0zBTNHCpNUllvEdyNYXXMlU0\nkNeqUxvKurn7LSgyCMzcMbfck9R82cA7dkXx4Scya7JrK5VWMd4jIe7jYPvmkSIWywYCgnlsgJke\ndg6iBsy/92hEQG9QqNzPxm6IbnVE1UTe/H2vixEbU5GZtbdGAnjGCjjOx5oo6JVANlnQK7VhZ005\nIjHkGymyJnNBC5pplAhLDPsXtUogs4k2qMEtl6xqIG4pMjO1iOaKzCe1aNeSgyKjhmhidRgICpAF\n751yr94SwnDYNB2shGs3hjAWc896dCsyHRsi1Y/h5qiIk45pISfyOsIiwwlLkU2XdFwwIGOmVC2w\nL9YoMsCsk33++Rzef2kCf3l5D/7y8TRmirrVDO1+vYMhEU/NqhiLSZAEhvP6JHzxhTx+a1vYMxDV\nDvO9fCiAx2cU/PhkCTdsClYGxI6EBRQ17luUt6eP2PWgjHUCsTH7+Zb+5VcNjoLK8XvjEWyNi/jA\n3vSq74M6lNEwnpAq44qcqks1OL7wQh4fuDSBTdFqu8L9p0v41pFqWszsITMVutfElXKlIbo1RWZf\n7NQGRNdjWs2+zabf28rZ+ViT1kXYaMRMdb+4qFbaUmwa1ciyqpkubHShUbTMSUMhEc/Oq/WBTPYP\nlMvFNByxippsZvYwv8/e73HRGhQcEs+SEVWGYeDaa6/F7/3e7633oZw1DIQE36ncssDwgUsTuDTe\n/p1zbkVmuL6cm2Nuw8fxrI6XDAdwIm8Wu+dKBkYjZoHdNnTUmj0AYEtUwhu3hXHJYAAXDwZwXq+M\n/YuaqxnaZiAkIJXWsMWaWfm2ZBQfv7IXr9wU9DRr1AaysZgInQPfOlLEqzZXU6iMMexISDiS9Rms\nXDTQE2CV5l/bem9TO/eyVRbKBnqD5t/2fZckkJAZ3vfIoucorXaxUOYYCAkIWA3LzoyZXQfdHJOw\nMSritNVAfzzrHiydUTlEAYjLgtXn6E67uWpkrSiygvtCwQt7fmNQbJ5aDInMHchqamT7LUXmJNwg\ntZhROARWVXteFDQDEZFhICR4BzKJrchc431cBhKy+bfk3Ay4jc0e/oqsZI23Cojs7Jjs8dnPfha7\nd+9e78M4qxgIinX1MScXDAQwFGj/yS8RqKafJgvV1CJgKjKnZf1EXsMlgzJKGseJnI6YNeh3NCLi\ndMEsvteaPQDg9gtiuH1PrPJve4mmOWfRfd/BkAAOVJTjJYMBnNcnY2ePGYRqT5r2VA8bxhheMhTA\ngLXuxsnWuFjXMA6Y6mOmaC7ltK/27cn3NnY/31KZLxmV+ZiSwPC+SxM4mNaWpe5aJetQk7XpRWcN\nczRS/fsey2mu1zdR0DFqnajtodZO7A3RUotDgycKOuIya6zIjOpkj9qG3f0L1Z7CmZKOXT2S68Ji\nsmBeVNlN8fsXVeyuVWQiq/SC1ZJVDYzF/Jv3AbumK5gKtWxgNOL+7EZlVtkZ1i6yKkfMqldHrPev\nkSKLSgxFnwn4tiJbje0CTjoikJ06dQo//vGPcfPNN6/3oZxV9IcEz/Uvq03C0UA5VdRrFJlYSSMC\nZv/Z1piELTERj88qGLKC3saoeRVc1M05brV7jmShflr34YxW2UXmZCAkICox9AXdt0ckAQMhsa4X\nzGtP2G/vCONd58fq0rRjMamuYRwAFhWzCL4pWh2RlVEMVyDrrVFkfifEWubL7v1VImPoDwnL7kv7\n1L4s7tmXrRsN5iSn8krjfEx2qwRnILNTi5xzs2+vVuFYf98Bz9SiqZ4CAmsptXi6oOP8PrnhUOjq\n0GB36kszOG7/1QKOVUxGBnb1SpWgWNY5sqqBgZCpHnOq2Y85FHYrJr8+MoObE0y2x6WGA65tl+2Q\n9fdcqxqZvXE+KjEslBsrMsGagO+lDG3XoleNTDN427Zyd0Qgu/vuu/GhD32oKzepdjNbYiJu3R1t\nfsc2Y1+x23P1NoTdNbJTDgv+8ayZ8tsSE/H4jILhcPXK/nRer6TRmrHdVmRqfWpxMChgLC56fv7G\nE2afnBO7gO0k2SPjpRuCdb9vbgioP1FNFXUMh0QMOoYWZxTumqLS59gNpxkcb/nRXCWYTeR1fHF/\nzvO1zperiqzyWB7miVZQdI7vHi9ia0zEJ/dlfZWQbdkGgJgsuCz4C47jMVOLOmZKpvGjoFWv5E0X\noDXMuSaQcc6h6EBANM1JOkfTVOlEXsf5/bLn3EYbuzfNrJFVbz+a1aAaqKh4nXNsiUkVs4fd+ygw\nc+PCSFisq48B/jWyvFptqJ9qoMjsBv5Bqy9tQ02gjMlsRSPBvMg6PocRyfzchJosxPSz4FcUmYdr\n8bPP5fCfR9vTOrBKi6db54c//CGGh4dx4YUX4sEHH2w6HiiVSq3RkZ0d7ALQ7C1t93uez0g4nQng\nyf3TEHgMp48ecv1cQhyPvXAIAYGjoCWwcOIwIuUgHpoP4soeBanUDFhGxoGshEhBQ9gIND1GjQOT\nhQR6UcKYoSClVINToiTg8qCIVKp+gWGfFsTeowxbi1X34qGMBL0UQCo12/S1GgrDoYUYUqkUDA78\nOi3j6l4VT2ckRI0AtHQGR3MSUqlJHJkNQjeAVGoKAKCkZ3C8ICGVmsC8ypDXEvjFc0exM6LjwQUZ\n/zoVxtXCRJ3rNDUTBHj1cQBAUsLYfyyNnsWl1TynFAG9YgQX6KeQEON49IXDGA7Un7Dm8zHMn15A\nas6AoETw4rFFhOfN9zg1GwR0hlRqCmWV4Xgmhl/vP44NUhC6LuDJ/YfQK3O8OBnChoCBVGoSSk7C\niflA5TWoBiCyBA4dPGi+HpbA/gMH61LjBjedn5wDE/kEegtTmEiHkErNeL6+QjmO08ePYLYoYiET\nAHrNz/uvFmUAETx2dBrSgooeMYLy/CROLprH9GxOQgLVz10CEQzpet3z5DSGnBKr+3xOKwLCLApk\n55AqC0ix03XHxjlQUBM4ffQQyjkJPVIYxw8fdN1nUWXIFOsffyWcng9jQNGQ0lUwNYoZRcD81ARS\nRX8HbsCI4rlDx1GOuC/aji8EUC4JmJ2cx0I24Hp/Ds1EcHqe46btkRUf87oHskceeQTf//738aMf\n/QilUgm5XA5/+Id/iM997nOe908mk2t8hGc3qVSq7e95ZkbB3lQep8L9OH+gjGRyxPXzrVMLCGzo\nhywwbJ3KYteuJC6OlvCfsxnsGu1HcjwKdUHFvQ8t4qgawt2XJJAc2tj0ecdOz+NYScDNWwaRHK6q\npySAV/j8zksTZdx3uIBkckvltsMnihhiCpLJTU2fcwfnyB2Zwebt4zid1/HP++fxqvP6IYoKtss6\nzh8N4rEDeSSTm/Hv+Qx290hIbo8glUph95ZRHDhaRDK52VyNcnAB5Z5RJLeF8d19WZR5ERjahmSf\n22CAYhbjcRFJxwlirJRFMCIiuXNpJ430tIIt6TySyVFsnl5AaLgfSY9FreVDM9iT3I6+oICRbAbx\nQRnJMXNf1Q/LWewIiUiOR2BwjtLhGeRiw9gtGlDmFPRt2orxHhml+UXsGQsjORoE0iq+m85W3uOM\nYiB0eK7yWZQPzGDrjp2VdTw2f/FoGq/cHMSefhnRw/N46Tlb8aXpBSST3p8PLTWD3eM7wBY1PFTK\nAyggmUzi+89ksadPw6IUQGRDBJsyeZy/bRA/e9Y8pueOFLBT0CrH946eMjZFRGyuceiqBkf54AzG\nx8ddil9fUNE/m8WerX04fLTo+nxV3lOdQzgwg3N3JTFQ1DEbLNZ9VwqagdKR2bZ+R/nsInZtCSO5\nIYjBuUUcnFawfctG13emluH5RfRsGEByxH2fxw8WMFLSsWM0CDGfd31n9KkFzLRJTa57avGDH/wg\nnn32WTz99NP44he/iJe//OW+QYw4M0gEGKaLBv75xTz++7n1qc3NURFHMhqOZ7XK9mv7v3YNYkdc\nwu/sDOOL1/dX+raasSMhIqPwOrNHI8YT5pgtZ6bATC229hhipTdOqzRN751WMF00U6qD4WoKbf+i\nuR7Hptdh9rDvYzc4H0xr2BoTsc9jConT7OF6rGWkFicdNcyRSP3KGsBM+9XVyBwnKGd7hMDMdNrD\nUwq2xkT0OVoMpotGpRVjICi63Ib5mnSuLHpb8E/mNfzkZBkTBQMjUfvxDd9ajD21onayRyqt4bVj\nIRzKaJgp6hgMiegPVd/DQxkNO+LVoPXS4WBdEAPsWq3Ze+jEbHxnGA4LlbFrtRQctdjhsIg/Oj9W\nd5+waNYL2zGS7N8OF/DUrOJKE9vP36hGBgA9AcGz57GoGb5mj0XFqJtss1zWPZARZx9xWcDJvI5L\nBmXsrrErA8ANm4L4pxfz+JdDhYqTcFPUPJkOW0XvkMTw33bHKss0W8E+8dTWyBphOxqd0yG8zB6N\nGItLOJ7V8dScgqtHAtg7o2C6aGA4bPZLzZYMFDQDp/Lm0F0b+yQMALMlHVtjIg5lNBic41BGw5u2\nhz0XUM6X9cr4MddjLcPs4TRgjISFukn/gFkHkYXqPri4zFxrdsz2iOr7tTEq4rl5FWNxyeyVs+5r\nzy4EgN6gaR6wa3Incxo2x6r1IZl5j6maKRnYO6PgUEbDxogIWTAb3r3s4fbJXxJss4d5u/3+vnw0\niLTCcTSrYyhsbkqfL5u13UMZDTt7WktoeVnwzTqUgOEGK4cKmlFnTKqFMYZog6brpfD1gwV8cG8a\nhzIaEpZr0e65bORaBIAembkm8tg02i6QVgzE5Pb4IjoqkF1zzTX4+te/vt6HQawyPQEBvQGG/35u\n/RUmAFwxHMQ/XNePnQkJlwyagS4iCbigX/a86m2VHdaJp9nJwQljDFcMB/Dz09Wp/HYBu1XGYiKO\nZDU8Pafi1t0xPDWr4nRex3BYsLYQAE/NqtiekFy74foC1RPnXMnAFcMBHM5oOJnXkQgwXDUSxL55\nFQbn+PrBAr5lzTGsdS0Cyzd72L1SgL8iy9aYVEyzh7drETADmQGYiswytBQ0A4rBK2PKBGvBq61E\nj1nuVRuv5ZoFzdx4fuGAjG8dKVSs/P3B+p40AK5G34DDfn/Ken97AgK2J0Q8Ml3GUEiwpuQzZFSO\nwxkdOxKtfRa9DB+VTeAyg865p4W+oLWWPTAt+CsLZAXNQFY18I+v6MebtocxHK4OPQZaUGRBP0VW\nNXs42xvM18xx8WBr2ZRmdFQgI84OwhLDv716sHKi8WI4LOIDl/XggoHqB/1vr+mr6xdbChVFtsSr\nwJu2RfDvR4oVl5yXa7ERYzEJPztVxmBIwPaEhM0xEQfSWuVkMRgS8MvJMs6tUachiUFi5klwtmRg\ne+Ntei8AABp7SURBVEJCVGL41UQZ4z0ShsMiohLDA6fL+OL+HH52qlwJerXvk1Pd1fKTkyV8JeW9\nUNSeJwjYgaz+McweMmcg87ffA8DGiIiIxDAYEiqKzFaozjrStrhYaSY/ltUwFq9+XgIeY6pmSwaG\nQiKu3xjE0ayjJ83qwaql5AhkzoboA2kNu3rMv8XOhISjWb3iGuwLmo3JPQHmmsLSCK9esqyVimWM\n+aqyQovKPyoJyK9wQdvJnI5NERGDIRHvOj9eeV/sQNpUkfmkFqtDmd1rcuyt7rV9l8uFAhmxLqxH\n/9pgSMDbk5ElKTIAOK9PQlRm2GvNa1yOIjtd0HGpdfX5kqEABFSHNg+EBDw0WfZMs/YFzT6euZLZ\ns7SzR8IPT5aQtE60Fw4E8JEnM7h1dwyptIb5sgGRsboreaeVv5ZUWsMvfRaa+tXIdF4dvWXvULNx\nNkRrBkdW5a4U8OaYiG1Wu0OvdVx2zdDJrh4ZBxatKSAeiqz23D1TNDAUFnDNSBAiM5UfYNbbvOYt\nKjoqw3Cdqa+DaQ3j1gnWVl1D1rH1BwXsnVYqP28Fr16yrFJtIB8OC57Dg72m0Hjh18O1FE7kdc9s\nR3TFNTLH0GBHatFeVzTeoqptBgUy4qyBMYZbz425Bhi3+ns3bQtX5gIutUZm13YusQLZS4cDGApX\nm9EHQwIWFY5z++q/1LZJY7akYzAoVhSCfQJ46bA5heQtO8PYkRDxy0mlLq3ofByv9pa5ko7UolZX\nw1B0jrRiVALukGV2UA2On50q42+eyAAwm6GdtY54oHpiTSumWnNeuLxkKIC/fklv5bgWy1VF5mRX\nr4QD1tbpY1kNWx2KTPZQZDNWjS0REPDuC+OVvq7+kHda1blny7lYM5VWK0rBrlnatbv+kIC9M8qS\nTsBeNbKMyit1qBGrJ7KWgkfzvheN5jm2yomcXjFUOWnd7OFXI7NnLbobou1luK3WGZtBgYwgWuA3\nNoXw7LyKhbJRt9OsGRFJwOu2hHCxVe/b0y/jb6/pq/x8ICQiJpvuxlpMk4aZWhwICRWFYCuCGzaF\n8MmreiEwhosGAvj56VKd0QMwT0gCY641JDZzZQMGgAM1PWam+aI6j1MSzJl/00Udj0yVK7MMszXD\njmNSVZEtlnldmlOwJo1UX59RaTB2ck6PjBcXNaQVA6oB1+vyqpHNlIyKcnr91nBFlfYH66eEAO4a\nmbOGcyitVQLVjoSERICh1zKr9AUFnMrrLlNOM6IeO8mcimx3r2y2V9TQ6gVTVPIeU1XSOO7Z13gf\nm82JXHXWaO1jA2YjeiMaKbKwyCBZvX2VQd+KOcjA67O6HCiQEUQLhCSGPf0y9s0pdbMWW+HPL0lU\nTlx2XcRmMCTg3Jr1HzZ9QQGTBR0l3TRC7OqR0BNgrjScrTAvGpDx1KzqqcgAoD/IPFNssyUDFw/I\neK7mZOqsj9mMRERM5A08NqNgxlo0WWv2cG43qK2P1dIbYBVFVptaHIkIUHSOJ2eVuskr3orMqIxy\ncjLgo8jskVdANdW9oDJo3PybAGaa9BuvGqy8x3Zbw1KUhLfZo6rILuiX8axHG0XeYwqNFzFZ8Jzu\n8eScgm8fLeJHJ+tXEdVyIqdji0dqMSKb9a1mWQzfQGYpMsbcO8nSivl5btc0JwpkBNEiF/bL2Dev\nLjm12IxrR4P4rz6jwnoDAg5mNAyEzO3aW2IS/vkVA54ngAv6ZTCgroes8liWuqtlrmTgutEgnpt3\n9/RM1szBBMxA9svJMuKyAFkw00l1isxRs/HaTODEnPDPTfVXo8gYM80APzlZctXHAHMZbK3SmCnV\nPwbgr8icZg8ACAoMR4rmTjrn++u8T19QqIyWapWwRyBzKrKxmIi8xiujymzMAdctKDIf1+LDUwqu\nGQngywfyDVUZ5+ZA7i0eGYGIZZ1vhv03r+3Xc6ZvnYaPdNlYUutMMyiQEUSLXDQQwL659geyDRER\n59VO57DoDwo4mNYw6FjR6xcYorKA8R7JN5B5rYWx5whesSGIZx3T3oH6zQSA2Uv245MlvGQ4gKGQ\ngJmSbtbIAtX3w2zSNWcoNlNkEcnsBzuRqzd7AMCuXrnSPO1E8hgcvFRF5jR7AOb/Hy2JDW31w2EB\nu3qlJdVZvRRZ1qHIGGPY01evyoqt2u+l+kDGOcfDU2Xcem4MoxERPzzhr8rmygaCIlyGncpjy0LT\n+hhg/j1iHitl7E3qAFx1skWFAhlBrAu7eiWcyOmYKxtLTi0ul96ggKNZrWK4aMabtodx4YB3UPQK\nZPMlAwNBESNhASJDZekl4O4hsxmJmOrh8qEAhsMiZopGnSJj1jT0rNo8kNnOxdlSvdkDAM7plaBx\nYGvcHVy87Pd+imwoJGKhbLiatAG3WgBM5XW4KLkmdtRy+VAAf31Fj+/PvTDt99Vj5Zy7pmcAppp+\nZs4dyAp6a6lFsyHa/dqOZk3pszUm4uZdUXzziP9wXr+0ImAOIrhlV2uDxb3SiyWHw9fpXEwrrQ37\nbhUKZATRIrLAcG6fhJzaXkXWiL6gAN1Rs2nG68bCuHDAu8nUtrorOkfKcgPaJhLGGM6vUQUTBe/U\nosSAiwdl0zZe1Cs9UU7sCfjOyfeNXmNPwDuFZfdz1Trqau33ZZ2joFUbqp2EJLOp/X6rqf2JGQXv\nfWgBj00rrucMiAzHmigygbXeP2ZjKrLqwZZ0c7Cx87n39Mt4pkaRtVoji8pC3WSPR6bLuHJDsPJ3\nPZHTfBeHHs/pnkYPwDzG39wabnoMgBXIHBdKnPPKGhcArjFVpmuxfd8hCmQEsQQu7DdPrGsZyAC0\nrMga0W8psu8cL+L9j6YBmPUxO0ie3191z2mGOb1ie81J/ZxeCbftiSEiCRgKiZgp1SsywLRjH7D6\n2po1sfcGhbr1JDajEQFv2RHGaE39RmamIpst6TiV1zBb0jEQEnxTfq/eHMKPTpTAOcfnX8hhR0LC\n8Zzmet6gwFAyGLYnWq9/tUJYElz2e6/365xeGcdzuivgteqO9aqRPTyl4KUbzAuagFXTO+GxTgiw\nHIse9bGlkqix4CsGIApVI41z59uiwim1SBDrxYUDAUgMrlFSq0mv9WVvVZE1fCxrTNV9h4uYLRrI\nKAbmytU+MaciO5zRsCEi1J1wI5JQWbsxZCmynMLrZub9j/Ni+NtnsnhxUW0ayPoCQmXPXC2MMfzx\nnnilBcDGHhr8b4eK+ODeDKaLRqXXy4srhgM4ldfwzSNFlHWO/3F+DJ+9th/vOKeaNguKwKBcv0F8\npdTWyDKKgUTN+xUQGcYTEl5crBpuWm2Irq2RHUprOJbTcYlDme9ISDjsM6D3UEaru2BZDrWpxdoJ\nOEHHzjdKLRLEOnJ+n4wbx1pLtbSDRIBBYGav2UrpCwp4dLqMqMRwXp+MQxlTydhBMtkj4WTeXCT5\n3ILqa0CxsUcreSmMCwYC+OhLexGVma/asmmkyPyQGIPKOY5kNZzO6/iXg4VKD5nn/QWGGzaH8Jln\nc3jHrqincguIDJuD/tual0ttH1ntJBSbTVH3LMvWJ3sIyDmU3N8/n8M7dkVcyzC3xyUcydQHMtXg\n2L+g4fz+xn/rVqgNZPNlAz2OYGX36nHOK5M92gUFMoJYAiGJ4T0Xxdfs+QTG0B8U2qLI+oICSjrw\n5h0RjPeY62nmSgb6LeteQGQY75Gxf0HDc/Mqzm8ayATMFE3XYtyj3rG7T8bXXjnY9Mr7lZuDeO1Y\naEmvRbZqZEeyGm6/IIaHp5WGigwAfnMsjJcMB3DdRu+9WkGBYdMqBLJa+31WMZDwqLMNhQXMOGYu\ntmy/dzz+o9NlTBZ0vKGmrrU9IVXmVjo5sKhhY1Rcct3Pi9rpHocy7m0O9hiwos4hstZs/a1CgYwg\nOpx7rur1HB+0VIbDAi4ekPGKTUHsTEiWIjNcQfL8PhnPLqh4bkHFniZX6UMhEZNFA5KwslRrskeu\nzI5sFUkwG6kzCsdrt4TwsuGA52QUJzsSEj72sl7fOtqmmIjxSPsDWW1q8YVF9wBkG7udwabVocER\nR43sn/bn8QfnxepmmW6Pi5Vddk6emVdwQRvUGFA/Af9QWnMZZ2yzx2KZVyaltAsKZATR4WyJSW2Z\ngBCRBHzq6j7IAsN4jxnI5kuGy0iyp1/GLyfKyKnc18lmE5LM3qFYG67ml0pAMIcdb42LEBjDX1/R\ngxu3Lk3V1XL7njj2xNqz6NHJUEhERjEwVdBhcI6fnirhNzbVH+uQ1c4AmI6/vMYRbaFeF7FmOU4V\ndJzK67hyQ71rdTQqIq0YdaOs9s2pvu0aS6U2tXgoo7lmUtqKrN09ZAAFMoI4K9keN117M44aGWBO\n+j+Q1nBun9xS0+9QWGzbcsSlIAkMB9Iqtls9X5LA6gwhnUJYYnjNlhD+/WgR++ZUxCTmafEfCguV\nBa6zJQMxibnqXH6IzLzfT0+VcMVwwHOzhMgYtsalSn8ZYC4QfXZ+9QLZ4UyNIrMme6T/X3v3GhPV\nmYcB/DlzkdsoWGQQRERxwAVBA3gpKqGG2IK6EbTx0lZsY7bS24pGV6MxbbJaYrNAN2pMNNmmJrQN\nErNxJS2dtQ1SpKUYt5hopZABYRRhYcpNBQb2AzIrMIAjw5x5x+f3yRkm4/+8nJmH85730t1nGcRk\nLwwyoueQu+r/gzCevA/j665EgKdi3Ptjg7QeI0c2OoJaMbBe39wxJi87k7S5nrhU9wD/qn2ApCDr\nV45+7ko0Pd6Bu6HTjFleNixMrJKgr3+I+JnW7/8Bjwd8tPfC9KgPxk4zatvN0Kgly15rE/XkPTLT\noz48NPcPWa1lcLDH4Mr39iTGWUBEdhc6TQVzP0Z0W26c5zlk6PZY/NyVkGD/+0rjGbwnZ+85X5Ml\nwEuJRS9Mwb8bHuFPEdZ3RveeMrA7wcPeftR3mjHLhvuiXmoJtR1mLNGO/nubO1WFfxoe4MzNDvRj\nYF5h1FP+np/Gk1dkgwM9njy3Blf2uNdltuvQe4BBRvTcmu+tsroG4aZ5nk/9HloPxYh9zBxhsPss\nRJArMgDYpvPE1CmS1aW4gIE/KPzcFY8neZvHHbzyJC+VhGhf9ZhXxzF+alS2dONQzDRoPRQorHs4\nYlfyidA8nmbQ29c/EGTDdghwV0q412XG1XuP8Lf46aO8y7MR5ywgIrtapp0y4RVKEgPdRt15ejKp\nFQNf3tYWCXZWf5iuxh/G6bL18xhYLaWh04zEUaYJWKNRKxDnN/bVlc5bjb8u9bE8tuUPlqehkCRM\nUw/saF7d1jtiNKSbAii68xBrZruPuQzYs2CQET2n5nurMd/GYe/DBWlUCLLeUzap1AoJc6fZZzSn\nM/F7vHGprVdkuyI08PeUP9QTA93xlzITHpr7sSFk6Fy2KUoJSgWwI/zpFiG2BYOMiISzcLrapl26\nReH3eLWUhs5ezLIhyOyxxJQ9/DlKg4u1D/GPWx0jun0jX1Bjb/S0UbtWJ8I5jp6IyAYBXsoRCwm7\nAj8PBSqauuGhlOAlw2jQiZIkCX8M8cD6Oe4jrpaDNSoEj7JdzESJ11JERC7Kz12J//y3B7Mm6Qvf\nURzd5csgIyJyEn4eCnT09NvUrUgMMiIipzE4CtOWgR7EICMichrT3RRQSQwyWzHIiIichEIamDA9\n3oLNNJTYdxSJiFzM31f6wNfOSzi5OgYZEZETsdcivs8T2WO/oaEB69evx7JlyxAfH4/Tp0/LXRIR\nEQlE9isylUqFo0ePIjo6Gh0dHUhMTMTq1asRFhYmd2lERCQA2a/I/P39ER0dDQDQaDQICwvD3bt3\nZa6KiIhEIXuQPam2thaVlZWIjY2VuxQiIhKE0wRZR0cH0tPTkZWVBY1GhuW0iYhISJLJZHL8rnjD\n9Pb2YvPmzUhKSkJGRsaYr62qqnJQVURENNl0Ot2E38Mpguztt9+Gr68vjh07JncpNExVVZVdTjSy\nDdtdHmx3McnetVhWVob8/HwUFxdj1apVSEhIgF6vl7ssIiIShOzD75cvX46Wlha5yyAiIkHJfkVG\nREQ0EQwyIiISGoOMiIiExiAjIiKhMciIiEhoDDIiIhIag4yIiITGICMiIqExyIiISGgMMiIiEhqD\njIiIhMYgIyIioTHIiIhIaAwyIiISGoOMiIiExiAjIiKhMciIiEhoDDIiIhIag4yIiITGICMiIqEx\nyIiISGgMMiIiEhqDjIiIhMYgIyIioTHIiIhIaAwyIiISGoOMiIiExiAjIiKhMciIiEhoDDIiIhIa\ng4yIiITmFEGm1+uxZMkSxMbGIjc3V+5yiIhIILIHWV9fH/bt24eCggKUlZXh/PnzuH37ttxlERGR\nIGQPsoqKCoSGhiI4OBhqtRobN25EYWGh3GUREZEgZA8yo9GIWbNmWR4HBgbCaDTKWBEREYlE9iCz\nRpIkuUugx3Q6ndwlPJfY7vJgu4tJ9iALDAxEfX295bHRaERAQICMFRERkUhkD7KYmBjU1NSgrq4O\n3d3dKCgoQHJystxlERGRIFRyF6BUKvHJJ58gLS0NfX19eOONNxAeHi53WUREJAjJZDL1y10EERHR\ns5K9a3E8nCztOFFRUVixYgVWrVqF1atXAwBMJhNSU1MRFxeHtLQ0/P777zJX6Rree+896HQ6xMfH\nW54bq63379+PmJgYrFy5Er/88oscJQvPWptnZWUhIiICCQkJSEhIgF6vt/wsOzsbMTExWLp0KS5f\nvixHyS6hoaEB69evx7JlyxAfH4/Tp08DsO/57tRBxsnSjqVQKHDp0iVcuXLF8sHNyclBYmIifv75\nZyQkJCAnJ0fmKl3Da6+9hoKCgiHPjdbW3377LQwGA65du4bc3Fzs2bNHjpKFZ63NAeCdd95BcXEx\niouLkZSUBAD49ddfceHCBfz000/Iz8/H3r170d/PzqtnoVKpcPToUfz4448oKirC2bNncfv2bbue\n704dZJws7Vj9/f3o6+sb8lxhYSG2bt0KANi6dSsuXbokR2ku58UXX4SPj8+Q54a39eC5XlhYiC1b\ntgAA4uLi0NbWhvv37zu2YBdgrc0BWA2owsJCbNy4ESqVCnPmzEFoaCgqKiocUabL8ff3R3R0NABA\no9EgLCwMRqPRrue7UwcZJ0s7liRJSEtLw0svvYTPP/8cAHD//n1otVoAAydkc3OznCW6tKampiFt\n3dTUBGDk5yAgIICfAzs6e/YsVq5ciffff9/SvcU2nxy1tbWorKxEXFzciO+WiZzvTh1k1nCy9OQp\nKirC999/j/z8fJw5cwalpaVsbyfF34t97Ny5E9evX0dJSQn8/f1x+PDhUV/LNp+Yjo4OpKenIysr\nCxqNxqb2HO+1Th1knCztWP7+/gCAGTNmYO3ataioqIBWq7Vc1jc2NsLPz0/OEl3aaG0dGBiIhoYG\ny+v4ObCfGTNmWL4kt2/fbuk+ZJvbV29vL9LT07F582asXbsWgH3Pd6cOMk6Wdpyuri50dHQAADo7\nO/Hdd98hMjISycnJyMvLAwB88cUXSElJkbNMlzL83syTbZ2Xl2dp6+TkZHz55ZcAgPLycnh7e1u6\nZMg2w9u8sbHR8u+LFy8iIiICwECbFxQUoLu7GwaDATU1NYiNjXVora7k3XffRXh4ODIyMizP2fN8\nd/p5ZHq9HgcOHLBMls7MzJS7JJdkMBjw+uuvQ5IkmM1mvPrqq8jMzERrayt27NiBhoYGBAUF4bPP\nPrN6w5xss3PnTpSUlKClpQVarRYHDhzAunXrkJ6ebrWt9+3bB71eD09PT5w8eRKLFy+W+QjEY63N\nr1y5gsrKSigUCgQHByM3N9fypZmdnY1z585BrVYjKyvLMiWFbFNWVoaUlBRERERAkiRIkoQjR44g\nNjZ21O8WW893pw8yIiKisTh11yIREdF4GGRERCQ0BhkREQmNQUZEREJjkBERkdAYZEREJDQGGRER\nCY1BRjRJrl69ipdffhnBwcGYN28ekpOTcf36deTl5XGFGiI7UsldAJEram9vx5YtW5Cbm4sNGzag\nu7sbV69exZQpU+Qujcjl8IqMaBJUV1dDkiSkpqZCkiS4ubkhMTERKpUKe/bsQXl5OYKCghASEgIA\n6O7uxuHDh7Fw4UKEh4dj7969ePToEQCgpKQEkZGRyM7ORmhoKBYtWoT8/HzL/1VUVITly5dj9uzZ\niIyMxIkTJ+Q4ZCLZMMiIJkFoaCiUSiUyMjKg1+thMpkAAGFhYcjOzsaSJUtQX18Pg8EAADhy5Ahq\namrwww8/4Nq1azAajTh+/Ljl/RobG9Ha2opbt27h1KlT2L17N6qrqwEAH3zwAT799FPcuXMHpaWl\nSEhIcPjxEsmJQUY0CaZOnYqvv/4aCoUCu3fvxvz587Ft2zbL5oHDnTt3DseOHYO3tze8vLyQmZmJ\n8+fPW34uSRIOHToEtVqNFStWYM2aNbhw4QIAQK1W49atW2hvb4e3t7dlN16i5wWDjGiS6HQ6nDx5\nEjdu3EBZWRnu3r2LgwcPjnhdc3Mzurq6kJiYiJCQEISEhGDTpk1obW21vMbHxwfu7u6Wx7Nnz8a9\ne/cADITgN998g6ioKKxbtw7l5eWTf3BEToRBRuQAg1dkN2/eHLHbra+vLzw9PVFWVgaDwQCDwYC6\nujrU1dVZXmMymfDgwQPL4/r6esycORMAsHjxYuTl5aG6uhopKSl48803HXNQRE6CQUY0CaqqqnDi\nxAkYjUYAA8FTUFCApUuXQqvVwmg0oqenB8BAt+H27dtx8OBBNDc3AxjYFffy5cuW9+vv78fHH3+M\nnp4elJaWoqioCKmpqejp6UF+fj7a2tqgVCqh0WigVCodf8BEMuLwe6JJoNFoUFFRgVOnTqGtrQ3e\n3t545ZVX8NFHH8HNzQ0LFixAWFgYlEolfvvtN3z44Yc4fvw4kpKS0NLSgsDAQLz11luWzRxnzpwJ\nHx8fLFiwAJ6ensjJyUFoaCh6enrw1VdfYf/+/TCbzdDpdDhz5ozMR0/kWNxYk8jJlZSUYNeuXbhx\n44bcpRA5JXYtEhGR0BhkREQkNHYtEhGR0HhFRkREQmOQERGR0BhkREQkNAYZEREJjUFGRERCY5AR\nEZHQ/gcVg5Y8qIgTQgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb725e615d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot losses\n",
    "\n",
    "with plt.style.context('fivethirtyeight'):\n",
    "    plt.plot(losses, linewidth = 1)\n",
    "    plt.xlabel('Steps')\n",
    "    plt.ylabel('Losses')\n",
    "    plt.ylim((0, 12))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.\n",
      "--------------------------------\n",
      "What' s your name\n",
      "Was ist dein Sohn \n",
      "--------------------------------\n",
      "2.\n",
      "--------------------------------\n",
      "My name is\n",
      "Meine Sohn \n",
      "--------------------------------\n",
      "3.\n",
      "--------------------------------\n",
      "What are you doing\n",
      "Was machst du denn \n",
      "--------------------------------\n",
      "4.\n",
      "--------------------------------\n",
      "I am reading a book\n",
      "Ich bin ein Frühstück \n",
      "--------------------------------\n",
      "5.\n",
      "--------------------------------\n",
      "How are you\n",
      "Wie sind du - \n",
      "--------------------------------\n",
      "6.\n",
      "--------------------------------\n",
      "I am good\n",
      "Ich bin gut \n",
      "--------------------------------\n",
      "7.\n",
      "--------------------------------\n",
      "Do you speak English\n",
      "Weißt du das \n",
      "--------------------------------\n",
      "8.\n",
      "--------------------------------\n",
      "What time is it\n",
      "Was ist denn denn \n",
      "--------------------------------\n",
      "9.\n",
      "--------------------------------\n",
      "Hi\n",
      "Hi \n",
      "--------------------------------\n",
      "10.\n",
      "--------------------------------\n",
      "Goodbye\n",
      "Wiedersehen \n",
      "--------------------------------\n",
      "11.\n",
      "--------------------------------\n",
      "Yes\n",
      "Ja \n",
      "--------------------------------\n",
      "12.\n",
      "--------------------------------\n",
      "No\n",
      "Nein \n",
      "--------------------------------\n"
     ]
    }
   ],
   "source": [
    "# let's test the model\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    \n",
    "    # placeholders\n",
    "    encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "    decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "    # output projection\n",
    "    size = 512\n",
    "    w_t = tf.get_variable('proj_w', [de_vocab_size, size], tf.float32)\n",
    "    b = tf.get_variable('proj_b', [de_vocab_size], tf.float32)\n",
    "    w = tf.transpose(w_t)\n",
    "    output_projection = (w, b)\n",
    "    \n",
    "    # change the model so that output at time t can be fed as input at time t+1\n",
    "    outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                                encoder_inputs,\n",
    "                                                decoder_inputs,\n",
    "                                                tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                                num_encoder_symbols = en_vocab_size,\n",
    "                                                num_decoder_symbols = de_vocab_size,\n",
    "                                                embedding_size = 100,\n",
    "                                                feed_previous = True, # <-----this is changed----->\n",
    "                                                output_projection = output_projection,\n",
    "                                                dtype = tf.float32)\n",
    "    \n",
    "    # ops for projecting outputs\n",
    "    outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "    # let's translate these sentences     \n",
    "    en_sentences = [\"What' s your name\", 'My name is', 'What are you doing', 'I am reading a book',\\\n",
    "                    'How are you', 'I am good', 'Do you speak English', 'What time is it', 'Hi', 'Goodbye', 'Yes', 'No']\n",
    "    en_sentences_encoded = [[en_word2idx.get(word, 0) for word in en_sentence.split()] for en_sentence in en_sentences]\n",
    "    \n",
    "    # padding to fit encoder input\n",
    "    for i in range(len(en_sentences_encoded)):\n",
    "        en_sentences_encoded[i] += (15 - len(en_sentences_encoded[i])) * [en_word2idx['<pad>']]\n",
    "    \n",
    "    # restore all variables - use the last checkpoint saved\n",
    "    saver = tf.train.Saver()\n",
    "    path = tf.train.latest_checkpoint('checkpoints')\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        # restore\n",
    "        saver.restore(sess, path)\n",
    "        \n",
    "        # feed data into placeholders\n",
    "        feed = {}\n",
    "        for i in range(input_seq_len):\n",
    "            feed[encoder_inputs[i].name] = np.array([en_sentences_encoded[j][i] for j in range(len(en_sentences_encoded))], dtype = np.int32)\n",
    "            \n",
    "        feed[decoder_inputs[0].name] = np.array([de_word2idx['<go>']] * len(en_sentences_encoded), dtype = np.int32)\n",
    "        \n",
    "        # translate\n",
    "        output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "        \n",
    "        # decode seq.\n",
    "        for i in range(len(en_sentences_encoded)):\n",
    "            print '{}.\\n--------------------------------'.format(i+1)\n",
    "            ouput_seq = [output_sequences[j][i] for j in range(output_seq_len)]\n",
    "            #decode output sequence\n",
    "            words = decode_output(ouput_seq)\n",
    "        \n",
    "            print en_sentences[i]\n",
    "            for i in range(len(words)):\n",
    "                if words[i] not in ['<eos>', '<pad>', '<go>']:\n",
    "                    print words[i],\n",
    "            \n",
    "            print '\\n--------------------------------'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# This model can be improved by using more training steps, better dataset or even with better selection of hyperparameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}