experiments/convergence/convergence_mnist_64workers_1000ms_seed1337_dmoe64x4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import threading\n",
    "from functools import partial\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torchvision import datasets, transforms\n",
    "from tqdm import tqdm\n",
    "\n",
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from dmoe_emulator import EmulatedDMoE, get_non_expert_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 1337\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "layer_dim = 512\n",
    "num_blocks = 4\n",
    "num_experts = 64\n",
    "num_active_experts = 4\n",
    "\n",
    "batch_size = 4\n",
    "num_trainers = 64\n",
    "\n",
    "delay_ms = 1000\n",
    "\n",
    "eval_interval = 1024\n",
    "total_steps = eval_interval * 20\n",
    "update_every_steps = 10\n",
    "\n",
    "in_features = 28 ** 2\n",
    "num_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/20480 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "class FeedforwardBlock(nn.Module):\n",
    "    def __init__(self, hid_dim):\n",
    "        super().__init__()\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(hid_dim, 4 * hid_dim), nn.LayerNorm(4 * hid_dim), nn.ReLU(inplace=True),\n",
    "            nn.Linear(4 * hid_dim, 4 * hid_dim), nn.LayerNorm(4 * hid_dim), nn.ReLU(inplace=True),\n",
    "            nn.Linear(4 * hid_dim, hid_dim),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return x + self.layers(x)\n",
    "    \n",
    "Optimizer=partial(torch.optim.Adam, lr=1e-3, amsgrad=True)\n",
    "emulate_latency = lambda: time.sleep(delay_ms / 1000. * np.random.weibull(1))\n",
    "\n",
    "model = nn.Sequential(\n",
    "    nn.Linear(in_features, layer_dim),\n",
    "    *(EmulatedDMoE(layer_dim, num_experts=num_experts, num_active=num_active_experts, \n",
    "                   update_every_inputs=batch_size, update_every_steps=update_every_steps,\n",
    "                   Expert=FeedforwardBlock, Optimizer=Optimizer) \n",
    "      for _ in range(num_blocks)),\n",
    "    nn.LayerNorm(layer_dim), nn.Linear(layer_dim, num_classes)\n",
    ").to(device)\n",
    "\n",
    "# create optimizer for non-expert params only\n",
    "non_expert_params = get_non_expert_params(model)\n",
    "opt = Optimizer(non_expert_params)\n",
    "\n",
    "# create initial gradients\n",
    "model(torch.zeros(1, in_features, device=device)).sum().backward()\n",
    "opt.zero_grad()\n",
    "\n",
    "\n",
    "lock_model = threading.Lock()\n",
    "need_to_eval = threading.Event()\n",
    "train_history, val_history = [], []\n",
    "progress = tqdm(total=total_steps)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "#20510\tloss=0.004901\tdelay=27: 20510it [1:21:20,  1.41s/it]"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr8AAAF1CAYAAADhgoKhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdZ3hc1bn28ftRc5G75IYLrmB6M8V2QkxLCCSQRg7pkELKSXJOOiQ5nJQ3hPSEE0hCEiCEACGUQILptjC44AbGuEtyk5ukkWVp1EbSrPfDjIwsq4ym7T2j/++6fDFlz96PCqN71n72WuacEwAAADAQ5HhdAAAAAJAuhF8AAAAMGIRfAAAADBiEXwAAAAwYhF8AAAAMGIRfAAAADBiEX/iemf3ezP4nzteWmNmnk10TAAwEZrbQzCq8rgNIpjyvC0B2M7Odkj7tnHs+3n045z6XvIoAAMBAxsgvPGVmfAADAPQbfz8QL8IvUsbM/ippqqR/mVnQzL5pZtPMzJnZp8xst6TF0W3/YWYHzOywmS01s1M67eceM/t/0dsLzazCzL5mZpVmtt/Mro+xnhwz+66Z7Yq+9l4zGxl9brCZ3WdmATOrNbPVZjY++tx1ZlZuZvVmtsPMPpLkbxUApIyZfcvMHu7y2G/M7Lbo7evNbHP0Pa7czD7bj33/xsz2mFmdma01s7d2ei7XzL5tZmXRfa81synR504xs+fMrMbMDprZt6OPH3m/j94/qu3CzHZGv57XJTWYWZ6Z3djpGJvM7L1davxMp69vk5mdbWbfMLNHumx3m5n9JtavHZmL8IuUcc59TNJuSe92zg1zzv2009Nvk3SSpHdE7z8labakcZLWSfpbL7ueIGmkpEmSPiXpdjMbHUNJ10X/XSRphqRhkn4bfe4T0X1OkVQk6XOSmsysUNJtkt7pnBsuab6k12I4FgD4xYOSrjCz4VIklEr6oKT7o89XSnqXpBGSrpf0KzM7O8Z9r5Z0pqQx0f39w8wGR5/7qqQPSboiuu9PSmqM1vG8pKclHSdplqQX+vH1fEjSlZJGOefaJJVJeqsi7+Hfl3SfmU2Mfq3XSPqepI9Ha7hKUkDSfZIuN7NR0e3yJF0r6d5+1IEMRfiFV77nnGtwzjVJknPuLudcvXOuRZE3qjM6RmW70SrpB865VufcIklBSSfGcMyPSPqlc67cOReUdJOka6Nveq2KhN5Zzrl259xa51xd9HVhSaea2RDn3H7n3MZ4v2gASDfn3C5FBhU6RkQvltTonFsZff5J51yZi3hR0rOKhMlY9n2fcy7gnGtzzv1C0iC9+X78aUnfdc5tje57vXMuoEjQPuCc+4Vzrjn63v9KP76k25xzezr9/fiHc26fcy7snPu7pO2SzutUw0+dc6ujNZQ653Y55/ZLWirpmuh2l0uqds6t7UcdyFCEX3hlT8eN6KmxW6Onreok7Yw+VdzDawPRT/sdGhUZxe3LcZJ2dbq/S5GLPsdL+qukZyQ9aGb7zOynZpbvnGuQ9B+KjATvN7MnzWxODMcCAD+5X5ERU0n6sN4c9ZWZvdPMVkZbEGoVGant6f33KGb29WhLweHoa0d2eu0URUZlu+rp8Vjt6XzHzD5uZq9FW9ZqJZ0aQw2S9BdJH43e/qgifwcwABB+kWouhsc/LOlqSZcq8sY5Lfq4JbmWfZKO73R/qqQ2SQejo8jfd86drEhrw7sUOU0m59wzzrnLJE2UtEXSH5NcFwCk2j8kLTSzyYqMAN8vSWY2SNIjkn4uabxzbpSkRYrh/Tfa3/tNRVooRkdfe7jTa/dImtnNS/co0nrWnQZJQzvdn9DNNkf+fpjZ8Yq8J39RUlG0hjdiqEGS/inpdDM7VZH3/N7a7ZBFCL9ItYPq+U2uw3BJLYr0YQ2VdEuKanlA0lfMbLqZDYse5+/OuTYzu8jMTov2wtUp0gYRNrPxZnZ1tPe3RZEWi3CK6gOAlHDOVUkqkXS3pB3Ouc3RpwoUaVWoktRmZu+U9PYYdztckQGEKkl5ZnazIn21Hf4k6YdmNtsiTjezIkn/ljTRzP7bzAaZ2XAzOz/6mtcU6U8eY2YTJP13HzUUKhKGq6TIxXuKjPx2ruHrZnZOtIZZ0cAs51yzpIcV+SCwyjm3O8avGxmO8ItU+7Gk70ZPR329h23uVaQFYa+kTZJWpqiWuxQ5rbVU0g5JzZK+FH1ugiJvgnWSNkt6MbptjiIXbeyTVKPIhXqfT1F9AJBK9ytyhu1Iy4Nzrl7SlyU9JOmQImfinohxf88octHaNkXew5t1dEvCL6P7fVaR99Y/SxoSPeZlkt4t6YAiPboXRV/zV0nrFWl/e1bS33srwDm3SdIvJK1QZLDlNEnLOj3/D0k/in7N9YqM9o7ptIu/RF9Dy8MAYs71dFYaAAAge5nZVEXa2SZ0usgZWY6RXwAAMOCYWceZvQcJvgNLn+HXzO6yyIIAb3R6bEx0curt0f/GMscqAACA56LXcdQp0n7xvx6XgzSLZeT3HkXmv+vsRkkvOOdmKzIx9Y1JrgsAACAlovPMD3POneKc29P3K5BNYur5NbNpkv7tnDs1en+rpIXOuf3RVVRKnHOxLDIAAAAAeCbent/x0dVRpMiVmuOTVA8AAACQMnmJ7sA558ysx+FjM7tB0g2SVFhYeM6cOSyOBSDzrF27tto5N9brOtKpuLjYTZs2zesy1NDQoMLCQq/LOIJ6euanWiR/1eOnWiR/1ZOKWnp7z443/B40s4md2h4qe9rQOXenpDslae7cuW7NmjVxHhIAvGNmu/reKrtMmzZNfnjPLikp0cKFC70u4wjq6ZmfapH8VY+fapH8VU8qauntPTvetocnJH0ievsTkh6Pcz8AAABA2sQy1dkDiqyccqKZVZjZpyTdKukyM9uuyGoxt6a2TAAAACBxfbY9OOc+1MNTlyS5FgAAACClWOENAAAAAwbhFwAAAAMG4RcAAAADBuEXAAAAAwbhFwAAAAMG4RcAAAADBuEXAAAAAwbhFwAAAANGn4tcAAAAIPOFw06lVUHVN7eqodV5XY5nCL8AAABZqLU9rDf2HtbqnTVateOQ1uyqUW1j65Hnv7fqec0eN0yzxw3TrHHDNGvccM0eP0xFhQUys4SP39LWrkAwpOpgixpD7RqSn6shBbkanJerwQU5GpKfq8H5uQkfp78IvwAAAFmgKdSuV3cf0qqdNVq9s0brdtWqqbVdkjStaKguO2m8zp0+RkWFBXp25esKDxur7ZVBPbJur4ItbUf2M3po/ptheNwwzR4fCccTRgxWsKVN1dFAGwi2qCoYUiDYEr0febzj+frmtp5KPUquSUOXPKNB+bkaEg3FQ/JzI/fzc/X5hTN1wYyipH2fCL8AAAAZ6HBjq1ZHg+6qnTXaUHFYbWEnM2nOhBH64NzJOnf6GJ03bYzGjRh81GtzD+Zr4cIzJEnOOR2oa1ZpZVDbDwa1vTKo0sp6PfXGfj3QaaQ4N8fUHu6+XWL00HwVDRuk4mEFOvm4ERo7bJCKCgtUPHyQiocN0tCCXLW0taspFFZTa7uao/+aQu3aWrZD4yZOPvJ4U6hdzW2R/9Y2htTWntwWDcIvAACAjzWG2rSzulE7qhu0ozqo8uoGbdpXp60H6+WclJ9rOn3yKH36rTN03vTROuf4MRo5JD/m/ZuZJo4cookjh+its8ceedw5p0BDSNsPRsLw/sPNGj20QEXDClQ8bJCKhhVo7LBBGl1YoPzc+OdQKMndq4ULT4779f1F+AUAAPBYa3tYFYeaIuG2qiEadCP/9h9uPmrbCSMGa/b4YbritIk6d9oYnTV1VEp6Z81MxcMiI7fzZiav7cBrhF8AAAAP/H31bj278aB2VDdod02j2jq1FIwYnKcZY4dp3owiTS8u1PSxhZpeXKhpRYUqHER8SwTfPQAAkBUaWtpUcahJobawQu3t2hxol7ZWRu+HFWoLq6Ut8t+Oxzrut7WHdc3cKTpxwvC01Nrc2q7/eXyjigsLdObUUXrnaRM0rahQM8YWanrxMI0emp+UGRdwLMIvAADIaM45PbF+n77/r02qaQgd/eTq1X2+viA3R63hsA7Wt+j/PnRWiqo82qu7axVqC+uH7zlVl5w0Pi3HRAThFwAAZKyKQ4367j/fUMnWKp05ZZS+d9UpGpKfq4K8HG3a8LrOm3u2BuXlaFBejgo6/uUefdvM9N8PvqqXS6vlnEvLiOuKsmrlmHTu9DEpPxaORvgFAABJVVXfoqXbqjRqaL4uOnGccnKSHybbw05/XbFTP31mqyTp5nedrE/Mn6bcTsdy+3J1zvGjY9rf/FnF+udr+7T1YL3mTBiR9Hq7Wl4W0GmTR2nE4NhnZUByEH4BAEBCnHPatL9OT5SF9KuNy7R+T+2R56YXF+pTb5mu9589WUMKkjMjwfaD9frWI69r3e5aXXjCWP3oPadqypihCe1zfnQ2g2WlgZSH38ZQm17bU6vPXDgjpcdB9wi/AACg35pb27WiLKDnNx/U4i2VR6bjOmOK9LXLTtBFc8ZpR3WD/vhSub77zzf0i2e36mMXHK+PzZumscMHxXXMUFtYd5SU6vYlpRo2KE+/+o8z9J4zJyWlTWHy6KE6vmioVpRV61NvmZ7w/nqzeuchtYXdkcCN9CL8AgCAmFTWNeuFLZV6YXOllpVWq6m1XUMLcvXW2cX6yqUnqKBmu97zjgVHtj910ki96/SJWrWjRn98aYf+b0mpfr+0XO87a5I+/dbpmjUu9pkV1u0+pBsfeV3bDgZ11RnH6eZ3n6ziYfGF6J7Mn1msf6/fp7b2sPISWLShL8vLqpWfa5p7PP2+XiD8AgCAbjnntHFfnZ7ffFAvbK7Uhr2HJUmTRg3RNXMn65KTxuv86WOOLLBQUlJ2zD7MTOfPKNL5M4pUVhXUn1/eoUfWVujB1Xt00Ylj9ZkLZ2jejKIeR28bWtr0s2e26i8rdmriiMG667q5unhOamZHWDCrSA+s2q3X9x7W2VNj6xWOx8qygM6aMjppbSDoH8IvAAA4oqXtzXaG5zdV6kBds8yks6eO1jfecaIuPWm8Thg/LK5Wg5ljh+mW956mr112gu5buVv3rtipD//xFZ1y3Ah95q0zdOXpE49aJrdka6W+89gb2ne4SR+74Hh98/I5GpbCBR7mzYi0ISwvrU5Z+D3c1KoNew/rSxfPTsn+0TfCLwAAA9yhhpCWbK3Uc5sOaum2KjWEIu0MF84eq0tPHq+LThyroiS2GBQNG6T/unS2Pvu2GXrs1b3600vl+u+/v6afPL1F1y+YpstPmahfPb9Nj726VzPHFuofn52nudNS3yJQNGyQTpo4QstKA/piisLpqh01Cjtl1XLBmYbwCwDAALSjukHPbzqo5zYf1JqdkUA2fsQgveesSbr05PGaN6PoSDtDqgzOz9WHzpuq/5g7RSXbKvXHpTt0y6ItumXRFuXnmr58yWz950UzNSgvfe0BC2YW6d6Vu9Tc2p6Sr395WbUG5eXorKmjkr5vxIbwCwDAANAednptzyE9t6lSz28+qNLKoCTppIkj9MWLZunSk8fr1ONGpmRO3r7k5JgunjNeF88Zrzf2Htazmw7qytMmpm2p4c4WzCrWn17eoTU7D+kts4uTvv8VZQGdO21MWgM9jkb4BQAgS9U1t+rl7dVasqVSi7dUKtAQUl6O6YIZRfro+VN1yUnjE54fN9lOnTRSp04a6dnxz5s+Rnk5pmVl1UkPv4Fgi7YcqNc33nFcUveL/iH8AgCQJZxz2nKgXiVbq7Rka6XW7jqk9rDTiMF5WnjiOF168ngtPHEsq4r1onBQns6cMkrLS6uTvu+V5TWS6Pf1GuEXAIAM1tTm9PQbB1SytVIlW6t0oC6y2MTJE0foc2+boYUnjtNZU0aldN7abDN/VrF+u3i7Dje1auSQ5H1QWFFerWGD8nS6hyPbIPwCAJBRnHPaXhnUki2RsLtqR6Pa3VoNH5Snt55QrIUnjNPbThyr8SMGe11qxlows0i3vbBdK8sDescpE5K23+VlgUhbBR9EPEX4BQAgA2yoOKwHV+9WydYq7a1tkiTNmTBc75iWr09cdo7OPn70UXPkIn5nTR2twfk5Wl5anbTwe6g5rPKqJn3o3KlJ2R/iR/gFAMCnnHNataNGt5eUaem2KhUW5Oots4v1pYtn6W0njtXEkUNUUlKi82fQQ5pMBXk5OnfaGC0rCyRtn5trwpLo9/UDwi8AAD7jnNOSrZW6fUmZ1u46pOJhg/Sty+fooxdM1XAuVkuLBbOKdetTW3SwrjkpLSSbA+0aOSRfJ08ckYTqkAjCLwAAPtEedlq0Yb/uKCnT5v11mjRqiH549Sm6Zu6UlC84gaMtmBmZ5mx5WbXee9bkhPe3uaZdF8wY58k8yjga4RcAAI+1tLXrsXV79fsXy7Qz0KiZYwv1i2vO0FVnHkcfr0dOPm6ERg7J1/LSQMLhd09No6qbnObPTP6iGeg/wi8AAB5pDLXpgVV79Mel5TpQ16zTJo3U7z96tt5+8gRGCD2Wm2OaN6NIy8sCcs7JLP6fx4po7/B8+n19gfALAFnAzC6X9BtJuZL+5Jy7tcvzx0u6S9JYSTWSPuqcq0h7oZAkHW5s1b0rduquZTt0qLFV508fo59+4HS9dXZxQiELybVgVpGe3nhAuwKNmlZcGPd+lpdVa0SBada4YUmsDvEi/AJAhjOzXEm3S7pMUoWk1Wb2hHNuU6fNfi7pXufcX8zsYkk/lvSx9Fc7sG09UK+H1+7RA6v2KNjSpkvmjNMXLpqpc44f43Vp6Mb8WZE2hWVl1XGHX+eclpcFdNKYHD7Y+AThFwAy33mSSp1z5ZJkZg9KulpS5/B7sqSvRm8vkfTPtFY4gFXVt+jx1/bqsVf3auO+OuXmmN556gR9YeEsnXwcV/772YziQk0YMVjLSwP6yPnHx7WP8uoGVda36IqpBUmuDvEi/AJA5pskaU+n+xWSzu+yzXpJ71OkNeK9koabWZFz7qiJTM3sBkk3SNL48eNVUlKSqppjFgwGfVFHh1jqCbU7rats1/J9bXqjul1hJ00bkaOPzCnQ+RPzNGJQnSq3rVPltvTUky5+qkVKTj0zh7XpxS37tXjJEuXEMXK7eHerJOn4wS1Z971JlnTXQvgFgIHh65J+a2bXSVoqaa+k9q4bOefulHSnJM2dO9ctXLgwjSV2r6SkRH6oo0NP9YTDTqt21ujRdRV6asMB1be0aeLIwfrs26bpfWdN0uzxw9Najxf8VIuUnHoCwyu07B/rNf7Es3XKcSP7/fqH/rZWx42s1bTinKz73iRLumsh/AJA5tsraUqn+5Ojjx3hnNunyMivzGyYpPc752rTVmEWK6sK6rF1kbaGvbVNKizI1eWnTtT7z56kC2YUMWtDhlsQ7ftdXhrod/gNh51WltfoohPHyexQKspDHAi/AJD5VkuabWbTFQm910r6cOcNzKxYUo1zLizpJkVmfkCcgi1tenRdhR5Zt1fr99Qqx6S3zB6rb7zjRL39lPEaWsCf12wxYeRgzRhbqGVl1frMhTP69dqtB+tV0xCKTHFWT/j1C/7vBIAM55xrM7MvSnpGkanO7nLObTSzH0ha45x7QtJCST82M6dI28N/elZwhmtqc/rQnSu1Ye9hzZkwXN+54iRdfeZxGpeEJXDhTwtmFuuRdRUKtYVVkBf7oiPLo/P7zptZpG2vlaaqPPQT4RcAsoBzbpGkRV0eu7nT7YclPZzuurJNqC2s377arC2HmvT7j56jy0+d4HVJSIP5M4v015W7tL6iVudOi31auhVlAU0rGqrjRg1REq5tRJKwZiIAADEIh52+8fB6bQyE9eP3nUbwHUDmzSySmbSstDrm17S1h/VKeUDzWNLYdwi/AADE4JZFm/X4a/v0gdn5+uDcKX2/AFlj1NACnXLcCC0vDfS9cdTGfXWqb2ljSWMfIvwCANCHO5eW6U8v79B186fpyhn5XpcDDyyYWaxX9xxSY6gtpu1XlEeC8gUzCL9+Q/gFAKAXj66r0C2LtujK0ybqf951MkvUDlDzZxWrtd1p1Y6amLZfXhbQCeOHaezwQSmuDP1F+AUAoAcvbqvSNx9+XfNmFOmX/3GGcpmzd8A6d9po5eeaVpT13foQagtr9Y4azaff15cIvwAAdGP9nlp9/r61mj1+uP7w8XM0KC/X65LgoaEFeTpr6mgtK+v7orf1FbVqam2n5cGnCL8AAHSxo7pB19+zWmMKC/SX68/ViMH0+SLS97txX51qG0O9breiLCAz6YIZsU+LhvQh/AIA0EllfbM+ftcrkqR7P3kei1fgiAWziuSc+mx9WF5WrVOOG6FRQwvSVBn6g/ALAEBUfXOrrrtrtarrQ7rrunM1Y+wwr0uCj5wxZZQKC3J7bX1obm3Xul219Pv6GOEXAABJLW3t+uxf12rbwXr97qNn68wpo7wuCT6Tn5uj86aP6XW+33W7DinUHtY8+n19K6Hwa2ZfMbONZvaGmT1gZpwbAgBknHDY6asPrdfysoB++oHTtfDEcV6XBJ9aMKtY5dUN2n+4qdvnl5cFlJtjOnc6/b5+FXf4NbNJkr4saa5z7lRJuZKuTVZhAACkg3NOP/j3Jj35+n7d9M45et/Zk70uCT42L7pi27IeRn+Xl1XrjMkjNWxQXjrLQj8k2vaQJ2mImeVJGippX+IlAQCQPr97sUz3LN+pTy6YrhsunOF1OfC5kyaM0JjCAi0vPbbvN9jSptcrDh8JyPCnuMOvc26vpJ9L2i1pv6TDzrlnk1UYAACp9o81e/TTp7fqqjOO03evPInV29CnnBzTvBlFWlZWLefcUc+t3lmjtrDjYjefS6TtYbSkqyVNl3ScpEIz+2g3291gZmvMbE1VVVX8lQIAkEQvbqvSjY9u0FtmFevn15yhHFZvQ4zmzyrSwboWlVU1HPX4irKACnJzdM7xoz2qDLFIpO3hUkk7nHNVzrlWSY9Kmt91I+fcnc65uc65uWPHjk3gcAAAJMfGfYf1hfvW6oTxw/W7j56tgjwmP0LsFkRHdpd3mfJseVm1zpo6SoPzWQ3QzxL5v323pAvMbKhFzhNdImlzcsoCACA19tY26fq7V2vEkHzdfd25Gs7qbein44uGatKoIVrWqe/3cGOrNu6ro+UhAyTS8/uKpIclrZO0IbqvO5NUFwAASXe4qVXX371KTaF23X39uZowkhk60X9mpvkzi7SiLKD2cKTvd+WOgJyLtETA3xI6z+Oc+1/n3Bzn3KnOuY8551qSVRgAAMkUagvr8/etVXlVg37/sXM0Z8IIr0tCBlswq1h1zW3auO+wpEi/75D8XJ0xmcVR/I4mJwBA1nPO6cZHXtfysoB+8v7TtWAWp6aRmPld5vtdURbQ3Gmj6R/PAPyEAABZ71fPb9ejr+7VVy87Qe8/h0UskLhxIwZr9rhhWl5Wrar6Fm09WE+/b4Yg/AIAstpDq/fothe264NzJ+tLF8/yuhxkkQWzirV6Z41e2h6ZynU+i1tkBMIvACBrvbitSjc9tkFvnV2sH733NBaxQFLNn1mk5tawfldSpuGD8nTKcfSRZwLCLwAgK3Wey/eOj5yt/Fz+5CG5zp9RpByTtlcGdf6MMcrjdywj8FMCAGSdfbVN+uQ9zOWL1Bo5JF+nTRopSZpHv2/GIPwCALJKXXOrrr97tRpbmMsXqTc/OnPIvBn0+2aKPK8LAAAgWTrm8i2rCuovnzyPuXyRcp9cMF2TRg3RSROHe10KYkT4BQBkBeecbnz0dS0rDejn15zBXL5Ii7HDB+mjFxzvdRnoB9oeAABZ4dfPb9ej6/bqK5eeoA8wly+AHhB+AQAZ76E1e/SbF7brmnMm68uXMJcvgJ4RfgEAGe3l7dX69qORuXxveR9z+QLoHeEXAJDRblm0WVOLhjKXL4CYZOS7RHvY6dfPb1N9c6vXpQAAPFTbGNLmA3V675mTmMsXQEwyMvw+9cZ+/fr57bpl0RavSwEAeOiVHTVyTrpgJnOsAohNRobfUFtYktTc2u5xJQAAL60sD2hwfo5OnzzS61IAZIiMDL8AAEjSyvIanXP8aA3Ky/W6FAAZgvALAMhItY0hbTlQx7KyAPqF8AsAyEhH+n0JvwD6gfALAMhIK8o6+n1HeV0KgAxC+AUAZKSV5QHNPX6MCvL4UwYgdnleF9Af33r4dY0cmq85E4Z7XQoAwEOHGkLacqBeX3/7RK9LAZBhMurj8t/X7NGdS8u9LgMA4LFXdtRIot8XQP9lVPgFAEDqPL8v/b4A+ofwCwDIOPT7AogX7xoAgIzS0e87jyWNAcSB8AsAyChv9vuO8bgSAJmI8AsAyCgrywMakp+r0ybR7wug/zIy/K6Kfuqva2r1uBIAQLqtLA9o7rTR9PsCiEtGvnM8uHqPJGnD3sMeVwIASKeaaL8vU5wBiFdGhl8AwMC0akdAEv2+AOJH+AUAZIyV5TX0+wJISEaHX+d1AQCAtKLfF0CiePcAAGQE+n0BJENGh1/H0C8ADBhv9vsSfgHEL6PDLwBg4FhRFpnf9/TJI70uBUAGI/wCADLCyvIazZ02Wvm5/OkCED/eQQAAvhcItmjrQfp9ASSO8AsA8L2OlT0JvwASRfgFAPjeynL6fQEkR4aHX6Z7AABJMrPLzWyrmZWa2Y3dPD/VzJaY2atm9rqZXeFFnfGi3xdAsvAuAgAZzsxyJd0u6Z2STpb0ITM7uctm35X0kHPuLEnXSrojvVXGry7ktPVgvebNpOUBQOIIvwCQ+c6TVOqcK3fOhSQ9KOnqLts4SSOit0dK2pfG+hKytaZdEv2+AJIjz+sCAAAJmyRpT6f7FZLO77LN9yQ9a2ZfklQo6dL0lJa4LTXtGlqQq9Mm0e8LIHGEXwAYGD4k6R7n3C/MbJ6kv5rZqc65cOeNzOwGSTdI0vjx41VSUpL+SrvYVN2qGSPytOylpV6XIkkKBoO++L508FM9fqpF8lc9fqpF8lc96a6F8AsAmW+vpCmd7k+OPtbZpyRdLknOuRVmNlhSsaTKzhs55+6UdKckzZ071y1cuDBFJcemOtii/U8/r49dOEsLF87ytJYOJSUl8v5eI9UAACAASURBVPr70pmf6vFTLZK/6vFTLZK/6kl3LRnd8+uY7AEAJGm1pNlmNt3MChS5oO2JLtvslnSJJJnZSZIGS6pKa5VxYH5fAMnm+/B7wnef0qf/ssbrMgDAt5xzbZK+KOkZSZsVmdVho5n9wMyuim72NUmfMbP1kh6QdJ1z/h9CWFke0KBc0e8LIGl83/YQagvr+c0HvS4DAHzNObdI0qIuj93c6fYmSQvSXVeiVpYHNHt0LvP7Akga3k0AAL5UHWzRtoNBnTSGP1UAkod3FACAL71SHun3nTMm1+NKAGQTwi8AwJdWlgdUWJCr40fwpwpA8mT0O4rvr9QAAMRtZXlAc6eNUV6OeV0KgCyS0eEXAJCdqoMt2l4ZZIozAElH+AUA+E5Hv+8FM8Z4XAmAbEP4BQD4Tke/76nM7wsgyQi/AADf6ej3ZX5fAMmW0LuKmY0ys4fNbIuZbTazeckqDAAwMHX0+86bSb8vgORLdIW330h62jn3geh68kOTUFPMMmBlTgBAP60sD0gSF7sBSIm4w6+ZjZR0oaTrJMk5F5IUSk5ZAICB6ki/73EjvC4FQBZKpO1huqQqSXeb2atm9iczK+y6kZndYGZrzGxNVVVVAoc71qHG1qTuDwDgvZXlNTp3+hjl0e8LIAUSeWfJk3S2pN85586S1CDpxq4bOefudM7Ndc7NHTt2bAKHAwBku6r6FpUyvy+AFEok/FZIqnDOvRK9/7AiYRgAgLi8soN+XwCpFXf4dc4dkLTHzE6MPnSJpE1JqQoAMCDR7wsg1RKd7eFLkv4WnemhXNL1iZf0pnCY2RwAYCCh3xdAqiUUfp1zr0mam6RajnG4iQvaAGCgqKxvVmllUNecM9nrUgBkMT5aAwB84ZXyGkn0+wJILcIvAMAXVpYHNGxQnk6h3xdAChF+AQC+sLI8oHOnjabfF0BK8Q4DAPBcdbBFZVUNOp+WBwApRvgFAHhu76EmSdLsccM8rgRAtiP8AgA8F2hokSQVDRvkcSUAsh3hFwDguepgSJJUVFjgcSUAsh3hFwDguUBH+B1G+AWQWoRfAIDnAsEWDS3I1dCCRBceBYDeEX4BAJ4LNIQY9QWQFoRfAIDnqoMtKirkYjcAqUf4BQB4LhAMqZiRXwBpQPgFAHgu0MDIL4D0GDDhd8Gti3XToxu8LgMA0IVzToEgPb8A0mPAhN+9tU16YNVur8sAAHRR19SmtrBjgQsAaTFgwi8AwJ+qo6u70fMLIB0yPvx+7aH1Wr2zxusyAABxOrLABT2/ANIg48PvI+sqdM3vV3hdBgAgToFgZOSXnl8A6ZDx4RcAkNmqG1jaGED6EH4BAJ7qGPkdM5TwCyD1CL8AAE8FgiGNHpqvvFz+JAFIPd5pAACeCjS0MM0ZgLQh/AIAPFUdDKmokJYHAOlB+AUAeCoQbFExI78A0oTwCwDwVKCBpY0BpA/hFwDgmdb2sGobW1ngAkDaEH4BAJ45xBy/ANKM8AsA8Ex1dGnjYsIvgDQh/AIAPBNo6FjamLYHAOlB+AUAeCYQHfllqjMA6UL4BQB4pjrIyC+A9CL8AgA8E2gIKT/XNGJwntelABggCL8AAM8Egi0qKhwkM/O6FAADBOEXAOCZQJAFLgCkF+EXAOCZ6oYQ/b4A0orwK+mh1Xv0+Gt7Y95+Z3WDdlY3pLAiABgYAsEWFTPTA4A0ypgrDDpWAUqFbz7yuiTp6jMnHfPclbe9pNMmjdSt7z/9yGMLf14iSdp565UpqwkABgLaHgCkm69Hfl2n22f98DlPati4r04Prt7jybEBIJs1htrU1NpO2wOAtPJ1+AUAZC8WuADgBcIvAMATHQtcFDPyCyCNCL8AAE8cGfml5xdAGhF+AQCeCDSwtDGA9CP8AgA8UU3PLwAPEH4BIAuY2eVmttXMSs3sxm6e/5WZvRb9t83Mar2os7NAMKRhg/I0OD/X61IADCAZM88vAKB7ZpYr6XZJl0mqkLTazJ5wzm3q2MY595VO239J0llpL7SLQEML/b4A0o6RXwDIfOdJKnXOlTvnQpIelHR1L9t/SNIDaamsF4FgiJYHAGmXVSO/r+4+pPKqBr3/nMlelwIA6TRJUufVeCoknd/dhmZ2vKTpkhb38PwNkm6QpPHjx6ukpCSphXa262CTiodYn8cIBoMpraO/qKdnfqpF8lc9fqpF8lc96a4lq8Lve+9YLkmEXwDo2bWSHnbOtXf3pHPuTkl3StLcuXPdwoULU1ZI87LnNWfaOC1ceHqv25WUlCiVdfQX9fTMT7VI/qrHT7VI/qon3bXQ9gAAmW+vpCmd7k+OPtada+WDlodw2KmmIaSiQqY5A5BehF8AyHyrJc02s+lmVqBIwH2i60ZmNkfSaEkr0lzfMQ43tao97LjgDUDaEX6TYNO+Ou2obvC6DAADlHOuTdIXJT0jabOkh5xzG83sB2Z2VadNr5X0oHPOeVFnZyxwAcArvu75Na8LiNEVt70kSdp565UeVwJgoHLOLZK0qMtjN3e5/7101tSbjgUuipntAUCaMfILAEi7QMfqboz8Akgzwi8AIO3ebHtg5BdAehF+B4h9tU3yQZsfAEiKtD2YSaOHEn4BpBfhN0YH65q1cd9hr8uIy/o9tZp/62Ldv2q316UAgCQpEGzRmKEFys3JlKs7AGSLhMOvmeWa2atm9u9kFORX829drCtve9nrMuJSWhmUJK3decjjSgAgIhAM0fIAwBPJGPn9L0Wm1slq7WFaBgAgWQINLSxwAcATCYVfM5ss6UpJf0pOOQCAgYCRXwBeSXTk99eSvikp3NMGZnaDma0xszVVVVUJHg4AkA2qgy0qZpozAB6IO/ya2bskVTrn1va2nXPuTufcXOfc3LFjx8Z7OABAlgi1hVXX3KYiFrgA4IFERn4XSLrKzHZKelDSxWZ2X1KqAgBkrZoGFrgA4J24w69z7ibn3GTn3DRF1otf7Jz7aNIqG2B2BRrUGGrzugwASLnqIAtcAPAO8/z6xNt+VqLr716dkn0zTwUAPwlER36LCb8APJCXjJ0450oklSRjXwPZKztqUnsA5pIH4AOBjpFfpjoD4AFGfvvp1d0sFAEAiQgEO3p+GfkFkH6E33567x3LvS4BADJadUOLCvJyNGxQUk4+AkC/EH4BAGkVCIZUXFggM3qxAKQf4XcAcI5L3gD4RyDYwjRnADxD+B1AjCveAPhAoIGljQF4J2vCbzJGN0+++Wkt2VqZhGr8hXFfAH4SCIaY6QGAZ7Im/D6wak+3j1ccatS0G5+MaR+NoXb98tltySzLV1LRXnf7klL9eNHm5O8YQFZyzqk62MIcvwA8kzXh95+v7T3mse0H6/WWnyzxoJqevbH3sN7/u+Vqbm33upSk+NkzW/WHpeVelwEgQzSE2tXSFqbtAYBnsib8dmfLgXqvSzjG9/+1UWt3HdLrFYe9LgUA0o4FLgB4LWvCb3lVg9clAAD6UM0CFwA8ljXhtzo6mgAA8K+Okd9ipjoD4BFfh19mKQCA7BJoYOQXgLd8HX6zGQtPIB6Vdc1avOWg12UAcesY+R1TSPgF4A3C70BAzs5YtY0hNYbajty/5g8r9Ml71nhYEZCY6mBIwwfnaVBertelABigBnz4rW9uPeq+S2JS/PZjG/TJe1Z3+5wXa9rHe8RH11WoJnqqEul15g+e02W/XHrk/q5Ao4fV+NPe2qYjo4nwv0BDiH5fAJ7K87oAr938+MaU7fv+V3anbN/psjvQqK8+tF7zZxbp/s9c4HU5A9Le2iavS/C1BbculiTtvPVKjytBLALBFhXR8gDAQwN+5NerEc1M6fkNtUcW4zhY1+xxJQCyQSAY4mI3AJ7K6vDrQWdBnyzu5oP4JbOVAwASEWhoURFtDwA8lNXh14+8DKJ+/DAAYOBoDzvVNIRUTNsDAA8Rfj3ixQVv8ciQ7gwAGaC2MaSwEyO/ADw14C54+85jG9TW7tTY2q4fv+80z+rIlJ7fDpkS1gH4FwtcAPCDARd+/9ZpBoazpoxK+/E96flNc85uCrWrpjGkSaOG6Ok3DmhQXo4umjMuvUVkgfZwzz845xwfSJBxOpahLypk5BeAd2h76CLDBmT7JZHg3Z+R6uvvWXVk+qnP3bdW1/cw1zF6t6y0+sjthpa2XrYEMkMgGBn5LWbkF4CHCL9IupXlNV6XkBXCnT5wXHf3Kg8rAZKjYzESen4BeInw24N9tU169/+97HUZSfX3NXu0oiwQ12u9PsVecahxQC/2sHrnIa9LABIWaAgpx6RRQ/K9LgXAAEb47cE9y3dqw97DXpeRdB/640pPjrtpX52aQu1xv/4tP1lypJUCQGaqDoY0pnCQcnLoVwfgnawNvy9sPqgv3v9qSo8Ragv7ajRy2o1P6n/++YbXZXTritte0lf+/prXZQDwUCDYQr8vAM9lbfiNNfjuCjTEfYxvPLz+qNHIkq2VPW67eMtBvbanNu5jxeqvK3f1uc1/3r9Oi7ccjGl/ybz+b+1u/5y631BxWKWV9V6XAQwogQaWNgbgvawNv7HaGWjs9vFYZjdYvOXosHvd3T3PavDJe9boPbcv619xMXhxW5Xu7zR9WyyefH2/PnnPmm6f23+4SS9vrz7m8Ww7Sfnu376sS3+51OsyeuV1nzWQbIFgC9OcAfBc1s7zm825YW9tkyrrmvXeO5YfeezD50/tcfv+jN5e8ZuXdKixVTtvvTKBCpGotvawbn685xYW5zLjd3zajU/qw+dP1S3v9W5BGfhHIMjILwDvDfiR30yzt7ZJC25dfFTw7Ut/5i4+1Nja/T5i3wWS4KXSau3q4axEpunvmQlkp+bWdtW3tKmYac4AeCxrw28sg2LdjZwlushFe9iptT181GOb9tUlttNOKuuak7avTLT1QOJ9ulX1LUmoJMUy7NPGP1/dq7f+dLHCvaxKh4GtpmNp40JGfgF4K2vDbyx6C7rxhuCrb39Zs7/z1FEh9YrbXopvZylW0xDS8rJj+3u745cz7O/4deJ9uocaQ0moBJ198+HXtaemSa3hcN8bY0DqWN2NBS4AeC1rw2+8Fwt1ZN4/vbyj72N089gbeyOjvB+/q/cVufwwPvbhP67Uh//4CqN1AFKuuqFjdTdGfgF4KyvD787q+KcvS5Z9Pc3/65chVElboi0EmXDhVKpcnoSRZL8Ih53+74XtOtzUfd824KWOkd9iZnsA4LGsDL8Lf14Sd8asa2pVS1v8K5H1KcMGWRPtgfa7LUnoIU4FF8cvyuItlfrFc9v0/Sc2pqCi5Htu08GE5tnu6qXtVUnbF5IvEGTkF4A/ZO1UZ7HobsRzb22TPvzHV2J6fV1zW7+PWV4djBy736/sv4aWNjWE2uIKUl0lY3Q424O01zoutGxMYBnpdPrMvWtkJu34cXKm1TtYlwEXMg5ggYaQBufnaGhBrtelABjgsnLkV5LqW/ofTDus3ZW6lciqo6f+OnLgdx7boNnfWZSSY11520s670cv9LkdoTTzZMuPjN+9gaM6usAFi7cA8JqvR35T/RaZyj+8sb7B/y1Fc6CWbK3scfW6Dmaxfw8IKf4Qy88sGSP9QLIFgiEV0/IAwAeyduQ3Fj/49yavS0iZ3pZaHuiyIcg3tx7b2jDQB9RiWZIc3gk0tDDNGQBfGNDhF7FLV7D6w4tlemjNnvQcLAN15Lvv/8tfF7Ux2oy+BIIhFrgA4AuEX/QYW/YfbjpmhPHvq3fr+rt7n8P4yH7jGIn78VNb9M2HX+/36/qjtyAfDjtfzHvc17dub09T6XnM/DSXH3zDORcJv4z8AvABX/f8wlvzfrxYY7qM1HzrkQ0xv357ZTDZJfVod6BRD6zerW++48SELqiZ8e1FOmvqKD32hQVJrC69OPsPv6lvaVOoPUzPLwBfIPymSLLbBNraw2ptjy/VJBKGahriXwo4nMYUdsNf12jLgXq9/+xJmjVueI/bfezPr+il7b0v6fzq7tpkl9dv8X3rBvaoK5nfv95c2pjwC8B7tD1kiM/+da1OuvnpuF6bjFCQjNPZ1cHUzcPaMcdtX/oKvv3V3Nqetgutelw1EPC5IwtcsLobAB8g/KZIbWNyl5h9YUulJGlXH9OXxavdJ72umeRQQ0hz/udp3b6kNKH9PP3Gfk278ck+R9n3H25O6DjpwIVv6E41I78AfITw65Fr71ypyrq+w0xTqP2oi5tSEX6dc5r57UV6/++X97xNHKEm23tPq6KjWY+/ti+h/fz55R2SpNJOPdKJfOu8+Lb3dWaAD1YDW6Ah8v9KMRe8AfABwq+HOkZze/OJu1dpwa2LU3L8rnHFD72u3SmvSt+Fc5nML/P8fvAPK/T5+9Ye9djBev+PWmc6M7vczLaaWamZ3djDNh80s01mttHM7k9XbR09v6OHMvILwHtc8OYzLW3tqm9uOzJCsmpHjSd1XP3bl4+6n44prHrq2734Fy+m/NiJSNeYpl/CbXc6nxnw6nd2IHdcmFmupNslXSapQtJqM3vCObep0zazJd0kaYFz7pCZjUtXfYFgi0YOyVdBHuMtALzHO5HPfPovazT3/z2f1mM+s/HgMY+trzic1hok6efPbE37MRPh4yzqGT/N87tqR42WlSb3AkcfO09SqXOu3DkXkvSgpKu7bPMZSbc75w5JknOu71NPSVLdEKLfF4Bv+HrkN9sHcrrriU3GbARbDtT1a/v/vH9dwsfsTn9HKsuqGlJSx0CT7l7rZzYeiHsavlT64B9WSJK+sHCmPnvhTI0cmu9xRSk1SVLnpRErJJ3fZZsTJMnMlknKlfQ959wxU8iY2Q2SbpCk8ePHq6SkJOHiyiqalOcU976CwWBS6kgW6umZn2qR/FWPn2qR/FVPumvxdfhN5dRYmSqWQPm6B6O23elPCFteWq3nNx87Ap0JShNczCPW71NfP3qvxlw/+9e1fW/koTtKylRV36KfXXOG16V4LU/SbEkLJU2WtNTMTnPOHdXs75y7U9KdkjR37ly3cOHChA/8o3Uvata4YVq48Jy4Xl9SUqJk1JEs1NMzP9Ui+aseP9Ui+auedNdC24OHqur7H+7jGdV7blPiobK2Kf7FLmLx4T+9ktL9p9riLckN7umaOzhd0tEO0duMJKEY54HOYHslTel0f3L0sc4qJD3hnGt1zu2QtE2RMJxyAdoeAPgI4ddDv3p+W79fE0tLQ9eYsXRbVb+P09XBuswdhd9yoE7Tbnwyrlkjpt34pG5+/I1un+s8Cr87gSno4rmQrfdsfOyT1cEWffyuVTqUwIp9XiuvCibldzlLrZY028ymm1mBpGslPdFlm38qMuorMytWpA2iPNWFtbWHdagxxAIXAHyD8Otj3QWVp9444EElme2xVyMDYN1d2BeLe1fs6nObWFpeE53r1jql5O4Cs/WSov/88g4t3Val+1ftTqgGL138ixf18btWeV2GLznn2iR9UdIzkjZLesg5t9HMfmBmV0U3e0ZSwMw2SVoi6RvOuUCqazvU2CrnpGJGfgH4hK/Dr3+uG/fGWT98Lq7X9RaC4tmuJ9sO1if0+nh94q5Vagq1p/WYfbUh/PDfm3p9fkd1g2Z8e5H+tT6xBTHSYW9tk6bd+KRW7/RoyrIEVBxq1B9eLDvm8YHwXuKcW+ScO8E5N9M596PoYzc7556I3nbOua865052zp3mnHswHXV1LHBRxAIXAHwi7vBrZlPMbEmnCdP/K5mFIX4HDjf1vVESvOf2ZWk5TlcvbqvSS9tjO/39zMYDqm3o/1LTD67arV93akt5YNWebnqnY49Um/dH2lUWbdh/zHOJLFscauu+l7Up1K6Wtvg+IKwoiwwGPpCBo8SfvGe1fvzUFu2rTc//A+hbxwIXRYWM/ALwh0Rme2iT9DXn3DozGy5prZk913lSdXjjuc2xTd/ZnuBp+ObW9I6+9tehhlDcMxHc+OiGo+5/+7HI/Z23Xtmv/eysbtDCn5foc2+b2eM2FYeODWrd/WS6Ru1tB+v19l8t1e0fPltXnj7xzdc66aSbn9aEEYO18tuXdPva7tQ0hDR8cGITwPR0MqEtHP8FZw+vrdAdS0pj2rahJfI7Gc6yCwYzWcesPYz8AvCLuEd+nXP7nXProrfrFekzm5SswhC/dJzidc4pweyckJqGkIItbb1u05pA4EqWxdElrB97tUJS7xeqdW6tiCW7bYhOafdCdIq4rj/3A3WxjyiHw05n//A5fe2h9TG/pjs91f2WnyxRW5wzLnz9H+tVXs0c0JmqY+SXnl8AfpGUnl8zmybpLElJna/Kz8u5+k1lffynzmP1zMY3L7Zr6yH5dndaP17tYaffd9O/KUVGZi/86ZKkHSsecc3S0Mt0XN94+PVeX9vRO5mInnqXOx799+up60nu6XcmHRLtb0f8Ag0tyssxjRic1YuMAMggCS9yYWbDJD0i6b+dc8fMw9V5taCpU6cmejj04Jrfr0j5Mbo7Pd+hMdSmplC7vvC35K0Wt2jDft361JYen6/pNBtGXXPvo8Be65gqrq2XaSF21/Q+Xdon71kT9/FTkf2WbqvSuBFHn8qu8nBhms65nq4H/wgEQxpTWKCcHD6AAPCHhMKvmeUrEnz/5px7tLttuq4WlMjx0LNdCcwzG4+u4eLeFbv0njMn9bpNfzXF2FPcFGqPa8EQLyRrsYVe2yeScoS+dTft2AfT8CEsHtm2aEgmqQ6G6PcF4CuJzPZgkv4sabNz7pfJKwmS4r5SX0pdu0hvAaK7p9J1xX1jqPtR3z8uTfn8/T3aUHFYtY2huINoLCG5877b2sP6+5o9x2yzt8vPoKcf4eGmvmfE2B1o1M+e2dLr70HX4wGBhhb6fQH4SiI9vwskfUzSxWb2WvTfFUmqK4rTZJni2U3HLr7x6XvjP00v9TyNV6z++NKOhF7fl95+O9/925e7bUWJZQDycFOrvvzAq/0q5M8v7+h2Geum6AeDvpYX/uAfIrX2Vt4Nf12j25eUZeTFZ/T8eicQDDHNGQBfibvtwTn3skinA1bXC7de3V2rTfsPx/TahpY2tYWdfvL0Fp0/fYyuPrP7SUK++8/ulxU+tpbOIr+SP3m6517hZGhrD+u+lb3Pg7u9sv/LKUtH9zLH/JrGN18Tzyn+0hhq7e+HkWRe/NhZOOy67R+lscGfAsEW2h4A+IqvV3gbyPbUxH/6OFWfSP7fk5t7fT7WC7IuuOUFnfH9Z3X/K7v1Xw++lnBd//vExqPut4edflfS/SwR3bnqty/HvO3+6AIidy3bobuW9X9kubfZHvqjt4HMDXvf/BDSNQd3PfqL26r0oyd7mJq7j9f2JZkXP3b2+6U9/2wZ4PWXplC7GkLtKqLtAYCPJDzbA1Lj0l++6HUJvUrk+qH6Pubn7Y+WtnY9+XpiI4yvV8Q2Yi1J+2qbNXHkENU29n/VOCm271s8+a1zW0N3U4r1FAo/0c1Fa30d3+t8+dru2h6f62nU3OuaB6qO6fmKCxn5BeAfvh75ZRTHv/xy8fxPntqalP08vLYipu2q+jmfctcWhGR+31Ixg0Esu0zlj/7V3YcU6GO6tN6O3xjq/kLRR1/dq8/fF99qf4jfkaWNGfkF4CO+Dr+IT08BIJPc1GV54Z50N6PEN/tYLKI7a3fVxLRdbxfRxTJDRyxtD7F86OscUnu7mK3r0RLJyzuiF7r9+eXUXUj43juW66rfLkt4P919nU+9cexFmUitjpFfen4B+AnhNwttOVDvdQk92hnjTAEPrOr9YrIOT288OtAs2rBfj6yLbRQ3Hh2jrd1lyBO/+7Qef22vwgmuZNbXzAzdvqbTS7oLfsk8ibKstDqu173n9mXdXjTXGGrT4U5tJB3Tpd2+pFSb9x+zbo427j3c7ai3X85G4E3VHSO/zPYAwEcIv0irHz/V+0VzieppOeR0eW7TQf1oUWq/xr50nrM3FZ1D8YbMLQfqtbvm2A8/F/28RGf84Fm1dprb2Dmnnz2zVVd3Mwq873Cz7o9+OGLxCn+j7QGAH/n6gjdafrNPd1llqy9GqmP/bXulPKCV5YEen79v5a4en4vpgrcY2x6e7nQav6eXdD1cezj26crandP+w03asr9eozuN3CW7F79j6efvdZm1Q+p5sY9t0d8ZWhn8LRBs0dCCXA0t8PWfGgADDO9IiMvrFT1fcd+b5zYfuxDDO369NNFyYtJbaDOTFty6OIZ9mP7jzpW9btPSy3y4zklLtlRqdGGB6ppau10iOJY5d1d0Ct+9fV2PrK3QTVecpIpoK8Fti0v11bef2Ou+l26vkiQ9/to+Pf7aPknSRSeO7bOmRK0o6/4DhVnPU7alaxVBxCfQEGLUF4DvEH4Rl437ju3FjEWqz1L3drFfb6HSFNvSvMk4zX79PaslSVecNqHb53/9wvaEj9HhD0vLddMVJx05/RyLjsCbSpv68fvT27e883PJmkMZyVMdbFER05wB8Blf9/yyJKl/ZWLMiHU6s970FfpLtlYlfIxuFi/rU6b9r3LFbS8d81h/lk3uCL0EXn8LBEMqZuQXgM/4OvzCv3747x5WBctQ5VWxBa/eWhokKdjHAh6dw1pre/fBLaefSTaWrRMNx50/iPopZ6/f0/0CJX96uTzNlaA7gQZGfgH4D+EX0NE9tMl0TK9qp/vPbTq2/7myrrnf4bLduX61k1TV976IRCrFWufp3382pu2e3ND96n73ruj+osNbFm1We4JT0SE2zjkFgvT8AvAfwi/gIw+t2dPvkd9H1+3VHSU9T/HWGGo7KnR+4PfL4y0vbeqbYx9BP/JYDJn2zqXl+vRfVsdbFvqhrqlNbWHHAhcAfMfXF7z56fQqkAwxjTkm+Rf/5JufOer+rkCjwmGnnHiai5NgQ0X3rQrpsiQJfdnoW3V0dTd6fgH4DSO/QBqF+xiedC49H/qe3dS/+XG7W5ktXu/+7csJ76O7b2PFIaY9fOTfzgAADQZJREFU85MjC1zQ8wvAZwi/gM/0t+0hHv3t+32505LGfp2FxevV/XC0QDDyO0bPLwC/IfwCKRTPhXTpyJb/8/ixq6mlw9uTtKDJ317ZrebWnud0hveqG1jaGIA/+Tr8+nSACYjZ4i2V/dr+F89tU0Mf06VlsmQuctLbRX7wXsfI75ihhF8A/uLr8Atkm1d3970s9HqPLwjLFLclcSU8JF8gGNLoofnKy+XPDAB/4V0JGKA+8Lv4pjyraYh9qWQMXIGGFqY5A+BLhF9ggFqz61Bcrzvc1JrkSpCNqoMhFRXS8gDAfwi/AICkCwRbVMzILwAf8nX4TebFMQCSZ9qNT3pdAnwu0MDSxgD8ydfht68FAQAA/tPaHlZtYysLXADwJV+H31FMkQMAGecQc/wC8DFfh98xXCwBABmnOrq0cTHhF4AP+Tr8AgAyT6ChY2lj2h4A+A/hFwCQVIHoyC9TnQHwo4wJv4u/9javSwAAxKA6yMgvAP/KmPA7Y+wwr0sAAMQg0BBSfq5pxOA8r0sBgGNkTPgFAGSGQLBFRYWDZGZelwIAxyD8AgCSKhBkgQsA/kX4BQAkVXVDiH5fAL5F+AUAJFUg2KJiZnoA4FOEXwBAUtH2AMDPCL8AgKRpDLWpqbWdtgcAvkX4BQAkDQtcAPC7jAq/N1w4w+sSAAC96FjgopiRXwA+lVHhd3B+rtclAAB6cWTkl55fAD6VUeF3wojBPT53xpRRaawEANCdQANLGwPwt4wIv8MHRZbIvPbcKfrCwplHHt9565VelQQA6EY1Pb8AfM73C6+/8u1LjrQ75OSYvv72E3VHSZnHVQEAuhMIhjRsUB5tagB8y/fhd3yXVoecHNOKmy7W4DzeWAHAbwINLfT7AvA134ff7kwcOcTrEgAA3QgEQ7Q8APC1jOj5BQBkhupgCxe7AfC1rA2/pT96p9clAMCAc+qkkTprKrPvAPCvrAm/33rHiUdu3/GRs5WXmzVfGoAka25t97qErPXza87QFxbO8roMAOhR1iTE+bOKj9y+4rSJHlYCwO92Bhq8LgEA4JGsCb8d3nHK+CO3H//PBR5WAgAAAL/JqvC7/n/frt9++Owj98+YMkprvnuphxUB8CPnvK4AAOCVrAq/I4fkK79Lr28xVx0DGADM7HIz22pmpWZ2YzfPX2dmVWb2WvTfp72oEwC8lpHz/AIA3mRmuZJul3SZpApJq83sCefcpi6b/t0598W0FwgAPpLx4feOj5ytljau3AYwoJ0nqdQ5Vy5JZvagpKsldQ2/ADDgZXz4ZWYHANAkSXs63a+QdH43273fzC6UtE3SV5xze7rZBgCyWsaH31j843Pz9MjaCj24+tj3+QkjButAXbMHVQHwygC94O1fkh5wzrWY2Wcl/UXSxV03MrMbJN0gSePHj1dJSUlai+xOMBj0RR0dqKdnfqpF8lc9fqpF8lc96a5lQITfc6eNUY5Zt+H37uvP1Tt/85IHVQHwSjj70u9eSVM63Z8cfewI51yg090/Sfppdztyzt0p6U5Jmjt3rlu4cGFSC41HSUmJ/FBHB+rpmZ9qkfxVj59qkfxVT7prSWi2h76uLvaTc44frSVfX3jk/q3vO02rvnOJhg8eEPkfQHZbLWm2mU03swJJ10p6ovMGZta5R+wqSZvTWB8A+Ebc4bfT1cXvlHSypA+Z2cnJKiwVphcXav7MIk0dM1TXnjdV44YP1uTRQ/XQZ+cds+1N75zjQYUA0mHYoOz60Ouca5P0RUnPKBJqH3LObTSzH5jZVdHNvmxmG81svaQvS7rOm2oBwFuJ/AXIyKuL7//MBcc8dt70MXr2Kxfq7b9aeuSxz75tpn781JZ0lgYgTQbn53pdQtI55xZJWtTlsZs73b5J0k3prgsA/CaR8Bvr1cUZ4YTxw1V2yxV6YfNBFf//9s4vVq6ijuOfb1rtAxZphWA1alujJrwgtQ99AB7EFNoI+C9aY0IRXzSaSIwxaBPDazVqYjQ0MTZSg1gRib4YqQblxZZwy6UUobTFa2JT2vBHq8GIlZ8PZ5aeXffsvXv37J7ZPd9PcrJz55w95/f7zszvzu7M7KwuNsb4zieuRIgbr3wL98/9lZdefuX/OsSHvnYdd/3+JPv+uMCrMzeN0JjZZNXKmdrfxxhjzBCMfeyvvHIY+KekY8u4zaXA8/VZVR9v3j3xR2arRUNYjwtYi24q9Vi7vHb7jlGMmUbm5uael/SXpu0gv7pte6rJyRbIy56cbIG87BmHLZUxe5TO76Kri6F75fBykfRoRGwe5R6zgrXoxnpcwFp0Yz1GJyIua9oGyK8sbU81OdkCedmTky2Qlz2TtmWUsb9FVxcbY4wxxhiTE8v+5jcizkvqrC5eAeyNiCdrs8wYY4wxxpiaGWnOb7/VxWNipGkTM4a16MZ6XMBadGM9ZofcytL2VJOTLZCXPTnZAnnZM1FbFLO305ExxhhjjDF98e/9GGOMMcaY1pB153eatk8eBUkLkp6QNC/p0ZS3VtIBScfT65qUL0nfTZockbSpdJ+d6frjknY25c+wSNor6ayko6W82vyX9L6k74n0Xk3Ww6VTocWdkk6l+jEvaXvp3FeTX8ckXV/K79t20gLVQyl/f1qsmi2S3ibpIUl/SruTfTHlt7J+zBIDyra2+j6kPdnEYUnvKfk/L+mcpNsnqU1FLGqk3VXY8k1JT6fnPSDpkpS/XtK/ShrtWeyZVX4NaU8jcbrClv0lOxYkzU9CG01TvI6ILA+KRXQngY3A64HHgSuatmtMvi4Al/bkfQO4I6XvAHan9Hbg14CALcChlL8WeDa9rknpNU37tkT/rwU2AUfH4T/wSLpW6b3bmvZ5SC3uBL7c59orUrtYBWxI7WXFoLYD/AzYkdJ7gM817fMieqwDNqX0auCZ5Hcr68csHQPKtrb6PqQ9C2QYh5N/z1H8ZunEtKmIRY20uwpbtgIrU3p3yZb15et67tP3mVV+DWlPbWXDEHG6ny09578FfH0S2jBF8Trnb35f2z45Il4BOtsnt4WbgbtT+m7gQ6X8fVFwELhE0jrgeuBARLwYES8BB4AbJm30coiIh4EXe7Jr8T+duzgiDkbRcvaV7pUdFVpUcTPw04j4d0T8GThB0W76tp30Cfn9wM/T+8u6ZklEnI6Iwyn9D+Apit0lW1k/ZokBZVvFUPW9JjNziMPXAScjYtBmI7Vrk1Nc7mdLRDwYEefTnwcp9hqoZJFnVvm1ZHsGMNY4PciWdK+PA/cOMrAubaYpXufc+e23ffKgwDjNBPCgpDkVO+IBXB4Rp1P6OeDylK7SZdb0qsv/t6Z0b/608YU0LLS3NOw0rBZvAv5W+ocxVVpIWg9cBRzC9WOm6ClbqKe+D0uucXgH3Z2XJrTpkGu7u43iW8AOGyQ9JukPkq4p2Vj1zCq/hiW3OH0NcCYijpfyJqJN7vE6585vm7g6IjYB24DPS7q2fDJ9wmntz3K03X/gLuCdwHuB0xTDWK1C0huA+4HbI+Jc+Zzrx3TTp2ybqu/ZxeE01/Mm4L6UlU0syKXdSdoFnAfuSVmngbdHxFXAl4CfSLp4qfcbwa9syqbEJ+n+4DQRbaYhXufc+V3S9smzQEScSq9ngQcohkPOpK/4O0MSZ9PlVbrMml51+X+K7uGwqdMlIs5ExH8j4lXgBxT1A4bX4gWKYaWVPflZI+l1FIH0noj4Rcp2/ZgB+pVtjfV9KDKNw9uAwxFxJtnWiDYlsmp3km4FPgh8KnWqSNMLXkjpOYp5te9e5JlVfi2Z3OJ0ev9HgP0lG8euzbTE65w7v63YPlnSRZJWd9IUk/iPUvjaWeG4E/hlSv8KuCWtktwC/D0NJ/wG2CppTRpu2ZryppVa/E/nzknakuY/3VK611TQCRqJD1PUDyi02CFplaQNwLsoFgP0bTvpn8NDwMfS+8u6Zkkqsx8CT0XEt0unXD+mnKqyrau+D2lLrnG465u7JrTpIZt2J+kG4CvATRHxcin/MkkrUnojhRbPLvLMKr+GsSe3OP0B4OmIeG2awLi1map4HSOsQh33QbES8BmKTye7mrZnTD5upFjl+TjwZMdPink/vwOOA78F1qZ8Ad9PmjwBbC7d6zaKyfQngE837dsQGtxLMRzzH4o5PJ+p039gM0UgOgl8j7S5S45HhRY/Tr4eScFiXen6XcmvY5RWvVa1nVTfHkka3QesatrnRfS4mmKI7Agwn47tba0fs3QMKNva6vsQtmQXh4GLKL4FfGMpb2LaVMSiRtpdhS0nKOaFdurOnnTtR1MZzgOHgRsXe2aVX0Pa00ic7mdLyv8R8Nmea8eqDVMUr73DmzHGGGOMaQ05T3swxhhjjDGmVtz5NcYYY4wxrcGdX2OMMcYY0xrc+TXGGGOMMa3BnV9jjDHGGNMa3Pk1xhhjjDGtwZ1fY4wxxhjTGtz5NcYYY4wxreF/lsO3s0aurZYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "#20541\tloss=0.030536\tdelay=41: 20541it [1:21:21,  7.18it/s]"
     ]
    }
   ],
   "source": [
    "def trainer_thread_method():\n",
    "    \"\"\" train model on batches, emulate network latency \"\"\"\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        datasets.MNIST('../data', train=True, download=True,\n",
    "                       transform=transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize((0.1307,), (0.3081,)),\n",
    "                           transforms.Lambda(lambda x: x.view(-1))\n",
    "                       ])),\n",
    "        batch_size=batch_size, shuffle=True,\n",
    "    )\n",
    "\n",
    "    while True:\n",
    "        for xb, yb in train_loader:\n",
    "            xb, yb = xb.to(device), yb.to(device)\n",
    "\n",
    "            with lock_model:\n",
    "                model.train(True)\n",
    "                initial_step_index = len(train_history)\n",
    "                logits = model(xb)\n",
    "                loss = F.cross_entropy(logits, yb)\n",
    "                \n",
    "                opt.zero_grad()\n",
    "                loss.backward()\n",
    "                grads = [param.grad.clone() if param.grad is not None else None\n",
    "                         for param in non_expert_params]\n",
    "\n",
    "            emulate_latency()\n",
    "\n",
    "            with lock_model:\n",
    "                model.train(True)\n",
    "                opt.zero_grad()\n",
    "                for param, grad in zip(non_expert_params, grads):\n",
    "                    param.grad[...] = grad\n",
    "                opt.step()\n",
    "                train_history.append(dict(\n",
    "                    loss=loss.item(), \n",
    "                    delay_steps=len(train_history) - initial_step_index,\n",
    "                ))\n",
    "                progress.desc = f'#{len(train_history)}\\tloss={loss.item():4f}\\tdelay={train_history[-1][\"delay_steps\"]}'\n",
    "                progress.update(1)\n",
    "                \n",
    "                if len(train_history) % eval_interval == 0 or len(train_history) >= total_steps:\n",
    "                    need_to_eval.set()\n",
    "            if len(train_history) >= total_steps:\n",
    "                return\n",
    "                \n",
    "\n",
    "def evaluate():\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "            datasets.MNIST('../data', train=False, transform=transforms.Compose([\n",
    "                transforms.ToTensor(),\n",
    "                transforms.Normalize((0.1307,), (0.3081,)),\n",
    "                transforms.Lambda(lambda x: x.view(-1))\n",
    "            ])),\n",
    "            batch_size=batch_size, num_workers=4, pin_memory=True,\n",
    "    )\n",
    "    with lock_model, torch.no_grad():\n",
    "        model.train(False)\n",
    "        loss_numerator = acc_numerator = denominator = 0.0\n",
    "        for xb, yb in test_loader:\n",
    "            xb, yb = xb.to(device), yb.to(device)\n",
    "            logits = model(xb)\n",
    "            loss_numerator += F.cross_entropy(logits, yb).item() * len(yb)\n",
    "            acc_numerator += (logits.argmax(-1).to(yb.dtype) == yb).to(torch.float32).sum()\n",
    "            denominator += len(yb)\n",
    "    return dict(loss=loss_numerator / denominator,\n",
    "                acc=acc_numerator / denominator,\n",
    "                num_updates=len(train_history))\n",
    "            \n",
    "\n",
    "# finally, run training\n",
    "trainers = [threading.Thread(target=trainer_thread_method) for i in range(num_trainers)]\n",
    "for trainer in trainers:\n",
    "    trainer.start()\n",
    "    \n",
    "while len(train_history) < total_steps:\n",
    "    need_to_eval.wait(), need_to_eval.clear()\n",
    "    val_metrics = evaluate()\n",
    "    val_history.append(val_metrics)\n",
    "        \n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=[12, 6])\n",
    "    plt.subplot(1, 2, 1); plt.title('train loss'); plt.ylim(0, 10)\n",
    "    plt.plot([info['loss'] for info in train_history])\n",
    "    \n",
    "    \n",
    "    plt.subplot(1, 2, 2); plt.title('val accuracy'); plt.grid()\n",
    "    plt.plot(*zip(*((info['num_updates'], info['acc']) for info in val_history)))\n",
    "    plt.show()\n",
    "\n",
    "for trainer in trainers:\n",
    "    trainer.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logs/delay1000ms_dmoe4outof64experts_seed1337.pkl\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pickle\n",
    "!mkdir -p logs\n",
    "\n",
    "num_files = len(os.listdir('logs'))\n",
    "fname = f'logs/delay{delay_ms}ms_dmoe{num_active_experts}outof{num_experts}experts_seed{seed}.pkl'\n",
    "print(fname)\n",
    "with open(fname, 'wb') as f_out:\n",
    "    pickle.dump(dict(train_history=train_history, val_history=val_history), f_out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py38",
   "language": "python",
   "name": "py38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}