diff --git a/benchmark/python/metric/benchmark_metric.py b/benchmark/python/metric/benchmark_metric.py index 3c9abf6e3cc0..fc0f8da5d451 100644 --- a/benchmark/python/metric/benchmark_metric.py +++ b/benchmark/python/metric/benchmark_metric.py @@ -66,7 +66,7 @@ def data(self): def run_metric(name, data_gen_cls, i, n, c, pred_ctx, label_ctx, **kwargs): """ Helper function for running one metric benchmark """ - metric = mx.gluon.metric.create(name, **kwargs) + metric = mx.metric.create(name, **kwargs) data_gen = data_gen_cls(n, c, pred_ctx, label_ctx) try: label, pred = data_gen.data() @@ -105,7 +105,7 @@ def test_metric_performance(): output_dims = [128, 1024, 8192] ctxs = [mx.cpu(), mx.gpu()] - print("\nmx.gluon.metric benchmarks", file=sys.stderr) + print("\nmx.metric benchmarks", file=sys.stderr) print( "{:15}{:10}{:12}{:12}{:15}{:15}{}".format( 'Metric', 'Data-Ctx', 'Label-Ctx', 'Data Size', 'Batch Size', 'Output Dim', 'Elapsed Time'), diff --git a/benchmark/python/sparse/sparse_end2end.py b/benchmark/python/sparse/sparse_end2end.py new file mode 100644 index 000000000000..d032f9d6c38e --- /dev/null +++ b/benchmark/python/sparse/sparse_end2end.py @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import time +import argparse +import os +import multiprocessing +from mxnet.test_utils import * + +MAX_NUM_BATCH = 99999999 +COMP = "compute" +COMM = "communication" +IO = "io" + +parser = argparse.ArgumentParser(description="Run sparse linear regression " \ + "with distributed kvstore", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--profiler', type=int, default=0, + help='whether to use profiler') +parser.add_argument('--num-epoch', type=int, default=1, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=512, + help='number of examples per batch') +parser.add_argument('--num-batch', type=int, default=MAX_NUM_BATCH, + help='number of batches per epoch') +parser.add_argument('--dummy-iter', type=int, default=0, + help='whether to use dummy iterator to exclude io cost') +parser.add_argument('--kvstore', type=str, default=None, + help='what kvstore to use [local, dist_sync, etc]') +parser.add_argument('--sparse-log-level', type=str, default='DEBUG', + help='logging level [DEBUG, INFO, ERROR]') +parser.add_argument('--dataset', type=str, default='avazu', + help='what test dataset to use') +parser.add_argument('--num-gpu', type=int, default=0, + help='number of gpus to use. 0 means using cpu(0);' + 'otherwise, use gpu(0),...,gpu(num_gpu-1)') +parser.add_argument('--output-dim', type=int, default=4, + help='number of columns of the forward output') +parser.add_argument('--dummy-metric', type=int, default=0, + help='whether to call update_metric') +parser.add_argument('--enable-logging-for', default="0", + help="Enable logging for the specified list of workers") +parser.add_argument('--measure-only', default=None, + help="Measure only", + choices=[IO, COMP, COMM]) +parser.add_argument('--omit-row-sparse-push', action='store_true', + help="omit row_sparse_push") + +class DummyIter(mx.io.DataIter): + "A dummy iterator that always return the same batch, used for speed testing" + def __init__(self, real_iter): + super(DummyIter, self).__init__() + self.real_iter = real_iter + self.provide_data = real_iter.provide_data + self.provide_label = real_iter.provide_label + self.batch_size = real_iter.batch_size + + for batch in real_iter: + self.the_batch = batch + break + + def __iter__(self): + return self + + def next(self): + return self.the_batch + +# testing dataset sources +avazu = { + 'data_name': 'avazu-app.t', + 'data_origin_name': 'avazu-app.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/avazu-app.t.bz2", + 'feature_dim': 1000001, + 'lc': 1719304, +} + +kdda = { + 'data_name': 'kdda.t', + 'data_origin_name': 'kdda.t.bz2', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdda.t.bz2", + 'feature_dim': 20216831, + 'lc': 510302, +} + +criteo = { + 'data_name': 'criteo.t', + 'data_origin_name': 'criteo.t.bz2', + 'url': "https://s3-us-west-2.amazonaws.com/sparse-dataset/criteo.t.bz2", + 'feature_dim': 8388621, + 'lc': 548787, +} + +datasets = { 'kdda' : kdda, 'avazu' : avazu , 'criteo': criteo } + + +def get_sym(feature_dim): + inputs = mx.symbol.Variable("data", stype='csr') + norm_init = mx.initializer.Normal(sigma=0.01) + weights = mx.symbol.Variable("w", shape=(feature_dim, args.output_dim), + init=norm_init, stype='row_sparse') + embed = mx.symbol.sparse.dot(inputs, weights) + softmax_output = mx.symbol.Variable("softmax_label") + model = mx.symbol.SoftmaxOutput(data=embed, label=softmax_output, name="out") + return model + + +def row_sparse_push(kv, param_arrays, grad_arrays, param_names): + for index, pair in enumerate(zip(param_arrays, grad_arrays)): + arg_list, grad_list = pair + if grad_list[0] is None: + continue + name = param_names[index] + kv.push(name, grad_list, priority=-index) + + +def row_sparse_pull(kv, key, data, slices, weight_array, priority): + # if have kvstore, need to pull corresponding rows of + # the weights to each context + # column indices (NDArray type) of the csr data + # used as the row_idx of the weight row-sparse matrix + row_indices = data.indices + if len(slices) == 1: + kv.row_sparse_pull(key, weight_array, priority=priority, row_ids=row_indices) + else: # more than one slices, multi-GPU training. Need to retain weight rows according to data slices + # TODO(junwu): + # the following line blocks, may need to pre-compute + # and cache it outside the for loop + indptr = data.indptr.asnumpy() + row_idx_array = [] + for s in slices: + row_idx_array.append(row_indices[indptr[s.start]:indptr[s.stop]]) + kv.row_sparse_pull(key, weight_array, priority=priority, row_ids=row_idx_array) + + +if __name__ == '__main__': + + # arg parser + args = parser.parse_args() + num_epoch = args.num_epoch + num_batch = args.num_batch + kvstore = args.kvstore + profiler = args.profiler > 0 + batch_size = args.batch_size if args.num_gpu == 0 else args.num_gpu * args.batch_size + dummy_iter = args.dummy_iter + dataset = args.dataset + log_level = args.sparse_log_level + measure_only = args.measure_only + num_cores = multiprocessing.cpu_count() + omit_row_sparse_push = args.omit_row_sparse_push + if measure_only == COMP or measure_only == IO: + assert not kvstore, "when compute_only or io_only is set, kvstore should be None" + num_batch = datasets[dataset]['lc'] / batch_size if num_batch == MAX_NUM_BATCH else num_batch + if measure_only == COMM: + assert (kvstore == "dist_async"), "when communication_only is set kvstore should be dist_async" + num_batch = datasets[dataset]['lc'] / batch_size if num_batch == MAX_NUM_BATCH else num_batch + + + contexts = mx.context.cpu(0) if args.num_gpu < 1\ + else [mx.context.gpu(i) for i in range(args.num_gpu)] + + # create kvstore when there are gpus + kv = mx.kvstore.create(kvstore) if kvstore else None + rank = kv.rank if kv is not None else 0 + num_worker = kv.num_workers if kv is not None else 1 + + # only print log for rank 0 worker + import logging + if log_level == 'ERROR': + log_level = logging.ERROR + elif log_level == 'DEBUG': + log_level = logging.DEBUG + else: + log_level = logging.INFO + + # Only log if it is in the list of workers to be logged + logging_workers_list = [int(i) for i in args.enable_logging_for.split(",")] + log_level = log_level if rank in logging_workers_list else logging.CRITICAL + + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=log_level, format=head) + + # dataset + assert(dataset in datasets), "unknown dataset " + dataset + metadata = datasets[dataset] + feature_dim = metadata['feature_dim'] + if logging: + logging.debug('preparing data ... ') + data_dir = os.path.join(os.getcwd(), 'data') + path = os.path.join(data_dir, metadata['data_name']) + if not os.path.exists(path): + get_bz2_data(data_dir, metadata['data_name'], metadata['url'], + metadata['data_origin_name']) + assert os.path.exists(path) + + # data iterator + train_data = mx.io.LibSVMIter(data_libsvm=path, data_shape=(feature_dim,), + batch_size=batch_size, num_parts=num_worker, + part_index=rank) + if dummy_iter or measure_only == COMP or measure_only == COMM: + train_data = DummyIter(train_data) + + # model + model = get_sym(feature_dim) + + # module + mod = mx.mod.Module(symbol=model, data_names=['data'], + label_names=['softmax_label'], context=contexts) + mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) + mod.init_params(initializer=mx.init.Uniform(scale=.1)) + sgd = mx.optimizer.SGD(momentum=0.0, clip_gradient=5.0, + learning_rate=0.1, rescale_grad=1.0/batch_size/num_worker) + mod.init_optimizer(optimizer=sgd, kvstore=kv) + # use accuracy as the metric + metric = mx.metric.create('acc') + + index = mod._exec_group.param_names.index('w') + # weight_array bound to executors of the contexts + weight_array = mod._exec_group.param_arrays[index] + + mx.nd.waitall() # sync point for initialization + # start profiler + if profiler: + device = 'cpu' + if args.num_gpu > 0: + device = 'gpu' + str(args.num_gpu) + name = 'profile_' + args.dataset + '_' + device + '_nworker' + str(num_worker)\ + + '_batchsize' + str(args.batch_size) + '_outdim' + str(args.output_dim) + '.json' + mx.profiler.set_config(profile_all=True, filename=name) + mx.profiler.set_state('run') + + logging.debug('start training ...') + start = time.time() + data_iter = iter(train_data) + time_cost_epoch = 0. + sum_cost_epoch = 0. + average_cost_epoch = 0. + + for epoch in range(num_epoch): + start_time_epoch = time.time() + nbatch = 0 + end_of_batch = False + metric.reset() + next_batch = next(data_iter) + if kv is not None: + row_sparse_pull(kv, 'w', next_batch.data[0], mod._exec_group.slices, weight_array, -index) + while not end_of_batch: + nbatch += 1 + batch = next_batch + + if measure_only != IO and measure_only != COMM: + mod.forward_backward(batch) + # update parameters + mod.update() + if measure_only == COMM: + if nbatch == 1: + mod.forward_backward(batch) + mod.update() + elif not omit_row_sparse_push: + row_sparse_push(kv, mod._exec_group.param_arrays, mod._exec_group.grad_arrays, mod._exec_group.param_names) + + + try: + # pre fetch next batch + next_batch = next(data_iter) + if nbatch == num_batch: + raise StopIteration + if kv is not None: + row_sparse_pull(kv, 'w', next_batch.data[0], mod._exec_group.slices, weight_array, -index) + except StopIteration: + end_of_batch = True + # accumulate prediction accuracy + if args.dummy_metric == 0: + mod.update_metric(metric, batch.label) + else: # call waitall to replace update_metric as sync point + mx.nd.waitall() # sync point for the current minibatch + logging.info('epoch {}, {}'.format(epoch, metric.get())) + end_time_epoch = time.time() + if epoch == 0: + logging.debug("num_batches = {}".format(nbatch)) + logging.info('|device|num_worker|average_cost_epoch|rank|') + time_cost_epoch = end_time_epoch - start_time_epoch + if epoch > 0: + sum_cost_epoch = sum_cost_epoch + time_cost_epoch + average_cost_epoch = float(sum_cost_epoch) / epoch + logging.info('num_worker = {}, time cost per epoch = {}'.format(str(num_worker), str(time_cost_epoch))) + if args.num_gpu < 1: + logging.info('|cpu/{} cores| {} | {} | {} |'.format(str(num_cores), str(num_worker), str(average_cost_epoch), rank)) + data_iter.reset() + if profiler: + mx.profiler.set_state('stop') + end = time.time() + time_cost = end - start + logging.info('num_worker = {}, rank = {}, time cost = {}'.format(str(num_worker), str(rank), str(time_cost))) diff --git a/example/adversary/adversary_generation.ipynb b/example/adversary/adversary_generation.ipynb index 0dda371a8f41..76c5f4cff569 100644 --- a/example/adversary/adversary_generation.ipynb +++ b/example/adversary/adversary_generation.ipynb @@ -168,7 +168,7 @@ "epoch = 3\n", "for e in range(epoch):\n", " train_loss = 0.\n", - " acc = mx.gluon.metric.Accuracy()\n", + " acc = mx.metric.Accuracy()\n", " for i, (data, label) in enumerate(train_data):\n", " data = data.as_in_context(ctx)\n", " label = label.as_in_context(ctx)\n", @@ -223,7 +223,7 @@ " l = loss(output, label)\n", "l.backward()\n", "\n", - "acc = mx.gluon.metric.Accuracy()\n", + "acc = mx.metric.Accuracy()\n", "acc.update(label, output)\n", "\n", "print(\"Validation batch accuracy {}\".format(acc.get()[1]))" @@ -256,7 +256,7 @@ "\n", "output = net(data_perturbated) \n", "\n", - "acc = mx.gluon.metric.Accuracy()\n", + "acc = mx.metric.Accuracy()\n", "acc.update(label, output)\n", "\n", "print(\"Validation batch accuracy after perturbation {}\".format(acc.get()[1]))" diff --git a/example/autoencoder/variational_autoencoder/VAE_example.ipynb b/example/autoencoder/variational_autoencoder/VAE_example.ipynb new file mode 100755 index 000000000000..964e13725c69 --- /dev/null +++ b/example/autoencoder/variational_autoencoder/VAE_example.ipynb @@ -0,0 +1,1204 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import mxnet as mx\n", + "import numpy as np\n", + "import os\n", + "import logging\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.cm as cm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building a Variational Autoencoder in MXNet\n", + "\n", + "#### Xiaoyu Lu, July 5th, 2017\n", + "\n", + "This tutorial guides you through the process of building a variational encoder in MXNet. In this notebook we'll focus on an example using the MNIST handwritten digit recognition dataset. Refer to [Auto-Encoding Variational Bayes](https://arxiv.org/abs/1312.6114/) for more details on the model description.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prerequisites\n", + "\n", + "To complete this tutorial, we need following python packages:\n", + "\n", + "- numpy, matplotlib " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Loading the Data\n", + "\n", + "We first load the MNIST dataset, which contains 60000 training and 10000 test examples. The following code imports required modules and loads the data. These images are stored in a 4-D matrix with shape (`batch_size, num_channels, width, height`). For the MNIST dataset, there is only one color channel, and both width and height are 28, so we reshape each image as a 28x28 array. See below for a visualization:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "60000 784\n" + ] + } + ], + "source": [ + "mnist = mx.test_utils.get_mnist()\n", + "image = np.reshape(mnist['train_data'],(60000,28*28))\n", + "label = image\n", + "image_test = np.reshape(mnist['test_data'],(10000,28*28))\n", + "label_test = image_test\n", + "[N,features] = np.shape(image) #number of examples and features\n", + "print(N,features)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAACWCAYAAAA7UIUvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFI5JREFUeJzt3X+wVfO/x/H3u9+JfikVnb5FaYTEnLmqe5mLDDKJwfGVaYzfVJRBP+5FgzHjd6YoMkyFcbvUJDToxnW7GDql6YpSFwnpVKRfqDM+94+2O+f9Waf94+xfa+3P8zHTnF6rtfd6n/a73afde62lzjkBAAAAQtSs3AUAAAAA5cJiGAAAAMFiMQwAAIBgsRgGAABAsFgMAwAAIFgshgEAABAsFsMAAAAIFothAAAABCuvxbCqnq+q61V1o6pOLlRRAAAAQCloU+9Ap6rNReQrETlXRL4XkRUicqVz7otDPaZLly6ud+/eTToe4uXbb7+V7du3azGemz6pLCtXrtzunOtajOemVyoH7ynIFu8pyEYu7ykt8jjOP4jIRufc1yIiqvpvIjJSRA65GO7du7fU1tbmcUjERXV1ddGemz6pLKq6qVjPTa9UDt5TkC3eU5CNXN5T8hmTOEZENjfI36e2Gap6o6rWqmrttm3b8jgcKhl9gmzRK8gGfYJs0Sso+gl0zrnZzrlq51x1165F+V8NVAD6BNmiV5AN+gTZoleQz2L4BxGpapB7prYBAAAAiZDPYniFiPRT1T6q2kpE/i4iiwtTFgAAAFB8TT6BzjlXr6rjROQdEWkuIi8459YWrDIAAACgyPK5moQ455aIyJIC1QIAAACUFHegAwAAQLBYDAMAACBYLIYBAAAQLBbDAAAACBaLYQAAAASLxTAAAACCxWIYAAAAwWIxDAAAgGCxGAYAAECw8roDHYDiue2220yeMWNGZJ8OHTqYvG7dOpO7d+9e+MIAAKggfDIMAACAYLEYBgAAQLBYDAMAACBYzAwXwdatW00eNGiQyUOHDjV5wYIFRa8J8bdq1SqTn376aZObNYv+23X37t0m19XVmczMMJBcf/zxh8nz5883+Zprrkn7+OXLl5s8ePBgk/ft2xd5TJs2bUw+cOCAyf77UOvWrdPWACQBnwwDAAAgWCyGAQAAECwWwwAAAAgWM8NFMHv2bJP9GeJFixaVshzEVH19vcnz5s3L+TmOP/54k3v16pVXTQDKY+/evZFtU6dONfnJJ580WVXTPuftt99u8mmnnWbyc889F3nMZZddZvL7779vcu/evU32z4Hxz1O48cYbTe7UqdOhCwbKhE+GAQAAECwWwwAAAAgWi2EAAAAEi5nhEnDOlbsExNAnn3xi8owZM3J+jsmTJ5vcsWPHvGoCUB6XXnppZNvSpUtNzjQj7Fu5cqXJtbW1GZ/vtddeS/ucO3bsyOk53333XZOXLFkSeU6uVZzehg0bTO7fv39knxEjRpg8a9Ysk48++ujCF+bZuXOnyX6v+OdTZTJu3LjItqqqqtwLywKfDAMAACBYLIYBAAAQLBbDAAAACBYzw0Wwdu1ak/0ZqksuuaSU5SAm/vzzT5OnTJmS0+NXrFgR2XbyySfnVRMyu+WWW0z+4osvTL7qqqtMHjhwoMmDBw8uTmFItLlz55r8wQcflKmS4vr6669N/uOPPyL7MDOc3vz5801ubNb7jTfeMNk/J8WfGT7qqKNMHjVqVNoaPvvss8g2v2d//PFHk7dt22ayf/5Uphn48847L7KNmWEAAACgwFgMAwAAIFgshgEAABAsZoaLwJ/F8TV2P3hUPn/m6sMPP0y7/xFHHGFyY9eJbNmyZf6FwfCvs/rss8+m3X/58uUm+3NwLVpE32b9uWL/epp9+vRJ+xz79u0zediwYWlrRPn512CdOnWqyQcOHChlOQXjnwMzcuRIky+//HKT27RpU/Sakqa+vt7k1atXm/zEE0/k/Jz+vK6fff41rXOd7y2GIUOGlOxYfDIMAACAYLEYBgAAQLAyLoZV9QVVrVPVzxts66yqS1V1Q+prp+KWCQAAABReNjPDc0TkKRGZ12DbZBFZ5px7SFUnp/KkwpeXDPv37zd548aNZaoEcfbNN9/ktP8pp5xicvfu3QtZDg7Bn+dt27atyb/99lvax/uzdo3Ngq5cudLka665xuRmzeznFP68nn+Mww47zOQuXbpEjvnRRx+ZTD+V1po1a0z2/97wX1OR6LXJ/b7IpF27dia3b9/e5MauLeu/7/j8GeBWrVrlVBOidu3aZfLpp5+edv+amprItuHDh5vsn6OyaNEikzdt2pRLiSXhfw+NnW9RLBn/ZDnn/ktEfvY2jxSRv64YPldELi5wXQAAAEDRNXVmuJtzbkvq5z+JSLcC1QMAAACUTN4n0LmD/7cT/f+dFFW9UVVrVbU206U9EC76BNmiV5AN+gTZolfQ1MXwVlXtISKS+lp3qB2dc7Odc9XOuequXbs28XCodPQJskWvIBv0CbJFr6Cp08mLReRqEXko9fX1glWUQNu3bzf57bffNrmxEyMQnrfeeiun/R955JEiVYJ0jj/+eJP9P98zZ8402T/5xffMM89EtmX69Mk/cSqTPXv2pM0i0e9r4cKFJnPjjtKqq7OfITV2UwP/hDn/RMkrrrjC5NGjR5t87LHHmlxVVZVznSi8vXv3mnz33Xfn9Pjrr78+su2cc84x2e+FBx980OQtW7ZIvt59912T77zzTpMznWzsn5zsv1eW8qZS2Vxa7RUR+VhE+qvq96p6nRxcBJ+rqhtEZFgqAwAAAImS8ZNh59yVh/ilcw6xHQAAAEgE7kAHAACAYJXuisYV7OOPPza5sdkvhOfXX381edmyZTk9vrq6upDloIn8ubY77rgjp8dPmhS9H1GmmeBXXnnF5Ew38vEvqL9hw4bIPv4ccaZ5PhSWPyfaFL169TJ5xowZJvu9inj6/fffTZ4/f77J/nlGN998s8n+fHA2/N7w58kzWbduXWTbmDFjcnqODh06mPzhhx+afMwxx+T0fIXEJ8MAAAAIFothAAAABIvFMAAAAILFzHABnHrqqSb714Js1aqVyW3atCl6TSg//9qRP/zwQ9r9u3WzdzVn9rwy+O8H2bjhhhty2v/ee+81ee7cuZF9xo0bZ/LkyZNNHjFiRE7HRG78vyeOPPJIk3fs2JHxOdavX2/yrFmzTB45cqTJxx13XC4lokTat29v8oIFC0wePny4yY8//njRa/K9+eabJtfU1ET2yfR3lD+n7M8IDxgwoInVFR6fDAMAACBYLIYBAAAQLBbDAAAACBYzwwXw2Wefmbxv3z6T+/TpY3JTZggRf/X19Sb7830+f0b4pZdeMrlZM/6tiuy0a9fO5FGjRkX28WeGJ0yYUNSaYHXv3t3kfv36mZzNzLBv4sSJJk+dOtXkRx991GT/erUoj5YtW5o8ZMgQk9977z2TS3H96Hfeecfkiy66yORszmHxr8E+fvx4k8t5HeFM+NsWAAAAwWIxDAAAgGCxGAYAAECwmBkuAP/aef59xS+//PJSloMymT59usnLli1Lu/+JJ55o8tlnn13wmhCmZ555JuM+/gwrSuu1114zubq6OrKPf23yTOcR+OerjB071uS77rrL5BUrVkSeo2/fvia3aMEyodhat25t8uDBg4t+zO+++87kiy++OO3+Rx11VGTbLbfcYrI/w56keyrwyTAAAACCxWIYAAAAwWIxDAAAgGAxDFQA/uydfz2+QYMGlbIclEk2c5oNnXDCCUWqBKHZuXOnydOmTYvs48/vVVVVFbUmpOf/vTFv3rzIPqNHjzZ569ateR3Tnyk+6aSTIvv417g988wz8zom4sG/jrA/77t//36Tjz76aJM//fTTyHP6+yQZnwwDAAAgWCyGAQAAECwWwwAAAAgWM8MF4M/n+dcZXr16tckjRowoek2IP/oAheLPq2/bti2yj99vnMsQL41dZ9y/hr1/DeBiuOKKK0xesGCByUOHDi16Dcjfrl27TPZnwb/99tu0j9+7d6/J/nkJIswMAwAAABWBxTAAAACCxWIYAAAAwWIxDAAAgGBxAl0B+DfZ8DMqz6ZNmyLbfvzxx7SPad68ucmnn356QWtCOH7++WeTp0+fbnLPnj0jj3n55ZeLWhMKz78xyubNm01euHChyffff7/JO3bsyPmY/smXY8eONfmjjz4yuW3btjkfA8W3Zs0akx977DGTM61TZs2aZfKAAQMKU1hM8ckwAAAAgsViGAAAAMFiMQwAAIBgMTNcAP6Fp3/66SeTL7roolKWgxLwX2MRkd9++y3tY8aMGWNy+/btC1oTKtf+/ftNfvjhh032+/HCCy+MPMfhhx9e+MJQVP55Bj169DC5Y8eOJvs3WiiE1q1bp60J8VBbW2tyrjd1qqmpMfmCCy7Iu6Yk4ZNhAAAABCvjYlhVq1T1fVX9QlXXqur41PbOqrpUVTekvnYqfrkAAABA4WTzyXC9iNzhnBsgIoNFZKyqDhCRySKyzDnXT0SWpTIAAACQGBlnhp1zW0RkS+rnu1X1SxE5RkRGisg/p3abKyL/KSKTilJlzK1atcpk//p9nTrxoXmlefbZZ3N+zFNPPWXyTTfdZPIJJ5yQV02oXFu2bDH5kUceMdm/1usDDzxQ9JqQ3u7du02eOHGiye3atTO5W7dukedwzpn86quvmuz/3ZPJn3/+aXKzZpk/DzvssMNyOgaK7/XXX49su/baa03OND/euXNnk6dNm2Zyhw4dmlhdMuU0M6yqvUXkVBH5RES6pRbKIiI/iUj0TzIAAAAQY1kvhlX1cBFZICITnHPmnxzu4D9f3SEed6Oq1qpqrX9nG+Av9AmyRa8gG/QJskWvIKvFsKq2lIML4Zedc3/d/3GrqvZI/XoPEalr7LHOudnOuWrnXHXXrl0LUTMqEH2CbNEryAZ9gmzRK8g4M6wHB2CfF5EvnXNPNPilxSJytYg8lPoaHWIJRN++fU32r/nZq1evUpaDmPLn//w+YWYYh7Jz506T/fMSHnvsMZMHDRpU9JqQ3ubNm01+7rnncn4O/z3Df939nIk/I9zY4/33IX8+vVWrVjkdE/nbtGmTyffcc09kn19++cXkTL0xbNgwk0M/tymbm278o4iMFpH/UdXVqW3/IgcXwf+uqteJyCYRqTnE4wEAAIBYyuZqEv8tIof6J8Y5hS0HAAAAKB3uQAcAAIBgZTMmgQz8a4D6szr79u0zmes2Jl+PHj1yfow/a3fWWWcVqhxUmPr6epOnTp1qcsuWLU0eOnRo0WtCbvzruF533XUmP//886Usp1Ht27ePbFuyZInJVVVVpSoHKXV19noEp512msn+fLBIdL7cd8YZZ5j84osvmtyiRdjLQT4ZBgAAQLBYDAMAACBYLIYBAAAQrLCHRApkz549JvvzpMwIV55bb701sm3mzJkm+/d2nz59elFrQuVYu3atyYsXLzb5wgsvNJnrCsdP9+7dTX7wwQdNfvPNN03eunVr0WsaP368yaNGjYrsw4xw6flriP79+5u8a5e56W9W15e+7777TJ4wYYLJoc8I+/hkGAAAAMFiMQwAAIBgsRgGAABAsBgaKQD/PuH+rBgqT2OvcWPXfgSa4oEHHjDZn+/z508Rfx07djT5q6++MnnMmDGRx7z00ks5HaO6utrkadOmmTxkyJCcng/FsXfvXpMvu+wyk/0Z4Wz4M8KTJk0y2b/OPSw+GQYAAECwWAwDAAAgWCyGAQAAECwWwwAAAAgWJ9AVwP3331/uEgBUsDPOOMPkgQMHlqkSNJV/EqSf586dG3lMY9uQfO3atTO5rq4up8fX1NREtk2ZMsVkbqqRGz4ZBgAAQLBYDAMAACBYLIYBAAAQLIZKAAAAymTx4sUmjxs3zuS2bduaPGfOnMhzMCOcHz4ZBgAAQLBYDAMAACBYLIYBAAAQLIZMACBm/JnBhQsXlqkSAMXWs2dPkxctWlSmSsLFJ8MAAAAIFothAAAABIvFMAAAAIKlzrnSHUx1m4hsEpEuIrK9ZAduGmpM72/Oua7FeOKE9YlIMuqkV8qPGtMrRZ+I8DoUSqX3Cq9B4ZSrzqz7pKSL4f8/qGqtc6665AfOATWWX1K+vyTUmYQa85GE748a4yEJ3yM1ll8Svr8k1CiSjDoZkwAAAECwWAwDAAAgWOVaDM8u03FzQY3ll5TvLwl1JqHGfCTh+6PGeEjC90iN5ZeE7y8JNYokoM6yzAwDAAAAccCYBAAAAIJV0sWwqp6vqutVdaOqTi7lsdNR1RdUtU5VP2+wrbOqLlXVDamvncpcY5Wqvq+qX6jqWlUdH8c6CyWOvUKfxE8c+0SEXokjeqXJ9QXVJyLx7JW490mqnsT2SskWw6raXESeFpELRGSAiFypqgNKdfwM5ojI+d62ySKyzDnXT0SWpXI51YvIHc65ASIyWETGpn7/4lZn3mLcK3OEPomNGPeJCL0SK/RKXoLpE5FY98ociXefiCS5V5xzJfkhIkNE5J0GeYqITCnV8bOor7eIfN4grxeRHqmf9xCR9eWu0av3dRE5N+51Vlqv0Cfx+RHnPqFX4vWDXqFPKqFXktQnSeuVUo5JHCMimxvk71Pb4qqbc25L6uc/iUi3chbTkKr2FpFTReQTiXGdeUhSr8T2958+iZ3Yvgb0SuzE8jUIoE9EktUrsX0NktYrnECXBXfwnzOxuOyGqh4uIgtEZIJzblfDX4tTnSGK0+8/fRJvcXoN6JV4i8trQJ/EW5xegyT2SikXwz+ISFWD3DO1La62qmoPEZHU17oy1yOq2lIONtjLzrmFqc2xq7MAktQrsfv9p09iK3avAb0SW7F6DQLqE5Fk9UrsXoOk9kopF8MrRKSfqvZR1VYi8ncRWVzC4+dqsYhcnfr51XJw9qVsVFVF5HkR+dI590SDX4pVnQWSpF6J1e8/fRLbPhGJ2WtAr9Ar2QisT0SS1Suxeg0S3SslHqYeLiJficj/isi/lntgukFdr4jIFhE5IAfng64TkSPl4FmPG0TkP0Skc5lr/Cc5+F8La0RkderH8LjVWcm9Qp/E70cc+4ReiecPeoU+SXKvxL1Pkt4r3IEOAAAAweIEOgAAAASLxTAAAACCxWIYAAAAwWIxDAAAgGCxGAYAAECwWAwDAAAgWCyGAQAAECwWwwAAAAjW/wEgPmufEARJLAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "nsamples = 5\n", + "idx = np.random.choice(len(mnist['train_data']), nsamples)\n", + "_, axarr = plt.subplots(1, nsamples, sharex='col', sharey='row',figsize=(12,3))\n", + "\n", + "for i,j in enumerate(idx):\n", + " axarr[i].imshow(np.reshape(image[j,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can optionally save the parameters in the directory variable 'model_prefix'. We first create data iterators for MXNet, with each batch of data containing 100 images." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "model_prefix = None\n", + "\n", + "batch_size = 100\n", + "latent_dim = 5\n", + "nd_iter = mx.io.NDArrayIter(data={'data':image},label={'loss_label':label},\n", + " batch_size = batch_size)\n", + "nd_iter_test = mx.io.NDArrayIter(data={'data':image_test},label={'loss_label':label_test},\n", + " batch_size = batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Building the Network Architecture\n", + "\n", + "### 2.1 Gaussian MLP as encoder\n", + "Next we constuct the neural network, as in the [paper](https://arxiv.org/abs/1312.6114/), we use *Multilayer Perceptron (MLP)* for both the encoder and decoder. For encoder, a Gaussian MLP is used as follows:\n", + "\n", + "\\begin{align}\n", + "\\log q_{\\phi}(z|x) &= \\log \\mathcal{N}(z:\\mu,\\sigma^2I) \\\\\n", + "\\textit{ where } \\mu &= W_2h+b_2, \\log \\sigma^2 = W_3h+b_3\\\\\n", + "h &= \\tanh(W_1x+b_1)\n", + "\\end{align}\n", + "\n", + "where $\\{W_1,W_2,W_3,b_1,b_2,b_3\\}$ are the weights and biases of the MLP.\n", + "Note below that `encoder_mu`(`mu`) and `encoder_logvar`(`logvar`) are symbols. So, we can use `get_internals()` to get the values of them, after which we can sample the latent variable $z$.\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "## define data and loss labels as symbols \n", + "data = mx.sym.var('data')\n", + "loss_label = mx.sym.var('loss_label')\n", + "\n", + "## define fully connected and activation layers for the encoder, where we used tanh activation function.\n", + "encoder_h = mx.sym.FullyConnected(data=data, name=\"encoder_h\",num_hidden=400)\n", + "act_h = mx.sym.Activation(data=encoder_h, act_type=\"tanh\",name=\"activation_h\")\n", + "\n", + "## define mu and log variance which are the fully connected layers of the previous activation layer\n", + "mu = mx.sym.FullyConnected(data=act_h, name=\"mu\",num_hidden = latent_dim)\n", + "logvar = mx.sym.FullyConnected(data=act_h, name=\"logvar\",num_hidden = latent_dim)\n", + "\n", + "## sample the latent variables z according to Normal(mu,var)\n", + "z = mu + mx.symbol.broadcast_mul(mx.symbol.exp(0.5 * logvar), \n", + " mx.symbol.random_normal(loc=0, scale=1, shape=(batch_size, latent_dim)),\n", + " name=\"z\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 Bernoulli MLP as decoder\n", + "\n", + "In this case let $p_\\theta(x|z)$ be a multivariate Bernoulli whose probabilities are computed from $z$ with a feed forward neural network with a single hidden layer:\n", + "\n", + "\\begin{align}\n", + "\\log p(x|z) &= \\sum_{i=1}^D x_i\\log y_i + (1-x_i)\\log (1-y_i) \\\\\n", + "\\textit{ where } y &= f_\\sigma(W_5\\tanh (W_4z+b_4)+b_5)\n", + "\\end{align}\n", + "\n", + "where $f_\\sigma(\\dot)$ is the elementwise sigmoid activation function, $\\{W_4,W_5,b_4,b_5\\}$ are the weights and biases of the decoder MLP. A Bernouilli likelihood is suitable for this type of data but you can easily extend it to other likelihood types by parsing into the argument `likelihood` in the `VAE` class, see section 4 for details." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# define fully connected and tanh activation layers for the decoder\n", + "decoder_z = mx.sym.FullyConnected(data=z, name=\"decoder_z\",num_hidden=400)\n", + "act_z = mx.sym.Activation(data=decoder_z, act_type=\"tanh\",name=\"activation_z\")\n", + "\n", + "# define the output layer with sigmoid activation function, where the dimension is equal to the input dimension\n", + "decoder_x = mx.sym.FullyConnected(data=act_z, name=\"decoder_x\",num_hidden=features)\n", + "y = mx.sym.Activation(data=decoder_x, act_type=\"sigmoid\",name='activation_x')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3 Joint Loss Function for the Encoder and the Decoder\n", + "\n", + "The variational lower bound also called evidence lower bound (ELBO) can be estimated as:\n", + "\n", + "\\begin{align}\n", + "\\mathcal{L}(\\theta,\\phi;x_{(i)}) \\approx \\frac{1}{2}\\left(1+\\log ((\\sigma_j^{(i)})^2)-(\\mu_j^{(i)})^2-(\\sigma_j^{(i)})^2\\right) + \\log p_\\theta(x^{(i)}|z^{(i)})\n", + "\\end{align}\n", + "\n", + "where the first term is the KL divergence of the approximate posterior from the prior, and the second term is an expected negative reconstruction error. We would like to maximize this lower bound, so we can define the loss to be $-\\mathcal{L}$(minus ELBO) for MXNet to minimize." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# define the objective loss function that needs to be minimized\n", + "KL = 0.5*mx.symbol.sum(1+logvar-pow( mu,2)-mx.symbol.exp(logvar),axis=1)\n", + "loss = -mx.symbol.sum(mx.symbol.broadcast_mul(loss_label,mx.symbol.log(y)) \n", + " + mx.symbol.broadcast_mul(1-loss_label,mx.symbol.log(1-y)),axis=1)-KL\n", + "output = mx.symbol.MakeLoss(sum(loss),name='loss')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Training the model\n", + "\n", + "Now, we can define the model and train it. First we will initilize the weights and the biases to be Gaussian(0,0.01), and then use stochastic gradient descent for optimization. To warm start the training, one may also initilize with pre-trainined parameters `arg_params` using `init=mx.initializer.Load(arg_params)`. \n", + "\n", + "To save intermediate results, we can optionally use `epoch_end_callback = mx.callback.do_checkpoint(model_prefix, 1)` which saves the parameters to the path given by model_prefix, and with period every $1$ epoch. To assess the performance, we output $-\\mathcal{L}$(minus ELBO) after each epoch, with the command `eval_metric = 'Loss'` which is defined above. We will also plot the training loss for mini batches by accessing the log and saving it to a list, and then parsing it to the argument `batch_end_callback`." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# set up the log\n", + "nd_iter.reset()\n", + "logging.getLogger().setLevel(logging.DEBUG) \n", + "\n", + "# define function to trave back training loss\n", + "def log_to_list(period, lst):\n", + " def _callback(param):\n", + " \"\"\"The checkpoint function.\"\"\"\n", + " if param.nbatch % period == 0:\n", + " name, value = param.eval_metric.get()\n", + " lst.append(value)\n", + " return _callback\n", + "\n", + "# define the model\n", + "model = mx.mod.Module(\n", + " symbol = output ,\n", + " data_names=['data'],\n", + " label_names = ['loss_label'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:Epoch[0] Train-loss=373.547317\n", + "INFO:root:Epoch[0] Time cost=5.020\n", + "INFO:root:Epoch[1] Train-loss=212.232684\n", + "INFO:root:Epoch[1] Time cost=4.651\n", + "INFO:root:Epoch[2] Train-loss=207.448528\n", + "INFO:root:Epoch[2] Time cost=4.665\n", + "INFO:root:Epoch[3] Train-loss=205.369479\n", + "INFO:root:Epoch[3] Time cost=4.758\n", + "INFO:root:Epoch[4] Train-loss=203.651983\n", + "INFO:root:Epoch[4] Time cost=4.672\n", + "INFO:root:Epoch[5] Train-loss=202.061007\n", + "INFO:root:Epoch[5] Time cost=5.087\n", + "INFO:root:Epoch[6] Train-loss=199.348143\n", + "INFO:root:Epoch[6] Time cost=5.056\n", + "INFO:root:Epoch[7] Train-loss=196.266242\n", + "INFO:root:Epoch[7] Time cost=4.813\n", + "INFO:root:Epoch[8] Train-loss=194.694945\n", + "INFO:root:Epoch[8] Time cost=4.776\n", + "INFO:root:Epoch[9] Train-loss=193.699284\n", + "INFO:root:Epoch[9] Time cost=4.756\n", + "INFO:root:Epoch[10] Train-loss=193.036517\n", + "INFO:root:Epoch[10] Time cost=4.757\n", + "INFO:root:Epoch[11] Train-loss=192.555736\n", + "INFO:root:Epoch[11] Time cost=4.678\n", + "INFO:root:Epoch[12] Train-loss=192.020813\n", + "INFO:root:Epoch[12] Time cost=4.630\n", + "INFO:root:Epoch[13] Train-loss=191.648876\n", + "INFO:root:Epoch[13] Time cost=5.158\n", + "INFO:root:Epoch[14] Train-loss=191.057798\n", + "INFO:root:Epoch[14] Time cost=4.781\n", + "INFO:root:Epoch[15] Train-loss=190.315835\n", + "INFO:root:Epoch[15] Time cost=5.117\n", + "INFO:root:Epoch[16] Train-loss=189.311271\n", + "INFO:root:Epoch[16] Time cost=4.707\n", + "INFO:root:Epoch[17] Train-loss=187.285967\n", + "INFO:root:Epoch[17] Time cost=4.745\n", + "INFO:root:Epoch[18] Train-loss=185.271324\n", + "INFO:root:Epoch[18] Time cost=4.692\n", + "INFO:root:Epoch[19] Train-loss=183.510888\n", + "INFO:root:Epoch[19] Time cost=4.762\n", + "INFO:root:Epoch[20] Train-loss=181.756008\n", + "INFO:root:Epoch[20] Time cost=4.838\n", + "INFO:root:Epoch[21] Train-loss=180.546818\n", + "INFO:root:Epoch[21] Time cost=4.764\n", + "INFO:root:Epoch[22] Train-loss=179.479776\n", + "INFO:root:Epoch[22] Time cost=4.791\n", + "INFO:root:Epoch[23] Train-loss=178.352077\n", + "INFO:root:Epoch[23] Time cost=4.981\n", + "INFO:root:Epoch[24] Train-loss=177.385084\n", + "INFO:root:Epoch[24] Time cost=5.292\n", + "INFO:root:Epoch[25] Train-loss=175.920123\n", + "INFO:root:Epoch[25] Time cost=5.097\n", + "INFO:root:Epoch[26] Train-loss=174.377171\n", + "INFO:root:Epoch[26] Time cost=4.907\n", + "INFO:root:Epoch[27] Train-loss=172.590589\n", + "INFO:root:Epoch[27] Time cost=4.484\n", + "INFO:root:Epoch[28] Train-loss=170.933683\n", + "INFO:root:Epoch[28] Time cost=4.348\n", + "INFO:root:Epoch[29] Train-loss=169.866807\n", + "INFO:root:Epoch[29] Time cost=4.647\n", + "INFO:root:Epoch[30] Train-loss=169.182084\n", + "INFO:root:Epoch[30] Time cost=5.034\n", + "INFO:root:Epoch[31] Train-loss=168.121719\n", + "INFO:root:Epoch[31] Time cost=5.615\n", + "INFO:root:Epoch[32] Train-loss=167.389992\n", + "INFO:root:Epoch[32] Time cost=4.733\n", + "INFO:root:Epoch[33] Train-loss=166.189067\n", + "INFO:root:Epoch[33] Time cost=5.041\n", + "INFO:root:Epoch[34] Train-loss=163.783392\n", + "INFO:root:Epoch[34] Time cost=5.168\n", + "INFO:root:Epoch[35] Train-loss=162.167959\n", + "INFO:root:Epoch[35] Time cost=5.019\n", + "INFO:root:Epoch[36] Train-loss=161.192039\n", + "INFO:root:Epoch[36] Time cost=5.064\n", + "INFO:root:Epoch[37] Train-loss=160.307114\n", + "INFO:root:Epoch[37] Time cost=5.180\n", + "INFO:root:Epoch[38] Train-loss=159.591957\n", + "INFO:root:Epoch[38] Time cost=5.440\n", + "INFO:root:Epoch[39] Train-loss=159.109593\n", + "INFO:root:Epoch[39] Time cost=5.119\n", + "INFO:root:Epoch[40] Train-loss=158.463844\n", + "INFO:root:Epoch[40] Time cost=5.299\n", + "INFO:root:Epoch[41] Train-loss=158.037287\n", + "INFO:root:Epoch[41] Time cost=4.856\n", + "INFO:root:Epoch[42] Train-loss=157.598576\n", + "INFO:root:Epoch[42] Time cost=5.227\n", + "INFO:root:Epoch[43] Train-loss=157.097344\n", + "INFO:root:Epoch[43] Time cost=5.237\n", + "INFO:root:Epoch[44] Train-loss=156.594472\n", + "INFO:root:Epoch[44] Time cost=4.783\n", + "INFO:root:Epoch[45] Train-loss=156.177069\n", + "INFO:root:Epoch[45] Time cost=4.834\n", + "INFO:root:Epoch[46] Train-loss=155.825302\n", + "INFO:root:Epoch[46] Time cost=4.902\n", + "INFO:root:Epoch[47] Train-loss=155.318117\n", + "INFO:root:Epoch[47] Time cost=4.966\n", + "INFO:root:Epoch[48] Train-loss=154.890766\n", + "INFO:root:Epoch[48] Time cost=5.012\n", + "INFO:root:Epoch[49] Train-loss=154.504158\n", + "INFO:root:Epoch[49] Time cost=4.844\n", + "INFO:root:Epoch[50] Train-loss=154.035214\n", + "INFO:root:Epoch[50] Time cost=4.736\n", + "INFO:root:Epoch[51] Train-loss=153.692903\n", + "INFO:root:Epoch[51] Time cost=5.057\n", + "INFO:root:Epoch[52] Train-loss=153.257554\n", + "INFO:root:Epoch[52] Time cost=5.044\n", + "INFO:root:Epoch[53] Train-loss=152.849715\n", + "INFO:root:Epoch[53] Time cost=4.783\n", + "INFO:root:Epoch[54] Train-loss=152.483047\n", + "INFO:root:Epoch[54] Time cost=4.842\n", + "INFO:root:Epoch[55] Train-loss=152.091617\n", + "INFO:root:Epoch[55] Time cost=5.044\n", + "INFO:root:Epoch[56] Train-loss=151.715490\n", + "INFO:root:Epoch[56] Time cost=5.029\n", + "INFO:root:Epoch[57] Train-loss=151.362293\n", + "INFO:root:Epoch[57] Time cost=4.873\n", + "INFO:root:Epoch[58] Train-loss=151.003241\n", + "INFO:root:Epoch[58] Time cost=4.729\n", + "INFO:root:Epoch[59] Train-loss=150.619678\n", + "INFO:root:Epoch[59] Time cost=5.068\n", + "INFO:root:Epoch[60] Train-loss=150.296043\n", + "INFO:root:Epoch[60] Time cost=4.458\n", + "INFO:root:Epoch[61] Train-loss=149.964152\n", + "INFO:root:Epoch[61] Time cost=4.828\n", + "INFO:root:Epoch[62] Train-loss=149.694102\n", + "INFO:root:Epoch[62] Time cost=5.012\n", + "INFO:root:Epoch[63] Train-loss=149.290113\n", + "INFO:root:Epoch[63] Time cost=5.193\n", + "INFO:root:Epoch[64] Train-loss=148.934186\n", + "INFO:root:Epoch[64] Time cost=4.999\n", + "INFO:root:Epoch[65] Train-loss=148.657502\n", + "INFO:root:Epoch[65] Time cost=4.810\n", + "INFO:root:Epoch[66] Train-loss=148.331948\n", + "INFO:root:Epoch[66] Time cost=5.201\n", + "INFO:root:Epoch[67] Train-loss=148.018539\n", + "INFO:root:Epoch[67] Time cost=4.833\n", + "INFO:root:Epoch[68] Train-loss=147.746825\n", + "INFO:root:Epoch[68] Time cost=5.187\n", + "INFO:root:Epoch[69] Train-loss=147.406399\n", + "INFO:root:Epoch[69] Time cost=5.355\n", + "INFO:root:Epoch[70] Train-loss=147.181831\n", + "INFO:root:Epoch[70] Time cost=4.989\n", + "INFO:root:Epoch[71] Train-loss=146.860770\n", + "INFO:root:Epoch[71] Time cost=4.934\n", + "INFO:root:Epoch[72] Train-loss=146.604369\n", + "INFO:root:Epoch[72] Time cost=5.283\n", + "INFO:root:Epoch[73] Train-loss=146.351628\n", + "INFO:root:Epoch[73] Time cost=5.062\n", + "INFO:root:Epoch[74] Train-loss=146.102506\n", + "INFO:root:Epoch[74] Time cost=4.540\n", + "INFO:root:Epoch[75] Train-loss=145.828805\n", + "INFO:root:Epoch[75] Time cost=4.875\n", + "INFO:root:Epoch[76] Train-loss=145.571626\n", + "INFO:root:Epoch[76] Time cost=4.856\n", + "INFO:root:Epoch[77] Train-loss=145.365383\n", + "INFO:root:Epoch[77] Time cost=5.003\n", + "INFO:root:Epoch[78] Train-loss=145.101047\n", + "INFO:root:Epoch[78] Time cost=4.718\n", + "INFO:root:Epoch[79] Train-loss=144.810765\n", + "INFO:root:Epoch[79] Time cost=5.127\n", + "INFO:root:Epoch[80] Train-loss=144.619876\n", + "INFO:root:Epoch[80] Time cost=4.737\n", + "INFO:root:Epoch[81] Train-loss=144.399066\n", + "INFO:root:Epoch[81] Time cost=4.742\n", + "INFO:root:Epoch[82] Train-loss=144.220090\n", + "INFO:root:Epoch[82] Time cost=4.810\n", + "INFO:root:Epoch[83] Train-loss=143.904279\n", + "INFO:root:Epoch[83] Time cost=5.176\n", + "INFO:root:Epoch[84] Train-loss=143.734935\n", + "INFO:root:Epoch[84] Time cost=4.921\n", + "INFO:root:Epoch[85] Train-loss=143.499403\n", + "INFO:root:Epoch[85] Time cost=4.692\n", + "INFO:root:Epoch[86] Train-loss=143.304287\n", + "INFO:root:Epoch[86] Time cost=4.778\n", + "INFO:root:Epoch[87] Train-loss=143.096145\n", + "INFO:root:Epoch[87] Time cost=4.962\n", + "INFO:root:Epoch[88] Train-loss=142.877920\n", + "INFO:root:Epoch[88] Time cost=4.815\n", + "INFO:root:Epoch[89] Train-loss=142.677429\n", + "INFO:root:Epoch[89] Time cost=5.127\n", + "INFO:root:Epoch[90] Train-loss=142.499622\n", + "INFO:root:Epoch[90] Time cost=5.463\n", + "INFO:root:Epoch[91] Train-loss=142.300291\n", + "INFO:root:Epoch[91] Time cost=4.639\n", + "INFO:root:Epoch[92] Train-loss=142.111362\n", + "INFO:root:Epoch[92] Time cost=5.064\n", + "INFO:root:Epoch[93] Train-loss=141.912848\n", + "INFO:root:Epoch[93] Time cost=4.894\n", + "INFO:root:Epoch[94] Train-loss=141.723130\n", + "INFO:root:Epoch[94] Time cost=4.635\n", + "INFO:root:Epoch[95] Train-loss=141.516580\n", + "INFO:root:Epoch[95] Time cost=5.063\n", + "INFO:root:Epoch[96] Train-loss=141.362380\n", + "INFO:root:Epoch[96] Time cost=4.785\n", + "INFO:root:Epoch[97] Train-loss=141.178878\n", + "INFO:root:Epoch[97] Time cost=4.699\n", + "INFO:root:Epoch[98] Train-loss=141.004168\n", + "INFO:root:Epoch[98] Time cost=4.959\n", + "INFO:root:Epoch[99] Train-loss=140.865592\n", + "INFO:root:Epoch[99] Time cost=5.155\n" + ] + } + ], + "source": [ + "# training the model, save training loss as a list.\n", + "training_loss=list()\n", + "\n", + "# initilize the parameters for training using Normal.\n", + "init = mx.init.Normal(0.01)\n", + "model.fit(nd_iter, # train data\n", + " initializer=init,\n", + " # if eval_data is supplied, test loss will also be reported\n", + " # eval_data = nd_iter_test,\n", + " optimizer='sgd', # use SGD to train\n", + " optimizer_params={'learning_rate':1e-3,'wd':1e-2}, \n", + " # save parameters for each epoch if model_prefix is supplied\n", + " epoch_end_callback = None if model_prefix==None else mx.callback.do_checkpoint(model_prefix, 1),\n", + " batch_end_callback = log_to_list(N/batch_size,training_loss), \n", + " num_epoch=100,\n", + " eval_metric = 'Loss')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:matplotlib.font_manager:findfont: Matching :family=sans-serif:style=normal:variant=normal:weight=normal:stretch=normal:size=12.0 to DejaVu Sans ('/usr/local/lib/python3.5/dist-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf') with score of 0.050000\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmYXFWd//H3t6t676S7kw5kX4AAsgbMBMQNFQUZZ9AZxsFlwJVxfjjO4vxQ0BkUxVHHGZfRB0WG38AMi7tEh0cEFRjHYQmrbEIgW4ck3Ul3J7133brf3x/3VlPp1K0Kla6uTufzep56UnXurapz63bO957lnmPujoiIyEtVU+0MiIjIwUkBREREyqIAIiIiZVEAERGRsiiAiIhIWRRARESkLAogMqnM7Jtm9veTve9MYGaNZvYTM9ttZt+rwvdfbmbXHui+ZnammXVObu4S87HRzM6aiu+Sly5d7QzI9GFmG4EPuPud5X6Gu3+oEvvOEOcDhwNz3T2Y6i93989VYt+XwsyWAxuA2mr8BjK5VAOR/WZmh9QFRwWOdxnwTDkF56H228vBQQFEADCz/wCWAj8xswEzu9TMlpuZm9n7zWwz8Mt43++Z2fa4KeYeMzs+73P+3cw+Gz8/08w6zeyjZtZlZtvM7L1l7js3bv7ZY2YPmNlnzezXRY7nVWb2GzPrM7MtZvaeOP0uM/tA3n7vyf+c+HgvMbNngWfN7Goz+9KEz77VzP42fr7QzH5gZt1mtsHMPpKQn08D/wD8afz7vt/Maszsk2a2KT7mG8ysNd6/4G8/4TNzv9mleb/ZW83sXDN7xsx6zOzyvP0/ZWb/OeHzLzKzzWa208w+UWjfIr/x5fH7NprZu/LSf9/MHo7P1RYz+1Te2+6J/+2Lf4dXxO/5oJk9ZWb9ZvakmZ2a955VZvZY/Pf2HTNryPuut5jZI/F5/o2ZnZS37WNmtjX+zN+Z2RuKHY+Uwd310AN3B9gInJX3ejngwA1AM9AYp78PmAXUA18BHsl7z78Dn42fnwkEwJVALXAuMAS0l7HvLfGjCTgO2AL8OuE4lgH9wDviz5oLrIq33UXUTJfb9z35nxMf7x3AHKAReE38XRZvbweGgYVEF2APEgWGOuAI4Hng7IR8fQr4z7zX7wPWx+9rAX4I/Eex337C5+V+s3+Ij/ODQDdwU3x+jo/zumLi9+d9/rfj4zwZGAVeViivCd/7L/HfwGuBQeCYvO0nxr/PScAO4K0Tvjed93l/AmwFfg8w4ChgWd7f5P3x7z0HeAr4ULztFKALOA1IARfF+9cDx8TnbWHe9x5Z7f9jM+2hGojsj0+5+6C7DwO4+3Xu3u/uo0QFzcm5K+cCMsCV7p5x99uAAaL/3Pu9r5mlgD8GrnD3IXd/Eri+SH7fCdzp7jfHn7XL3R95Ccf7j+7eEx/vfxMVeK+Ot50P/K+7v0BU4M1z9yvdfczdnycqkC/Yz+95F/Av7v68uw8AlwEXTGiu2uu3LyADXOXuGaIA2wF8NT4/TwBPEgWHJJ9292F3fxR4tMS+E/29u4+6+93AfwFvB3D3u9z9t+4euvtjwM1EQSbJB4AvuvsDHlnv7pvytn/N3V9w9x7gJ8CqOP1i4Fvufp+7Z939eqIgeDqQJQokx5lZrbtvdPfnXsKxyX5QAJH9sSX3xMxSZvZ5M3vOzPYQXfFBVHAVssv3bvMfIrrafin7ziMa8LElb1v+84mWAAdSWIx/trs7UcH8jjjpncCN8fNlwMK4+aTPzPqAy4k6yvfHQiC/oNxEdJz57y92nBD9Ztn4eS7I7MjbPkzy7w2wPe95sXMzUa+7D+a93kR0PJjZaWb2q7hZbzfwIZL/PqD0+UrK4zLgoxN+/yVEtY71wF8TXeB0mdktZrZwP49N9pMCiORLmpo5P/2dwHnAWUArUdMARE0PldJN1GSyOC9tSZH9twBHJmwbJGoGy5lfYJ+Jv8PNwPlmtoyoueQHed+zwd3b8h6z3P3cInnL9wJRIZizlOg48wPAdJ0uu93MmvNeLyU6Hoia0NYCS9y9FfgmL/59FDqeYuermC1Eta/837/J3W8GcPeb3P1VRL+xA18o4zukCAUQybeDqD2+mFlEzQS7iAriigz3zBdfYf8Q+JSZNZnZscCFRd5yI3CWmb3dzNJxB3yu2eMR4I/izzkKeP9+fP/DwE7gWuB2d++LN90P9MedtY1x7ewEM/u9/Ty0m4G/MbMVZtZC9Ft+xw+e4a2fNrM6M3s18BYgd2/LLKDH3UfMbA3RRUdONxCy99/ZtcDfmdnLLXJUHKxL+TbwobjGY2bWHHfgzzKzY8zs9WZWD4wQ1cTCAzxemUABRPL9I/DJuDng7xL2uYGouWIrUfv6vVOUtw8T1Xi2A/9BVPiOFtrR3TcTdcJ/FOghChq5tv0vA2NEwfJ6XmyOKuUmolrXTXnfkyUqOFcR3duQCzJJ/UETXRcfyz3x+0eAv9zP91bbdqCXqNZxI1HH9tPxtv8DXGlm/UQd/N/Nvcndh4CrgP+J/85Od/fvxWk3EQ1++DFRh3lR7r6OaODA1+O8rCcaFAFR/8fnic7JduAwoj4mmUS5kSUiBxUz+wIw390vqnZeRA5VqoHIQcHMjjWzk+KmijVETU8/qna+RA5lurtVDhaziJqtFhI1P/0zcGtVcyRyiFMTloiIlEVNWCIiUpYZ3YTV0dHhy5cvr3Y2REQOKg8++OBOd59Xar8ZHUCWL1/OunXrqp0NEZGDipltKr2XmrBERKRMCiAiIlIWBRARESmLAoiIiJRFAURERMqiACIiImVRABERkbLM6PtAREQOBkNjATv7x+gbHqOjpZ7DZzeQqonW4BocDdi2e4TB0YCxbMhoJmTPSIaewTH6hsYIQqcuXUNdqoa6dA2pGiNdY8ybVc/rj93fxTHLowAiIhIbC0I6e4foHwnIZEPGsiFhCKE7oTtmRo1ByozhTJau/lG69ozSP5IhlTJSZoQOA6MZBkYCRjIhqVRUoNeYMZLJMpzJMjSWZc9wht3xY2gsu1c+alPG/NYG9gwH7B7OlHUspyxtUwAREUni7uwaHGPTriG69oxgFhXWZjAahAyPRQX2SCbL8FiWoUyWgZGAPSOZ8SCRDZ0gdLbtHmZr7zBhGfPLNtamXgwyGC0NaVrq0zTU1hCETpCNtjXWpmisS9FYm2LJnCZOaKyltbGWuS11dLTU09ZYS/fAKFt6hnmhb5jWxloWtDWwsLWRWQ1p6tMp6tI1zG5M095UR1tTLemaGjLZkNEgJMg7nlwNppIUQESkYsaCkL6hMXqGxkiZ0VyfprkuTf9ohm27R3ihb5gg6zTXp2iuTzM0lmXzriE29wyxbfcw3QNj7OyPrvBDj2oC7kRX9DVGJhvuc/VeTLrGmNWQZlZDLbMa0lGTj0WftWpJO29btYhlc5tpb44K5nTKqE3VEJXFUYEcupONm40On93AvJZ66tLV7U5O1aRoqE1N+fcqgIjIPsLQ6R0ao3coM15oB2HISCa6qh8aC+jPu5Lfndcc0zs4xq7BMXYOjNI/Ut7y7rMb0ixqb6KjpY4jO5qZ3VhLjRmpuJzOhpANQ1I1NSyZ08jSOU0saG2Mt0VX+w210ZV+Q11N9G9titqUxg1NJgUQkRlmJJMlCJ3muhRm0VXz0FjAroExegbHxgv6gdGA0UyWkSBk93CGLT1DdPZGTSe7BsfIvoS2nOa6FLPj5pg5zXWcsKiVuc11zG2uo725jvamOkJ3BkcDBkYDWurTLGhrZGFrA3XpGgZGAwZHszTU1rBsTjOtTbWV+nlkEimAiEwz7k7P4BizGmrHm0bC0NnaN8z6rgH6hscYHI1qAcNjISNB1L7f2TvMs139bO4Zwh1qDJrr0wRZZzhTvJmnNmUsbGtkcXsjZx4zj3mz6pnXUk97cx2puAO4xqChNkVTXZrG2hSzG9PMbqilpSGtK/tDlAKISAXlgkHv0BhjgZPJhgyOBWzri9r/dw6MjnfajmSyPNc9wLM7BugfjZp+OlrqmNNcx9beYQYT2vprU0Z9OsWC1gZOWNjK205ZRFNdiv6RqJkpVWN0tNQzt6WOOU11tDZFNYWokzdFQ20NDekUNVPQ6SoziwKISAm5kT4DIwFB3L6+s3+UzT1RZ2+und8MRjMhuwZH2TkwRnf/KF39I2SyyU1BsxvSpOOr91SNcURHM287NerI7R/JsGPPCDsHxjjjyA6OPnwWRx/ewtyWeprrX6wJTMVoG5FCFEDkkDQ4GjA4Go3TH8pEN2p19kbDOHcPZ8bb6l/oG2ZLz1Di1X9uVA+AA3WpGua21NPRUseKjmYOn93A4bPrmdNcR326htpU1KG7oK2RBa0NVRk5IzJZFEBkRtqxZ4SHNvXy9Pb+8fHxQ5ksG7oHWd89QHf/aMH31aaM1sba8eGmC9saOf2IuSyb20RbUy2pmmjYZ1tTbTzyp2G8BiFyqFEAkRlhcDTg1+t38qunu/j1+p109g4DUbNSbaqGdI1Rn65h2dxmXnv0PI6Y18zshtrxPoD5sxtY3N7EYbPq1Rcgsp8UQOSglMmGPNa5m/99bif/s34XD27qZSwb0lKf5pVHzeU9Zyzn5cvaOX5ha9Vv8hKZqRRAZNpzd17YPcKjW/p4ZEsfD2/u5bHO3YwGIQDHLZjNRWcs43XHHMbq5XMUMESmiAKITBuDowEPbOzh3ud72NI7xMBI1JG9adcQOweiPou6VA3HL5rNu09fxsuXtXP6EXOZ01xX5ZyLHJoUQKQqBkcDfvF0F09t28PmXUNs3DXI77b3E4RObcpY0t7ErIY0zfVpXnN0B6uWtHHS4jZetmAW9WmNXBKZDhRAZMqMBSG/eW4ntz7yAj97fDvDmSzpGmNxeyNL5zbzwdfM44wj5/LyZe001elPU2S60/9SqRh3p7N3mIe39HHnkzv41dNd9I8GzG5I89ZTFvG2UxZx6tI2DYMVOUgpgMikCkPnrme6uOm+LTy8uZddg2MAtDfVcs4J8zn7+Pm8amWHbqATmQEUQGS/BdmQrv5Rtu8ZYdfAGLsGRtkzkqEuVUNTXZqB0YAb79vEc92DzJ/dwOuOPYyTl7Rx8uJWjlswWzUNkRlGAUQS7R7KcNczXfziqS7WbexhR/9oySm+T1g0m69esIpzT1ygGVpFZjgFEBn34KZebvjfjWzbPcKOPdHcUNnQmdtcxxlHdbBsThML4zmcOlrqmdNSR2tjLWNByHAmSxg6i9sbx9egEJGZTQFExn3xZ0/z+NbdHL+wlZMWt3HeyQt57TGHsWpJW/EZX+unLo8iMn0ogBxCdg9n+PkT2/mv327jpMVt/O0bjx7ftnNglAc29vDh1x3F377pmCrmUkQOFgogM1AmG7Jh5yDP7Ohn066h8Rv1Ht7cx1g2pKG2ht+s38V7zlg+fhf3HU/uIHQ454QFVc69iBwsFECmscHRgK19w4xmQjJhyFgQsnNglK49o+wcGCWTDcmGkA1DeocydPWP0NU/ypaeob0WMepoqWfpnEbeffoy/uDkBTTVpTn7K/fww4c6+cCrjwDgZ49vZ9ncJl62YFa1DldEDjIKINPIWBDy40e28qOHtvJc9wBdCWtWQLR6XV2qJl6vGtqb65jXUs+x82dx9vHzOfrwFlYeNosj5jUXvKv71KVt3HT/Zt7/qhXsGQn4zXM7ed8rV6gDXET2mwJIFWzYOcj1v9nII1v6OOqwFo6dPwt3uO5/NrBt9wgrD2vhNUfPY0VHM0vmNNFYmyKdigLGnOY6Dp/dQFtj7QGtW3HBmqVc+v3HeGBjL529UY3lnBPmT+JRishMpwAyhe7f0MM3736OXz7dRW3KWLWkjbuf6eb7D3YCsGb5HD73Rydy5tHzKl4TeMtJC/jMT57klvs3MzAaMH92Aycvbqvod4rIzKIAMgUe6+zjSz9/hnue6aajpY6/esNK3nX6Ug6b1QBEI6B2D2c4cl7LlOWpqS7NH65aOB683rFmqVbiE5GXpCoBxMz+CfgDYAx4Dnivu/fF2y4D3g9kgY+4++1x+jnAV4EUcK27f74aed8f7s4zOwb41e+6+OXTXdy/oYf2plouP/dY/uz05TTW7T0PVEdLPR0tU38zxTvWLOXG+zYDcPbxar4SkZemWjWQO4DL3D0wsy8AlwEfM7PjgAuA44GFwJ1mlrtZ4RvAG4FO4AEzW+vuT1Yh7/vYM5Lh1oe38tDmPp7fOciG7gH2jARAtFre/z37GC58xTJmNdRWOad7O2FRKycuauWFvmHWrJhT7eyIyEGmKgHE3X+e9/Je4Pz4+XnALe4+Cmwws/XAmnjbend/HsDMbon3rVoAcXce2dLHzfdv5iePbmM4k2VhawNHzGvhvFWLOH7hbM485jDmtzZUK4v75V/fcQr9I0HxO81FRAqYDn0g7wO+Ez9fRBRQcjrjNIAtE9JPK/RhZnYxcDHA0qVLJzWjADv2jPD9Bzv5wUOdPN89SFNdireespB3rlnGiYtbJ/37Km15R3O1syAiB6mKBRAzuxMo1LD+CXe/Nd7nE0AA3DhZ3+vu1wDXAKxevbr41LEvUVf/CGd/5R76hjKsWTGHP3/NEZx74oJp1zQlIjIVKhZA3P2sYtvN7D3AW4A3uHuuoN8KLMnbbXGcRpH0KfPZnz7F0GiWn/7lqzhh0cFX2xARmUxVWbAhHlF1KfCH7j6Ut2ktcIGZ1ZvZCmAlcD/wALDSzFaYWR1RR/vaqczzPc90s/bRF/iLM49U8BARoXp9IF8nmgT8jviGuXvd/UPu/oSZfZeoczwALnH3LICZfRi4nWgY73Xu/sRUZXYkk+WTP36cIzqa+Yszj5yqrxURmdaqNQrrqCLbrgKuKpB+G3BbJfOV5F9/+Sybe4a46YOnaS1vEZGY1hwtIQydb//3Bv7g5IWccWRHtbMjIjJtKICUMJaNplE/dr6mORcRyacAUkIQRgPEalO60U5EJJ8CSAlBNgQgXaOfSkQkn0rFEnIr+6kGIiKyNwWQEoIwroGk9FOJiORTqVhCENdA0ppsUERkLwogJWRyfSBqwhIR2YsCSAm5UVjqRBcR2ZtKxRICdaKLiBSkAFLCeCe6aiAiIntRqVhCbhiv+kBERPamAFJC7kbCWg3jFRHZi0rFEl7sRFcNREQknwJICS8O49VPJSKST6ViCRqFJSJSmAJICRqFJSJSmErFEjSZoohIYQogJWgyRRGRwlQqlpDRZIoiIgUpgJTwYie6fioRkXwqFUt4sQlLNRARkXwKICWMd6JrFJaIyF5UKpYQaD0QEZGCFEBKyE1lklInuojIXhRASlAnuohIYSoVSwjCEDPVQEREJlIAKSGTdXWgi4gUoJKxhCAbqgNdRKQABZASgtB1F7qISAEKICVksqE60EVEClDJWEKQdTVhiYgUoABSQiYMtRaIiEgBKhlLCLKutUBERApQACkhCEOtBSIiUoBKxhIyWY3CEhEpRAGkhECjsEREClLJWEIQahSWiEghCiAlZLKhpjIRESmgKiWjmX3GzB4zs0fM7OdmtjBONzP7mpmtj7efmveei8zs2fhx0VTlVfeBiIgUVq1L639y95PcfRXwU+Af4vQ3Ayvjx8XA1QBmNge4AjgNWANcYWbtU5HRTOiaiVdEpICqBBB335P3shnw+Pl5wA0euRdoM7MFwNnAHe7e4+69wB3AOVORV3Wii4gUlq7WF5vZVcCFwG7gdXHyImBL3m6dcVpSesVlNZmiiEhBFbu0NrM7zezxAo/zANz9E+6+BLgR+PAkfu/FZrbOzNZ1d3cf8OdpMkURkcIqVgNx97P2c9cbgduI+ji2Akvyti2O07YCZ05Ivyvhe68BrgFYvXq1F9rnpdAwXhGRwqo1Cmtl3svzgKfj52uBC+PRWKcDu919G3A78CYza487z98Up1VckHVNpigiUkC1+kA+b2bHACGwCfhQnH4bcC6wHhgC3gvg7j1m9hnggXi/K929ZyoyGjVhqQYiIjJRVQKIu/9xQroDlyRsuw64rpL5KkRNWCIihaltpoRMVuuBiIgUopKxBK0HIiJSmAJICVoPRESksJIlo5mdYGY35O6tMLPrzeykqchctbk7maxTqxsJRUT2UTSAxDf9/Yjonov3xY+7gR/kbgicybJhdBuJaiAiIvsqNQrrSuCN7r4xL+0xM/slcGv8mLGC8QCiGoiIyESlLq3TE4IHAHFabSUyNJ1ksiGA1gMRESmgVMkYmNnSiYlmtgwIKpOl6SPIqgYiIpKkVBPWFcCdZvY54ME4bTXwceBjlczYdJAJoxqIZuMVEdlX0QDi7j82sw3AR4G/jJOfBN7u7o9WOnPV9mINRE1YIiITlZzKJA4UF05BXqad8VFYqoGIiOyj1DDeDjO7wsw+YmYtZnZ1vKbHrWZ21FRlslrGO9FVAxER2UepkvEmoJ5ojfL7gQ3A+UTrmF9b2axVn4bxiogkK9WEdbi7X25mBmxy9y/G6U+bWcFZc2eSXA1EkymKiOyrVMmYhfFp1ndO2BZWJEfTSK4TXZMpiojsq1QN5AgzWwtY3nPi1ysqmrNpIMgN41UfiIjIPkoFkPz5rr40YdvE1zNOJlcD0SgsEZF9lLoP5O6kbWb2HaKJFWcs3QciIpLsQErGV0xaLqap8TvR1QciIrIPXVoXMd6JrlFYIiL7KNqEZWanJm3iEJiNN8iqBiIikqRUJ/o/F9n29GRmZDrKhBrGKyKSpFQn+uumKiPTUaAbCUVEEpWaC+vSvOd/MmHb5yqVqelC64GIiCQrdWl9Qd7zyyZsO2eS8zLt5EZhaTJFEZF9lSoZLeF5odczTq4GktKNhCIi+ygVQDzheaHXM05uNl4N4xUR2VepUVgnm9keotpGY/yc+HVDRXM2DWgYr4hIslKjsFJTlZHpSOuBiIgkU9tMEeMrEqoJS0RkHyoZiwiyTo1BjTrRRUT2oQBSRCYMNROviEgClY5FBFnXWiAiIgkUQIoIsqqBiIgkUelYRCZ0TaQoIpJAAaSIIBtqIkURkQQqHYsIsq57QEREEiiAFBE1YeknEhEpRKVjEVETlmogIiKFKIAUkcm6RmGJiCSoauloZh81Mzezjvi1mdnXzGy9mT2Wvya7mV1kZs/Gj4umIn9BGGoUlohIglKz8VaMmS0B3gRszkt+M7AyfpwGXA2cZmZzgCuA1UTTyD9oZmvdvbeSeQyyrrVAREQSVLMG8mXgUvZeV+Q84AaP3Au0mdkC4GzgDnfviYPGHUzBioiZbKiJFEVEElSldDSz84Ct7v7ohE2LgC15rzvjtKT0Qp99sZmtM7N13d3dB5TPbKhhvCIiSSrWhGVmdwLzC2z6BHA5UfPVpHP3a4BrAFavXn1AqyZmQqdJnegiIgVVLIC4+1mF0s3sRGAF8KiZASwGHjKzNcBWYEne7ovjtK3AmRPS75r0TE8QZENNpigikmDKL6/d/bfufpi7L3f35UTNUae6+3ZgLXBhPBrrdGC3u28DbgfeZGbtZtZOVHu5vdJ51Z3oIiLJqjYKK8FtwLnAemAIeC+Au/eY2WeAB+L9rnT3nkpnRuuBiIgkq3oAiWshuecOXJKw33XAdVOULUDrgYiIFKPL6yK0HoiISDKVjkVoPRARkWQKIEVoPRARkWQqHYvQKCwRkWQKIEVkwlDrgYiIJFDpWESQda0HIiKSQAEkgbsThFoPREQkiUrHBEEYTaOl+0BERApTAEkQZKMAohqIiEhhKh0TZMIQQH0gIiIJFEASZMdrIAogIiKFKIAkGK+BqAlLRKQglY4Jcn0g6kQXESlMASSBOtFFRIpT6Zgg14SlyRRFRApTAEkwXgPRZIoiIgWpdEyQyeY60VUDEREpRAEkwfid6AogIiIFKYAkCHI1EDVhiYgUpNIxQUY3EoqIFKUAkiAYH4Wln0hEpBCVjgleHIWlGoiISCEKIAlyo7BUAxERKUylY4LcKCz1gYiIFKYAkiCjUVgiIkWpdEygPhARkeIUQBJk1YQlIlKUAkiCjIbxiogUpdIxgZqwRESKUwBJ8OJkivqJREQKUemYQJMpiogUpwCSQJMpiogUp9IxQW4yRdVAREQKUwBJEIQhqRrDTAFERKQQBZAEQdY1AktEpAgFkASZrOseEBGRIlRCJgjCUHehi4gUoQCSIJN1jcASESlCJWSCIBtqBJaISBFVCSBm9ikz22pmj8SPc/O2XWZm683sd2Z2dl76OXHaejP7eKXzGISuJiwRkSLSVfzuL7v7l/ITzOw44ALgeGAhcKeZHR1v/gbwRqATeMDM1rr7k5XKXCYbUqsmLBGRRNUMIIWcB9zi7qPABjNbD6yJt6139+cBzOyWeN+KBZAg66Q0jFdEJFE1L7E/bGaPmdl1ZtYepy0CtuTt0xmnJaXvw8wuNrN1Zrauu7u77MxFo7BUAxERSVKxEtLM7jSzxws8zgOuBo4EVgHbgH+erO9192vcfbW7r543b17ZnxOErk50EZEiKtaE5e5n7c9+ZvZt4Kfxy63AkrzNi+M0iqRXhO5EFxEprlqjsBbkvXwb8Hj8fC1wgZnVm9kKYCVwP/AAsNLMVphZHVFH+9pK5jGTVROWiEgx1epE/6KZrQIc2Aj8OYC7P2Fm3yXqHA+AS9w9C2BmHwZuB1LAde7+RCUzGIROQ60CiIhIkqoEEHf/syLbrgKuKpB+G3BbJfOVL8iGpOun2yA1EZHpQ5fYCaLJFNUHIiKSRAEkQRCGmgtLRKQIlZAJgqymMhERKUYBJEEmDLUeiIhIESohE+g+EBGR4hRAEmSyrvtARESKUAmZIAi1HoiISDEKIAkCrUgoIlKUSsgEGa1IKCJSlAJIAq1IKCJSnAJIAe5ONnRSasISEUmkErKAIHQAajWMV0QkkQJIAUE2CiAaxisikkwlZAGZMARQJ7qISBEKIAWM10DUhCUikkgBpIBUjfH7Jy5gxbyWamdFRGTa0opJBbQ21vKNd51a7WyIiExrqoGIiEhZFEBERKQsCiAiIlIWBRARESmLAoiIiJRFAURERMqiACIiImVRABERkbKYu1c7DxVjZt3ApgP4iA5g5yRl52BxKB4zHJrHfSgeMxyax/0mnpXeAAAFlUlEQVRSj3mZu88rtdOMDiAHyszWufvqaudjKh2KxwyH5nEfiscMh+ZxV+qY1YQlIiJlUQAREZGyKIAUd021M1AFh+Ixw6F53IfiMcOhedwVOWb1gYiISFlUAxERkbIogIiISFkUQAows3PM7Hdmtt7MPl7t/FSKmS0xs1+Z2ZNm9oSZ/VWcPsfM7jCzZ+N/26ud18lmZikze9jMfhq/XmFm98Xn/DtmVlftPE42M2szs++b2dNm9pSZvWKmn2sz+5v4b/txM7vZzBpm4rk2s+vMrMvMHs9LK3huLfK1+PgfM7OyV89TAJnAzFLAN4A3A8cB7zCz46qbq4oJgI+6+3HA6cAl8bF+HPiFu68EfhG/nmn+Cngq7/UXgC+7+1FAL/D+quSqsr4K/MzdjwVOJjr+GXuuzWwR8BFgtbufAKSAC5iZ5/rfgXMmpCWd2zcDK+PHxcDV5X6pAsi+1gDr3f15dx8DbgHOq3KeKsLdt7n7Q/HzfqICZRHR8V4f73Y98Nbq5LAyzGwx8PvAtfFrA14PfD/eZSYecyvwGuDfANx9zN37mOHnmmjZ7kYzSwNNwDZm4Ll293uAngnJSef2POAGj9wLtJnZgnK+VwFkX4uALXmvO+O0Gc3MlgOnAPcBh7v7tnjTduDwKmWrUr4CXAqE8eu5QJ+7B/HrmXjOVwDdwP+Lm+6uNbNmZvC5dvetwJeAzUSBYzfwIDP/XOckndtJK+MUQAQzawF+APy1u+/J3+bROO8ZM9bbzN4CdLn7g9XOyxRLA6cCV7v7KcAgE5qrZuC5bie62l4BLASa2beZ55BQqXOrALKvrcCSvNeL47QZycxqiYLHje7+wzh5R65KG//bVa38VcArgT80s41EzZOvJ+obaIubOWBmnvNOoNPd74tff58ooMzkc30WsMHdu909A/yQ6PzP9HOdk3RuJ62MUwDZ1wPAynikRh1Rp9vaKuepIuK2/38DnnL3f8nbtBa4KH5+EXDrVOetUtz9Mndf7O7Lic7tL939XcCvgPPj3WbUMQO4+3Zgi5kdEye9AXiSGXyuiZquTjezpvhvPXfMM/pc50k6t2uBC+PRWKcDu/Oaul4S3YlegJmdS9ROngKuc/erqpylijCzVwH/DfyWF/sDLifqB/kusJRoOvy3u/vEDrqDnpmdCfydu7/FzI4gqpHMAR4G3u3uo9XM32Qzs1VEAwfqgOeB9xJdRM7Yc21mnwb+lGjE4cPAB4ja+2fUuTazm4EziaZt3wFcAfyYAuc2DqZfJ2rOGwLe6+7ryvpeBRARESmHmrBERKQsCiAiIlIWBRARESmLAoiIiJRFAURERMqiACIyTZnZmbnZgkWmIwUQEREpiwKIyAEys3eb2f1m9oiZfStea2TAzL4cr0XxCzObF++7yszujddh+FHeGg1HmdmdZvaomT1kZkfGH9+St4bHjfFNYCLTggKIyAEws5cR3en8SndfBWSBdxFN3LfO3Y8H7ia6MxjgBuBj7n4S0QwAufQbgW+4+8nAGUSzx0I0Q/JfE61NcwTRXE4i00K69C4iUsQbgJcDD8SVg0aiSetC4DvxPv8J/DBek6PN3e+O068Hvmdms4BF7v4jAHcfAYg/735374xfPwIsB35d+cMSKU0BROTAGHC9u1+2V6LZ30/Yr9w5g/LnaMqi/7MyjagJS+TA/AI438wOg/F1qJcR/d/Kzfj6TuDX7r4b6DWzV8fpfwbcHa8G2Wlmb40/o97Mmqb0KETKoKsZkQPg7k+a2SeBn5tZDZABLiFasGlNvK2LqJ8Eomm1vxkHiNyMuBAFk2+Z2ZXxZ/zJFB6GSFk0G69IBZjZgLu3VDsfIpWkJiwRESmLaiAiIlIW1UBERKQsCiAiIlIWBRARESmLAoiIiJRFAURERMry/wGCuVFXyLXDyQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "ELBO = [-training_loss[i] for i in range(len(training_loss))]\n", + "plt.plot(ELBO)\n", + "plt.ylabel('ELBO');plt.xlabel('epoch');plt.title(\"training curve for mini batches\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As expected, the ELBO is monotonically increasing over epoch, and we reproduced the results given in the paper [Auto-Encoding Variational Bayes](https://arxiv.org/abs/1312.6114/). Now we can extract/load the parameters and then feed the network forward to calculate $y$ which is the reconstructed image, and we can also calculate the ELBO for the test set. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "arg_params = model.get_params()[0]\n", + "nd_iter_test.reset()\n", + "test_batch = nd_iter_test.next()\n", + "\n", + "# if saved the parameters, can load them using `load_checkpoint` method at e.g. 100th epoch\n", + "# sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, 100)\n", + "# assert sym.tojson() == output.tojson()\n", + "\n", + "e = y.bind(mx.cpu(), {'data': test_batch.data[0],\n", + " 'encoder_h_weight': arg_params['encoder_h_weight'],\n", + " 'encoder_h_bias': arg_params['encoder_h_bias'],\n", + " 'mu_weight': arg_params['mu_weight'],\n", + " 'mu_bias': arg_params['mu_bias'],\n", + " 'logvar_weight':arg_params['logvar_weight'],\n", + " 'logvar_bias':arg_params['logvar_bias'],\n", + " 'decoder_z_weight':arg_params['decoder_z_weight'],\n", + " 'decoder_z_bias':arg_params['decoder_z_bias'],\n", + " 'decoder_x_weight':arg_params['decoder_x_weight'],\n", + " 'decoder_x_bias':arg_params['decoder_x_bias'], \n", + " 'loss_label':label})\n", + "\n", + "x_fit = e.forward()\n", + "x_construction = x_fit[0].asnumpy()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAADACAYAAADhh27FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmUXWWZ7/HfAxnIPJC6SQgJCRAwYQzUihG4t72SjggyZNnQAkKiMngXAi7TKtpqR1qQq40IF5cNNpNCqyBCEIIKNAi0QFPYQYaQhEwmUCEVMlUChCS894+zo4e8z0l21Rn32d/PWrVy6tnTu8956q03u/azXwshCAAAAMijPerdAAAAAKBeGAwDAAAgtxgMAwAAILcYDAMAACC3GAwDAAAgtxgMAwAAILcYDDcAM/s3M/tavdsBlMPMxppZMLMeJZa/ZGYfrnGzABf9LrKOPrdyjOcMS2a2qejbvpK2SNqefH9hCOGO2rcKeWZmyySdF0J4uN5tScvMxkpaKqlnCGFbfVuDRke/i0ZCn5tv7v8m8iaE0H/H6zQ/EGbWg8RD1pHHqCf6XeQNOdy4uE0iBTP7tpn9wsx+Zmadkj5lZreb2eyidaYmHfqO7/c1s3vMrMPMlprZRbvY/1/2tWM/ZvbVZNvXzexkM/u4mS0ys7Vm9uWibT9kZk+b2Xozazez68ysZ9Hyj5nZQjPbYGb/z8z+08xmFi0/z8xeMbN1ZvagmY2u0NuGKklyYV7ymf/BzA4vWnaZmS02s04ze9nMphctm5l8/teY2ZuSZiexJ83sX5IcWGpmHyvaZpCZ3ZTk1mvJz8KeybI9k+3WmNkSSSftpt3LzGxq8nq2md2V5H6nmb1gZgcleb/azFaY2bSibT9tZvOTdZeY2YU77fvLSRtfT3I6mNmBybLeSTv/bGZvmNm/mlmf8j4FVBv9LhoFfW7z97kMhtObLunfJQ2S9ItdrWhme0i6X9KzkkZJ+ltJXzKz41Mea18VPpt9JP2zpJskfVLSJEkflnS5mY1J1t0m6VJJwyQdK+kESRcm7fgfku6U9KVk+VJJk4va+Ylk2amSWiQ9k5wjGpSZTZJ0swqf8d6SbpB0n5n1TlZZLOl/qpCn35J0u5mNLNrFByUtkTRc0hVFsQUq5Mh3Jd1kZpYsu1WFHDtQhfybJum8ZNn5kj6exFsl/V0XT+dkST+VNETSf0v6rQp5P0rS5cm57bA6OdZASZ+WdI2ZHZW8JydI+qKkqUk7P7zTca6SdJCkI5PloyR9s4ttRX3Q76Ku6HNz0ueGEPgq+pK0TNLUnWLflvQfO8VulzS76PupkpYlr4+VtGSn9b8h6ccljvmXfSX72SRpz+T7IZKCpKOL1n9e0sdL7OsfJN2VvP6MpCeKlpmkdkkzk+8fkjSjaHkPFe7bG1XvzyHvX14eJvEfSfrnnWILJP1Nif3Mk3Rq8nqmpD/vtHympFeLvu+b5NsIFTrvLZL6FC0/U9Kjyev/kPS5omXTkm177O6cJM2W9FDRspN3yvsByb4Gl9jXvZIuTV7fLOk7RcsOTLY9MMn5zZIOKFr+IUlL6/0Z87XrfKff5aveOZjE6XND8/e53DOc3oourLufpDFmtr4otqekx1JuvyaEsKOQ5O3k3zeKlr8tqb8kmdkHJF0t6WgVfqh6qHClQSpc4fhLu0MIwcxW7tTOH5rZtUWx91S4QvJayraitvaTNMPMLi6K9VLhs5aZnavC/9jHJsv6q3D1YQcvj1fteBFCeCu5QNFf0lBJPSW1//WihfYo2sf78kvS8i6ey8457eV9f0nrkz8j/pMKVxv2UCHXXyhqR1vRvorb1JKs+1zROZgKP49ofPS7qDf63Bz0uQyG09v5sRubVfjAdxhR9HqFpEUhhAlVb1XhzxpPS/r7EMImM/sHFf60IRWuRhTfA2Qq/LmiuJ3fCCHs8s+PaCgrJF0RQrhi5wVmtp+kH0s6XtJTIYTtZjZPhY5oh648PmaFClcphgW/6KNdUvG9jmOcdcqW/DnybknnSpoTQthqZvfqr+fVrsJAYofiNq1RoZM/JITAQCN76HdRb/S5OehzuWe4++ZJOsnMhiT3B11StOwpSe+a2Swz2yu56f0wMzu6Cu0YIGmDpM1mNkHJfWuJ+yUdZYVCkB4q3OPWUrT8XyX9Y7KdzGywmXX1HiRUT88kf3Z89VCh4/2cmX3QCvqZ2UlmNkBSPxU63g6pUAAh6dDuHjyE0C7pd5KuNrOBZraHmR1gZn+TrHKnpEusULQ0RNJlZZzrrvSS1FuF89qWXLGYVrT8TkmfNrMJZtZXhT+N7ziH91R4z65J7uWUmY0ys49Wqa2oLvpdVBN9bkHu+lwGw913q6T5KvyZ4jeSfr5jQfI/uhNVKJpYpsL/lG5Q4Ub0SpslaYakzuQYf7naEEJ4Q9LfS/q+pDclHaDCTfNbkuV3JcvuMrONkv4kqaETNmfmqvA/7B1fs0MIbSoUUVwvaZ2kV1W4B00hhJdV+NPtUyr8OewwSf9ZZhvOVaFjfDk53i8l7SgO+bEKBRjPS/qjpF+VeSxXCKFThUHPnUkbzpJ0X9HyByVdJ+lRFd6Pp5NFW5J/v7IjnuT5w5IOrkZbUXW3in4X1UOfq3z2uUy6kSNWeDzL65L+LoTwRL3bA1RDcsXtRUm9S/ypEagZ+l00u2boc7ky3OTM7ITkz3C9VfhTxlZJ/1XnZgEVZWbTrfBsyyGS/q+kX2e1U0b20e+i2TVbn8tguPkdp8IzDjtU+FPc9BDCll1vAmTOhSo8F3OxClP6/p/6Ngc5R7+LZtdUfS63SQAAACC3uDIMAACA3CrrOcNWmJLvWhUepvxvIYSrdrX+sGHDwtixY8s5JKDnnntuTQihZfdrVg65i3ItW7ZMa9assd2vWTnkLSqBPhdZlTZ3uz0YTipkf6jC/O8rJT1rZvcljxpxjR07Vm1tbaUWA6mYWVdn3SkbuYtytba21vyY5C0qgT4XWZU2d8u5TWKyCvNrLwkhvKvC8x5PLWN/AAAAQE2VMxgepffPR71S759yUpJkZheYWZuZtXV0dJRxOKC2yF1kEXmLrCJ3US9VL6ALIdwYQmgNIbS2tNT0liOgLOQusoi8RVaRu6iXcgbDr0kaXfT9vkkMAAAAyIRyBsPPShpvZuPMrJekT6po7moAAACg0XX7aRIhhG1m9nlJv1Xh0Wo3hxBeqljLAAAAgCor6znDIYS5kuZWqC0AAABATTEDHQAAAHKrrCvDAAAA+KsQQr2b8D5mNZ34MpO4MgwAAIDcYjAMAACA3GIwDAAAgNxiMAwAAIDcYjAMAACA3OJpEgAAAN1QjSdH8PSH2uPKMAAAAHKLwTAAAAByi8EwAAAAcovBMAAAAHKLAjoAAIBdqOUUy2mPRaFd5XBlGAAAALnFYBgAAAC5xWAYAAAAucVgGAAAALlVVgGdmS2T1Clpu6RtIYTWSjQKQP7UskCl0ihkAbIpbb9Taj0vvn379lSxUrz+pEePeLjmrefFSrX9vffeS7XuHnvE1029WFf6wUbrMyvxNIn/HUJYU4H9AAAAADXFbRIAAADIrXIHw0HS78zsOTO7wFvBzC4wszYza+vo6CjzcEDtkLvIIvIWWUXuol7KHQwfF0I4StLHJF1kZv9r5xVCCDeGEFpDCK0tLS1lHg6oHXIXWUTeIqvIXdRLWfcMhxBeS/5dbWb3SJos6fFKNAxoduUUjHWlmMMrkvBUo6DBa081iizqXbhR7nki5r2nb775ZhSbM2eOu/0ll1wSxd56661Ux+7Xr18Uu+6669x1zz333CjmFTuh/rycShsrVQC3devWKLZhw4ZUsVL9eO/evVPFevXqFcW8fqfU7wAv3rNnzyjWt2/fVMf2iupKtanRdPvKsJn1M7MBO15LmibpxUo1DAAAAKi2cv77OlzSPcmIv4ekfw8h/KYirQIAAABqoNuD4RDCEklHVLAtAAAAQE3xaDUAAADkVm7u8n/66aej2LXXXuuuO2rUqCjWp0+fKDZjxowoNnToUHefpeLIB69QwYt5RRpbtmyJYp2dne5xNm3alGr7rhSIpG3TO++8E8XefffdKLbnnnu6x/EKMrxijr322iuKjRgxIooNGzbMPY63z7QzOaE6lixZEsW+/e1vR7HbbrutrOOUKvDZ2dtvvx3Fzj//fHfdhQsXRrErr7yy28dGZaQtjEvbN3v9myStWLEiij377LNRbOXKlVHM6/MkaeDAgVFs7733jmLeuMIr3vT6Ycnv271+c/To0VFs8ODBUSzLOZ7dlgMAAABlYjAMAACA3GIwDAAAgNxiMAwAAIDcYjAMAACA3MrN0yS8Jz8sWrSorH1eccUVUWzQoEHuulOmTCnrWLUwduzYKPbVr37VXXfMmDFVbk02lZr2ctu2bVHMm8Zz3bp1UeyVV16JYk8++aR7nD//+c9RbOPGjamOU2qq2s2bN6eKlTPVreRXUHtPfvB+xiZPnhzFZs6c6R7Hy12mz62NUtPPfulLX4pi9957b8WP7z3JxIuVqr73fO9734tixxxzTBQ75ZRTUu8T6XVlavq0T5Pwnpbj9a2SdOedd0axF154IYp5fZn3O1fyn9Tg9ZvePr0noZRqu/d7YMiQIVHMe+qF155S/WgWnszDlWEAAADkFoNhAAAA5BaDYQAAAOQWg2EAAADkVm6qRrxijHnz5rnrHnLIIVHspZdeimLPPPNMFJszZ467z9/+9rdRbNy4cVFs6dKl7vZpeTewjxw5Mop5U0h6St3g/5WvfKVL7cqLUsUc3rSXXsGZV+jwu9/9Loo9+uij7nHa29tTHSftVNCleNt75967d+8o5k2nLElr166NYhs2bIhiXju9IsHjjjvOPY433ToFdJXn5cMNN9zgrpu2WK5Pnz5R7NBDD3XXnTVrVhT7yEc+EsW8aW4vu+yyKOYVypVyxx13RLGPfvSjUcz7+UD1eP2WV9zsFZbNnTvX3ef9998fxbwitkmTJkWxAw880N2nl9NeUZ1X2Ob1owsWLHCP4z1EwOufW1paotg+++yTatus4MowAAAAcovBMAAAAHKLwTAAAAByi8EwAAAAcmu3VSNmdrOkj0taHUI4NIkNlfQLSWMlLZN0RgghvuO8gUyYMCFVrJTDDz88ip155plR7KqrrnK3X7ZsWRTzCuiWLFmSuk0e74Z6r4DOO3ZHR0cU+8AHPlBWe1CaV8yxxx7x/0+9Ipthw4a5+/SKlrziMK8YwyuSkPwCI2/WLm+WIW+2OG+GI8kvaH3ggQeimFcgkrZIsFqyMMNSrXnv/0UXXZR6ey/HvvnNb0axL3/5y11rWAqzZ8+OYt5MY5K0fPnyKPbLX/4yinkz0J199tldbxxSSTvbnDfboPf7+vHHH3eP4xU9jx49OopNnDgxinkzZ0p+/963b98o5vU7XSmEXr16dRTzZt8bMWJEFPMKAgcMGOAex/u91mh9Zporw7dKOmGn2GWSHgkhjJf0SPI9AAAAkCm7HQyHEB6XtPOlmFMl3Za8vk3SaRVuFwAAAFB13b1neHgIYccDTVdJGl5qRTO7wMzazKzN+zM80KjIXWQReYusIndRL2UX0IXCjTn+TAOF5TeGEFpDCK2l7kkEGhG5iywib5FV5C7qpbvTLr1hZiNDCO1mNlJSfBd2TpWagSVtIVpXivrS8mbKW7NmTRT74Ac/GMWmTZtW8fbkkVdA4BVEeLOjHX/88anWk6TNmzdHMa8Yw8vHUr98vHam5RXvebMzSdLWrVuj2G9+85so5r2Xw4fHf5zyij6k9MV/qK/LL788ilWjWM7j9eN33323u25ra2uqfV555ZVR7NRTT41i/fv3T7U/7Fran2mv3/EK2UvNDtuzZ88otv/++0exD33oQ1HM67ckPwe8fittsZxXFCdJ69evj2LerJ/eeKGzszOKlfod4v0e8Aoc69kPd/fK8H2SZiSvZ0jy5yAGAAAAGthuB8Nm9jNJT0k62MxWmtlnJV0l6W/NbJGkqcn3AAAAQKbs9jaJEEL8MN2C+G+3AAAAQIYwAx0AAAByi8EwAAAAcqu7T5NAg/KeJjB9+vQo5k1L+YMf/CCK9enTpzINywnvSQeSXwnsTZ3tTX3sPc1hzJgx7nG86uKBAwdGMW/aTK/iV0pf4Zt2+tNSlc0rV66MYl4+e+/HUUcdFcW8acgl/zx5mkTlvfjii6nX9abovvjiiyvZnC7xcvSss84qa5+vvPJKFPvud78bxbynaKB6vD7Km/J927Zt7vZeP+7lsxfr3bu3u89Sv0fStGnjxo1RzHtqhCStW7cuinlT23uxd955J4qVerqF97vBi3lq1TdzZRgAAAC5xWAYAAAAucVgGAAAALnFYBgAAAC5RQFdk7n11luj2KpVq6KYV6i13377VaNJuVLqZv+0BRFeoZ0XK1XY6BUleAVj5U5JnLb4wSuoKDWt6R/+8Ico9u6770axww8/PIqdeOKJUWzw4MHucSiWq42XX3459br1LGr0iotmzpwZxRYuXFjxY3v99ezZs9110/YhedOVPPHW9fpCr6jOW0/y+7hNmzZFMe/3sFfcLPn9u9cmb5rk+fPnR7H29nb3OB0dHVHM69u9Yjmvby5VZOi9R+X+fFe6f+CnCwAAALnFYBgAAAC5xWAYAAAAucVgGAAAALlFAV1GLV682I1/8YtfTLX9U089FcVGjBhRVptQmnezf9qCGG+9UsUcaY9dDV6Bx4YNG6LYnDlz3O2XLFkSxbwCE69Y7uCDD45iPXv2dI+D2mhtbU29rlfI85Of/CSKfe5zn0u9T68QyCs4+vrXvx7Ffv3rX6c+TjnOOOOMKEaBZ/V4763XTwwbNiyKlSp282are+mll6LYgw8+GMVeffVVd5977bVXFNu6dWsU6+zsjGLez9Ly5cvd43jFo9579Pbbb0cxb3bQUrOLejPteb/XvBgz0AEAAABVxmAYAAAAucVgGAAAALnFYBgAAAC5tdsCOjO7WdLHJa0OIRyaxGZLOl/Sjju1vxZCmFutRiJWqsDDu8n+9NNPj2L7779/xduErimnMKBUAV3ameEqva3kzz70wgsvRLHHHnvM3d6bpWjy5MlR7OSTT45igwYNimKlChQpUKqNcvuYW265JYqNHz8+ih1wwAHu9tdff30Uu+aaa8pqUzm8HD3nnHOiGPlZPV6f0KtXryh26KGHpopJfgGdV6jp9XsLFixw9zlgwIAo5hWh9e3bN4p551NqZjhvZjkv/7y+2Suq88Yfkl9cXe7vm0pLc2X4VkknOPFrQghHJl8MhAEAAJA5ux0MhxAelxT/1wcAAADIuHLuGf68mf3JzG42syGlVjKzC8yszczavOffAY2K3EUWkbfIKnIX9dLdwfCPJB0g6UhJ7ZKuLrViCOHGEEJrCKG1paWlm4cDao/cRRaRt8gqchf10q0Z6EIIb+x4bWY/lnR/xVqEiHdT+j333OOu691k/53vfCeKdWUGM5TPK0rwCgjSFs+UKj5Ie5y025ba3iuIWLVqVRS7/fbbo9iKFSvc44wcOTKKfeITn4hi48aNi2LeLFIUItWXV6x09dX+dZNZs2ZFsba2tig2bdq08hvWTUcccYQbf/7551NtP3Xq1NT7RPnSzvrZo0c8DBo7dmwUO+2009zjeL9zX3nllSjmFat5hWml2uT952D06NGp2vPmm2+6x0k745s3I16fPn2iWJbHFd26Mmxmxb+1pkt6sTLNAQAAAGonzaPVfibpw5KGmdlKSf8k6cNmdqSkIGmZpAur2EYAAACgKnY7GA4hnOmEb6pCWwAAAICaYgY6AAAA5BaDYQAAAORWt54mgdq66ab4rpQnnnjCXfess86KYky93JjKedpBV578UM56pdZ96623otgdd9wRxR5++OHUxzn++OOjmFeB701BmrYqGrXjvf+XXHKJu6431e3Pf/7zVDFvWthSxx8+fHgU86b87spTUA455BA3vrNLL7001XqoHi8nvCcgDBw4MIpNmTLF3aeXU4sWLYpi3hMdSvVRw4YNi2LeU3S89To7O6PY0qVL3eP069cvinnvx5Ah8VQS3nvkPdWn1D4brX/myjAAAAByi8EwAAAAcovBMAAAAHKLwTAAAAByiwK6BjNv3rwodvHFF0exwYMHu9tffvnlFW8Tmk9Xpm3etm1bFHvuueeimFdAt379+ih29NFHu8c555xzopg3BWkWijHg8wodJb9Q0ot94xvfiGKvvfaau09vSluvWC6tMWPGuPEDDzwwim3YsCGKHXbYYd0+NqrHy0mvEGzQoEHu9hMnToxi3nTOW7ZsiWLetPaSP/1x2sJhbz2vAE7yC+i86Zy9ftgroPO2LdXORit65sowAAAAcovBMAAAAHKLwTAAAAByi8EwAAAAcosCujryZk4688wzo9j27duj2Nlnn+3uk9nm8i1tAYJXLFeqmGPVqlVR7Prrr49i3ixHXiHI9OnT3eMcdNBBUcwrgvLOkQK6fNhvv/1SxaqhVIGp1z97RUxewREak9efeH2RlL4Ar1T/mvb43nHS7tMr3it1HC9PvVifPn2iWLkz0NWzb+fKMAAAAHKLwTAAAAByi8EwAAAAcovBMAAAAHJrtwV0ZjZa0k8kDZcUJN0YQrjWzIZK+oWksZKWSTojhLCuek3NNu9G95NOOimKLViwIIpNmDAhin3rW9+qTMOQS14xkFfQKUl33XVXFPv9738fxbZu3RrFvNnmTjnlFPc43uxF5RQEUlSHSlq8eLEb9wpHb7jhhmo3BzVWqj9JO5NaVwro0vL6vc2bN0exdev8oZlX8Ja2WC5tcXOpeKP1z2muDG+TNCuEMFHSFEkXmdlESZdJeiSEMF7SI8n3AAAAQGbsdjAcQmgPIfwxed0pab6kUZJOlXRbstptkk6rViMBAACAaujSPcNmNlbSJEnPSBoeQmhPFq1S4TYKb5sLzKzNzNo6OjrKaCpQW+Qusoi8RVaRu6iX1INhM+sv6W5JXwghbCxeFgo3rrhPJA8h3BhCaA0htLa0tJTVWKCWyF1kEXmLrCJ3US+pZqAzs54qDITvCCH8Kgm/YWYjQwjtZjZS0upqNbIZrF27Noo99thjqbb96U9/GsWGDh1abpOQE16RhVfsNn/+fHf7uXPnRrFNmzZFscGDB0exT33qU1Fsn332cY+TdpYioB66UhT3wAMPRLHzzjuvks1Bg0jbR3mFdl3hFeB5sXfeeSeKlWqjV7Ts9cOetIWDu4p3d71q2O2nY4XW3SRpfgjh+0WL7pM0I3k9Q9KcyjcPAAAAqJ40V4aPlXSOpBfMbF4S+5qkqyTdaWaflbRc0hnVaSIAAABQHbsdDIcQnpRU6tr18ZVtDgAAAFA7zEAHAACA3GIwDAAAgNxK9TQJpLdhwwY3PmXKlFTb33777VFs0qRJZbUJ+eE9OWL79u1RzMvTxx9/3N3nihUrolj//v2j2LRp06KYN/Vyr1693ONUupLYey9K4akVqKQnn3wyiq1fvz6KeU9gQfaV25+UM7289zSIUk+I8J5G4U2z7D05oiv9aznnUytcGQYAAEBuMRgGAABAbjEYBgAAQG4xGAYAAEBuUUBXYbfccosbX7JkSartjzvuuCjWaDeao/5KFS94BRFbtmyJYsuXL49iCxcudPfpTdl52GGHRbHTTz89ig0ZMiSKlTstaTWKMbJQ4IHsWLt2bRTzfr4mT55ci+agAZTbn3h9u9dvbd26NYqV6nP33nvvKDZw4MAo5hXgecfuSlFdo+HKMAAAAHKLwTAAAAByi8EwAAAAcovBMAAAAHKLAroyLFq0KIrNnj279g1BJqUtNuhKoYJXPOHNfPX8889HsXXr1rn79IrgjjjiiCh28MEHR7GuzFzkFZhQ2AYA6fXs2TOKjRkzxl1348aNUaylpSWKjRs3Lop5hdWlCvWy0GdzZRgAAAC5xWAYAAAAucVgGAAAALnFYBgAAAC5tdsCOjMbLeknkoZLCpJuDCFca2azJZ0vqSNZ9WshhLnVamgjeuKJJ6KYd0N6KRMmTIhiffr0KatNyI60BWNebNu2be4+N2/eHMVWrFgRxRYvXhzFNm3a5O6zb9++Ucybucgrnkg7a1K9ZaHAA/U1ffp0N37PPfdEsbfeeiuK7bvvvhVvE/LD66N69IiHcMOHD49iU6dOdfd51FFHRbH+/ftHsdGjR0exAQMGRDFvpjrJb3uj9blpniaxTdKsEMIfzWyApOfM7KFk2TUhhH+pXvMAAACA6tntYDiE0C6pPXndaWbzJY2qdsMAAACAauvSPcNmNlbSJEnPJKHPm9mfzOxmM4sfRlrY5gIzazOzto6ODm8VoCGRu8gi8hZZRe6iXlIPhs2sv6S7JX0hhLBR0o8kHSDpSBWuHF/tbRdCuDGE0BpCaPUe5gw0KnIXWUTeIqvIXdRLqsGwmfVUYSB8RwjhV5IUQngjhLA9hPCepB9Lmly9ZgIAAACVl+ZpEibpJknzQwjfL4qPTO4nlqTpkl6sThObwzHHHBPFHnrooSjG0ySwM6/qttS0l95UnN50ygcddFDq4/fr1y+KHXvssVFs8ODBUaxUOz1pp14ud4rmRqtiRjaUqsh//fXXa9wS5JHXb3lPb/Ce/rP//vu7+/SmWfaeUOEdx4uV6luz0OemeZrEsZLOkfSCmc1LYl+TdKaZHanC49aWSbqwKi0EAAAAqiTN0ySelOQN63P1TGEAAAA0H2agAwAAQG4xGAYAAEBupblnGCV85jOfSRUD0ko7bWWpggRvikyvWG78+PFRzJs6WUo/fXK5bS93ewBoVmkL6EpNiVzpYzcbrgwDAAAgtxgMAwAAILcYDAMAACC3GAwDAAAgtyxtcUxFDmbWIWl58u0wSWtqdvDq43xqZ78QQk0nri/K3UZ+X7qD86mdeuat1NjvTXdwPrVD7lYW51M7qXK3poPh9x3YrC2E0FqXg1cB55MPzfa+cD750WzvDeeTH8323nA+jYfbJAAAAJBbDIYBAACQW/UcDN9Yx2NXA+eTD832vnA++dFs7w3nkx/N9t5wPg2mbvcMAwAAAPXGbRIAAADILQbDAAAAyK2aD4Zu3yMbAAACn0lEQVTN7AQzW2Bmr5rZZbU+frnM7GYzW21mLxbFhprZQ2a2KPl3SD3b2BVmNtrMHjWzl83sJTO7NIln9pyqhdxtLORuOlnPW6m5cpe8TS/rudtMeSs1d+7WdDBsZntK+qGkj0maKOlMM5tYyzZUwK2STtgpdpmkR0II4yU9knyfFdskzQohTJQ0RdJFyWeS5XOqOHK3IZG7u9EkeSs1V+6Styk0Se7equbJW6mJc7fWV4YnS3o1hLAkhPCupJ9LOrXGbShLCOFxSWt3Cp8q6bbk9W2STqtpo8oQQmgPIfwxed0pab6kUcrwOVUJudtgyN1UMp+3UnPlLnmbWuZzt5nyVmru3K31YHiUpBVF369MYlk3PITQnrxeJWl4PRvTXWY2VtIkSc+oSc6pgsjdBkbultSseSs1wedM3u5Ss+ZuU3zOzZa7FNBVWCg8qy5zz6szs/6S7pb0hRDCxuJlWT0ndE1WP2dyF1n8nMlbZPVzbsbcrfVg+DVJo4u+3zeJZd0bZjZSkpJ/V9e5PV1iZj1VSOw7Qgi/SsKZPqcqIHcbELm7W82at1KGP2fyNpVmzd1Mf87Nmru1Hgw/K2m8mY0zs16SPinpvhq3oRrukzQjeT1D0pw6tqVLzMwk3SRpfgjh+0WLMntOVULuNhhyN5VmzVspo58zeZtas+ZuZj/nps7dEEJNvySdKGmhpMWS/rHWx69A+38mqV3SVhXuYfqspL1VqKBcJOlhSUPr3c4unM9xKvxJ40+S5iVfJ2b5nKr4XpG7DfRF7qZ+nzKdt8k5NE3ukrddeq8ynbvNlLfJ+TRt7jIdMwAAAHKLAjoAAADkFoNhAAAA5BaDYQAAAOQWg2EAAADkFoNhAAAA5BaDYQAAAOQWg2EAAADk1v8HLgAFn4S3bUIAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# learning images on the test set\n", + "f, ((ax1, ax2, ax3, ax4)) = plt.subplots(1,4, sharex='col', sharey='row',figsize=(12,3))\n", + "ax1.imshow(np.reshape(image_test[0,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax1.set_title('True image')\n", + "ax2.imshow(np.reshape(x_construction[0,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax2.set_title('Learned image')\n", + "ax3.imshow(np.reshape(image_test[99,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax3.set_title('True image')\n", + "ax4.imshow(np.reshape(x_construction[99,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax4.set_title('Learned image')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('loss', 140.17346005859375)]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# calculate the ELBO which is minus the loss for test set\n", + "metric = mx.metric.Loss()\n", + "model.score(nd_iter_test, metric)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. All together: MXNet-based class VAE" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from VAE import VAE" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One can directly call the class `VAE` to do the training:\n", + "\n", + "```VAE(n_latent=5,num_hidden_ecoder=400,num_hidden_decoder=400,x_train=None,x_valid=None,\n", + "batch_size=100,learning_rate=0.001,weight_decay=0.01,num_epoch=100,optimizer='sgd',model_prefix=None,\n", + "initializer = mx.init.Normal(0.01),likelihood=Bernoulli)```\n", + "\n", + "The outputs are the learned model and training loss." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:Epoch[0] Train-loss=383.478870\n", + "INFO:root:Epoch[0] Time cost=5.075\n", + "INFO:root:Epoch[1] Train-loss=211.923867\n", + "INFO:root:Epoch[1] Time cost=4.741\n", + "INFO:root:Epoch[2] Train-loss=206.789445\n", + "INFO:root:Epoch[2] Time cost=4.601\n", + "INFO:root:Epoch[3] Train-loss=204.428186\n", + "INFO:root:Epoch[3] Time cost=4.865\n", + "INFO:root:Epoch[4] Train-loss=202.417322\n", + "INFO:root:Epoch[4] Time cost=4.606\n", + "INFO:root:Epoch[5] Train-loss=200.635136\n", + "INFO:root:Epoch[5] Time cost=4.711\n", + "INFO:root:Epoch[6] Train-loss=199.009614\n", + "INFO:root:Epoch[6] Time cost=5.159\n", + "INFO:root:Epoch[7] Train-loss=197.565788\n", + "INFO:root:Epoch[7] Time cost=4.588\n", + "INFO:root:Epoch[8] Train-loss=196.524507\n", + "INFO:root:Epoch[8] Time cost=4.905\n", + "INFO:root:Epoch[9] Train-loss=195.725745\n", + "INFO:root:Epoch[9] Time cost=4.426\n", + "INFO:root:Epoch[10] Train-loss=194.902025\n", + "INFO:root:Epoch[10] Time cost=4.685\n", + "INFO:root:Epoch[11] Train-loss=194.026873\n", + "INFO:root:Epoch[11] Time cost=4.622\n", + "INFO:root:Epoch[12] Train-loss=193.350646\n", + "INFO:root:Epoch[12] Time cost=4.712\n", + "INFO:root:Epoch[13] Train-loss=192.737502\n", + "INFO:root:Epoch[13] Time cost=4.618\n", + "INFO:root:Epoch[14] Train-loss=192.338165\n", + "INFO:root:Epoch[14] Time cost=4.763\n", + "INFO:root:Epoch[15] Train-loss=191.888625\n", + "INFO:root:Epoch[15] Time cost=5.168\n", + "INFO:root:Epoch[16] Train-loss=191.170650\n", + "INFO:root:Epoch[16] Time cost=4.809\n", + "INFO:root:Epoch[17] Train-loss=190.307264\n", + "INFO:root:Epoch[17] Time cost=4.622\n", + "INFO:root:Epoch[18] Train-loss=188.988063\n", + "INFO:root:Epoch[18] Time cost=4.543\n", + "INFO:root:Epoch[19] Train-loss=187.616311\n", + "INFO:root:Epoch[19] Time cost=5.154\n", + "INFO:root:Epoch[20] Train-loss=186.352783\n", + "INFO:root:Epoch[20] Time cost=4.661\n", + "INFO:root:Epoch[21] Train-loss=185.428020\n", + "INFO:root:Epoch[21] Time cost=5.193\n", + "INFO:root:Epoch[22] Train-loss=184.543097\n", + "INFO:root:Epoch[22] Time cost=4.519\n", + "INFO:root:Epoch[23] Train-loss=184.029907\n", + "INFO:root:Epoch[23] Time cost=4.732\n", + "INFO:root:Epoch[24] Train-loss=183.643270\n", + "INFO:root:Epoch[24] Time cost=5.011\n", + "INFO:root:Epoch[25] Train-loss=183.246912\n", + "INFO:root:Epoch[25] Time cost=4.706\n", + "INFO:root:Epoch[26] Train-loss=183.065233\n", + "INFO:root:Epoch[26] Time cost=4.673\n", + "INFO:root:Epoch[27] Train-loss=182.680542\n", + "INFO:root:Epoch[27] Time cost=4.628\n", + "INFO:root:Epoch[28] Train-loss=182.428677\n", + "INFO:root:Epoch[28] Time cost=4.772\n", + "INFO:root:Epoch[29] Train-loss=182.219946\n", + "INFO:root:Epoch[29] Time cost=4.571\n", + "INFO:root:Epoch[30] Train-loss=182.070927\n", + "INFO:root:Epoch[30] Time cost=4.603\n", + "INFO:root:Epoch[31] Train-loss=181.837968\n", + "INFO:root:Epoch[31] Time cost=4.559\n", + "INFO:root:Epoch[32] Train-loss=181.624303\n", + "INFO:root:Epoch[32] Time cost=5.069\n", + "INFO:root:Epoch[33] Train-loss=181.534547\n", + "INFO:root:Epoch[33] Time cost=4.654\n", + "INFO:root:Epoch[34] Train-loss=181.239556\n", + "INFO:root:Epoch[34] Time cost=4.776\n", + "INFO:root:Epoch[35] Train-loss=181.098188\n", + "INFO:root:Epoch[35] Time cost=4.571\n", + "INFO:root:Epoch[36] Train-loss=180.820560\n", + "INFO:root:Epoch[36] Time cost=4.815\n", + "INFO:root:Epoch[37] Train-loss=180.828095\n", + "INFO:root:Epoch[37] Time cost=4.455\n", + "INFO:root:Epoch[38] Train-loss=180.495569\n", + "INFO:root:Epoch[38] Time cost=5.096\n", + "INFO:root:Epoch[39] Train-loss=180.389106\n", + "INFO:root:Epoch[39] Time cost=4.797\n", + "INFO:root:Epoch[40] Train-loss=180.200965\n", + "INFO:root:Epoch[40] Time cost=5.054\n", + "INFO:root:Epoch[41] Train-loss=179.851014\n", + "INFO:root:Epoch[41] Time cost=4.642\n", + "INFO:root:Epoch[42] Train-loss=179.719933\n", + "INFO:root:Epoch[42] Time cost=4.603\n", + "INFO:root:Epoch[43] Train-loss=179.431740\n", + "INFO:root:Epoch[43] Time cost=4.341\n", + "INFO:root:Epoch[44] Train-loss=179.235384\n", + "INFO:root:Epoch[44] Time cost=4.638\n", + "INFO:root:Epoch[45] Train-loss=179.108771\n", + "INFO:root:Epoch[45] Time cost=4.754\n", + "INFO:root:Epoch[46] Train-loss=178.714163\n", + "INFO:root:Epoch[46] Time cost=4.457\n", + "INFO:root:Epoch[47] Train-loss=178.508338\n", + "INFO:root:Epoch[47] Time cost=4.960\n", + "INFO:root:Epoch[48] Train-loss=178.288002\n", + "INFO:root:Epoch[48] Time cost=4.562\n", + "INFO:root:Epoch[49] Train-loss=178.083288\n", + "INFO:root:Epoch[49] Time cost=4.619\n", + "INFO:root:Epoch[50] Train-loss=177.791330\n", + "INFO:root:Epoch[50] Time cost=4.580\n", + "INFO:root:Epoch[51] Train-loss=177.570741\n", + "INFO:root:Epoch[51] Time cost=4.704\n", + "INFO:root:Epoch[52] Train-loss=177.287114\n", + "INFO:root:Epoch[52] Time cost=5.172\n", + "INFO:root:Epoch[53] Train-loss=177.122645\n", + "INFO:root:Epoch[53] Time cost=4.678\n", + "INFO:root:Epoch[54] Train-loss=176.816022\n", + "INFO:root:Epoch[54] Time cost=4.819\n", + "INFO:root:Epoch[55] Train-loss=176.670484\n", + "INFO:root:Epoch[55] Time cost=4.568\n", + "INFO:root:Epoch[56] Train-loss=176.459671\n", + "INFO:root:Epoch[56] Time cost=4.450\n", + "INFO:root:Epoch[57] Train-loss=176.174175\n", + "INFO:root:Epoch[57] Time cost=4.579\n", + "INFO:root:Epoch[58] Train-loss=175.935856\n", + "INFO:root:Epoch[58] Time cost=4.552\n", + "INFO:root:Epoch[59] Train-loss=175.739928\n", + "INFO:root:Epoch[59] Time cost=4.385\n", + "INFO:root:Epoch[60] Train-loss=175.579695\n", + "INFO:root:Epoch[60] Time cost=4.496\n", + "INFO:root:Epoch[61] Train-loss=175.403871\n", + "INFO:root:Epoch[61] Time cost=5.088\n", + "INFO:root:Epoch[62] Train-loss=175.157114\n", + "INFO:root:Epoch[62] Time cost=4.628\n", + "INFO:root:Epoch[63] Train-loss=174.953950\n", + "INFO:root:Epoch[63] Time cost=4.826\n", + "INFO:root:Epoch[64] Train-loss=174.743393\n", + "INFO:root:Epoch[64] Time cost=4.832\n", + "INFO:root:Epoch[65] Train-loss=174.554056\n", + "INFO:root:Epoch[65] Time cost=4.375\n", + "INFO:root:Epoch[66] Train-loss=174.366719\n", + "INFO:root:Epoch[66] Time cost=4.583\n", + "INFO:root:Epoch[67] Train-loss=174.160622\n", + "INFO:root:Epoch[67] Time cost=4.586\n", + "INFO:root:Epoch[68] Train-loss=173.981699\n", + "INFO:root:Epoch[68] Time cost=5.149\n", + "INFO:root:Epoch[69] Train-loss=173.751617\n", + "INFO:root:Epoch[69] Time cost=4.495\n", + "INFO:root:Epoch[70] Train-loss=173.548732\n", + "INFO:root:Epoch[70] Time cost=4.588\n", + "INFO:root:Epoch[71] Train-loss=173.380950\n", + "INFO:root:Epoch[71] Time cost=5.042\n", + "INFO:root:Epoch[72] Train-loss=173.158519\n", + "INFO:root:Epoch[72] Time cost=4.817\n", + "INFO:root:Epoch[73] Train-loss=172.970726\n", + "INFO:root:Epoch[73] Time cost=4.791\n", + "INFO:root:Epoch[74] Train-loss=172.782357\n", + "INFO:root:Epoch[74] Time cost=4.377\n", + "INFO:root:Epoch[75] Train-loss=172.581992\n", + "INFO:root:Epoch[75] Time cost=4.518\n", + "INFO:root:Epoch[76] Train-loss=172.385020\n", + "INFO:root:Epoch[76] Time cost=4.863\n", + "INFO:root:Epoch[77] Train-loss=172.198309\n", + "INFO:root:Epoch[77] Time cost=5.104\n", + "INFO:root:Epoch[78] Train-loss=172.022333\n", + "INFO:root:Epoch[78] Time cost=4.571\n", + "INFO:root:Epoch[79] Train-loss=171.816585\n", + "INFO:root:Epoch[79] Time cost=4.557\n", + "INFO:root:Epoch[80] Train-loss=171.643714\n", + "INFO:root:Epoch[80] Time cost=4.567\n", + "INFO:root:Epoch[81] Train-loss=171.460581\n", + "INFO:root:Epoch[81] Time cost=4.735\n", + "INFO:root:Epoch[82] Train-loss=171.284854\n", + "INFO:root:Epoch[82] Time cost=5.012\n", + "INFO:root:Epoch[83] Train-loss=171.113129\n", + "INFO:root:Epoch[83] Time cost=4.877\n", + "INFO:root:Epoch[84] Train-loss=170.947790\n", + "INFO:root:Epoch[84] Time cost=4.487\n", + "INFO:root:Epoch[85] Train-loss=170.766223\n", + "INFO:root:Epoch[85] Time cost=4.723\n", + "INFO:root:Epoch[86] Train-loss=170.602559\n", + "INFO:root:Epoch[86] Time cost=4.803\n", + "INFO:root:Epoch[87] Train-loss=170.448713\n", + "INFO:root:Epoch[87] Time cost=4.636\n", + "INFO:root:Epoch[88] Train-loss=170.273053\n", + "INFO:root:Epoch[88] Time cost=4.562\n", + "INFO:root:Epoch[89] Train-loss=170.099485\n", + "INFO:root:Epoch[89] Time cost=4.567\n", + "INFO:root:Epoch[90] Train-loss=169.934289\n", + "INFO:root:Epoch[90] Time cost=4.905\n", + "INFO:root:Epoch[91] Train-loss=169.768920\n", + "INFO:root:Epoch[91] Time cost=4.636\n", + "INFO:root:Epoch[92] Train-loss=169.620803\n", + "INFO:root:Epoch[92] Time cost=4.429\n", + "INFO:root:Epoch[93] Train-loss=169.448189\n", + "INFO:root:Epoch[93] Time cost=4.985\n", + "INFO:root:Epoch[94] Train-loss=169.295794\n", + "INFO:root:Epoch[94] Time cost=4.649\n", + "INFO:root:Epoch[95] Train-loss=169.143627\n", + "INFO:root:Epoch[95] Time cost=4.602\n", + "INFO:root:Epoch[96] Train-loss=168.989410\n", + "INFO:root:Epoch[96] Time cost=4.904\n", + "INFO:root:Epoch[97] Train-loss=168.841089\n", + "INFO:root:Epoch[97] Time cost=4.602\n", + "INFO:root:Epoch[98] Train-loss=168.694906\n", + "INFO:root:Epoch[98] Time cost=4.589\n", + "INFO:root:Epoch[99] Train-loss=168.527604\n", + "INFO:root:Epoch[99] Time cost=4.560\n", + "INFO:root:Epoch[100] Train-loss=168.385596\n", + "INFO:root:Epoch[100] Time cost=4.835\n", + "INFO:root:Epoch[101] Train-loss=168.246526\n", + "INFO:root:Epoch[101] Time cost=4.558\n", + "INFO:root:Epoch[102] Train-loss=168.093663\n", + "INFO:root:Epoch[102] Time cost=4.609\n", + "INFO:root:Epoch[103] Train-loss=167.938807\n", + "INFO:root:Epoch[103] Time cost=4.599\n", + "INFO:root:Epoch[104] Train-loss=167.814916\n", + "INFO:root:Epoch[104] Time cost=4.394\n", + "INFO:root:Epoch[105] Train-loss=167.676473\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:root:Epoch[105] Time cost=4.724\n", + "INFO:root:Epoch[106] Train-loss=167.560241\n", + "INFO:root:Epoch[106] Time cost=4.316\n", + "INFO:root:Epoch[107] Train-loss=167.424132\n", + "INFO:root:Epoch[107] Time cost=4.646\n", + "INFO:root:Epoch[108] Train-loss=167.284482\n", + "INFO:root:Epoch[108] Time cost=4.472\n", + "INFO:root:Epoch[109] Train-loss=167.184511\n", + "INFO:root:Epoch[109] Time cost=4.768\n", + "INFO:root:Epoch[110] Train-loss=167.037793\n", + "INFO:root:Epoch[110] Time cost=4.717\n", + "INFO:root:Epoch[111] Train-loss=166.916652\n", + "INFO:root:Epoch[111] Time cost=4.803\n", + "INFO:root:Epoch[112] Train-loss=166.796803\n", + "INFO:root:Epoch[112] Time cost=4.617\n", + "INFO:root:Epoch[113] Train-loss=166.655028\n", + "INFO:root:Epoch[113] Time cost=4.420\n", + "INFO:root:Epoch[114] Train-loss=166.561129\n", + "INFO:root:Epoch[114] Time cost=4.333\n", + "INFO:root:Epoch[115] Train-loss=166.434593\n", + "INFO:root:Epoch[115] Time cost=4.526\n", + "INFO:root:Epoch[116] Train-loss=166.322805\n", + "INFO:root:Epoch[116] Time cost=4.310\n", + "INFO:root:Epoch[117] Train-loss=166.195452\n", + "INFO:root:Epoch[117] Time cost=4.458\n", + "INFO:root:Epoch[118] Train-loss=166.073792\n", + "INFO:root:Epoch[118] Time cost=4.333\n", + "INFO:root:Epoch[119] Train-loss=165.967437\n", + "INFO:root:Epoch[119] Time cost=4.459\n", + "INFO:root:Epoch[120] Train-loss=165.876094\n", + "INFO:root:Epoch[120] Time cost=5.070\n", + "INFO:root:Epoch[121] Train-loss=165.748064\n", + "INFO:root:Epoch[121] Time cost=4.782\n", + "INFO:root:Epoch[122] Train-loss=165.656283\n", + "INFO:root:Epoch[122] Time cost=4.640\n", + "INFO:root:Epoch[123] Train-loss=165.540462\n", + "INFO:root:Epoch[123] Time cost=4.522\n", + "INFO:root:Epoch[124] Train-loss=165.448734\n", + "INFO:root:Epoch[124] Time cost=4.858\n", + "INFO:root:Epoch[125] Train-loss=165.347751\n", + "INFO:root:Epoch[125] Time cost=4.842\n", + "INFO:root:Epoch[126] Train-loss=165.230048\n", + "INFO:root:Epoch[126] Time cost=4.495\n", + "INFO:root:Epoch[127] Train-loss=165.147932\n", + "INFO:root:Epoch[127] Time cost=4.766\n", + "INFO:root:Epoch[128] Train-loss=165.036021\n", + "INFO:root:Epoch[128] Time cost=4.526\n", + "INFO:root:Epoch[129] Train-loss=164.977613\n", + "INFO:root:Epoch[129] Time cost=5.091\n", + "INFO:root:Epoch[130] Train-loss=164.881467\n", + "INFO:root:Epoch[130] Time cost=5.223\n", + "INFO:root:Epoch[131] Train-loss=164.785627\n", + "INFO:root:Epoch[131] Time cost=4.165\n", + "INFO:root:Epoch[132] Train-loss=164.707629\n", + "INFO:root:Epoch[132] Time cost=4.527\n", + "INFO:root:Epoch[133] Train-loss=164.598039\n", + "INFO:root:Epoch[133] Time cost=4.167\n", + "INFO:root:Epoch[134] Train-loss=164.502932\n", + "INFO:root:Epoch[134] Time cost=4.354\n", + "INFO:root:Epoch[135] Train-loss=164.422286\n", + "INFO:root:Epoch[135] Time cost=4.387\n", + "INFO:root:Epoch[136] Train-loss=164.344749\n", + "INFO:root:Epoch[136] Time cost=4.662\n", + "INFO:root:Epoch[137] Train-loss=164.264898\n", + "INFO:root:Epoch[137] Time cost=4.671\n", + "INFO:root:Epoch[138] Train-loss=164.178707\n", + "INFO:root:Epoch[138] Time cost=4.776\n", + "INFO:root:Epoch[139] Train-loss=164.109071\n", + "INFO:root:Epoch[139] Time cost=4.787\n", + "INFO:root:Epoch[140] Train-loss=163.993291\n", + "INFO:root:Epoch[140] Time cost=4.726\n", + "INFO:root:Epoch[141] Train-loss=163.956234\n", + "INFO:root:Epoch[141] Time cost=4.337\n", + "INFO:root:Epoch[142] Train-loss=163.845638\n", + "INFO:root:Epoch[142] Time cost=4.787\n", + "INFO:root:Epoch[143] Train-loss=163.790882\n", + "INFO:root:Epoch[143] Time cost=5.563\n", + "INFO:root:Epoch[144] Train-loss=163.723495\n", + "INFO:root:Epoch[144] Time cost=4.529\n", + "INFO:root:Epoch[145] Train-loss=163.634262\n", + "INFO:root:Epoch[145] Time cost=5.028\n", + "INFO:root:Epoch[146] Train-loss=163.552854\n", + "INFO:root:Epoch[146] Time cost=4.933\n", + "INFO:root:Epoch[147] Train-loss=163.501429\n", + "INFO:root:Epoch[147] Time cost=4.912\n", + "INFO:root:Epoch[148] Train-loss=163.444245\n", + "INFO:root:Epoch[148] Time cost=5.034\n", + "INFO:root:Epoch[149] Train-loss=163.348476\n", + "INFO:root:Epoch[149] Time cost=4.600\n", + "INFO:root:Epoch[150] Train-loss=163.256955\n", + "INFO:root:Epoch[150] Time cost=4.704\n", + "INFO:root:Epoch[151] Train-loss=163.216139\n", + "INFO:root:Epoch[151] Time cost=4.670\n", + "INFO:root:Epoch[152] Train-loss=163.144691\n", + "INFO:root:Epoch[152] Time cost=4.678\n", + "INFO:root:Epoch[153] Train-loss=163.050236\n", + "INFO:root:Epoch[153] Time cost=4.595\n", + "INFO:root:Epoch[154] Train-loss=162.991225\n", + "INFO:root:Epoch[154] Time cost=5.307\n", + "INFO:root:Epoch[155] Train-loss=162.907200\n", + "INFO:root:Epoch[155] Time cost=4.684\n", + "INFO:root:Epoch[156] Train-loss=162.838075\n", + "INFO:root:Epoch[156] Time cost=4.686\n", + "INFO:root:Epoch[157] Train-loss=162.759286\n", + "INFO:root:Epoch[157] Time cost=4.750\n", + "INFO:root:Epoch[158] Train-loss=162.725998\n", + "INFO:root:Epoch[158] Time cost=4.637\n", + "INFO:root:Epoch[159] Train-loss=162.635852\n", + "INFO:root:Epoch[159] Time cost=4.498\n", + "INFO:root:Epoch[160] Train-loss=162.563777\n", + "INFO:root:Epoch[160] Time cost=5.048\n", + "INFO:root:Epoch[161] Train-loss=162.527387\n", + "INFO:root:Epoch[161] Time cost=5.040\n", + "INFO:root:Epoch[162] Train-loss=162.395881\n", + "INFO:root:Epoch[162] Time cost=4.764\n", + "INFO:root:Epoch[163] Train-loss=162.353654\n", + "INFO:root:Epoch[163] Time cost=4.561\n", + "INFO:root:Epoch[164] Train-loss=162.285584\n", + "INFO:root:Epoch[164] Time cost=5.051\n", + "INFO:root:Epoch[165] Train-loss=162.204332\n", + "INFO:root:Epoch[165] Time cost=4.455\n", + "INFO:root:Epoch[166] Train-loss=162.147100\n", + "INFO:root:Epoch[166] Time cost=5.021\n", + "INFO:root:Epoch[167] Train-loss=162.051296\n", + "INFO:root:Epoch[167] Time cost=4.551\n", + "INFO:root:Epoch[168] Train-loss=161.978708\n", + "INFO:root:Epoch[168] Time cost=4.744\n", + "INFO:root:Epoch[169] Train-loss=161.927990\n", + "INFO:root:Epoch[169] Time cost=4.821\n", + "INFO:root:Epoch[170] Train-loss=161.883088\n", + "INFO:root:Epoch[170] Time cost=4.365\n", + "INFO:root:Epoch[171] Train-loss=161.785367\n", + "INFO:root:Epoch[171] Time cost=4.448\n", + "INFO:root:Epoch[172] Train-loss=161.716386\n", + "INFO:root:Epoch[172] Time cost=4.622\n", + "INFO:root:Epoch[173] Train-loss=161.656391\n", + "INFO:root:Epoch[173] Time cost=4.500\n", + "INFO:root:Epoch[174] Train-loss=161.598127\n", + "INFO:root:Epoch[174] Time cost=4.677\n", + "INFO:root:Epoch[175] Train-loss=161.518613\n", + "INFO:root:Epoch[175] Time cost=4.958\n", + "INFO:root:Epoch[176] Train-loss=161.418783\n", + "INFO:root:Epoch[176] Time cost=4.607\n", + "INFO:root:Epoch[177] Train-loss=161.407767\n", + "INFO:root:Epoch[177] Time cost=4.427\n", + "INFO:root:Epoch[178] Train-loss=161.319552\n", + "INFO:root:Epoch[178] Time cost=4.930\n", + "INFO:root:Epoch[179] Train-loss=161.234087\n", + "INFO:root:Epoch[179] Time cost=4.240\n", + "INFO:root:Epoch[180] Train-loss=161.187404\n", + "INFO:root:Epoch[180] Time cost=4.484\n", + "INFO:root:Epoch[181] Train-loss=161.123118\n", + "INFO:root:Epoch[181] Time cost=4.937\n", + "INFO:root:Epoch[182] Train-loss=160.999420\n", + "INFO:root:Epoch[182] Time cost=4.489\n", + "INFO:root:Epoch[183] Train-loss=160.955369\n", + "INFO:root:Epoch[183] Time cost=4.894\n", + "INFO:root:Epoch[184] Train-loss=160.908542\n", + "INFO:root:Epoch[184] Time cost=4.269\n", + "INFO:root:Epoch[185] Train-loss=160.846908\n", + "INFO:root:Epoch[185] Time cost=4.998\n", + "INFO:root:Epoch[186] Train-loss=160.765964\n", + "INFO:root:Epoch[186] Time cost=4.467\n", + "INFO:root:Epoch[187] Train-loss=160.687773\n", + "INFO:root:Epoch[187] Time cost=4.609\n", + "INFO:root:Epoch[188] Train-loss=160.652674\n", + "INFO:root:Epoch[188] Time cost=5.327\n", + "INFO:root:Epoch[189] Train-loss=160.551175\n", + "INFO:root:Epoch[189] Time cost=4.267\n", + "INFO:root:Epoch[190] Train-loss=160.477424\n", + "INFO:root:Epoch[190] Time cost=4.798\n", + "INFO:root:Epoch[191] Train-loss=160.501221\n", + "INFO:root:Epoch[191] Time cost=4.695\n", + "INFO:root:Epoch[192] Train-loss=160.370335\n", + "INFO:root:Epoch[192] Time cost=4.640\n", + "INFO:root:Epoch[193] Train-loss=160.279749\n", + "INFO:root:Epoch[193] Time cost=4.653\n", + "INFO:root:Epoch[194] Train-loss=160.242415\n", + "INFO:root:Epoch[194] Time cost=5.044\n", + "INFO:root:Epoch[195] Train-loss=160.197063\n", + "INFO:root:Epoch[195] Time cost=4.684\n", + "INFO:root:Epoch[196] Train-loss=160.132983\n", + "INFO:root:Epoch[196] Time cost=4.460\n", + "INFO:root:Epoch[197] Train-loss=160.083149\n", + "INFO:root:Epoch[197] Time cost=4.713\n", + "INFO:root:Epoch[198] Train-loss=160.025012\n", + "INFO:root:Epoch[198] Time cost=4.779\n", + "INFO:root:Epoch[199] Train-loss=159.945513\n", + "INFO:root:Epoch[199] Time cost=4.659\n" + ] + } + ], + "source": [ + "# can initilize weights and biases with the learned parameters as follows: \n", + "# init = mx.initializer.Load(params)\n", + "\n", + "# call the VAE, output model contains the learned model and training loss\n", + "out = VAE(n_latent=2, x_train=image, x_valid=None, num_epoch=200) " + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# encode test images to obtain mu and logvar which are used for sampling\n", + "[mu,logvar] = VAE.encoder(out,image_test)\n", + "# sample in the latent space\n", + "z = VAE.sampler(mu,logvar)\n", + "# decode from the latent space to obtain reconstructed images\n", + "x_construction = VAE.decoder(out,z)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAADACAYAAADhh27FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xm0XFWZ9/HfQxJCyETm4ZLkQgiahOYNcknC8NpR0jSDLMAWFbUNk9AuURHpBmmlozRL+qVB8JWlQAeChrZBZQgsWhtBRpkueYMEAoQkF5NwM0IGQshA9vtHnWh591Pk3NSt4dT5fta6K3WfOsM+Vfvu2jl1nvNYCEEAAABAHu1V6wYAAAAAtcJkGAAAALnFZBgAAAC5xWQYAAAAucVkGAAAALnFZBgAAAC5xWS4DpjZf5jZZbVuB1AOM2s2s2Bm3Us8/5KZTatyswAX4y6yjjG36xj3GZbM7J2iX/eVtFXS+8nv54cQbq9+q5BnZtYm6dwQwm9r3Za0zKxZ0lJJPUIIO2rbGtQ7xl3UE8bcfHP/N5E3IYQ+ux6n+YMws+50PGQd/Ri1xLiLvKEP1y8uk0jBzP7VzO4ws5+b2SZJXzCzOWY2s2iZ6cmAvuv3/c3sbjNbY2ZLzewrH7D9P21r13bM7FvJum+a2clm9gkzW2Rmb5nZPxWte6SZPW1m682s3cx+aGY9ip4/wcxeM7MNZvZ/zexJMzuz6PlzzewVM3vbzP7bzEZ10cuGCkn6wvzkPf+9mR1a9NylZrbYzDaZ2ctmdlrRc2cm7/8PzGydpJlJ7Akz+/ekDyw1sxOK1ulvZrOSvrUi+VvoljzXLVlvrZktkXTSbtrdZmbTk8czzewXSd/fZGYvmtnBSb9fbWbLzOy4onXPMrOFybJLzOz8Dtv+p6SNbyZ9OpjZQclzPZN2/tHMVpnZT8ysV3nvAiqNcRf1gjG38cdcJsPpnSbpPyX1l3THBy1oZntJul/Sc5KaJP2NpH80s2NT7mt/Fd6bkZKukDRL0mclHSZpmqTvmdnoZNkdkr4uabCkoyUdL+n8pB1DJd0p6R+T55dKmlzUzr9LnjtF0hBJzyTHiDplZodJukWF93iQpBslzTWznskiiyX9bxX66XclzTGzEUWbmCJpiaRhkq4sir2qQh/5P5JmmZklz81WoY8dpEL/O07SuclzX5L0iSTeIulTnTyckyX9TNIASf9P0m9U6PdNkr6XHNsuq5N99ZN0lqQfmNlHktfkeEkXSZqetHNah/1cJelgSZOS55skXd7JtqI2GHdRU4y5ORlzQwj8FP1IapM0vUPsXyU93CE2R9LMot+nS2pLHh8taUmH5b8j6eYS+/zTtpLtvCOpW/L7AElB0uFFy78g6RMltnWxpF8kj8+W9HjRcyapXdKZye8PSppR9Hx3Fa7ba6r1+5D3H68fJvEfS7qiQ+xVSX9dYjvzJZ2SPD5T0h87PH+mpNeLft836W/DVRi8t0rqVfT8GZJ+lzx+WNI/FD13XLJu990dk6SZkh4seu7kDv2+b7Kt/Ups6x5JX08e3yLp+0XPHZSse1DS5zdLGlv0/JGSltb6Pebng/s74y4/te6DSZwxNzT+mMs1w+kt68SyYySNNrP1RbFukh5Juf7aEMKuRJItyb+rip7fIqmPJJnZhyVdI+lwFf6ouqtwpkEqnOH4U7tDCMHMlndo5w1mdn1RbKcKZ0hWpGwrqmuMpBlm9tWi2N4qvNcysy+q8D/25uS5PiqcfdjF68crdz0IIbybnKDoI2mgpB6S2v980kJ7FW3jL/qXpDc6eSwd+7TX7/tIWp98jfgvKpxt2EuFvv5iUTtai7ZV3KYhybLPFx2DqfD3iPrHuItaY8zNwZjLZDi9jrfd2KzCG77L8KLHyyQtCiGMr3irCl9rPC3pMyGEd8zsYhW+2pAKZyOKrwEyFb6uKG7nd0IIH/j1I+rKMklXhhCu7PiEmY2RdLOkYyU9FUJ438zmqzAQ7dKZ28csU+EsxeDgJ320Syq+1nG0s0zZkq8jfyXpi5LuDSFsN7N79OfjaldhIrFLcZvWqjDITwwhMNHIHsZd1Bpjbg7GXK4Z3nPzJZ1kZgOS64O+VvTcU5K2mdk3zWyf5KL3vzKzwyvQjr6SNkjabGbjlVy3lrhf0keskAjSXYVr3IYUPf8TSf+crCcz28/MOnsNEiqnR9J/dv10V2Hg/Qczm2IFvc3sJDPrK6m3CgPvGqmQACHpkD3deQihXdL/SLrGzPqZ2V5mNtbM/jpZ5E5JX7NC0tIASZeWcawfZG9JPVU4rh3JGYvjip6/U9JZZjbezPZV4avxXcewU4XX7AfJtZwysyYz+9sKtRWVxbiLSmLMLcjdmMtkeM/NlrRQha8pfi3pv3Y9kfyP7kQVkibaVPif0o0qXIje1b4paYakTck+/nS2IYSwStJnJF0raZ2ksSpcNL81ef4XyXO/MLONkv4gqa47bM48oML/sHf9zAwhtKqQRPEjSW9Lel2Fa9AUQnhZha9un1Lh67C/kvRkmW34ogoD48vJ/n4paVdyyM0qJGC8IGmepLvK3JcrhLBJhUnPnUkbPidpbtHz/y3ph5J+p8Lr8XTy1Nbk30t2xZN+/ltJH6pEW1Fxs8W4i8phzFU+x1yKbuSIFW7P8qakT4UQHq91e4BKSM64LZDUs8RXjUDVMO6i0TXCmMuZ4QZnZscnX8P1VOGrjO2Snq1xs4AuZWanWeHelgMk/Zuk+7I6KCP7GHfR6BptzGUy3PiOUeEeh2tU+CrutBDC1g9eBcic81W4L+ZiFUr6frm2zUHOMe6i0TXUmMtlEgAAAMgtzgwDAAAgt8q6z7AVSvJdr8LNlP8jhHDVBy0/ePDg0NzcXM4uAT3//PNrQwhDdr9k16HvolxtbW1au3at7X7JrkO/RVdgzEVWpe27ezwZTjJkb1Ch/vtySc+Z2dzkViOu5uZmtba2lnoaSMXMOlt1p2z0XZSrpaWl6vuk36IrMOYiq9L23XIuk5isQn3tJSGEbSrc7/GUMrYHAAAAVFU5k+Em/WU96uX6y5KTkiQzO8/MWs2sdc2aNWXsDqgu+i6yiH6LrKLvolYqnkAXQrgphNASQmgZMqSqlxwBZaHvIovot8gq+i5qpZzJ8ApJo4p+3z+JAQAAAJlQzmT4OUnjzOwAM9tb0mdVVLsaAAAAqHd7fDeJEMIOM7tA0m9UuLXaLSGEl7qsZQAAAECFlXWf4RDCA5Ie6KK2AAAAAFVFBToAAADkVllnhgEAAFB9IYQu36ZZVYtk1g3ODAMAACC3mAwDAAAgt5gMAwAAILeYDAMAACC3mAwDAAAgt7ibBAAAQJ1Ie5cIbzkvVq07RGT5ThScGQYAAEBuMRkGAABAbjEZBgAAQG4xGQYAAEBukUAHIFMqUYLUk+VkEADVkXY82rlzZxR7//333WV37NiRallvm9641b27P9Xba6/4fGi3bt1SbTNtrJRy1+9qnBkGAABAbjEZBgAAQG4xGQYAAEBuMRkGAABAbpWVQGdmbZI2SXpf0o4QQktXNApA/pRTdamc5aT0iRsk1WXbhg0botjNN98cxebMmRPFXnjhBXebvXv3jmILFy6MYqNGjUrTRNSBtJXdJD+JzUuAe/fdd6NYe3u7u82XX345inl9av369VGsR48eUWzw4MHufpqbm6PYmDFjUq3ft2/fKNarVy93P14CX9rkPW85qevH4q64m8THQghru2A7AAAAQFVxmQQAAAByq9zJcJD0P2b2vJmd5y1gZueZWauZta5Zs6bM3QHVQ99FFtFvkVX0XdRKuZPhY0IIH5F0gqSvmNlHOy4QQrgphNASQmgZMmRImbsDqoe+iyyi3yKr6LuolbKuGQ4hrEj+XW1md0uaLOmxrmgYkEflVlerRHJZ2mQSL5Ek7XKd2Y+XOOElWZRKsPCWTRsrhcS62inVn7yEt/POi7/AnDdvXqr9lOoPW7ZsiWLLli2LYiTQ1ae0406panHbt2+PYhs3boxiixYtimIPPPCAu81HHnkkii1dujTVvvv06RPF+vfv7+5n7NixUezggw+OYhMnToxiH/rQh6LYyJEj3f3069cvinmJfp5SY2tXj7l7fGbYzHqbWd9djyUdJ2lBVzUMAAAAqLRyzgwPk3R3MjvvLuk/Qwi/7pJWAQAAAFWwx5PhEMISSf+rC9sCAAAAVBW3VgMAAEBudUXRjUx4+umno9j111/vLtvU1BTFvMoqM2bMiGIDBw50t1kqjnwoJwnNq2a0bds2dz9bt26NYu+9916q2ObNm1Nv00sa8ioseTFve6W26SWIeIkTw4cPj2Jecojk/33vu+++UcxL8CBRrra8222deOKJ7rJpE+PSKpVA5/3Ntra2RrGjjjoqinl/h9ddd527n1tvvTWKvfTSS1HMq/aFgnISjEslanpj1Lp166LY888/H8UWLPDTrFavXh3FvDmEVy3OS6Ar9Xnh9Wmvqt2bb74ZxYYOHRrFBg0a5O6n1GuXRrlJ5WlxZhgAAAC5xWQYAAAAucVkGAAAALnFZBgAAAC5xWQYAAAAuZWbtFPvzg9eecTOuPLKK6NYqbKHU6dOLWtf1dDc3BzFvvWtb7nLjh49usKtyaZSma9py3t6Wb8bNmyIYl5WveRnIb/++utR7I033ohiK1eudLe5atWqKOZlHHsZwz179oxi++yzj7sf7zXy9uM56KCDotjRRx/tLjt9+vQo5pXKTVsuFJWxadOmKHbEEUdEMa/0cSn77bdfFDvnnHOi2LRp06KYV35Wkm644YYo1tLSEsUWL14cxc4666wo9uSTT7r78Tz22GNR7OMf/3jq9VE+bxz3+u4777wTxXr37u1u0+vnH/7wh6NYqT7ZkTfeS/5dL7w75nh3KPHGx1J3MunWrVsU8+5kUcu79XBmGAAAALnFZBgAAAC5xWQYAAAAucVkGAAAALmVmwS6e+65J4rNnz/fXXbixIlRzCt7+cwzz0Sxe++9193mb37zmyh2wAEHRLGlS5e666flXcA+YsSIKJY26cRLqpOkSy65pFPtyru05T290p5vv/12FHvllVfc/XjlPRcuXBjFXnvttSjmJd9JfvlkLwmuX79+UWzYsGFRrG/fvu5+vLLTS5YsiWJeUp33uh144IHufkqVJu2I0svV4/19XHHFFVHMG7e85BxJOvPMM6PYhRdeGMUmTJiQooWlXX311VHMG+8vuOCCKPbCCy+k3s/pp58exbxEP5Tm/U17fc9brtR4kDYR2tuPV2JZ8hN6jzzyyCg2ZMiQKOYl6nljeKm497nk3RjAG++9svaSPy+pt6Q6zgwDAAAgt5gMAwAAILeYDAMAACC3mAwDAAAgt3abQGdmt0j6hKTVIYRDkthASXdIapbUJunTIYQ4y6eOjB8/PlWslEMPPTSKnXHGGVHsqquuctdva2uLYl4CnZcw1Bl77713FPMS6Lx9e1XNvKo3qBwvWcBLKvCSzSTp3XffjWJegseAAQOi2ODBg91teklwI0eOjGJesuXQoUOjWKkKdF6lPK8qnlcRz0ugK7UfL8nDS+boTBINyvPtb387il1zzTVRzHufvGQ1SfrYxz5WfsOKeIlFknTrrbdGMS9Rb8uWLVHM+5u78cYb3f2cfPLJUcwbG9A5lfib9vqKl0DXp08fd33vM9tLYvOqwHmfDRs3bnT3431e9OrVK4oNHz48ig0aNCjVupKfQJc2Wa6eEuhmSzq+Q+xSSQ+FEMZJeij5HQAAAMiU3U6GQwiPSXqrQ/gUSbclj2+TdGoXtwsAAACouD39jmVYCKE9ebxSUvw9asLMzjOzVjNr9b6GB+oVfRdZRL9FVtF3UStlX3AUChfCxBfD/Pn5m0IILSGEFu8G0UC9ou8ii+i3yCr6LmplTyvQrTKzESGEdjMbIckvXZVDpZJ20iaidSapLy2vUt7atWuj2JQpU6LYcccd1+XtyaO0iXFeokHv3r2jmJdgIUlNTU2ptuklkXnrStLBBx+calmvIpGX8ORVSCoV9xI/vOQUL5lj3Lhx7n68RBTvNUL1/PKXv0y1nFdVrqsT5SS/6uPZZ5/tLjt37txU2/Qmd88991wU86qPofZKJXKlraTmxbykOknatGlTFHvrrY5Xq/oJcIsWLYpib7zxhrsfbyz1EqG9PulVz+vZs6e7H298TZsYV08JdJ65kmYkj2dI8msQAwAAAHVst5NhM/u5pKckfcjMlpvZOZKukvQ3ZrZI0vTkdwAAACBTdvvdYAghvpluwbFd3BYAAACgqrhjNwAAAHKLyTAAAAByixTqBrN58+Yodtppp0UxL4v0uuuui2Klyiuic7yMWC8L2cu69d4D744IkjRmzJgoNnr06CjmZQd7pZMlv3Szd9cU7xi9UtBeprQkzZs3L4otX748inklxw855JAoNmHCBHc/3t050pZjRm194Qtf6PJteiW/r7jiiiiW9q4RknTCCSdEsR/+8IdRjDtHZEdn7ibhjeOdubPOggULoph3hxNvLFy2bFkUK1WOeezYsVHMG0u9uwd5n0ul7sqThdL2nBkGAABAbjEZBgAAQG4xGQYAAEBuMRkGAABAbpFA12Bmz54dxbwEEa98rZd8hc4plRRQquxmR2lLNPft29dd30vI6dOnTxTz3n8vsazU/r3j9I5x27ZtUWz+/Pnufu65554o5iWYeCXLp06dGsX2339/dz9e0gmy4Y477ohiRx55pLtsjx49opiX0HnJJZdEsTlz5qRuk5dIdPnll0exAw88MPU2UX86k0Dnlbv3xlcv4V2SlixZEsW8MsveWOaNmV4pcMkvqTxy5Mgo5n2GlFNiuR5xZhgAAAC5xWQYAAAAucVkGAAAALnFZBgAAAC5RQJdRi1evNiNX3TRRanWf+qpp6LY8OHDy2oTOidtsoGXoOElNEhSz549o5iX4JO2gpzkJx15y+7YsSOKrVixIordfvvt7n7a2tqimJcgMm3atCjmJVF5SSySn6SI2jrggAOi2Ouvvx7FfvKTn0SxF1980d3mT3/60yh29dVXR7G0yXKlKjQ+++yzUYzKcvnhjc9egrOXxFYqmddLglu/fn0US/sZUqqSrDcWep8h3nJZTpbz8KkAAACA3GIyDAAAgNxiMgwAAIDcYjIMAACA3NptAp2Z3SLpE5JWhxAOSWIzJX1J0ppksctCCA9UqpGI3XfffW58+/btUez000+PYlRDqq60Fds8XqWfUslh3jbTJj/s3LnT3WbaZTdt2hTFHn744Sj26KOPuvvxtulVm/vUpz4VxUaMGBHFvApkUuMlfjSCO++8M4oNGDAg1bpPPvmkGx87duwet+fcc8+NYldddZW7bNp2IttKjRve+OolOHtjlFcJVPIrw3m8MdNLZN6yZYu7/rp161It632ueLEsj61pzgzPlnS8E/9BCGFS8sNEGAAAAJmz28lwCOExSW9VoS0AAABAVZVzzfAFZvYHM7vFzEp+T2Rm55lZq5m1rlmzptRiQN2h7yKL6LfIKvouamVPJ8M/ljRW0iRJ7ZKuKbVgCOGmEEJLCKHFu+k0UK/ou8gi+i2yir6LWtmjCnQhhFW7HpvZzZLu77IWIeIlxd19993usl71mO9///tRzKuag9rzkjG896pU5aK0SXmeziQ/bNu2LYotXbo0is2dOzeKedWVJL8CopfIdPjhh0cxr8JSZyrNZTnxoxF4Fbu8KpmXXHJJFHviiSfcbZZKCE3j1FNPjWIkyuVbqbHVGzu8Cp/9+/ePYqWqGo4bNy6KeZ8DXnXQP/7xj1HMq14nSStXroxi3hn5ziQoZ9UenRk2s+JX5jRJC7qmOQAAAED1pLm12s8lTZM02MyWS/oXSdPMbJKkIKlN0vkVbCMAAABQEbudDIcQznDCsyrQFgAAAKCqqEAHAACA3GIyDAAAgNzao7tJoLpmzYqvSnn88cfdZT/3uc9FMUov1ycvC9mLeeWYS90poZwM+lK8jGUv49i7c8S8efOiWKlS0p/85Cej2Gc+85ko5pU65e4o2eb1+8mTJ0ex2bNnR7GJEye62yxVgjaNyy+/PIq1tLS4y3ILsGxLW2q4lHLG8f3228/dpnc3iZEjR0Yxb2x+7rnnotjChQvd/bz33ntRbMOGDVGsEp8r9YYzwwAAAMgtJsMAAADILSbDAAAAyC0mwwAAAMgtEujqzPz586PYV7/61ShW6sL7733ve13eJlSPlxjnJXOUKh9cTiKZl4wh+aU8vVK59913X6r9TJkyxY1/+ctfjmJeCVyS5fLLKx9bKlHu5JNPjmJeCdm77rorinmJn4cccoi7H6/8bc+ePd1l0XjSJuB546uXDCxJTU1NUWzUqFFRbMeOHVFs7dq1UWz16tXufrzPm+3bt0exziQUZhVnhgEAAJBbTIYBAACQW0yGAQAAkFtMhgEAAJBbJNDVkJf4ccYZZ0Qx78L7z3/+8+42qTbXeNJWOOoMLyGiVCLSq6++GsXuvffeKOYlN40ZMyaKnX766e5+vL5Lslx+vfPOO1Hs7LPPjmK9e/d21/cS4zzf+MY3otiPfvSjKOYlJkl+da9Jkyal2jfqU2eq0qVNlvOS3QYOHOhu06tq6CUTb968OYrtvffeqWKlpD32Rkuq48wwAAAAcovJMAAAAHKLyTAAAAByi8kwAAAAcmu3CXRmNkrSTyUNkxQk3RRCuN7MBkq6Q1KzpDZJnw4hvF25pmbbzp07o9hJJ50UxbxkpfHjx0ex7373u13TMOSSlyzX1tbmLnv//fdHsaeffjqKeQkeH/3oR6PYtGnT3P10JskDjc9LoHvllVeiWKkEOq+6lufaa6+NYocffngUO+uss9z1L7rooijmVWMs1U5UTznJYd5nuOQnxm3YsCGKvfvuu1GsVKXCffbZJ4p5SXlvvfVWFGtvb0/VHslP4POqNJabsJ0FaUaLHZK+GUKYIGmqpK+Y2QRJl0p6KIQwTtJDye8AAABAZux2MhxCaA8hzEseb5K0UFKTpFMk3ZYsdpukUyvVSAAAAKASOnXNsJk1SzpM0jOShoUQdp2PX6nCZRTeOueZWauZta5Zs6aMpgLVRd9FFtFvkVX0XdRK6smwmfWR9CtJF4YQNhY/FwoX2Lh3YA4h3BRCaAkhtHg3kgbqFX0XWUS/RVbRd1ErqSrQmVkPFSbCt4cQdpX1WWVmI0II7WY2QtLqSjWyEXgXuj/yyCOp1v3Zz34WxUpVrgE62rp1axTzqsX9/ve/d9f34l6y22GHHRbFTjnllChW6kMubcIT8sFLOBs9enQUW7Vqlbu+d2bR63telcNhw9wvOl0vv/xyqm0iO7xkue3bt7vLrl+/PoqtWLEiVaxU0rC3fy8Bz0tkfuqpp6KYN/+QpKFDh0axfv36RbE8jM27PUIrpBHOkrQwhFCcdjtX0ozk8QxJcX1WAAAAoI6lOTN8tKS/l/Simc1PYpdJukrSnWZ2jqQ3JH26Mk0EAAAAKmO3k+EQwhOSSt1k7tiubQ4AAABQPY1/IQgAAABQApNhAAAA5Faqu0kgvVJlD6dOnZpq/Tlz5kQxL0sf8HilQd9+O66S7mXAP/vss+42N27cGMUOPPDAKDZ58uQo5pUS98p9Svko+Yn0+vbtG8UuvvjiKPa1r33NXX/kyJFRzMuKP/roo6PYY489lqaJkvw7VHjldJFtpe4msW7duij22muvRbEFCxakWleSNm/eHMWWL18exZYuXZpq3TFjxrj7aWpqimLenVTKLdGchbGdM8MAAADILSbDAAAAyC0mwwAAAMgtJsMAAADILRLoutitt97qxpcsWZJq/WOOOSaKZeHic1SXV65T8pMnli1bFsXmz58fxdrb291tDhgwIIp5CXRTpkyJYl4SVGdKe9L3UWzGjBlRbN68ee6ys2fPjmLe382jjz5aVpt+/etfl7U+sqHUWOSVVPYSzrZu3RrFvERmSVq8eHEU8xKhvbHUK1k+bdo0dz/Tp0+PYl4CXffu8VTR23eWx2vODAMAACC3mAwDAAAgt5gMAwAAILeYDAMAACC3SKArw6JFi6LYzJkzq98QNDQv6cdLxpCk9evXR7GVK1dGMa/y0b777utu06uwNXHixCjmVTPyEi86I4QQxbKcpIHy9OnTJ4rdeOON7rLNzc1RbNasWVHMSzD1+vLDDz/s7mfEiBFuHPUn7djhJYd5iXKSNHTo0Ch2xBFHRLHBgwdHMa9KouQn0HljvrfNQw89NIp5yc2Sn2znfQ5069YtijXaOMyZYQAAAOQWk2EAAADkFpNhAAAA5BaTYQAAAOTWbrNbzGyUpJ9KGiYpSLophHC9mc2U9CVJa5JFLwshPFCphtajxx9/PIpt3Lgx9frjx4+PYr169SqrTcg2L2HMS6DbsWOHu74X9xI/vMSNnj17utscOHBgFJs0aVIU86rNeUkW3jGWindm/TTrovGUStL8zne+kyqGfPPGCS9hzItJ/rjZv3//KHbQQQdFsWOPPdbdpjfmpx33vOS/Um3PQ2JcWmlSvXdI+mYIYZ6Z9ZX0vJk9mDz3gxDCv1eueQAAAEDl7HYyHEJol9SePN5kZgslxfedAQAAADKmU9cMm1mzpMMkPZOELjCzP5jZLWY2oMQ655lZq5m1rlmzxlsEqEv0XWQR/RZZRd9FraSeDJtZH0m/knRhCGGjpB9LGitpkgpnjq/x1gsh3BRCaAkhtHg37wfqFX0XWUS/RVbRd1ErqSbDZtZDhYnw7SGEuyQphLAqhPB+CGGnpJslTa5cMwEAAICul+ZuEiZplqSFIYRri+IjkuuJJek0SQsq08TGcNRRR0WxBx98MIpxNwl05GX39ujRw1120KBBqZb1SoiWuhOKly3trb/PPvu461dDXjOgAXS9zownjD2NIc3dJI6W9PeSXjSz+UnsMklnmNkkFW631ibp/Iq0EAAAAKiQNHeTeEKS91+fXN1TGAAAAI2HCnQAAADILSbDAAAAyK001wyjhLPPPjtVDEirWqVB999//yiWttxnZ3jHU27CCQkrAICuxJlhAAAA5BaTYQAAAOQWk2EAAADkFpNhAAAA5JZVImmm5M7M1kh6I/l1sKS1Vdt55XGxTVv2AAADPUlEQVQ81TMmhFDVwvVFfbeeX5c9wfFUTy37rVTfr82e4Hiqh77btTie6knVd6s6Gf6LHZu1hhBaarLzCuB48qHRXheOJz8a7bXhePKj0V4bjqf+cJkEAAAAcovJMAAAAHKrlpPhm2q470rgePKh0V4Xjic/Gu214Xjyo9FeG46nztTsmmEAAACg1rhMAgAAALnFZBgAAAC5VfXJsJkdb2avmtnrZnZptfdfLjO7xcxWm9mCothAM3vQzBYl/w6oZRs7w8xGmdnvzOxlM3vJzL6exDN7TJVC360v9N10st5vpcbqu/Tb9LLedxup30qN3XerOhk2s26SbpB0gqQJks4wswnVbEMXmC3p+A6xSyU9FEIYJ+mh5Pes2CHpmyGECZKmSvpK8p5k+Zi6HH23LtF3d6NB+q3UWH2XfptCg/Td2Wqcfis1cN+t9pnhyZJeDyEsCSFsk/Rfkk6pchvKEkJ4TNJbHcKnSLoteXybpFOr2qgyhBDaQwjzksebJC2U1KQMH1OF0HfrDH03lcz3W6mx+i79NrXM991G6rdSY/fdak+GmyQtK/p9eRLLumEhhPbk8UpJw2rZmD1lZs2SDpP0jBrkmLoQfbeO0XdLatR+KzXA+0y//UCN2ncb4n1utL5LAl0XC4V71WXufnVm1kfSryRdGELYWPxcVo8JnZPV95m+iyy+z/RbZPV9bsS+W+3J8ApJo4p+3z+JZd0qMxshScm/q2vcnk4xsx4qdOzbQwh3JeFMH1MF0HfrEH13txq130oZfp/pt6k0at/N9PvcqH232pPh5ySNM7MDzGxvSZ+VNLfKbaiEuZJmJI9nSLq3hm3pFDMzSbMkLQwhXFv0VGaPqULou3WGvptKo/ZbKaPvM/02tUbtu5l9nxu674YQqvoj6URJr0laLOmfq73/Lmj/zyW1S9quwjVM50gapEIG5SJJv5U0sNbt7MTxHKPCVxp/kDQ/+Tkxy8dUwdeKvltHP/Td1K9TpvttcgwN03fpt516rTLddxup3ybH07B9l3LMAAAAyC0S6AAAAJBbTIYBAACQW0yGAQAAkFtMhgEAAJBbTIYBAACQW0yGAQAAkFtMhgEAAJBb/x851DqC1HpbSAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "f, ((ax1, ax2, ax3, ax4)) = plt.subplots(1,4, sharex='col', sharey='row',figsize=(12,3))\n", + "ax1.imshow(np.reshape(image_test[0,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax1.set_title('True image')\n", + "ax2.imshow(np.reshape(x_construction[0,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax2.set_title('Learned image')\n", + "ax3.imshow(np.reshape(image_test[146,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax3.set_title('True image')\n", + "ax4.imshow(np.reshape(x_construction[146,:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + "ax4.set_title('Learned image')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:matplotlib.font_manager:findfont: Matching :family=sans-serif:style=normal:variant=normal:weight=normal:stretch=normal:size=15.0 to DejaVu Sans ('/usr/local/lib/python3.5/dist-packages/matplotlib/mpl-data/fonts/ttf/DejaVuSans.ttf') with score of 0.050000\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAEICAYAAABLdt/UAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXuQJFd15r9T1VXqqepBgmx5tUbT2RIWbLQxL7URhLyWlsK2GMkrO0wIyy0Y1oTbKtiNMQFhHh1reTeiYQkcYFgZRIM1zKpqWWQEy2MF2EiyMCC09IDQogeyzHSPhBCaaUkITQ2a0fTZP7KylZWdj5uZNx+VeX4RJ2a6KjPrZlXmd0+ee+65xMwQBEEQykMt7wYIgiAIehFhFwRBKBki7IIgCCVDhF0QBKFkiLALgiCUDBF2QRCEkiHCLuQKEa0R0avzbocglAkRdmFsICImol/RdKwLiehBHccShKIhwi4IglAyRNiFwkBELyei24jocSL6CRFdTUTN4XtfH272fSJ6koheN3z9EiK6Y7jPt4joRY7jrRHR24noTiL6GRF9mogmiagN4MsAfnl4rCeJ6Jc92rObiO4mop8T0Y+J6O3D1y8kogeJ6N1EdGT4OQuO/S4mou8R0RNE9AAR/aXruL8xbOvjw/ffOHz9FCL6KyI6REQ/JaJriGiH1i9ZqAQi7EKROAngrQCmAbwSQAfAmwGAmX9zuM2LmXmKmT9NRC8FcC2APwVgAPgYgC8Q0SmOY14G4CIAZwF4EYA3MvNRAK8B8NDwWFPM/JBHe/4WwJ8y804ALwRws+O9M4btfC6APQBWiOgFw/eOAngDgNMAXAygS0S/BwBEZMLqVP47gNMBvATAHcP9/huA5w9f+5Xhsf9C8bsThC1E2IXCwMwHmPnbzPw0M6/BEuoLAnZZBPAxZr6dmU8y834ATwF4hWObDzPzQ8z8KIAvwhJNVU4AmCOiZzHzY8z8Xdf7/5mZn2LmWwH8H1idCJj5H5n5/zHzJjPfCeBTjvP4IwBfY+ZPMfMJZt5g5juIiIbn81ZmfpSZfw7gPQD+MEJ7BQGACLtQIIjo+UT0JSJ6mIiegCVs0wG7mADeNgxpPE5EjwPYBcAZVnnY8f8BgKkITfoDALsBrBPRrUT0Ssd7jw09f5t1+3OJ6DwiuoWIDhPRzwBc6TiPXQD+xeOzTgfQAnDAcS5fGb4uCJEQYReKxEcB3AvgHGZ+FoB3A6CA7R8AsMzMpzmsxcyfUvis0LKmzPwdZr4UwC8B+N8Arne8/exhrN5mBoAdzvmfAL4AYBcznwrgGsd5PADgeR4fdwTAMQC/6jiXU5k5SkckCABE2IVisRPAEwCeJKJ/A6Drev+nAM52/P1xAFcOPWQiovZw4HKnwmf9FIBBRKd6vUlETSJaIKJTmfnEsF2brs3+y3C7fwvgEgB/5ziPR5n5F0T0cljhF5s+gFcT0WVENEFEBhG9hJk3h+fzQSL6pWEbnktEv6NwLoIwggi7UCTeDksEfw5L5D7tev8vAewfhiouY+ZVAH8C4GoAjwG4H8AbVT6Ime+FFfv+0fB427JiALwewNowLHQlgAXHew8PP/MhWGJ95fCYgDXg+1+J6OewBj+3PH1mPgQrvPM2AI/CGjh98fDtdwzP4dvDz/waAHtAVhCUIVloQxCiQUQXAugx85l5t0UQvBCPXRAEoWSIsAuCIJQMCcUIgiCUDPHYBUEQSsaEjoMQ0WkAPgFr2jUD+GNmvs1v++npaZ6dndXx0YIgCJXhwIEDR5g5dNKaFmEH8CEAX2Hm1w6LNrWCNp6dncXq6qqmjxYEQagGRLSusl1iYR9O8PhNDPOHmfk4gONJjysIgiDEQ0eM/SwAhwHsG5Yq/YRrqjUAgIgWiWiViFYPHz6s4WMFQRAEL3QI+wSAlwH4KDO/FFbJ0ne6N2LmFWaeZ+b500+XukaCIAhpoUPYHwTwIDPfPvz7M7CEXhAEQciBxMLOzA8DeMCxyEAHwN1JjysIgiDEQ1ce+38C0CeiO2EtZPAeTccVhBH6/T5mZ2dRq9UwOzuLfr+fd5MEoXBoSXdk5jsAzOs4liD40e/3sbi4iMFgAABYX1/H4uIiAGBhYSFoV0GoFDLzVBgblpaWtkTdZjAYYGlpKacWCUIxEWEXCo0z9LK+7j0349ChQxm3ShCKjQi7UFjs0Mv6+jqCitXNzMxsbS/x92Ijv1FGMHPmdu6557IghGGaJsOqPeRrrVaLe70e93o9brVanu8J+uj1emyaJhMRm6YZ6fuV3yg5AFZZQWNF2IXCQkS+gu4WFr9OwDTNfE+iRCQVZvmNkqMq7LnUY5+fn2cpAiaEMTs76xlXN00Ta2trI6/VajXPcA0RYXPTvQa1EIcov4cX8hslh4gOMHNoBqLE2IXCsry8jFZrtFBoq9XC8vLytm3tOLvq60J0/AapVQev5TfKDhF2obAsLCxgZWUFpmmCiGCaJlZWVjxz1qN0AoJF1IHMpMIsv1GGqMRrdJvE2IU0SDKwVzX84uXdbtf3O/Tap9FosGEYyt+5/EbJgMTYBUHwwy9eTkQjcfBWqzXylNTv97F3715sbGx4Hte9vaAXibELguCLX1zc7eh5zew9duyY73EHgwH27t0rueo5I8IuCBUkyoDl+vr6ljh7lXVws7GxsTWpzK7nI+KeLSLsgpAiecy0dH7m9PQ0pqent33+7t27QUQj+7n/dmKLs19ZhyAGgwH27Nkj4p4lKoF43SaDp0Ke9Ho9NgxjawDQMIxUBvHSnGnpNwjp9Zlum5yc9JzwNTc3F7ifPUgatE2QOc9dBlHjAZl5KpSdOOLQ6/W40WhsE51ms6ldXNKaaRnUYaiUYfCzer0ee19Vs38nKS0QDxF2odR4CXSj0QgVhyDhs0XH7iwMw/BN5VPpVPy8WyLadi5ROqigDiNtYU5q9jmm0eFVARF2oRCk9cjtDKU4zTCMwP3CQglBYYyoBcf8BMwwjJHOw91BhYUskoRDwkQ3bWG3z8Xv84VgRNiF3NH9yK0aaggiaH+VUETQNm6P029CT7PZVBJAv++v3W5rF9xGo8ETExOpinpYuEg89nBE2IXc0XkDqwwKuoXdKSK2IPt5+jrMK8TiHqRV/fygkEUa7Y7SrrifEzTAKzF2NUTYhdzR+citKnJ2KCZKR2Bb0sFDZxjI6/OjiGJQyEK3dTqdzMIwdskC5/ctWTHqiLALuaPTY1cRHmdmS1xvN4nAOYU9ibdte69TU1OZCHueJp56NFSFXSYoCamhs5pf2EzJer2Oa6+9dqtGSdx1UJl5a6KOYRgwDANEhHq9HrrvxsbG1kSgKBN5ms0m2u321t87duwAABw9ejRi68cPWYw8JVTUX7eJx14ddGXFBIVWnF5ft9tVCqmEeeb2U0XS3HAV63Q6kcNGZTLJhlEHEooRyoaXaDs7i263q/z474z1BgluFrHnWq2Wu7jmaZINo46qsEsoRhgL+v0+9u/fj5MnT269Zod17PDLysqK0rH27NmD888/P3S7m266yfJ+UqbMy8IF1Z8BZKGN1FBRf90mHnt1iRuaURmI9XrfywzDqHToI2tz/t7OhTyCZvYK3kBCMULRSJK/rJI6mVV6oNh28xvX8AuzSC57PDIXdgB1AN8D8KWwbUXYq0mS9EeVfU855ZTcBa6qNjc3t61jDRJqmX0aD1Vh1xlj3wvgHo3HE0pGklXuw1In+/0+nnrqqeSNFGJxzz33jIxHEBH27NkDANvq0QfVdY+bpiq4UFH/MANwJoCbALwK4rELPiT10oLi8+NQ2bBqVqvVPCtwBtXKEY89GGTssf81gD8H4Du8T0SLRLRKRKuHDx/W9LHCOJF0wtLCwgLW1tawubmJtbW1kQWT46zsI6TL5uYmTpw4MfLaiRMncPz4cc/tJUNGH4mFnYguAfAIMx8I2o6ZV5h5npnnTz/99KQfK4whCwsLWFlZgWmaICKYpqltRXuVmaFCsdF1LQgAsSMuFusARO8F8HoATwOYBPAsAJ9l5iv89pmfn+fV1dVEnysITsLypYV0ISIk0RLTNLG2tqavQSWFiA4w83zYdok9dmZ+FzOfycyzAP4QwM1Boi4Iuun3+yLsOaMq6o1GA81mc+Q1CcHoR2aeCmPP0tJSIm9R8Ea1+Jl7HyfNZnOrkJppmti3bx+uvfbaVMJxwjMkDsXEQUIxgk5qtZoIu2ZM08Ty8jL27t2LjY2NyPseOnQIMzMzIyUfhOSohmImsmiMIKTJzMyMZMVopNVqYffu3VhcXMRgMIi0r8TKi4GEYgQAVpzaPZFkXPBKoxTiYU8suvHGGz1FPWgsQ2LlxUGEXUC/38fi4iLW19fBzFhfX8fi4uJYibu9OIWQDGbGjTfe6DsD1C/kFRQrH2enYWxRmcWk22TmabHwm7VZr9cLX3kvztqmYuEWZdHvoIUypNiXXiDVHQVVVKoi6rwZda2qxCylBNKyZrOpvABIUBkA1TISOq+JMiPCLnjidQOpiqOOOh66PTgp1ZudTUxMbKv9EvbbqZRb9rsmnLXbRewtRNgFZh4VcsMwthVgsm8g1XBGWh523E5DPPZszTCMbWIbpzib8/f22yZKGeCqIMIuKMef7ZtRNa6ahoftF6cNe0RXPceqryuqy9y/U9gTmMoTWpSnrqpXfxRhrwhu4XM+vvqtauNlUbz2JDdYFI9dNWzT6/WUzjXK9yHm/7s7rzmVlZPCOucoT11BA7VVACLs5UdnRkicWLWuNvs9AYR1AmFhJqfV63Vut9u5C+M4mVcoRNUBiCLAXteE3/UoHrsIe+nJO74cJRzjFmGVRYyDOhuvsJGItz7zG7xMMtAe5Ll7PXlKmuR2IMJeLrxuirwzQgzDUG57nKwH1UE1d5tUQzNi/qIcp7N1/rZe4TKvJ6put8vMzN1ud+s3q9fr3O12JQXSA4iwlwc/YYwyiSQt82uv84b0a2dY1kOUR3R3m/Lu9MbRGo1GqHiGeex2x+om6FrtdDqer9uiLzwDRNjLg9/NNDk56XuzvATgpwE+nLIYtNvtkbBK1EFYt7kf4d2dhMoxgkIGdrpe3iJaRHM/gXl5zGHjOkQ0IshRwjduq9frWdxeYwVE2PND9yNkVO+zXqvxNwB+mCh1YddtYYNuKk8ptVrNM9Zuf4/2b5P3uRbNVCYNOfPWg45l/05Jv2cJv4wCEfZ8SKM2RlSP5wqAD05M8Ad27Bg7YQ/LeihC+GlczTCMkVh20HevkpaaRecoA6ajQIQ9H6LkaavS6/W40WiwCVg/mYddMPycKYB/DPClAF+F9EMxOs1+jA962hFPO7q5s5BUMk6Cjhd2rafRfsECisIuZXs141fu1O91VYgIPwHwCpf9HYBjAB4YbvcXAO4B8PmQYxURZsb+/ft9ywf3+33UanLJRsEwDBw7dgwbGxtb3+n+/fuxZ88eGIaxtZ277LHf9+xcKm95eTnRtTQxobbOz8bGhpT6jYqK+us28dhH8Ur1UjnmxQCfBPgNw7+fD/BRgF84/PsqFNNj9wun+IUIbI8z73aPmwVlThmGERhDDzquTa/X46mpqUzOpeoTk2wgoZh8iBpj73a7nhdyp9MJFLNzAH4M4Ksdr30Z4L9x/H0ViifstVrN95zF9FncuQ6maQaGWJyzfr2yY9KaIGYP7Lo7E3cWTtmBCHt+RMmKiTORZgrguwD+BsATw9cuAvgJgJ8H8KlDey/AR4b/b6YsJFfBO/b/O67tOp2O9tisePPeIsisPw5uX8t+37kdw8/6nKempnzvM3eBO79c+3EAIuzFIEzkwy7YywE+CCvkcnD49w0APwTwGY7t9sJbWG1bSvnGugrWE8R5LnuWazt7EE/nZwfl81fRnGELnTOU7UHMsFBN2PtpmZf37tfJNJvNsRR3iLDnT9Djqi30QeVkLwf4SYwK9FMAnwD4/OHFaW/7XFiZMU7bB/Djw//PpHxTXQX1sI942OnZxMREZOdBxZzhxLBwS56zoolopGRwUKc2jnF7iLDnj8pjcFAo5iC8ve8nsN0z3umx/1UIF1vVUJDp0xaG1XGofJbz5svjpi+7OSdgxS3f7Gd2+CKKN57X72wLdljnMo4lgCHCnj+qF3a73d66+YiIJyYmGLDCL6xoF3gc9yoEiy2z/6Oq25rY3plcD/AA4LOHn3Vi+HnHAf4uwL+fw01dNbPDe7Kg9zNme+0q3924ARH2/FEduHKO+Dtv0IPwFvGDmm6ATqcTWxDcqZYLAL8V4AsB/l2AvzRsq4h7upakFktZTWW1LImxi7DHRtUb9itM5RVjf3L4ep43jleqpZd9C+DvFeBGL6u1221mjjYbVzx7S9THdaFsZCXsAHYBuAXA3QDuArA3bJ+yCrtqudoo5pUVk+dN4ZVq6WdvH7a7VoCbuYxme52qHrvUqrdsYmKCG43GyGvjkg+PDIX9XwN42fD/OwHcB2AuaJ+iC3vQOqLu3t15Y1VhUNAr1dLP3gardLAIe3rmHJsJ29YW9rzbXHQrsgePvEIxsMqU/FbQNkUWdpWBKOfU6zI/2rqfFq6HlW55vuL+3wL4QAHOoypWBcciKytqVUkoCjtZ2+qBiGYBfB3AC5n5Cdd7iwAWAWBmZubc9fV1bZ+rk9nZWai0zTAMTE1NKW07jlwO4OMA2o7XGMA/AniXa9u7AXwRwA0A7h3u8ycALgLwe8P3hGwgIui8p6uMaZpYW1vLuxkjENEBZp4P3U7XRUBEUwBuBbDMzJ8N2nZ+fp5XV1e1fK5uarWa3BgADgKYVdz2QgCvB/DvYMXlNgF8F8B7AHwlhbYJQhYQETY3N/Nuxgiqwq5WNzP8wxqwHLZ+mKgXnZmZmdJ64VGY8Xl9E0Dd4/VbU2yLkC31eh0nT57MuxmpMTk5iV/84heh2z3nOc/JoDXpkLi4NVkFmf8WwD3M/IHkTcqX5eVltFqtXNtQhHrpftXjk1WVF4pOq9XC/v37R+qulw0VUR93dKxacD6sJ/FXEdEdQ9ut4bip0O/3MTs7i1qthtnZ2W0F/BcWFrCysgLTNEFEME0T7Xbb52jp4BcKIqLM2vJuAEddrx0dvi6Ul8FggKWlJSwuLubdlNx59NFH825CfFRGWHVbXlkxKgv0+qU0qsxmy8I6nU5mn1W0HHqxbMyekl+Uaz4vK2LJAcjM01GCJmYErSZjU4SLPI2St2JibgtbbKMKNu7pjpUQ9rj55s7VYvK+0AApdyuWvtneehUdCGd1zCKKOrMI+whxvQ/7Ii/KJKQq3mxi2ZlzWn2Ue8a5LsA4W7fbjbT6WR5AhP0ZggQxaFEAeSQVK7NNTk5uiZi9YLizjEbe7cvaarXatk6qaDVkIML+DH7iXK/XfUsD2DG2cfOSbS9DwjZiYWYLutd7rVZra12AqptzVaa8gQj7MwQJt3Mbr0cwv06hCIOpfhdhULvFxFRt3JyaNK1erxei1C+qKOxe4uysvmhnxUT5Ufw6hbB1H/Mye8FhuSnFxNK1PDJnUDVhVx3kjPJj+HUKRY4/Rq3RLSYmFt+yznVH1YQ9ipCp/BhB4ZtxEE3DMAobLhIbbytLFoyfnQvwPoDvhTU5b1/AtlkviA1FYddRUqAQHDqkXsUkaFu75MAVV1yBwWAw8t5gMMCePXvGokjYxsZG4SrTCeNNvV5Ht9vFU089hV6vl3dzUuN8AL8B4DsAHg7ZdmbGr1xeCN/8JnDeecDkJHDWWcCHPxzvOH6oqL9uK4rH7rVSkko4R+LXYlW1RqMRmlgw7kaO/38H/h67agLGNv75n5nbbebXvY75ppuY3/te5nqd+eMfD9U5VC0Uo5qa2Gq1uNvtJk4HFHEXq6rZA/RFmZEd1UzAkj4Pu8C1rZ+w12q1LeH2Sy/2Hc9bXGQ+5xzmEyeeea3bZT7zTObNzUCdQ9WEfXjSoRekqleudIGU1GMREwuzcRlr8rImwOe57HqABwCf7drWT9idnZtbT0z4dxx8yy3Mu3Yxv+Mdo+J1663W+3feGaZx1RP2sIlIQdtENTucU6QMmbIPaomJpWEXwxokfYPHe0GhGD89cXYcrwCYb7uN+bWvZZ6cZP7BDyzZ3bdvVLweecR6/frrAzUOVRs8BYDdu73LwC8uLmJhYQEAtAx8tlotLC8vAwA+8pGPJD6eLt70pjfl3QRBGCvOAdAD8FEA/yPivv1+3zMR4ziA24f2E9MEjhwBbrgBuOYa4LTTrI3sf22e/Wzr38cei9gKH1TUX7dl7bE7B0vjxMYNwwgcFCnKI2lR2iE2XqZjzCmKERF3Op3cr9cpgO8C+BsAT/hsE+Sx+5UBt63VavHn3/9+5lNPZX7zmy2xePBByzP/3OdGxevECev1j30sUONQxVBMkGgTUegP4bdfWBGgoFrvYmJFNjtMmbXI2gOLWd437oVjbgf4IYDPCNgnSNjDbArg+xoNfuScc5iPH7fE4sknLdn95CdHRURzKKZUwp7WxakyW1WKbomNq+ksSx3lPsha1J8ELMkb2ibAfxmyXxRh9+s4zt6xY1Q/du1ifuc7RwXk61+32iWDp9tJu3Z6UG6qpD+KiWErZJl3O9x2EKOibttDGM2O2QnwNMB/MLT7Ab7Z8bff8cM6jpHZ7ouLzC94AfPTTz/z2lveYgm+pDt645wkkMYF4ue95x0vFBMrihXxXjgJb2F32wVD83vf7/gHfbZ3dhx8223MP/vZMxOULr+c+eabmd/3PuaJCZmgpEpQ+qM9EDo1NRXrwrUZx3xeSYsUS9M6nU5hVh2z7SC8hfegpuOrdhx8yy2WcPzTPzH/+q8zn3IKs2kyf+hDSpoGEXa1OuxxPHu78H5RS/eKiemyRqMRecENe0A277Y7zStU8uTwdR3HP+g6tm0HHdvoACLsFmH1G+J424ZhcKPRyP1ijdPuvNsgNj5Wq9W40+nEutaZOff2u809uKlL1O1jh3UcOmq3Q4RdjagDrkFrpBbZ5ubmCjmoJVYc8yrzHPeJdtzCkzosrOOwyxAkASLs6ji9eq9Ffd0evwikWNksjsPiNz4la6X6W1Igwp4eVfRGxMprdkw8isNie5/dbncrH71er8dKRqiSJV0zFSLs3ijXTA45hsw0FSuLxVkA3S+sIE+z6hZnzVRkKewALgLwQwD3A3hn2PZ5CbtKlowquuLs4xivFyuX2VleUceb/BaNF1O3qGumIithB1AH8C8AzgbQBPB9AHNB++Ql7GFFwsLQPfnJMAyJ2YsVwuyVkeIIdLPZHMsssSJY1DVTkaGwvxLAVx1/vwvAu4L2yUvY/QRU5cvVXa7A+aRQpJruYtU123NnlnEk3eY356XIHvtrAXzC8ffrAVztsd0igFUAqzMzM5FORhdJPHadF7rtqTuRkIxYUUyuRf3Wbre1hIFRNGF32rjF2KPMogvrAPwGncKeCBqNRiFKATSbTe50Orm3Qyxdk/CgftORuAEJxXgT9uW634+yRqpTtL3EL6wTCcqnd8Y/7fe9JpRkYXLTV9ckGyy+6QAZCvsEgB8BOAvPDJ7+atA+Rc1j9/Kao4hYs9n0zS7wW7AjSS8usXkxsfEwHbNOmTMUduuzsBvAfbCyY5bCti+qsOuIowd5NO5YftL0SxngEhMrvtVqNS11YpgzFvaoVlRhTzvEYGcd2B66XyegMpirEvefm5vL/aIOu+AlTU6szOaVKJEEiLBHx88D1iX4qkLmTL/0CtWopF7aC3DnfWGrXPgSsxcrm8Wd+BgGRNij4xcasQuBAdkMHDpF2f15KsWa7Fj/uAjmuLRTTCzMkmS8qAAR9ngEDWZm4QEnTWt0PvqNg8cuJlYmSxuIsOsnzLOM63naS/UlTWFUGZwVExNLx6LOIo0DFIW9BkGZmZkZ3/darRauvPJKmKYJIoJpmjAMI/SYrVYL+/fvx3XXXYdjx45hc3MzdJ92uw0i2nac5eXlrb/7/T6WlpYwGAxQr9cBAKZpotPpbP0tCIIe3Pdf7qiov24bN4/dWRjJyyv3G/n28pibzea2iUfM6mETv1DNKaecsvX/dru9bRt7rEA8eDExfZZ2TN0NJBSjh6BJSyo/qOoEJJUwjixkoM8WAf57gB8G+HGAvwHwbxWgXWLjY1mEXtxAhF0PSUv9Jv0csXRsHeCPAXwpwK8G+JOw1qr83QK0Taz4Zmee2eioA6MCRNj1kKTUbxRkoDNbMzxe+ybANxegbWLFtlqtNlIeROcCPmFAhF0PWXnszNYFIiVT9ZgJWJe3h13gs8/VAN9bgLYX3ewxJXeN8SotYu0U7iw1AiLsekirN/Z7dFMNyeRV2XFcrAnweS67HuABwGf77HMA4C8UoO1FNr9BePdEviqYLdxZPdUzi7D7EicWFmUflW2DZrjmfbGW1S6GFUN/g8/7/wGWN39hAdpaZAt6orRnTBPR2Dkecdobtgi4eOwpoFpjxcv79tvXS7DdNdLdtWDcx+/1elLTOmM7B+DHYIVavN5/GcBPAvzBArS1zGZ7tvb1X4T7wA4rBWlDmHBLjD0jYff7ov28DWfP6rWvX+64V7541AtALJ51Af4OwI8CfBTgO4evubebAvguWOmMEx7vnwXwTwD+PMC1ApxX2cyeVW0OF61JEn6MY41GYyREZHcmXk/TQc5bmHBLVkwGwh71QnHGwtK4yMIe2cSi27uHdgnArwJ4GVaoZWP470GALwf4BoAfAvgMj2OcDvB9AH8b4B0FOKcymn3tB4lj2ES+uJ69zpK5WQl3GKiysEet2eL02NOoNBg2yCKW3C4H+ARgXdJDe2r42vke27dhefz3ATxdgPaPu/ld2/bKQSrhDD/h7PV6nuWu7ZRDdzaZ7hroRQJlE/YoPabfRWQYRugjlW6vWiUtSszjhseoSDvtAo/tD/ps+wS2Z8fsBPirsIT/co/38z73Mlm73WZm9cwRv/Etd8jTPUGoKqBMwh51cCLssS+og4gSYw8z9zqnEmNXN9V0xTos7/sk/DsCt10Q8n7e5142C3JobI/ebw6H6thYVUCZhD1OOlGSmJif1+B1gTUajcAUMHcbJaUxnnmlK/4rhIv4wQK0Xczf7MHNOA5PGnniRQdlEvYsJwCE4ddhqLZRwjG5FRJlAAAR4UlEQVTRzS9dsQ7wubA88M8AvIlRUX8SVqgl7/aLBVvcnHfx2Mdc2LOcAOCFivev2sY8bhx3RcharcadTmfknLrd7kjpXxXLovxBWLqi066H5dU7s2LyEisxfaYyNlYVUCZhz3ICgMpn2+GXqJOfirQO6cTEhGdd+F6vp7R/rVZLtISfqgWlK7rtYlie+vMK8P2K6THVsbGqgDIJO3N+eaQqoRP3xQd4T4QochgmSvZOs9lMpYO6HJanbXvc18PKXPFKV/Sy9wB8DJKTXhYrc9piXFA2Yc8LVQELml1qi2beN0qSc0jbLocVE2eHbcIqo+uVrvh/AX4LrFrqrwH4AwAfB/i9BfgexZKbnS0jjAIRdj2oetkqBYGKUBsjzGyCamSkYQcxKupBdgHAKwD/EFY5gcMAfwvghQJ8f2L6TNgORNj1oOppF2V2qdcMPVUjom2Pvlmdj18e+skCCIxY9lav17fdhxJjF2H3Jc4FEpb9oRqfTmu9UvcgaBJP253Fk5XXfhDewn6wACIjlo8579kk6w6XCWQh7ADeD+BeAHcC+ByA01T2y0vY42bXRLmwgjx8ItIeu/ZK+UwSI7e9drvzc6+Sk5Z5xdglD7265vTYw5yLKqU+IiNh/20AE8P/vw/A+1T2y0vYk+TDq9Rjt18PughVPGrTNJVyxMPKKsQRZ6+c4azMnRUjol5ts+8nlXBgVSYrIetQDIDfB9BX2TYvYdc9g9XvCcBvJp3TCwm6SP2O7TT7M9yLFng9mkYJp0xOTuZ+Q4uJ2RZUK8ZpVSkvgByE/YsArlDZdhw99ijH8/OQnQXB/C5WZ5qX07uPMojp9OTHIc1STCzIVJ4ixWOPKOwAvgbgBx52qWObJVgxdgo4ziKAVQCrMzMzsU8saXEvnTNYg54Aut3uiDftFHW7Le4MlkajEalcgcoNMQ4plmJiQeYc97H/dr4vMfYUPHYAbwRwG4CW6j5xPXYdwqwzbSrpE4BqW/JOoRQTy8L8nBD3/VTl1EdkNHh6EYC7AZweZb+4wq47lJKUtGvYZD1JSEwsb3M7Me41DaoOMhL2+wE8AOCOoV2jsl9cYS9S+V6btLyHLKb1y5OAWNYWdM35vVelUEsYUBR2srbNlvn5eV5dXY283+zsLNbX17e9bpom1tbWNLSsOPidKwDUajXs2LEDR48eTfQZzWYTjUYj8XEEQQUiQq1Ww8mTJyPvW8Z7PA5EdICZ58O2q2XRGF0sLy+j1WqNvNZqtbC8vJxTi9Lj0KFDvu9tbm5CR4d8/PhxTE9PwzTNxMcShDCYOZaoA8H3g7CdsRL2hYUFrKyswDRNEBFM08TKygoWFhbybpp2ZmZmAt8fDAbKx2o2m77vHTp0CLt371Y+liDkQdj9ILhQidfotnEqApYXXiuzxzE77h93+TExsSKY1Ga3gGKMfaw89iqxsLCAnTt3JjpGq9XC7t27sbS0hM3NTU0tE4Ts2djYwBVXXIHp6Wn0+/28m1N4RNgLQr/fx+zsLGq1GmZnZ9Hv97GxsRH5OHa8vF6vYzAY4JprrvEdhBWEIkJEvu9tbGxgcXFxm7h73T+VRsWt120SihnFLx8+ajqiYRiZLDAtJpa3Oeeu5LkmctagjOmOZWV6ejqWd+6k2WyCmXHixAlNrRKE4kJEW+HFKqVBlzLdsYzEDbk0Gg0YhrGVHbRz504RdWHsqdfrSts5s2T8Qo1VTpEUYc+ZpaUl3/cMw9iWt2+/vm/fPhw5cgTXXXcdACh3Do1GA+12O15jBSFlTp48GZiea3PkyBFMT08HxuMrnSKpEq/RbeMcY9ddQgABccRerxf4eaplB9yrPWVRfybJ2qti1TXDMLZdO7VaLfKykl7r95YByJqn+kljkMavol2tVgvdV0WgvfJ/064RY3cgUjK4mhZ3ScWgRTXiXEtlBCLs+kmjumSSCzNIoIOeJtL02JvN5sgiH3ktsxckHlmt41plsx0e1e1rtZrWjK6yLrwBEXb9pFFdMkln4bdvvV4PDBWpriMZZu70Snt2q2ma3O12t9pXFM/dfnrJu8OJGlYYV8sr9basqY7MIuypkIbHniS8oyJQfsfqdrvKN4qXh9tsNtkwDCYiz7ioVzt03OhEFLmjqNfr276DXq+Xi/BURdSzNvc4UlmBCLt+0poIkXS5P3tf1RVonPuGiZthGJ41a+LUnlHpAILMPo+oHnfQE5WquOuo2yOWjlWpjgxE2NMhqghnuYxXklCRX6el28OMK5B2Bxonq8cOTdmhI+dvoRKScoaVxNRtcnIyk88pc+jFDUTY8yfrqc5prMGq8waMW2HSbku329Wa0dNoNEKP1263mVnWnS26lXWw1A1E2PMn6zVa0+hIdN14Ojx1VXG1t9MxaGsYBjOzxMbHwKrgtUOEPX/yWKNVd+gnSgzaGeawwxf230kGKk3TjBwKsT87qVjYE13yFi2xcKtCSAYi7PmTtseeRfy+1+ttG/BsNBrbhNsr68T5ftKbNo5I6/DY43QqYslscnIydjpq2UMyEGHPnzRj7FnG7+MMGKtmvxiGoSScUUM5umLinU4nd6GrmnW73dhPSWk+DRcBiLAXg7S86qzj91FQDbs44+c6a8tkPdCpa8KXnzWbzUjzDopojUYj8nUR1OH7DcQX4fpPE4iwl5s84vdeRM2k8erkdNaVUQ2b6FoDtt1upx6qabfbY72AirP9qh2gfX34zaHodruVWVzDCUTYy00RPHa/cFDQDWsLuFPYdQmj3amFHU9SF/VaGpO37N/SPYnOORmp2+1uXU/1ep273W5m135eQIS93BRhOTA/AVX1hu32Rk1j9DP7xh7HLJZxLXNsC63usYgwB6UI138eQIS9/ITVak87YyZIaFVDK6qpkK1WayQTp91ub3UgtVqN2+32SKplEbzyiYkJ5e9gHDsjpwDrDEcRUaj3XYQn1jyACHt1ycqbCbqZ7bx2lRs5zMO3i3g5OytnZUlVEW80Gtpi6zpFTOX7TNOSfifM0Wfm2tlQROQ5+Svsei3KGFPWQFHYZWm8ErK0tITBYDDy2mAw8F2Gr9/vY3Z2FrVaDbOzs+j3+0qfs7y87Ll0HwAcP34cAAKXLgOAWq22tSixH/v37wcALC4uYn19HcyMjY2NreUAres9nBMnToR+Vhj1eh1EpLw2ZxjO5dt2794d+n2lQavVQqPRSHwMVYgIl1122dbfx44d27bNYDDAnj17fK9Fv2XvKr0cnhMV9Q8zAG+D1WNOq2wvHnu6RPFmonr37hBP2ml4uhdgSGLO70VHqKfRaBR6URJVi3INEBF3Oh3lMQW/a1Fi7CmHYgDsAvBVAOsQYS8EUeKPUbbNqgJkUc0pGjrCJs7VpsZ1dmutVouUqtrtdiN31H5x8ywrpxYFZCjsnwHwYgBrEGEvBFG8mSjevZ/4JPFe2+32WGSE2MXAgr7jOGaPH+ga7C3KalV+FqcDK3vcPArIQtgBXArgQ8P/ryFA2AEsAlgFsDozM5PBV1BtVLyZoIlBXl5SGpkm9qzKooRb/MxrMQddqzDpWl1qHCzONVT2TJcoQJewA/gagB942KUAbgdwKisIu9PEY8+fII/Tz7v387aSeon2jRslVuteb9XP5ubmtImS+3vRGRc3DGNsY+xRf+sonVgV4uZRQNoeO4BfA/AILEFfA/A0gEMAzgjbV4Q9f4Ieif2WGvML8XhN745idt6yijfnvtGz9nSd3qPuuHgVVmqy01ZVZqtWJW4eBWSdxw7x2AuDShgmTESDshH86r3EFSW7jnvUG1138TAVc8Z7dYemnEXRyui9O8cpwq4XCb94AxH2aqI6cKoiwnFuLlVvzNk2Fa87SkZPFtbpdFL5fPs8k3SUft9z3mMZkraYHMjM02qimr6o4hWGZSN4ee9Ra7GrZIREzejJyubm5rR71u7vPMmx7EW8nb+NfX2kMQO33W4H/tZRriPBGxH2ihJ1clLcx2E/T0vFI3TH8MM8U7+6ITo82qSdg87BWa/vPO7AtH1eTlFPowqjiokHrg8R9ooSpzhSnMfhOKLqV1o17OkhaILKOOTAq5p73dgohcGctXncnVWj0cjt6cbO0xf0IMJeUeLGLKM+DscViqBBWb99/EJCUbzQoPYWoTBYrVbb1kmpPgE5xVNnXF7H9yKTi/Qiwl5hsohZJhEQPw886tNGlDaE5ci7OwidYm8/qcQJq6hkDDnFU1eb3QuWB7U9KDddslv0IsIupEpY+CRICII88ChPG1EW6AiaJWrn0Ts7Q93ZIyrfWVDbgs7VKZ46Swo4jxv220h2SzaIsAupExYDjlKuwHlM1aeNKB67fSw/gXS3SWdMul6vb52bs8NQeSpwpj96dTZu8dTZGbk74LDfRrJb0keEXciEqCGZqBOfgojiAdsipSpiqudlL2Yd1hHEnXTkHmwOW+dTZ4xdwijFQ4RdyAQVwbLzqe36Ll6zVuM+xrs7hLBYb5Q8f5WMG2eH4PfZ9mpBScVV5Xvy2sYr28a9GpXXwK143MVDhF3IjLB8eDvG7SdKUQdNg7x7nbFgleqNzjYGCXvc0I7K0nlenVKcpx8JoxQfEXYhc4KEJ+i9KJOqvAYSvbxW3bFglQ4h6Dz8zj/Mm3fO2KzqOp/CM4iwC5kTJH5xRM/LE1Ud/AxrZxzvNGy/oPNQeZLwysl3Lp8X9clGKB8i7EIu+IlfEtGzCQv3qLYvrbQ8FfEO6hjCxgckpVAQYRcKRVgIRcWLVs3lDiJtrzdJrFol1CKx8Gojwi4UBi9P0554EwU/UbYHZ1UocpxaQi1CGKrCXoMgpMzS0hIGg8HIa8yMG2+8MdJxlpeX0Wq1Rl4jIlx55ZVYWFhQOsbMzEyk17PE6/xarRaWl5dzapEwtqiov24Tj71a6PSSk4YiksbB0ybvzxeKDSQUIxSFooUY/MRTBieFoqMq7GRtmy3z8/O8urqa+ecK+dDv97G4uDgSjmm1WlhZWVEOoWTB7Ows1tfXt71umibW1tayb5AguCCiA8w8H7adxNiF1FlYWMDKygpM0wQRwTTNwok6ABw6dCjS64JQVMRjF4Qh4rELRUc8dkGIiGSlCGVBhF0QhoxLyEgQwpBQjCAIwpggoRhBEISKIsIuCIJQMkTYBUEQSoYIuyAIQskQYRcEQSgZuWTFENFhANtngpSLaQBH8m5Ejsj5y/nL+evHZObTwzbKRdirABGtqqQllRU5fzl/Of/8zl9CMYIgCCVDhF0QBKFkiLCnx0reDcgZOf9qI+efIxJjFwRBKBnisQuCIJQMEXZBEISSIcKeMkT0NiJiIprOuy1ZQkTvJ6J7iehOIvocEZ2Wd5uygIguIqIfEtH9RPTOvNuTJUS0i4huIaK7ieguItqbd5vygIjqRPQ9IvpSXm0QYU8RItoF4LcBVHFttX8A8EJmfhGA+wC8K+f2pA4R1QH8DYDXAJgDcDkRzeXbqkx5GsDbmHkOwCsAvKVi52+zF8A9eTZAhD1dPgjgz2GteF8pmPnvmfnp4Z/fBnBmnu3JiJcDuJ+Zf8TMxwH8LwCX5tymzGDmnzDzd4f//zkscXtuvq3KFiI6E8DFAD6RZztE2FOCiC4F8GNm/n7ebSkAfwzgy3k3IgOeC+ABx98PomLCZkNEswBeCuD2fFuSOX8Ny5nbzLMRE3l++LhDRF8DcIbHW0sA3g0rDFNags6fmT8/3GYJ1iN6P8u2CflBRFMAbgDwZ8z8RN7tyQoiugTAI8x8gIguzLMtIuwJYOZXe71ORL8G4CwA3yciwApDfJeIXs7MD2fYxFTxO38bInojgEsAdLgaEyZ+DGCX4+8zh69VBiJqwBL1PjN/Nu/2ZMz5AP49Ee0GMAngWUTUY+Yrsm6ITFDKACJaAzDPzJWpdkdEFwH4AIALmPlw3u3JAiKagDVQ3IEl6N8B8EfMfFeuDcsIsryY/QAeZeY/y7s9eTL02N/OzJfk8fkSYxfS4moAOwH8AxHdQUTX5N2gtBkOFv9HAF+FNXB4fVVEfcj5AF4P4FXD3/yOofcqZIx47IIgCCVDPHZBEISSIcIuCIJQMkTYBUEQSoYIuyAIQskQYRcEQSgZIuyCIAglQ4RdEAShZPx/7EEwhkbXQPkAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAACPCAYAAAAfidZ8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXmsXdWV5r8FMYHEmHjCGGP8jHHwQAzGDjghjkgYhEMqgNLEga4qSk11ulsVpao7ippUq1v1T3VXSqpqtVrd6oqqUpCESlIRICBBIsSYyQNgYwYP2MZ4wMYDZjSEEAi7/3jXO99e9e72uffd6Zzz/SSLdd8+7979znf2uZu7vruWhRAghBBCCCFEHTmu3xMQQgghhBCiX2gzLIQQQgghaos2w0IIIYQQorZoMyyEEEIIIWqLNsNCCCGEEKK2aDMshBBCCCFqizbDQgghhBCitmgzXBAb5jtm9krj33fMzPo9L9E6ZvY5M1tpZm+Y2a5+z0e0j5l9y8w2mtkRM9tpZt/q95xEe5jZfzSzF8zsTTN7ycz+p5l9qN/zEu1hZieY2RYz29vvuYj2MLO/MLP3zOwt+ndWv+fVDbQZLs7XAFwD4DwACwD8HoB/19cZiXZ5G8D3AGjjVH4MwB8CGA/gSgBfN7Ov9ndKok3uBnBBCGEcgHMxfK/9Rn+nJEbBtwC83O9JiFHzkxDCWPr3Qr8n1A20GXaY2XL3f0HvmtmDAG4E8DchhL0hhH0A/gbAH/VzriJPMy1DCI+HEH4AoJKLuopktPzrEMKTIYT3QwhbAdwF4OJ+z1c0J6PljhDC60cPA/ABgLP7OFWRIfNeCTObCeD3AfyPvk5SFCKnZV3QZtgRQoj/FwTgdAxvmH4EYD6Ap+nQpxs/EwNKRktRMopo2bAtLQWwqQ9TFAXJaWlmN5jZmwAOY/iT4b/r30xFjmOsyf8N4M8BvNOv+YniHEPL3zOzV81sk5n9h/7NsrtoM9wEMzsOwD8BeDCE8HcAxgJ4gw55A8BY+YYHnxG0FCXlGFr+BYbvaf/Y63mJ1hlJyxDCPzVsEh8H8P8AHOzjFEUBvI5mdi2A40MId/Z5aqJFRliT/wxgLoDJAP4tgP9mZtf3cYpdQ5vh5vwlgJPxO8/aWwDG0fg4AG+FEEKvJyZaxmspysuIWprZ1zHsHb4qhPBuPyYmWqbpugwhbMfwJ/z/t9eTEi0TdTSzjwL4a+heW1aSNRlC2BxCeCmE8NsQwmoA/wvAv+rnBLuFvqk7Ao0v4FwP4JMhhPcaP96E4bTd443H50Hp2IGniZaihDTT0sz+DYCbAXw2hKBvrpeAguvyQwBm9W5WolW8jmY2H8AQgEcaSdMTAJxiZgcALAkh7OrXXEWegmsyYNjPXzn0ybDDzBZi2O90TQiBvwn7fQD/ycymmdnpAL4J4JY+TFEUpJmWZnacmZ0IYMzwQzvRzE7o1zzFsclo+a8B/HcAl1f1W85VI6PlH5vZqY14HoBvA1jRn1mKY9FEx40ApgM4v/HvjzFsdTkfwIv9mKc4Npk1ebWZjW+Ulr0Qw58Y39WveXYTfTL8L7kaw2WaHiU78CMAvgDgLADPNn7299CXOwadZlp+B8BKOu4dAA8BuKSXkxMt0UzLOQAmAniCfv7DEMK/7/kMRVGaaXkAwF+a2VgMl+T6KYD/2pcZiiKMqGMIYdnRB2b2KoAPQggH+jA/UZxma/J1DJch/TCAvQC+E0K4tS8z7DImy6sQQgghhKgrskkIIYQQQojaos2wEEIIIYSoLdoMCyGEEEKI2jKqzbCZXWlmW83seTO7uVOTEkIIIYQQohe0/QU6MzsewDYAl2P4W4ZPALg+hLC5c9MTQgghhBCie4ymtNqFAJ4/WtvTzH6M4fIcTTfDkyZNCkNDQ6N4SdEuu3btwuHDhztWLFta9o9Oaikd+8v69esPhxAmd+K5pGV/kZbVQVpWg1beK0ezGZ6GtIj2XgAX5X5haGgI69atG8VLinZZvHhxR59PWvaPTmopHfuLme3u1HNJy/4iLauDtKwGrbxXdv0LdGb2NTNbZ2brXn755WP/ghhYpGU1kI7VQVpWB2lZHaRl+RjNZngfhtsuHuWMxs8SQgjfDSEsDiEsnjy5I1kH0SekZTWQjtVBWlYHaVkdpGX5GM1m+AkAs81sppmdAOCrAO7uzLSEEEIIIYToPm17hkMI75vZ1wHcB+B4AN8LIWzq2MyEEEIIIYToMqP5Ah1CCPcCuLdDcxFCCCGEEKKnjGozLIQQnSBX79yP8WOOzdIKOscdd1zTMdFZiurnj/vggw9G/B3Wzj+Wlp2l3V4D7T5/M809Xmd+nLsGdH38jk5r67XLre2cDkXXcy+1VDtmIYQQQghRW7QZFkIIIYQQtUU2CSFEX8il2PixT80VTa0PSvqtihS1rgCpXrmxdvWSlsUomjLvRGq9nfXrydkkBjHN3i86rVdu7Le//W0yxo9zNomcha1Z7Om2lvpkWAghhBBC1BZthoUQQgghRG3RZlgIIYQQQtQWeYZR3O/2m9/8JsaHDh1Kjtu7d2+MP/zhDydjs2bNivHYsWOTseOPPz7GKh0zeor6UN97770Yv/baa8lxe/bsifH777+fjM2YMSPGEydOTMbGjBkTY3nahimqR27d5TyGufOc88G141uso47NNGpFy6Lll3LnupXSe0Wog5Y5HTrxnP752D/67rvvxvj1119Pjnv11Vdj/NZbbyVj/H74kY98JBmbNGlSjE855ZQY+/dbfg5PWbXtxPXfyv2Q3/fefvvtGPt9z4EDB2Ls30dZl/Hjxydjp59+eoz5ffTEE09MjvvQh5pvUTutpT4ZFkIIIYQQtUWbYSGEEEIIUVtkk2gBTq1v2bIlGbvvvvti7FMCy5cvj7FP/eRSOs3IpTvKmgZql1ZSf81sEjt27EiOu/POO2PM1hgA+NKXvhTjCy+8MBnjlE5RjaqgZdGyaLmyPLkxb1Xh88zWFL+WciWBeF6+nA8/D8e50m2tlHUbZIraVXLlslrpUtXs/LaS0uVji2pU1TJ8ne44lrse/Lp84403YvzCCy/EeO3atclxzz333Ii/41+PrRAAMG/evBh/9rOfjfHs2bOT4z760Y/GuEzaebptjcjdY1955ZUYb9q0KcYPPfRQchy/d3o7DK+xU089NRlbsGBBjJcuXRrjc845JzmObaW5+3sndNYnw0IIIYQQorZoMyyEEEIIIWqLNsNCCCGEEKK21NIz3K7P9Ne//nWMn3zyyeQ49kF534sYDJp5hrdu3Zoct23bthh731rR9rFVJ9dulf1nHHv/NfsF2aMGAC+//HKMubQPAIwbN27E2Pvx2U/steIx/3v+cTOKelwH/TrJeaubacnrB0hLZPlyWfzYexNPOumkGPN5z5VYyrV29WW2TjjhBBShaCnEMpErY5c7Lre2uWTaSy+9lIyxn3TNmjUx3rx5c3Icr+133nknGePX89cA36e5xNd1112XHMfeYn89lJVOeMH92ub7sddyxYoVMX744Ydj7Pc9fA/39wSe88knn5yM7d+/P8Z8DXi9eC/VzverWkGfDAshhBBCiNqizbAQQgghhKgtlbJJdLqsjH9OTtdyKh0ADh48GGMuGwKkKYJcSZ9OpOeKpsbKRLu2llz6jy0v3ibBXXW46xEATJkyJca+O06VtWylxBKnyzgFxp2nAGD79u0x5vI9QKoBWxqAVIPTTjstxj6tymk1rxXr6sv+cGo91wGprOQ6h/lUJ6fFjxw5EmNOdQOplrt3707G3nzzzRh72wLrwLp6exL/nk+XciktX9ayWaeyKtwXgfbtOUU7yf3qV79KxjZu3Bjj22+/PRl74oknYsxpcJ+ez9laOHXv7TZ8jfFzslUKSLub8TVVB4p2BQTSsmh33HFHMvbAAw/EeOfOnSM+H5DanHx3XX5f8K/9/PPPx5jX6LRp05Lj+LG/VtSBTgghhBBCiA6hzbAQQgghhKgtpc8B9rLjDqd+nnnmmeQ4tlBwmgZI0wfeJiFGphO65rTklL3vJsjpOZ9m45S6T9c2S9v4n5e58sBRiqZZeV3s3bs3Oe7ZZ5+NMadAgdSecNZZZyVjs2bNivGECRNi7KtVcIWKXKcr/01nTrvmrC9l0Qoorpe3SbA1gr9xzilxIK0Y8NprryVjfP8788wzkzE+95xy9Vryc/oxtkn46gecWs11K6wiRTuY+dQ33/8ee+yxZOz73/9+0zG2VLDmvF6B1BrjK7ewrcpbEfft2xdj7nC3evXq5LiLL754xNcCqm978tc/rxXfafXWW2+N8b333puMcbUO1ogrdQDpeuZ7MZBq6V97165dI475yiMXXXRRjL0FqtNaamcmhBBCCCFqizbDQgghhBCitmgzLIQQQgghakvpDTTs2+tGaTX2U23YsCHG3v/I3qT58+cnY1wWqNu+wzL5GHsNa8mlXXxpNT6HH//4x5OxnP+7ndJqueMGWcuiZZvYN+ZLbvEa8l63mTNnxvjTn/50MnbGGWfEmL2f7G/1r82lD4HUSzd16tSm889RFT8x43Vgzzd32Xz66aeT49jP6b19Z599dowvuOCCZIw9+Ozp9WW1uBQil90D0nXty+tNnDgxxr70UxVo5fsIPMZlr3yHxwcffDDG7BEGUp8wawKknlH2euZ8pr4s2uuvvx5jvy5XrlwZY76XeJ/p2rVrY7x48eJkrCqe4WY+Ye+n37NnT4x/+MMfJmNcTo09wkBa7ozvv4sWLUqOmzt3boz9uufvaaxfvz4Z4/s2+4f9vorv2957zs/RifvtMT8ZNrPvmdkhM9tIP5tgZveb2fbGf8fnnkMIIYQQQohBpIhN4hYAV7qf3QxgRQhhNoAVjcdCCCGEEEKUimPmDEIID5vZkPvx1QAuacS3AngQwH/u4LzaouhH5a3YKbjU0OOPPx5jn8Y755xzYuxLQdWhjE+nKZqGbwXWctWqVTH23bQ4HeO7CeYsL0Upawq9GblygWxd8Ckw7kzmSyCde+65MZ4xY0Yyxim8XIqQux5xWTAgTZdyGt8/J1M13UbC/+2sH5ez8lYFLos2Z86cZGzJkiUxnj59ejLGZdF4XftrivXyKV0uk8jPBwBDQ0MxbqZrVfH3SbZGcBk0thUAwG233RZjXz4tV3Zy2bJlI8b+/ZA18jrz83st2YrDNgm2VgCp7c2/T/N1Wqb1nCsTyu9rfl3+5Cc/ibHvGMj3Y29xuPTSS2N85ZW/+yzUvx+yNcZbUPjc+5KNL774Yoz5vsLla/0c/XP4jnSjpd0v0E0JIRyd9QEA9ep5KIQQQgghKsGoq0mE4f9lafpRnZl9zczWmdk6/wmcKBfSshpIx+ogLauDtKwO0rJ8tLsZPmhmUwGg8d9DzQ4MIXw3hLA4hLB48uTJbb6cGASkZTWQjtVBWlYHaVkdpGX5aLfOyN0AbgTwV43/3tWxGfUZ781hXyO3IPUtLNknx+V8gNQXJZ/p6ClaTs+PsbdwzZo1MWZvKZB63KZNm5aMsZa50mo5qqBlrqwY+9m4ja73g3HpM9/Kk8sq+ZJY7MFnjdkHCaQle3bu3JmMcYtR36q5bt7S3PXIvj/+hMvf/1g/X1qSS+F5nXkNsafVw9eR939z+SUu1Qak1xjr2o0ynIOGv475PrdxYywOhR/84AfJcewT5vMHAKeffnqMly9fnoxdc801MWavtvd25u7fvC75OYDUo8z3AD9Hvh68Z7gqG1Nef+yZvueee5LjfvrTn8aYfbpA2gr9iiuuSMa+8pWvxPj888+PsS+Fxz5hryXr7PdErDP/Lb6VO++//D2n0xQprfYjAGsAnGNme83sJgxvgi83s+0ALms8FkIIIYQQolQUqSZxfZOhS5v8XAghhBBCiFJQjXYsHcR/1M8pOS7zwSW2AOAzn/lMjH0XpKI2iSqkzwcJn1bZsWNHjLds2RJjb3dYuHBhjDmVBBTvelNnLbkEzuHDh2Psv0jCtgZ/nrmskj+XnE5niwN3FQRSjbdt25aMcQrWd9Li66Yq6fSi1iKfWue0Jevlj/vYxz4WY06lA6nNxZeZ5LlwaTxvXeFOWl5nThP7jpGcQs/pyo9z6d4y4e9/zbqRsWUMSHX2tpMbbrhhxBhIdffvj83InWu+poA0Rc/XkbfXcEc9X46rrPjzxJYXLvnKXeWA9D3P70u4fNr116efeXLnPj7vuTKa/p7AFgq/hnitc5x7/m6vw1FXkxBCCCGEEKKsaDMshBBCCCFqizbDQgghhBCittTSM+y9J+zH8b6XdevWxZi9SL5F7KJFi2Ls2xLmylCJ7uG9ZI888kiM2ZPoPY7s//bet06UySsjrbTHZj8b+4RfeeWV5Dhea74EEvtAvY+MfaDsiVu9enVy3DPPPBNjLrMGpCW+fEk29rCx79J7MMu6rnNz9X8j68D3P1+OkH/Pt8X2jxl+Hm6zzH5vIPUJc/tWIP17eI7+MXvDfbmv3Lrm63vQdeY1xaUkgbQd7y9/+csY+3bG3O78qquuSsa4nBqXzAOAMWPGxLjoefL3DtbBv4/ytcL3du9DZ28sz6nM+PcyLhXJ5dP4ngek54b3KEBaPu2iiy5Kxvg7HDkfb25t8Jhfl7zW+d7vvzvC16LXudPok2EhhBBCCFFbtBkWQgghhBC1pZY2iRw+/bdq1aoY88f+S5YsSY7jTmXtdiYTncWn/1hLTutyKTUAmDVrVoxzlpc6k7MWcUqPY58u57Jdzz33XDLGKTFfEoh15a5KW7duTY7j1Lov1cWdjXJdj7hTkl/XVSyZ6G0Szbo++fPJOnCpJyA9n3498bnn5/AdAzn9y+X6gFQjn45la86RI0difNJJJyXH8fWWSwsPOrzenn766WTs3nvvjTGXDPVWggULFsT42muvTcbYGtFtO6C3L3Fqna9Lb3lh25vXuayWF15DAHD//ffHeOXKlTH21z/bOb/4xS8mYxdffHGMfZfPduyAuRKY3trE92bec40fPz45bvr06TGWTUIIIYQQQoguoc2wEEIIIYSoLbW0SfhvsHI64uDBg8kYp/w4ncQVB4A0VTfo6Zcqw+kz36lq8+bNMWYtP/WpTyXHcapGWrYOnzNOv/kU2P79+2O8YcOGZIxT5j4dy6lATqX6tCqn6fya5/S8t0nwt/D52828xkd6zrLCenmLwMSJE2PMWvq/ndeWrwyycePGGHstuTII/x5bGoDUluFtOVwlwl8DbKngVLPvbuatOGXB68BrgytGAKmNiM8ZawykFkDf0a8TFSNyY5wy3759ezK2e/fuGHPKfMqUKclxbHvjTpaDjj8X3D2P/3YAeOCBB2LMexZfjYGtENxxDkjXQCesQX5dsp1t06ZNyRhXGeK1N2/evOQ4trx0276kT4aFEEIIIURt0WZYCCGEEELUFm2GhRBCCCFEbamlZ9jDPlNfjob9g+x59B1b2vFSidHjfVZcvmvFihXJGPsQJ02aFGPv/5aWo4NLHbHn0PvB2C/K/mEg9Xr68kj8/OwX9CWWWEfvuWN8OUV+nPPTVuXa4L/Dd1zkkpGsn/cFsz9/7969yRh7sr2HkzXi2PtA2f+9b9++ZCzX/Y6vMcaXaSpzOTWGS8n5bmTeh32U2bNnJ4/5OxS5kltFyX1Hh73LQFpi8ec//3kyxvcI9jmzLxYAli5dGmN/PZcJPjePPfZYMsba8vk888wzk+MuvPDCGPtOq7wG/L2sWZk87wvmtefLLT766KMxXrt2bTLG99izzz47xqwdkL5Pq7SaEEIIIYQQXUKbYSGEEEIIUVtqY5PgVE0utc7dXIA0PccpX+7EA1QnZVoGct3POE3oUzPMueeeG+OZM2cmY6ylv1ak8zA5+wCnVvncLlu2LDmOU2C7du1KxjiN5kuajRs3LsannHJKjP21wL/nS25xWSH//BMmTBjxON+pq6ypdX8N89/hS4zxfY5TmHzegbRTlO/8yM/vSz/xuefn9OltLs3kSybmOtyxfny9eetNOx23BgF/zbO9yNtV+Fj+++fMmZMcxxYVf40XLSfIx3GJMCC1zaxZsyYZu/POO2O8fv36ZIx1Wbx4cYy//OUvJ8dx1zWfWh9kbf25ZduBL03GlhdeQ/y+BgBz586NsV/bufe5Zjp7GxLbSLncGwD8+Mc/jrG/v/O6vOyyy2LsO/vyddpt7cp5NxdCCCGEEKIDaDMshBBCCCFqizbDQgghhBCitlTWM9xKC0j2MHFpFyD1oC1YsCDGZWrzWEaK6seebgDYsmVLjPfs2ZOMsWdq0aJFMfY+RtEa3pfH5/nUU0+NsT/P7E3k9stAWrqLPcJA6pHj9Xno0KHkOH5O9gH7eXH5MD8v/lty/sNcaaJBJ1dajc8b+/d86bPzzjsvxtxqFUjXq9eSy+GxluwDBoADBw7E2Hsf2ZPKvmAg/Z4H/y05/3eZtMy18PXw9cta+vcyfg5fdpBfz58XHuPf8x7ve+65J8a/+MUvkjEugeh1Zj/p8uXLY8z3cv97ZfL151pr+5bx/HfxdzR8a21eU/57E7n7F5dM43l47+99990X45/97GfJGOvu7/1XXHFFjK+77roYn3baaU3nL8+wEEIIIYQQXUKbYSGEEEIIUVsqa5PIwSkAANi5c2eMfScsTifNnz8/xr6ED6NyXO3RTtken8Z7/PHHY+y7T3E5l/PPPz/GOS3FyBTtzMapWZ+C53QmawOkJXx8+Sgm18WO072+Ox2n032ZRE4bd7vr0SDQTC8g1ZatBb4cHZ9Pn47NlUJs1o3Md7jzncoYtl74Lltsh+H5567ZQb9f5+6T/H7FfzuQ2sb4vL/00kvJcc8++2yMfSktTnf7MS6px2XRHn744eS4J598MsY+/c9rz3d5vfHGG2PMpdV8Cj7XWW3QyJV85cf+/sX3Tj7OlzXctm1b09dme4VfX2xL4m53/P4KABs2bGj62tyx9+qrr07GbrrpphhzBzr/d/bS5nLMVzKz6Wa20sw2m9kmM/vTxs8nmNn9Zra98d/xx3ouIYQQQgghBoki2+73AXwzhDAPwBIAf2Jm8wDcDGBFCGE2gBWNx0IIIYQQQpSGY26GQwj7QwhPNuIjALYAmAbgagC3Ng67FcA13ZqkEEIIIYQQ3aAls6SZDQFYCOAxAFNCCEdNegcATGnyawNBzmfKHiY/xqVKuNVhrqzMoPuUyob3UrHnm9svA6n/2+tw5plnxnhoaKjpcaI1cucv5y1m2F8G5Ms7sSeVvW7cvhRIr5NcaTXvV27HJ1yVayhXViznLWbfvfeGsw6+FCJry7p63zEfx15HIC3z5v3f7K2sikaM/5v4uuZSoACwffv2GPNaWbVqVXIcn3u+TwKp79r7urmkHntVfcnDXCm8z33uczFmXymQfmeHvdFlarncCnydex24dTl/V2L16tXJcayRb2/NeO82t/LmEpX+Hsvnmr2/AHDNNb/7fPSGG25Ixnidsk+4FS9/p3Uu7E42s7EAbgfwZyGEpAhkGN6tjOjqN7Ovmdk6M1vn60+KciEtq4F0rA7SsjpIy+ogLctHoc2wmY3B8Eb4thDCHY0fHzSzqY3xqQAOjfS7IYTvhhAWhxAWT548uRNzFn1CWlYD6VgdpGV1kJbVQVqWj2PaJGz4s+h/ALAlhPC3NHQ3gBsB/FXjv3d1ZYbHoGg5Lk7V+RIg3PXGl9nij/59Skf0Bq8xp1p9WSDuXOXLP7GWnJbPpVty11dV0nGdptl58eeSU2J+3eXSY5xm5RSpL8uT05g7HfluZLk5N5tjHeC/N6elPy885tOgXOKLrwGvJdtcuKucH/N2m9ycmx1XJl39XPnvv+yyy5IxTnevWbMmxocPH06Oe+ihh2LMpbOAf3lPZbjUGp9r3xVt9uzZMf785z+fjC1btizGU6dOTcbYfpOzMpW1M6SfG9u3uPseAOzYsSPGK1eujLF/P+QSaV47Xm/evsT7JT7vvlvnJz/5yRhfe+21Tcf8uuTXzt07ekkRz/DFAP4AwLNm9lTjZ3+O4U3wP5vZTQB2A/hKd6YohBBCCCFEdzjmZjiE8CiAZtv1Szs7HSGEEEIIIXpHZVtv+U5H/LH/q6++moy9/fbbMfbfLJ8xY0aMuTuOUuujJ3eecl2rOB3HaSD/e96rNWvWrBizlr3sclM3iq6FVr4Rzt+y5vSeT+HxN6L983GVAU+zOWvtNqdoajqnA997fSe5s846K8ZshQLS9K+32/B9P9ftq0zkrB9cZcF3cOMqEXzOuOMckJ7f3DXP91AAmD59eozZkrZo0aLkOJ5XrvpHK9apZscNOrm/ibU877zzkjGuEsE2L18xgi0wbC8D0vc97uIIpBYVfu1LLrkkOY619R0P2V7RbsfHXmqpXYAQQgghhKgt2gwLIYQQQojaos2wEEIIIYSoLaXzDBf1mebGfNke7kzmPSoXXHBBjLmET87jWKZyLoNEM09fTldfLoa9TtyZCkg7Mp188skxlpadpdN+zKIeVO9L43XO/jsgX6aJHw9K2Z9BoF1di66nXBc79jT6Mk2sl/eZNnutnIexTPh587nwJc2uuuqqGM+dOzfGTz31VHIcd/Hk72cA6f3W31/nzJkTY/6ujdcrt/Zy662sGhXF/318LfuyrpdffnmMuVQdd9MF0hJsvmMg+7N9GbuFCxfGmP36/jtVOS07/T7abf31ybAQQgghhKgt2gwLIYQQQojaUjqbRFH8R+pcfmRoaCgZ+8Y3vhFjX8aLU61FS4WI9mh2Dv255pT30qVLkzHueuN/j1N80rL/FO0OloPXtS/1xKlatsUAaRkoX4KP1zyn/uqWtgU6Y3kp+hx8Pn1XQO4YeOTIkWSMNfKluthewWnnqmiX+zu8ZYTXAFvG5s+fnxzH5ej8+yFrWbTTYCvp805TJp1zc/Xrga/rT3ziEzGeN29echyXnvTrkB/798Bm+rVbIq1dVFpNCCGEEEKIHqDNsBBCCCGEqC3aDAshhBBCiNpSKc9wzl/CvhfvYfKl1tp5ftFZcq16+bEvwcSteos+v+gsRc9tUf+hp5nf218LvrQUw9eJL7vG/ry6l1Zr52/23sScN5zPL5933x6Wy1/6MlN8//ZZ5+vrAAAESElEQVRlvJq1aq7q9wSK/h25MnPenyr6Qzttiv17pb8niubok2EhhBBCCFFbtBkWQgghhBC1pXQ2iVzqoBNlgKqSLisDOtf1pmgaMGeh4NS3tztxKancc7STjhTN8ecsV8Kpmc6+syR3y/K0U8ZLugohGH0yLIQQQgghaos2w0IIIYQQorZoMyyEEEIIIWpL6TzDOeQDE6J61LENcpVoxxvuvb/tPr8QQhRBnwwLIYQQQojaos2wEEIIIYSoLdaJcmSFX8zsZQC7AUwCcLhnL9ycOs1jRghhcqeerKHl26jP+StKqbTUmmxKr+YhLbuPtBw9dZtHp7UclPdKoF5aFtaxp5vh+KJm60IIi3v+wppHRxmUeQ/KPIDBmksrDMq8NY/RMyhz1zxGz6DMXfMYHYM070GZy6DM4yiySQghhBBCiNqizbAQQgghhKgt/doMf7dPr+vRPEbHoMx7UOYBDNZcWmFQ5q15jJ5BmbvmMXoGZe6ax+gYpHkPylwGZR4A+uQZFkIIIYQQYhCQTUIIIYQQQtSWnm6GzexKM9tqZs+b2c09fu3vmdkhM9tIP5tgZveb2fbGf8d3eQ7TzWylmW02s01m9qf9mEcn6JeWg6Bj4zWl5ehfV1p2kLrfXxuvKS1H/9p917IqOgK6v5ZFy55ths3seAD/B8AyAPMAXG9m83r1+gBuAXCl+9nNAFaEEGYDWNF43E3eB/DNEMI8AEsA/EnjHPR6HqOiz1regv7rCEjLTnALpGVH0P01Ii1Hzy3ov5al1xHou5a3oP86AmXRMoTQk38APgXgPnr8bQDf7tXrN15zCMBGerwVwNRGPBXA1h7P5y4Al/d7HmXTctB0lJbSst//+q2jtJSW0nHwtBw0HQdZy17aJKYBeJEe7238rJ9MCSHsb8QHAEzp1Qub2RCAhQAe6+c82mTQtOzr+ZOWHUVatseg6QhIy3aRlkSJdQQGT0utySboC3QNwvD/nvSktIaZjQVwO4A/CyG82a95VJFenz9p2T2kZXWQltVB75XVQGsypZeb4X0AptPjMxo/6ycHzWwqADT+e6jbL2hmYzB8QdwWQrijX/MYJYOmZV/On7TsCtKyPQZNR0Batou0RCV0BAZPS63JJvRyM/wEgNlmNtPMTgDwVQB39/D1R+JuADc24hsx7GXpGmZmAP4BwJYQwt/2ax4dYNC07Pn5k5ZdQ1q2x6DpCEjLdqm9lhXRERg8LbUmm9Fj4/QXAGwDsAPAf+nxa/8IwH4A72HYt3MTgIkY/hbjdgC/BDChy3P4DIZTAc8AeKrx7wu9nkeZtRwEHaWltBzEf3W/v0rL6mhZFR37qeUg6FgmLdWBTgghhBBC1BZ9gU4IIYQQQtQWbYaFEEIIIURt0WZYCCGEEELUFm2GhRBCCCFEbdFmWAghhBBC1BZthoUQQgghRG3RZlgIIYQQQtQWbYaFEEIIIURt+f+UtoefRFpU0wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "z1 = z[:,0]\n", + "z2 = z[:,1]\n", + "\n", + "fig = plt.figure()\n", + "ax = fig.add_subplot(111)\n", + "ax.plot(z1,z2,'ko')\n", + "plt.title(\"latent space\")\n", + "\n", + "#np.where((z1>3) & (z2<2) & (z2>0))\n", + "#select the points from the latent space\n", + "a_vec = [2,5,7,789,25,9993]\n", + "for i in range(len(a_vec)):\n", + " ax.plot(z1[a_vec[i]],z2[a_vec[i]],'ro') \n", + " ax.annotate('z%d' %i, xy=(z1[a_vec[i]],z2[a_vec[i]]), \n", + " xytext=(z1[a_vec[i]],z2[a_vec[i]]),color = 'r',fontsize=15)\n", + "\n", + "\n", + "f, ((ax0, ax1, ax2, ax3, ax4,ax5)) = plt.subplots(1,6, sharex='col', sharey='row',figsize=(12,2.5))\n", + "for i in range(len(a_vec)):\n", + " eval('ax%d' %(i)).imshow(np.reshape(x_construction[a_vec[i],:],(28,28)), interpolation='nearest', cmap=cm.Greys)\n", + " eval('ax%d' %(i)).set_title('z%d'%i)\n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Above is a plot of points in the 2D latent space and their corresponding decoded images, it can be seen that points that are close in the latent space get mapped to the same digit from the decoder, and we can see how it evolves from left to right." + ] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/caffe/caffe_net.py b/example/caffe/caffe_net.py new file mode 100644 index 000000000000..803efda9b68e --- /dev/null +++ b/example/caffe/caffe_net.py @@ -0,0 +1,145 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Generate helper functions to load Caffe into MXNet""" +import argparse +import mxnet as mx +from data import get_iterator +import train_model + + +def get_mlp(): + """Get multi-layer perceptron""" + data = mx.symbol.Variable('data') + fc1 = mx.symbol.CaffeOp(data_0=data, num_weight=2, name='fc1', + prototxt="layer{type:\"InnerProduct\" inner_product_param{num_output: 128} }") + act1 = mx.symbol.CaffeOp(data_0=fc1, prototxt="layer{type:\"TanH\"}") + fc2 = mx.symbol.CaffeOp(data_0=act1, num_weight=2, name='fc2', + prototxt="layer{type:\"InnerProduct\" inner_product_param{num_output: 64} }") + act2 = mx.symbol.CaffeOp(data_0=fc2, prototxt="layer{type:\"TanH\"}") + fc3 = mx.symbol.CaffeOp(data_0=act2, num_weight=2, name='fc3', + prototxt="layer{type:\"InnerProduct\" inner_product_param{num_output: 10}}") + if use_caffe_loss: + label = mx.symbol.Variable('softmax_label') + mlp = mx.symbol.CaffeLoss(data=fc3, label=label, grad_scale=1, name='softmax', + prototxt="layer{type:\"SoftmaxWithLoss\"}") + else: + mlp = mx.symbol.SoftmaxOutput(data=fc3, name='softmax') + return mlp + + +def get_lenet(): + """LeCun, Yann, Leon Bottou, Yoshua Bengio, and Patrick + Haffner. "Gradient-based learning applied to document recognition." + Proceedings of the IEEE (1998) + """ + data = mx.symbol.Variable('data') + + # first conv + conv1 = mx.symbol.CaffeOp(data_0=data, num_weight=2, + prototxt="layer{type:\"Convolution\" " + "convolution_param { num_output: 20 kernel_size: 5 stride: 1} }") + act1 = mx.symbol.CaffeOp(data_0=conv1, prototxt="layer{type:\"TanH\"}") + pool1 = mx.symbol.CaffeOp(data_0=act1, + prototxt="layer{type:\"Pooling\" pooling_param { pool: MAX kernel_size: 2 stride: 2}}") + + # second conv + conv2 = mx.symbol.CaffeOp(data_0=pool1, num_weight=2, + prototxt="layer{type:\"Convolution\" " + "convolution_param { num_output: 50 kernel_size: 5 stride: 1} }") + act2 = mx.symbol.CaffeOp(data_0=conv2, prototxt="layer{type:\"TanH\"}") + pool2 = mx.symbol.CaffeOp(data_0=act2, + prototxt="layer{type:\"Pooling\" pooling_param { pool: MAX kernel_size: 2 stride: 2}}") + + fc1 = mx.symbol.CaffeOp(data_0=pool2, num_weight=2, + prototxt="layer{type:\"InnerProduct\" inner_product_param{num_output: 500} }") + act3 = mx.symbol.CaffeOp(data_0=fc1, prototxt="layer{type:\"TanH\"}") + + # second fullc + fc2 = mx.symbol.CaffeOp(data_0=act3, num_weight=2, + prototxt="layer{type:\"InnerProduct\"inner_product_param{num_output: 10} }") + if use_caffe_loss: + label = mx.symbol.Variable('softmax_label') + lenet = mx.symbol.CaffeLoss(data=fc2, label=label, grad_scale=1, name='softmax', + prototxt="layer{type:\"SoftmaxWithLoss\"}") + else: + lenet = mx.symbol.SoftmaxOutput(data=fc2, name='softmax') + return lenet + + +def get_network_from_json_file(file_name): + network = mx.sym.load(file_name) + return network + + +def parse_args(): + """Parse the arguments""" + parser = argparse.ArgumentParser(description='train an image classifier on mnist') + parser.add_argument('--network', type=str, default='lenet', + help='the cnn to use (mlp | lenet | ') + parser.add_argument('--caffe-loss', type=int, default=0, + help='Use CaffeLoss symbol') + parser.add_argument('--caffe-data', action='store_true', + help='Use Caffe input-data layer only if specified') + parser.add_argument('--data-dir', type=str, default='mnist/', + help='the input data directory') + parser.add_argument('--gpus', type=str, + help='the gpus will be used, e.g "0,1,2,3"') + parser.add_argument('--num-examples', type=int, default=60000, + help='the number of training examples') + parser.add_argument('--batch-size', type=int, default=128, + help='the batch size') + parser.add_argument('--lr', type=float, default=.1, + help='the initial learning rate') + parser.add_argument('--model-prefix', type=str, + help='the prefix of the model to load/save') + parser.add_argument('--save-model-prefix', type=str, + help='the prefix of the model to save') + parser.add_argument('--num-epochs', type=int, default=10, + help='the number of training epochs') + parser.add_argument('--load-epoch', type=int, + help="load the model on an epoch using the model-prefix") + parser.add_argument('--kv-store', type=str, default='local', + help='the kvstore type') + parser.add_argument('--lr-factor', type=float, default=1, + help='times the lr with a factor for every lr-factor-epoch epoch') + parser.add_argument('--lr-factor-epoch', type=float, default=1, + help='the number of epoch to factor the lr, could be .5') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + use_caffe_loss = args.caffe_loss + use_caffe_data = args.caffe_data + + data_shape = () + if args.network == 'mlp': + data_shape = (784, ) + net = get_mlp() + elif args.network == 'lenet': + if not use_caffe_data: + data_shape = (1, 28, 28) + net = get_lenet() + else: + net = get_network_from_json_file(args.network) + + # train + if use_caffe_loss: + train_model.fit(args, net, get_iterator(data_shape, use_caffe_data), mx.metric.Caffe()) + else: + train_model.fit(args, net, get_iterator(data_shape, use_caffe_data)) diff --git a/example/caffe/train_model.py b/example/caffe/train_model.py new file mode 100644 index 000000000000..d7dfd5d7a31e --- /dev/null +++ b/example/caffe/train_model.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Train module using Caffe operator in MXNet""" +import os +import logging +import mxnet as mx + + +def fit(args, network, data_loader, eval_metrics=None, batch_end_callback=None): + """Train the model using Caffe operator in MXNet""" + # kvstore + kv = mx.kvstore.create(args.kv_store) + + # logging + head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s' + if 'log_file' in args and args.log_file is not None: + log_file = args.log_file + log_dir = args.log_dir + log_file_full_name = os.path.join(log_dir, log_file) + if not os.path.exists(log_dir): + os.mkdir(log_dir) + logger = logging.getLogger() + handler = logging.FileHandler(log_file_full_name) + formatter = logging.Formatter(head) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + logger.info('start with arguments %s', args) + else: + logging.basicConfig(level=logging.DEBUG, format=head) + logging.info('start with arguments %s', args) + + # load model + model_prefix = args.model_prefix + if model_prefix is not None: + model_prefix += "-%d" % (kv.rank) + model_args = {} + if args.load_epoch is not None: + assert model_prefix is not None + tmp = mx.model.FeedForward.load(model_prefix, args.load_epoch) + model_args = {'arg_params' : tmp.arg_params, + 'aux_params' : tmp.aux_params, + 'begin_epoch' : args.load_epoch} + # save model + save_model_prefix = args.save_model_prefix + if save_model_prefix is None: + save_model_prefix = model_prefix + checkpoint = None if save_model_prefix is None else mx.callback.do_checkpoint(save_model_prefix) + + # data + (train, val) = data_loader(args, kv) + + # train + devs = mx.cpu() if args.gpus is None else [ + mx.gpu(int(i)) for i in args.gpus.split(',')] + + epoch_size = args.num_examples / args.batch_size + + if args.kv_store == 'dist_sync': + epoch_size /= kv.num_workers + model_args['epoch_size'] = epoch_size + + if 'lr_factor' in args and args.lr_factor < 1: + model_args['lr_scheduler'] = mx.lr_scheduler.FactorScheduler( + step=max(int(epoch_size * args.lr_factor_epoch), 1), + factor=args.lr_factor) + + if 'clip_gradient' in args and args.clip_gradient is not None: + model_args['clip_gradient'] = args.clip_gradient + + # disable kvstore for single device + if 'local' in kv.type and ( + args.gpus is None or len(args.gpus.split(',')) is 1): + kv = None + + mod = mx.mod.Module(network, context=devs) + + if eval_metrics is None: + eval_metrics = ['accuracy'] + # TopKAccuracy only allows top_k > 1 + for top_k in [5, 10, 20]: + eval_metrics.append(mx.metric.create('top_k_accuracy', top_k=top_k)) + + if batch_end_callback is not None: + if not isinstance(batch_end_callback, list): + batch_end_callback = [batch_end_callback] + else: + batch_end_callback = [] + batch_end_callback.append(mx.callback.Speedometer(args.batch_size, 50)) + + mod.fit(train_data=train, eval_metric=eval_metrics, eval_data=val, optimizer='sgd', + optimizer_params={'learning_rate':args.lr, 'momentum': 0.9, 'wd': 0.00001}, + num_epoch=args.num_epochs, batch_end_callback=batch_end_callback, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + kvstore=kv, epoch_end_callback=checkpoint, **model_args) diff --git a/example/capsnet/capsulenet.py b/example/capsnet/capsulenet.py new file mode 100644 index 000000000000..4d455dbc504c --- /dev/null +++ b/example/capsnet/capsulenet.py @@ -0,0 +1,373 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Generate MXNet implementation of CapsNet +Reference 1: https://www.cs.toronto.edu/~fritz/absps/transauto6.pdf +Reference 2: https://arxiv.org/pdf/1710.09829.pdf +""" +import os +import re +import gzip +import struct +import numpy as np +import scipy.ndimage as ndi +import mxnet as mx +from capsulelayers import primary_caps, CapsuleLayer + +from mxboard import SummaryWriter + + +def margin_loss(y_true, y_pred): + loss = y_true * mx.sym.square(mx.sym.maximum(0., 0.9 - y_pred)) +\ + 0.5 * (1 - y_true) * mx.sym.square(mx.sym.maximum(0., y_pred - 0.1)) + return mx.sym.mean(data=mx.sym.sum(loss, 1)) + + +def capsnet(batch_size, n_class, num_routing, recon_loss_weight): + """Create CapsNet""" + # data.shape = [batch_size, 1, 28, 28] + data = mx.sym.Variable('data') + + input_shape = (1, 28, 28) + # Conv2D layer + # net.shape = [batch_size, 256, 20, 20] + conv1 = mx.sym.Convolution(data=data, + num_filter=256, + kernel=(9, 9), + layout='NCHW', + name='conv1') + conv1 = mx.sym.Activation(data=conv1, act_type='relu', name='conv1_act') + # net.shape = [batch_size, 256, 6, 6] + + primarycaps = primary_caps(data=conv1, + dim_vector=8, + n_channels=32, + kernel=(9, 9), + strides=[2, 2], + name='primarycaps') + primarycaps.infer_shape(data=(batch_size, 1, 28, 28)) + # CapsuleLayer + kernel_initializer = mx.init.Xavier(rnd_type='uniform', factor_type='avg', magnitude=3) + bias_initializer = mx.init.Zero() + digitcaps = CapsuleLayer(num_capsule=10, + dim_vector=16, + batch_size=batch_size, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + num_routing=num_routing)(primarycaps) + + # out_caps : (batch_size, 10) + out_caps = mx.sym.sqrt(data=mx.sym.sum(mx.sym.square(digitcaps), 2)) + out_caps.infer_shape(data=(batch_size, 1, 28, 28)) + + y = mx.sym.Variable('softmax_label', shape=(batch_size,)) + y_onehot = mx.sym.one_hot(y, n_class) + y_reshaped = mx.sym.Reshape(data=y_onehot, shape=(batch_size, -4, n_class, -1)) + y_reshaped.infer_shape(softmax_label=(batch_size,)) + + # inputs_masked : (batch_size, 16) + inputs_masked = mx.sym.linalg_gemm2(y_reshaped, digitcaps, transpose_a=True) + inputs_masked = mx.sym.Reshape(data=inputs_masked, shape=(-3, 0)) + x_recon = mx.sym.FullyConnected(data=inputs_masked, num_hidden=512, name='x_recon') + x_recon = mx.sym.Activation(data=x_recon, act_type='relu', name='x_recon_act') + x_recon = mx.sym.FullyConnected(data=x_recon, num_hidden=1024, name='x_recon2') + x_recon = mx.sym.Activation(data=x_recon, act_type='relu', name='x_recon_act2') + x_recon = mx.sym.FullyConnected(data=x_recon, num_hidden=np.prod(input_shape), name='x_recon3') + x_recon = mx.sym.Activation(data=x_recon, act_type='sigmoid', name='x_recon_act3') + + data_flatten = mx.sym.flatten(data=data) + squared_error = mx.sym.square(x_recon-data_flatten) + recon_error = mx.sym.mean(squared_error) + recon_error_stopped = recon_error + recon_error_stopped = mx.sym.BlockGrad(recon_error_stopped) + loss = mx.symbol.MakeLoss((1-recon_loss_weight)*margin_loss(y_onehot, out_caps)+recon_loss_weight*recon_error) + + out_caps_blocked = out_caps + out_caps_blocked = mx.sym.BlockGrad(out_caps_blocked) + return mx.sym.Group([out_caps_blocked, loss, recon_error_stopped]) + + +def download_data(url, force_download=False): + fname = url.split("/")[-1] + if force_download or not os.path.exists(fname): + mx.test_utils.download(url, fname) + return fname + + +def read_data(label_url, image_url): + with gzip.open(download_data(label_url)) as flbl: + magic, num = struct.unpack(">II", flbl.read(8)) + label = np.fromstring(flbl.read(), dtype=np.int8) + with gzip.open(download_data(image_url), 'rb') as fimg: + magic, num, rows, cols = struct.unpack(">IIII", fimg.read(16)) + image = np.fromstring(fimg.read(), dtype=np.uint8) + np.reshape(image, len(label), (rows, cols)) + return label, image + + +def to4d(img): + return img.reshape(img.shape[0], 1, 28, 28).astype(np.float32)/255 + + +class LossMetric(mx.metric.EvalMetric): + """Evaluate the loss function""" + def __init__(self, batch_size, num_gpus): + super(LossMetric, self).__init__('LossMetric') + self.batch_size = batch_size + self.num_gpu = num_gpus + self.sum_metric = 0 + self.num_inst = 0 + self.loss = 0.0 + self.batch_sum_metric = 0 + self.batch_num_inst = 0 + self.batch_loss = 0.0 + self.recon_loss = 0.0 + self.n_batch = 0 + + def update(self, labels, preds): + """Update the hyper-parameters and loss of CapsNet""" + batch_sum_metric = 0 + batch_num_inst = 0 + for label, pred_outcaps in zip(labels[0], preds[0]): + label_np = int(label.asnumpy()) + pred_label = int(np.argmax(pred_outcaps.asnumpy())) + batch_sum_metric += int(label_np == pred_label) + batch_num_inst += 1 + batch_loss = preds[1].asnumpy() + recon_loss = preds[2].asnumpy() + self.sum_metric += batch_sum_metric + self.num_inst += batch_num_inst + self.loss += batch_loss + self.recon_loss += recon_loss + self.batch_sum_metric = batch_sum_metric + self.batch_num_inst = batch_num_inst + self.batch_loss = batch_loss + self.n_batch += 1 + + def get_name_value(self): + acc = float(self.sum_metric)/float(self.num_inst) + mean_loss = self.loss / float(self.n_batch) + mean_recon_loss = self.recon_loss / float(self.n_batch) + return acc, mean_loss, mean_recon_loss + + def get_batch_log(self, n_batch): + print("n_batch :"+str(n_batch)+" batch_acc:" + + str(float(self.batch_sum_metric) / float(self.batch_num_inst)) + + ' batch_loss:' + str(float(self.batch_loss)/float(self.batch_num_inst))) + self.batch_sum_metric = 0 + self.batch_num_inst = 0 + self.batch_loss = 0.0 + + def reset(self): + self.sum_metric = 0 + self.num_inst = 0 + self.loss = 0.0 + self.recon_loss = 0.0 + self.n_batch = 0 + + +class SimpleLRScheduler(mx.lr_scheduler.LRScheduler): + """A simple lr schedule that simply return `dynamic_lr`. We will set `dynamic_lr` + dynamically based on performance on the validation set. + """ + + def __init__(self, learning_rate=0.001): + super(SimpleLRScheduler, self).__init__() + self.learning_rate = learning_rate + + def __call__(self, num_update): + return self.learning_rate + + +def do_training(num_epoch, optimizer, kvstore, learning_rate, model_prefix, decay): + """Perform CapsNet training""" + summary_writer = SummaryWriter(args.tblog_dir) + lr_scheduler = SimpleLRScheduler(learning_rate) + optimizer_params = {'lr_scheduler': lr_scheduler} + module.init_params() + module.init_optimizer(kvstore=kvstore, + optimizer=optimizer, + optimizer_params=optimizer_params) + n_epoch = 0 + while True: + if n_epoch >= num_epoch: + break + train_iter.reset() + val_iter.reset() + loss_metric.reset() + for n_batch, data_batch in enumerate(train_iter): + module.forward_backward(data_batch) + module.update() + module.update_metric(loss_metric, data_batch.label) + loss_metric.get_batch_log(n_batch) + train_acc, train_loss, train_recon_err = loss_metric.get_name_value() + loss_metric.reset() + for n_batch, data_batch in enumerate(val_iter): + module.forward(data_batch) + module.update_metric(loss_metric, data_batch.label) + loss_metric.get_batch_log(n_batch) + val_acc, val_loss, val_recon_err = loss_metric.get_name_value() + + summary_writer.add_scalar('train_acc', train_acc, n_epoch) + summary_writer.add_scalar('train_loss', train_loss, n_epoch) + summary_writer.add_scalar('train_recon_err', train_recon_err, n_epoch) + summary_writer.add_scalar('val_acc', val_acc, n_epoch) + summary_writer.add_scalar('val_loss', val_loss, n_epoch) + summary_writer.add_scalar('val_recon_err', val_recon_err, n_epoch) + + print('Epoch[%d] train acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, train_acc, train_loss, + train_recon_err)) + print('Epoch[%d] val acc: %.4f loss: %.6f recon_err: %.6f' % (n_epoch, val_acc, val_loss, val_recon_err)) + print('SAVE CHECKPOINT') + + module.save_checkpoint(prefix=model_prefix, epoch=n_epoch) + n_epoch += 1 + lr_scheduler.learning_rate = learning_rate * (decay ** n_epoch) + + +def apply_transform(x, transform_matrix, fill_mode='nearest', cval=0.): + """Apply transform on nd.array""" + x = np.rollaxis(x, 0, 0) + final_affine_matrix = transform_matrix[:2, :2] + final_offset = transform_matrix[:2, 2] + channel_images = [ndi.interpolation.affine_transform( + x_channel, + final_affine_matrix, + final_offset, + order=0, + mode=fill_mode, + cval=cval) for x_channel in x] + x = np.stack(channel_images, axis=0) + x = np.rollaxis(x, 0, 0 + 1) + return x + + +def random_shift(x, width_shift_fraction, height_shift_fraction): + tx = np.random.uniform(-height_shift_fraction, height_shift_fraction) * x.shape[2] + ty = np.random.uniform(-width_shift_fraction, width_shift_fraction) * x.shape[1] + shift_matrix = np.array([[1, 0, tx], + [0, 1, ty], + [0, 0, 1]]) + x = apply_transform(x, shift_matrix, 'nearest') + return x + + +def _shuffle(data, idx): + """Shuffle the data.""" + shuffle_data = [] + + for idx_k, idx_v in data: + shuffle_data.append((idx_k, mx.ndarray.array(idx_v.asnumpy()[idx], idx_v.context))) + + return shuffle_data + + +class MNISTCustomIter(mx.io.NDArrayIter): + """Create custom iterator of mnist dataset""" + def __init__(self, data, label, batch_size, shuffle): + self.data = data + self.label = label + self.batch_size = batch_size + self.shuffle = shuffle + self.cursor = None + + def reset(self): + """Reset class MNISTCustomIter(mx.io.NDArrayIter):""" + # shuffle data + if self.is_train: + np.random.shuffle(self.idx) + self.data = _shuffle(self.data, self.idx) + self.label = _shuffle(self.label, self.idx) + + if self.last_batch_handle == 'roll_over' and self.cursor > self.num_data: + self.cursor = -self.batch_size + (self.cursor % self.num_data) % self.batch_size + else: + self.cursor = -self.batch_size + + def set_is_train(self, is_train): + """Set training flag""" + self.is_train = is_train + + def next(self): + """Generate next of iterator""" + if self.iter_next(): + if self.is_train: + data_raw_list = self.getdata() + data_shifted = [] + for data_raw in data_raw_list[0]: + data_shifted.append(random_shift(data_raw.asnumpy(), 0.1, 0.1)) + return mx.io.DataBatch(data=[mx.nd.array(data_shifted)], label=self.getlabel(), + pad=self.getpad(), index=None) + else: + return mx.io.DataBatch(data=self.getdata(), label=self.getlabel(), pad=self.getpad(), index=None) + + else: + raise StopIteration + + +if __name__ == "__main__": + # Read mnist data set + path = 'http://yann.lecun.com/exdb/mnist/' + (train_lbl, train_img) = read_data(path + 'train-labels-idx1-ubyte.gz', path + 'train-images-idx3-ubyte.gz') + (val_lbl, val_img) = read_data(path + 't10k-labels-idx1-ubyte.gz', path + 't10k-images-idx3-ubyte.gz') + + # set batch size + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--batch_size', default=100, type=int) + parser.add_argument('--devices', default='gpu0', type=str) + parser.add_argument('--num_epoch', default=100, type=int) + parser.add_argument('--lr', default=0.001, type=float) + parser.add_argument('--num_routing', default=3, type=int) + parser.add_argument('--model_prefix', default='capsnet', type=str) + parser.add_argument('--decay', default=0.9, type=float) + parser.add_argument('--tblog_dir', default='tblog', type=str) + parser.add_argument('--recon_loss_weight', default=0.392, type=float) + args = parser.parse_args() + for k, v in sorted(vars(args).items()): + print("{0}: {1}".format(k, v)) + contexts = re.split(r'\W+', args.devices) + for i, ctx in enumerate(contexts): + if ctx[:3] == 'gpu': + contexts[i] = mx.context.gpu(int(ctx[3:])) + else: + contexts[i] = mx.context.cpu() + num_gpu = len(contexts) + + if args.batch_size % num_gpu != 0: + raise Exception('num_gpu should be positive divisor of batch_size') + + # generate train_iter, val_iter + train_iter = MNISTCustomIter(data=to4d(train_img), label=train_lbl, batch_size=int(args.batch_size), shuffle=True) + train_iter.set_is_train(True) + val_iter = MNISTCustomIter(data=to4d(val_img), label=val_lbl, batch_size=int(args.batch_size), shuffle=True) + val_iter.set_is_train(False) + # define capsnet + final_net = capsnet(batch_size=int(args.batch_size/num_gpu), + n_class=10, + num_routing=args.num_routing, + recon_loss_weight=args.recon_loss_weight) + # set metric + loss_metric = LossMetric(args.batch_size/num_gpu, 1) + + # run model + module = mx.mod.Module(symbol=final_net, context=contexts, data_names=('data',), label_names=('softmax_label',)) + module.bind(data_shapes=train_iter.provide_data, + label_shapes=val_iter.provide_label, + for_training=True) + + do_training(num_epoch=args.num_epoch, optimizer='adam', kvstore='device', learning_rate=args.lr, + model_prefix=args.model_prefix, decay=args.decay) diff --git a/example/ctc/lstm_ocr_train.py b/example/ctc/lstm_ocr_train.py new file mode 100644 index 000000000000..49d9531920ae --- /dev/null +++ b/example/ctc/lstm_ocr_train.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" An example of using WarpCTC loss for an OCR problem using LSTM and CAPTCHA image data""" + +from __future__ import print_function + +import argparse +import logging +import os + +from captcha_generator import MPDigitCaptcha +from hyperparams import Hyperparams +from ctc_metrics import CtcMetrics +import lstm +import mxnet as mx +from ocr_iter import OCRIter + + +def get_fonts(path): + fonts = list() + if os.path.isdir(path): + for filename in os.listdir(path): + if filename.endswith('.ttf'): + fonts.append(os.path.join(path, filename)) + else: + fonts.append(path) + return fonts + + +def parse_args(): + """Parse command line arguments""" + parser = argparse.ArgumentParser() + parser.add_argument("font_path", help="Path to ttf font file or directory containing ttf files") + parser.add_argument("--loss", help="'ctc' or 'warpctc' loss [Default 'ctc']", default='ctc') + parser.add_argument("--cpu", + help="Number of CPUs for training [Default 8]. Ignored if --gpu is specified.", + type=int, default=8) + parser.add_argument("--gpu", help="Number of GPUs for training [Default 0]", type=int) + parser.add_argument("--num_proc", help="Number CAPTCHA generating processes [Default 4]", type=int, default=4) + parser.add_argument("--prefix", help="Checkpoint prefix [Default 'ocr']", default='ocr') + return parser.parse_args() + + +def main(): + """Program entry point""" + args = parse_args() + if not any(args.loss == s for s in ['ctc', 'warpctc']): + raise ValueError("Invalid loss '{}' (must be 'ctc' or 'warpctc')".format(args.loss)) + + hp = Hyperparams() + + # Start a multiprocessor captcha image generator + mp_captcha = MPDigitCaptcha( + font_paths=get_fonts(args.font_path), h=hp.seq_length, w=30, + num_digit_min=3, num_digit_max=4, num_processes=args.num_proc, max_queue_size=hp.batch_size * 2) + try: + # Must call start() before any call to mxnet module (https://github.com/apache/incubator-mxnet/issues/9213) + mp_captcha.start() + + if args.gpu: + contexts = [mx.context.gpu(i) for i in range(args.gpu)] + else: + contexts = [mx.context.cpu(i) for i in range(args.cpu)] + + init_states = lstm.init_states(hp.batch_size, hp.num_lstm_layer, hp.num_hidden) + + data_train = OCRIter( + hp.train_epoch_size // hp.batch_size, hp.batch_size, init_states, captcha=mp_captcha, name='train') + data_val = OCRIter( + hp.eval_epoch_size // hp.batch_size, hp.batch_size, init_states, captcha=mp_captcha, name='val') + + symbol = lstm.lstm_unroll( + num_lstm_layer=hp.num_lstm_layer, + seq_len=hp.seq_length, + num_hidden=hp.num_hidden, + num_label=hp.num_label, + loss_type=args.loss) + + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + module = mx.mod.Module( + symbol, + data_names=['data', 'l0_init_c', 'l0_init_h', 'l1_init_c', 'l1_init_h'], + label_names=['label'], + context=contexts) + + metrics = CtcMetrics(hp.seq_length) + module.fit(train_data=data_train, + eval_data=data_val, + # use metrics.accuracy or metrics.accuracy_lcs + eval_metric=mx.metric.np(metrics.accuracy, allow_extra_outputs=True), + optimizer='sgd', + optimizer_params={'learning_rate': hp.learning_rate, + 'momentum': hp.momentum, + 'wd': 0.00001, + }, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + num_epoch=hp.num_epoch, + batch_end_callback=mx.callback.Speedometer(hp.batch_size, 50), + epoch_end_callback=mx.callback.do_checkpoint(args.prefix), + ) + except KeyboardInterrupt: + print("W: interrupt received, stopping...") + finally: + # Reset multiprocessing captcha generator to stop processes + mp_captcha.reset() + + +if __name__ == '__main__': + main() diff --git a/example/deep-embedded-clustering/autoencoder.py b/example/deep-embedded-clustering/autoencoder.py new file mode 100644 index 000000000000..c75634475e3a --- /dev/null +++ b/example/deep-embedded-clustering/autoencoder.py @@ -0,0 +1,205 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=missing-docstring, arguments-differ +from __future__ import print_function + +import logging +import numpy as np +import mxnet as mx +import model +from solver import Solver, Monitor + + +class AutoEncoderModel(model.MXModel): + def setup(self, dims, sparseness_penalty=None, pt_dropout=None, + ft_dropout=None, input_act=None, internal_act='relu', output_act=None): + self.N = len(dims) - 1 + self.dims = dims + self.stacks = [] + self.pt_dropout = pt_dropout + self.ft_dropout = ft_dropout + self.input_act = input_act + self.internal_act = internal_act + self.output_act = output_act + + self.data = mx.symbol.Variable('data') + for i in range(self.N): + if i == 0: + decoder_act = input_act + idropout = None + else: + decoder_act = internal_act + idropout = pt_dropout + if i == self.N-1: + encoder_act = output_act + odropout = None + else: + encoder_act = internal_act + odropout = pt_dropout + istack, iargs, iargs_grad, iargs_mult, iauxs = self.make_stack( + i, self.data, dims[i], dims[i+1], sparseness_penalty, + idropout, odropout, encoder_act, decoder_act + ) + self.stacks.append(istack) + self.args.update(iargs) + self.args_grad.update(iargs_grad) + self.args_mult.update(iargs_mult) + self.auxs.update(iauxs) + self.encoder, self.internals = self.make_encoder( + self.data, dims, sparseness_penalty, ft_dropout, internal_act, output_act) + self.decoder = self.make_decoder( + self.encoder, dims, sparseness_penalty, ft_dropout, internal_act, input_act) + if input_act == 'softmax': + self.loss = self.decoder + else: + self.loss = mx.symbol.LinearRegressionOutput(data=self.decoder, label=self.data) + + def make_stack(self, istack, data, num_input, num_hidden, sparseness_penalty=None, + idropout=None, odropout=None, encoder_act='relu', decoder_act='relu'): + x = data + if idropout: + x = mx.symbol.Dropout(data=x, p=idropout) + x = mx.symbol.FullyConnected(name='encoder_%d'%istack, data=x, num_hidden=num_hidden) + if encoder_act: + x = mx.symbol.Activation(data=x, act_type=encoder_act) + if encoder_act == 'sigmoid' and sparseness_penalty: + x = mx.symbol.IdentityAttachKLSparseReg( + data=x, name='sparse_encoder_%d' % istack, penalty=sparseness_penalty) + if odropout: + x = mx.symbol.Dropout(data=x, p=odropout) + x = mx.symbol.FullyConnected(name='decoder_%d'%istack, data=x, num_hidden=num_input) + if decoder_act == 'softmax': + x = mx.symbol.Softmax(data=x, label=data, prob_label=True, act_type=decoder_act) + elif decoder_act: + x = mx.symbol.Activation(data=x, act_type=decoder_act) + if decoder_act == 'sigmoid' and sparseness_penalty: + x = mx.symbol.IdentityAttachKLSparseReg( + data=x, name='sparse_decoder_%d' % istack, penalty=sparseness_penalty) + x = mx.symbol.LinearRegressionOutput(data=x, label=data) + else: + x = mx.symbol.LinearRegressionOutput(data=x, label=data) + + args = {'encoder_%d_weight' % istack: mx.nd.empty((num_hidden, num_input), self.xpu), + 'encoder_%d_bias' % istack: mx.nd.empty((num_hidden,), self.xpu), + 'decoder_%d_weight' % istack: mx.nd.empty((num_input, num_hidden), self.xpu), + 'decoder_%d_bias' % istack: mx.nd.empty((num_input,), self.xpu), } + args_grad = {'encoder_%d_weight' % istack: mx.nd.empty((num_hidden, num_input), self.xpu), + 'encoder_%d_bias' % istack: mx.nd.empty((num_hidden,), self.xpu), + 'decoder_%d_weight' % istack: mx.nd.empty((num_input, num_hidden), self.xpu), + 'decoder_%d_bias' % istack: mx.nd.empty((num_input,), self.xpu), } + args_mult = {'encoder_%d_weight' % istack: 1.0, + 'encoder_%d_bias' % istack: 2.0, + 'decoder_%d_weight' % istack: 1.0, + 'decoder_%d_bias' % istack: 2.0, } + auxs = {} + if encoder_act == 'sigmoid' and sparseness_penalty: + auxs['sparse_encoder_%d_moving_avg' % istack] = mx.nd.ones(num_hidden, self.xpu) * 0.5 + if decoder_act == 'sigmoid' and sparseness_penalty: + auxs['sparse_decoder_%d_moving_avg' % istack] = mx.nd.ones(num_input, self.xpu) * 0.5 + init = mx.initializer.Uniform(0.07) + for k, v in args.items(): + init(mx.initializer.InitDesc(k), v) + + return x, args, args_grad, args_mult, auxs + + def make_encoder(self, data, dims, sparseness_penalty=None, dropout=None, internal_act='relu', + output_act=None): + x = data + internals = [] + N = len(dims) - 1 + for i in range(N): + x = mx.symbol.FullyConnected(name='encoder_%d'%i, data=x, num_hidden=dims[i+1]) + if internal_act and i < N-1: + x = mx.symbol.Activation(data=x, act_type=internal_act) + if internal_act == 'sigmoid' and sparseness_penalty: + x = mx.symbol.IdentityAttachKLSparseReg( + data=x, name='sparse_encoder_%d' % i, penalty=sparseness_penalty) + elif output_act and i == N-1: + x = mx.symbol.Activation(data=x, act_type=output_act) + if output_act == 'sigmoid' and sparseness_penalty: + x = mx.symbol.IdentityAttachKLSparseReg( + data=x, name='sparse_encoder_%d' % i, penalty=sparseness_penalty) + if dropout: + x = mx.symbol.Dropout(data=x, p=dropout) + internals.append(x) + return x, internals + + def make_decoder(self, feature, dims, sparseness_penalty=None, dropout=None, + internal_act='relu', input_act=None): + x = feature + N = len(dims) - 1 + for i in reversed(range(N)): + x = mx.symbol.FullyConnected(name='decoder_%d'%i, data=x, num_hidden=dims[i]) + if internal_act and i > 0: + x = mx.symbol.Activation(data=x, act_type=internal_act) + if internal_act == 'sigmoid' and sparseness_penalty: + x = mx.symbol.IdentityAttachKLSparseReg( + data=x, name='sparse_decoder_%d' % i, penalty=sparseness_penalty) + elif input_act and i == 0: + x = mx.symbol.Activation(data=x, act_type=input_act) + if input_act == 'sigmoid' and sparseness_penalty: + x = mx.symbol.IdentityAttachKLSparseReg( + data=x, name='sparse_decoder_%d' % i, penalty=sparseness_penalty) + if dropout and i > 0: + x = mx.symbol.Dropout(data=x, p=dropout) + return x + + def layerwise_pretrain(self, X, batch_size, n_iter, optimizer, l_rate, decay, + lr_scheduler=None, print_every=1000): + def l2_norm(label, pred): + return np.mean(np.square(label-pred))/2.0 + solver = Solver(optimizer, momentum=0.9, wd=decay, learning_rate=l_rate, + lr_scheduler=lr_scheduler) + solver.set_metric(mx.metric.CustomMetric(l2_norm)) + solver.set_monitor(Monitor(print_every)) + data_iter = mx.io.NDArrayIter({'data': X}, batch_size=batch_size, shuffle=True, + last_batch_handle='roll_over') + for i in range(self.N): + if i == 0: + data_iter_i = data_iter + else: + X_i = list(model.extract_feature( + self.internals[i-1], self.args, self.auxs, data_iter, X.shape[0], + self.xpu).values())[0] + data_iter_i = mx.io.NDArrayIter({'data': X_i}, batch_size=batch_size, + last_batch_handle='roll_over') + logging.info('Pre-training layer %d...', i) + solver.solve(self.xpu, self.stacks[i], self.args, self.args_grad, self.auxs, + data_iter_i, 0, n_iter, {}, False) + + def finetune(self, X, batch_size, n_iter, optimizer, l_rate, decay, lr_scheduler=None, + print_every=1000): + def l2_norm(label, pred): + return np.mean(np.square(label-pred))/2.0 + solver = Solver(optimizer, momentum=0.9, wd=decay, learning_rate=l_rate, + lr_scheduler=lr_scheduler) + solver.set_metric(mx.metric.CustomMetric(l2_norm)) + solver.set_monitor(Monitor(print_every)) + data_iter = mx.io.NDArrayIter({'data': X}, batch_size=batch_size, shuffle=True, + last_batch_handle='roll_over') + logging.info('Fine tuning...') + solver.solve(self.xpu, self.loss, self.args, self.args_grad, self.auxs, data_iter, + 0, n_iter, {}, False) + + def eval(self, X): + batch_size = 100 + data_iter = mx.io.NDArrayIter({'data': X}, batch_size=batch_size, shuffle=False, + last_batch_handle='pad') + Y = list(model.extract_feature( + self.loss, self.args, self.auxs, data_iter, X.shape[0], self.xpu).values())[0] + return np.mean(np.square(Y-X))/2.0 diff --git a/example/deep-embedded-clustering/dec.py b/example/deep-embedded-clustering/dec.py new file mode 100644 index 000000000000..8fb3891e3e99 --- /dev/null +++ b/example/deep-embedded-clustering/dec.py @@ -0,0 +1,178 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +from __future__ import print_function + +import os +import logging +import numpy as np +from sklearn.cluster import KMeans +from scipy.spatial.distance import cdist +import mxnet as mx +import data +import model +from autoencoder import AutoEncoderModel +from solver import Solver, Monitor + + +def cluster_acc(Y_pred, Y): + from sklearn.utils.linear_assignment_ import linear_assignment + assert Y_pred.size == Y.size + D = max(Y_pred.max(), Y.max())+1 + w = np.zeros((D, D), dtype=np.int64) + for i in range(Y_pred.size): + w[Y_pred[i], int(Y[i])] += 1 + ind = linear_assignment(w.max() - w) + return sum([w[i, j] for i, j in ind])*1.0/Y_pred.size, w + + +class DECModel(model.MXModel): + class DECLoss(mx.operator.NumpyOp): + def __init__(self, num_centers, alpha): + super(DECModel.DECLoss, self).__init__(need_top_grad=False) + self.num_centers = num_centers + self.alpha = alpha + + def forward(self, in_data, out_data): + z = in_data[0] + mu = in_data[1] + q = out_data[0] + self.mask = 1.0/(1.0+cdist(z, mu)**2/self.alpha) + q[:] = self.mask**((self.alpha+1.0)/2.0) + q[:] = (q.T/q.sum(axis=1)).T + + def backward(self, out_grad, in_data, out_data, in_grad): + q = out_data[0] + z = in_data[0] + mu = in_data[1] + p = in_data[2] + dz = in_grad[0] + dmu = in_grad[1] + self.mask *= (self.alpha+1.0)/self.alpha*(p-q) + dz[:] = (z.T*self.mask.sum(axis=1)).T - self.mask.dot(mu) + dmu[:] = (mu.T*self.mask.sum(axis=0)).T - self.mask.T.dot(z) + + def infer_shape(self, in_shape): + assert len(in_shape) == 3 + assert len(in_shape[0]) == 2 + input_shape = in_shape[0] + label_shape = (input_shape[0], self.num_centers) + mu_shape = (self.num_centers, input_shape[1]) + out_shape = (input_shape[0], self.num_centers) + return [input_shape, mu_shape, label_shape], [out_shape] + + def list_arguments(self): + return ['data', 'mu', 'label'] + + def setup(self, X, num_centers, alpha, save_to='dec_model'): + sep = X.shape[0]*9//10 + X_train = X[:sep] + X_val = X[sep:] + ae_model = AutoEncoderModel(self.xpu, [X.shape[1], 500, 500, 2000, 10], pt_dropout=0.2) + if not os.path.exists(save_to+'_pt.arg'): + ae_model.layerwise_pretrain(X_train, 256, 50000, 'sgd', l_rate=0.1, decay=0.0, + lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1)) + ae_model.finetune(X_train, 256, 100000, 'sgd', l_rate=0.1, decay=0.0, + lr_scheduler=mx.lr_scheduler.FactorScheduler(20000, 0.1)) + ae_model.save(save_to+'_pt.arg') + logging.log(logging.INFO, "Autoencoder Training error: %f"%ae_model.eval(X_train)) + logging.log(logging.INFO, "Autoencoder Validation error: %f"%ae_model.eval(X_val)) + else: + ae_model.load(save_to+'_pt.arg') + self.ae_model = ae_model + + self.dec_op = DECModel.DECLoss(num_centers, alpha) + label = mx.sym.Variable('label') + self.feature = self.ae_model.encoder + self.loss = self.dec_op(data=self.ae_model.encoder, label=label, name='dec') + self.args.update({k: v for k, v in self.ae_model.args.items() if k in self.ae_model.encoder.list_arguments()}) + self.args['dec_mu'] = mx.nd.empty((num_centers, self.ae_model.dims[-1]), ctx=self.xpu) + self.args_grad.update({k: mx.nd.empty(v.shape, ctx=self.xpu) for k, v in self.args.items()}) + self.args_mult.update({k: k.endswith('bias') and 2.0 or 1.0 for k in self.args}) + self.num_centers = num_centers + + def cluster(self, X, y=None, update_interval=None): + N = X.shape[0] + if not update_interval: + update_interval = N + batch_size = 256 + test_iter = mx.io.NDArrayIter({'data': X}, batch_size=batch_size, shuffle=False, + last_batch_handle='pad') + args = {k: mx.nd.array(v.asnumpy(), ctx=self.xpu) for k, v in self.args.items()} + z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0] + kmeans = KMeans(self.num_centers, n_init=20) + kmeans.fit(z) + args['dec_mu'][:] = kmeans.cluster_centers_ + solver = Solver('sgd', momentum=0.9, wd=0.0, learning_rate=0.01) + + def ce(label, pred): + return np.sum(label*np.log(label/(pred+0.000001)))/label.shape[0] + solver.set_metric(mx.metric.CustomMetric(ce)) + + label_buff = np.zeros((X.shape[0], self.num_centers)) + train_iter = mx.io.NDArrayIter({'data': X}, {'label': label_buff}, batch_size=batch_size, + shuffle=False, last_batch_handle='roll_over') + self.y_pred = np.zeros((X.shape[0])) + + def refresh(i): + if i%update_interval == 0: + z = list(model.extract_feature(self.feature, args, None, test_iter, N, self.xpu).values())[0] + p = np.zeros((z.shape[0], self.num_centers)) + self.dec_op.forward([z, args['dec_mu'].asnumpy()], [p]) + y_pred = p.argmax(axis=1) + print(np.std(np.bincount(y_pred)), np.bincount(y_pred)) + print(np.std(np.bincount(y.astype(np.int))), np.bincount(y.astype(np.int))) + if y is not None: + print(cluster_acc(y_pred, y)[0]) + weight = 1.0/p.sum(axis=0) + weight *= self.num_centers/weight.sum() + p = (p**2)*weight + train_iter.data_list[1][:] = (p.T/p.sum(axis=1)).T + print(np.sum(y_pred != self.y_pred), 0.001*y_pred.shape[0]) + if np.sum(y_pred != self.y_pred) < 0.001*y_pred.shape[0]: + self.y_pred = y_pred + return True + self.y_pred = y_pred + solver.set_iter_start_callback(refresh) + solver.set_monitor(Monitor(50)) + + solver.solve(self.xpu, self.loss, args, self.args_grad, None, + train_iter, 0, 1000000000, {}, False) + self.end_args = args + if y is not None: + return cluster_acc(self.y_pred, y)[0] + else: + return -1 + + +def mnist_exp(xpu): + X, Y = data.get_mnist() + if not os.path.isdir('data'): + os.makedirs('data') + dec_model = DECModel(xpu, X, 10, 1.0, 'data/mnist') + acc = [] + for i in [10*(2**j) for j in range(9)]: + acc.append(dec_model.cluster(X, Y, i)) + logging.log(logging.INFO, 'Clustering Acc: %f at update interval: %d'%(acc[-1], i)) + logging.info(str(acc)) + logging.info('Best Clustering ACC: %f at update_interval: %d'%(np.max(acc), 10*(2**np.argmax(acc)))) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + mnist_exp(mx.gpu(0)) diff --git a/example/distributed_training-horovod/gluon_mnist.py b/example/distributed_training-horovod/gluon_mnist.py index c2e6f0bdc533..7b39f5776a42 100644 --- a/example/distributed_training-horovod/gluon_mnist.py +++ b/example/distributed_training-horovod/gluon_mnist.py @@ -104,7 +104,7 @@ def conv_nets(): # Function to evaluate accuracy for a model def evaluate(model, data_iter, context): data_iter.reset() - metric = mx.gluon.metric.Accuracy() + metric = mx.metric.Accuracy() for _, batch in enumerate(data_iter): data = batch.data[0].as_in_context(context) label = batch.label[0].as_in_context(context) @@ -149,7 +149,7 @@ def evaluate(model, data_iter, context): # Create loss function and train metric loss_fn = gluon.loss.SoftmaxCrossEntropyLoss() -metric = mx.gluon.metric.Accuracy() +metric = mx.metric.Accuracy() # Train model for epoch in range(args.epochs): diff --git a/example/distributed_training-horovod/module_mnist.py b/example/distributed_training-horovod/module_mnist.py new file mode 100644 index 000000000000..4fcb02a46996 --- /dev/null +++ b/example/distributed_training-horovod/module_mnist.py @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import logging +import os +import zipfile + +import horovod.mxnet as hvd +import mxnet as mx +from mxnet.test_utils import download + +# Training settings +parser = argparse.ArgumentParser(description='MXNet MNIST Example') +parser.add_argument('--batch-size', type=int, default=64, + help='training batch size (default: 64)') +parser.add_argument('--dtype', type=str, default='float32', + help='training data type (default: float32)') +parser.add_argument('--epochs', type=int, default=5, + help='number of training epochs (default: 5)') +parser.add_argument('--lr', type=float, default=0.05, + help='learning rate (default: 0.05)') +parser.add_argument('--momentum', type=float, default=0.5, + help='SGD momentum (default: 0.5)') +parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training (default: False)') +args = parser.parse_args() + +if not args.no_cuda: + # Disable CUDA if there are no GPUs. + if mx.context.num_gpus() == 0: + args.no_cuda = True + +logging.basicConfig(level=logging.INFO) +logging.info(args) + + +# Function to get mnist iterator given a rank +def get_mnist_iterator(rank): + data_dir = "data-%d" % rank + if not os.path.isdir(data_dir): + os.makedirs(data_dir) + zip_file_path = download('http://data.mxnet.io/mxnet/data/mnist.zip', + dirname=data_dir) + with zipfile.ZipFile(zip_file_path) as zf: + zf.extractall(data_dir) + + input_shape = (1, 28, 28) + batch_size = args.batch_size + + train_iter = mx.io.MNISTIter( + image="%s/train-images-idx3-ubyte" % data_dir, + label="%s/train-labels-idx1-ubyte" % data_dir, + input_shape=input_shape, + batch_size=batch_size, + shuffle=True, + flat=False, + num_parts=hvd.size(), + part_index=hvd.rank() + ) + + val_iter = mx.io.MNISTIter( + image="%s/t10k-images-idx3-ubyte" % data_dir, + label="%s/t10k-labels-idx1-ubyte" % data_dir, + input_shape=input_shape, + batch_size=batch_size, + flat=False, + num_parts=hvd.size(), + part_index=hvd.rank() + ) + + return train_iter, val_iter + +# Step 1: initialize Horovod +hvd.init() + +# Horovod: pin context to process +context = mx.cpu(hvd.local_rank()) if args.no_cuda else mx.gpu(hvd.local_rank()) + +# Step 2: load data +train_iter, val_iter = get_mnist_iterator(hvd.rank()) + +# Step 3: define network +def conv_net(): + # placeholder for data + data = mx.sym.var('data') + # first conv layer + conv1 = mx.sym.Convolution(data=data, kernel=(5, 5), num_filter=10) + relu1 = mx.sym.Activation(data=conv1, act_type='relu') + pool1 = mx.sym.Pooling(data=relu1, pool_type='max', kernel=(2, 2), + stride=(2, 2)) + # second conv layer + conv2 = mx.sym.Convolution(data=pool1, kernel=(5, 5), num_filter=20) + relu2 = mx.sym.Activation(data=conv2, act_type='relu') + pool2 = mx.sym.Pooling(data=relu2, pool_type='max', kernel=(2, 2), + stride=(2, 2)) + # first fully connected layer + flatten = mx.sym.flatten(data=pool2) + fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=50) + relu3 = mx.sym.Activation(data=fc1, act_type='relu') + # second fully connected layer + fc2 = mx.sym.FullyConnected(data=relu3, num_hidden=10) + # softmax loss + loss = mx.sym.SoftmaxOutput(data=fc2, name='softmax') + return loss + +net = conv_net() +model = mx.mod.Module(symbol=net, context=context) + +# Step 4: initialize parameters +initializer = mx.init.Xavier(rnd_type='gaussian', factor_type="in", + magnitude=2) +model.bind(data_shapes=train_iter.provide_data, + label_shapes=train_iter.provide_label) +model.init_params(initializer) + +# Horovod: fetch and broadcast parameters +(arg_params, aux_params) = model.get_params() +if arg_params is not None: + hvd.broadcast_parameters(arg_params, root_rank=0) +if aux_params is not None: + hvd.broadcast_parameters(aux_params, root_rank=0) +model.set_params(arg_params=arg_params, aux_params=aux_params) + +# Step 5: create optimizer +optimizer_params = {'learning_rate': args.lr * hvd.size(), + 'rescale_grad': 1.0 / args.batch_size} +opt = mx.optimizer.create('sgd', **optimizer_params) + +# Horovod: wrap optimizer with DistributedOptimizer +opt = hvd.DistributedOptimizer(opt) + +# Step 6: fit and train model +batch_cb = None +if hvd.rank() == 0: + batch_cb = mx.callback.Speedometer(args.batch_size * hvd.size()) +model.fit(train_iter, # train data + kvstore=None, # no kvstore + eval_data=val_iter, # validation data + optimizer=opt, # use SGD to train + eval_metric='acc', # report accuracy during training + batch_end_callback=batch_cb, # report training speed + num_epoch=args.epochs) # train for at most 10 dataset passes + +# Step 7: evaluate model accuracy +acc = mx.metric.Accuracy() +model.score(val_iter, acc) + +if hvd.rank() == 0: + print(acc) + assert acc.get()[1] > 0.96, "Achieved accuracy (%f) is lower than \ + expected (0.96)" % acc.get()[1] diff --git a/example/distributed_training-horovod/resnet50_imagenet.py b/example/distributed_training-horovod/resnet50_imagenet.py index cdf17a8912e0..251e64f0749a 100644 --- a/example/distributed_training-horovod/resnet50_imagenet.py +++ b/example/distributed_training-horovod/resnet50_imagenet.py @@ -283,8 +283,8 @@ def evaluate(epoch): return val_data.reset() - acc_top1 = mx.gluon.metric.Accuracy() - acc_top5 = mx.gluon.metric.TopKAccuracy(5) + acc_top1 = mx.metric.Accuracy() + acc_top5 = mx.metric.TopKAccuracy(5) for _, batch in enumerate(val_data): data, label = batch_fn(batch, context) output = net(data.astype(args.dtype, copy=False)) @@ -318,7 +318,7 @@ def evaluate(epoch): # Create loss function and train metric loss_fn = gluon.loss.SoftmaxCrossEntropyLoss() - metric = mx.gluon.metric.Accuracy() + metric = mx.metric.Accuracy() # Train model for epoch in range(args.num_epochs): @@ -368,6 +368,92 @@ def evaluate(epoch): evaluate(epoch) +def train_module(): + # Create input symbol + data = mx.sym.var('data') + if args.dtype == 'float16': + data = mx.sym.Cast(data=data, dtype=np.float16) + net.cast(np.float16) + + # Create output symbol + out = net(data) + if args.dtype == 'float16': + out = mx.sym.Cast(data=out, dtype=np.float32) + softmax = mx.sym.SoftmaxOutput(out, name='softmax') + + # Create model + mod = mx.mod.Module(softmax, context=context) + + # Initialize parameters + if args.use_pretrained: + arg_params = {} + for x in net.collect_params().values(): + x.reset_ctx(mx.cpu()) + arg_params[x.name] = x.data() + else: + arg_params = None + aux_params = None + mod.bind(data_shapes=train_data.provide_data, + label_shapes=train_data.provide_label) + mod.init_params(initializer, arg_params=arg_params, aux_params=aux_params) + + # Horovod: fetch and broadcast parameters + (arg_params, aux_params) = mod.get_params() + if arg_params is not None: + hvd.broadcast_parameters(arg_params, root_rank=0) + if aux_params is not None: + hvd.broadcast_parameters(aux_params, root_rank=0) + mod.set_params(arg_params=arg_params, aux_params=aux_params) + + # Create optimizer + # Note that when using Module API, we need to specify rescale_grad since + # we create optimizer first and wrap it with DistributedOptimizer. For + # Gluon API, it is handled in Trainer.step() function so there is no need + # to specify rescale_grad (see above train_gluon() function). + optimizer_params = {'wd': args.wd, + 'momentum': args.momentum, + 'rescale_grad': 1.0 / batch_size, + 'lr_scheduler': lr_sched} + if args.dtype == 'float16': + optimizer_params['multi_precision'] = True + opt = mx.optimizer.create('sgd', **optimizer_params) + + # Horovod: wrap optimizer with DistributedOptimizer + opt = hvd.DistributedOptimizer(opt) + + # Setup validation data and callback during training + eval_data = None + if args.eval_epoch: + eval_data = val_data + batch_callback = None + if args.log_interval > 0 and rank == 0: + batch_callback = mx.callback.Speedometer(batch_size * num_workers, + args.log_interval) + + epoch_callback = None + if args.save_frequency > 0: + epoch_callback = mx.callback.do_checkpoint( + '%s-%d' % (args.model, rank), + period=args.save_frequency) + + # Train model + mod.fit(train_data, + eval_data=eval_data, + num_epoch=args.num_epochs, + kvstore=None, + batch_end_callback=batch_callback, + epoch_end_callback=epoch_callback, + optimizer=opt) + + # Evaluate performance if not using synthetic data + if args.use_rec: + acc_top1 = mx.metric.Accuracy() + acc_top5 = mx.metric.TopKAccuracy(5) + res = mod.score(val_data, [acc_top1, acc_top5]) + for name, val in res: + logging.info('Epoch[%d] Rank[%d] Validation-%s=%f', + args.num_epochs - 1, rank, name, val) + if __name__ == '__main__': train_gluon() diff --git a/example/distributed_training/cifar10_dist.py b/example/distributed_training/cifar10_dist.py index c89619d595f2..b66845702137 100644 --- a/example/distributed_training/cifar10_dist.py +++ b/example/distributed_training/cifar10_dist.py @@ -121,7 +121,7 @@ def evaluate_accuracy(data_iterator, network): ---------- tuple of array element """ - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() # Iterate through data and label for i, (data, label) in enumerate(data_iterator): diff --git a/example/distributed_training/cifar10_kvstore_hvd.py b/example/distributed_training/cifar10_kvstore_hvd.py index ff679864f7c3..e6780e5db85e 100644 --- a/example/distributed_training/cifar10_kvstore_hvd.py +++ b/example/distributed_training/cifar10_kvstore_hvd.py @@ -123,7 +123,7 @@ def evaluate(data_iterator, network, context): ---------- tuple of array element """ - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() # Iterate through data and label for i, (data, label) in enumerate(data_iterator): @@ -208,7 +208,7 @@ def __len__(self): optimizer_params={'learning_rate': args.lr}, kvstore=store) -train_metric = mx.gluon.metric.Accuracy() +train_metric = mx.metric.Accuracy() # Run as many epochs as required for epoch in range(args.epochs): diff --git a/example/fcn-xs/solver.py b/example/fcn-xs/solver.py new file mode 100644 index 000000000000..e99b31a13055 --- /dev/null +++ b/example/fcn-xs/solver.py @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import numpy as np +import mxnet as mx +import time +import logging +from collections import namedtuple +from mxnet import optimizer as opt +from mxnet.optimizer import get_updater +from mxnet import metric + +# Parameter to pass to batch_end_callback +BatchEndParam = namedtuple('BatchEndParams', ['epoch', 'nbatch', 'eval_metric']) +class Solver(object): + def __init__(self, symbol, ctx=None, + begin_epoch=0, num_epoch=None, + arg_params=None, aux_params=None, + optimizer='sgd', **kwargs): + self.symbol = symbol + if ctx is None: + ctx = mx.cpu() + self.ctx = ctx + self.begin_epoch = begin_epoch + self.num_epoch = num_epoch + self.arg_params = arg_params + self.aux_params = aux_params + self.optimizer = optimizer + self.kwargs = kwargs.copy() + + def fit(self, train_data, eval_data=None, + eval_metric='acc', + grad_req='write', + epoch_end_callback=None, + batch_end_callback=None, + kvstore='local', + logger=None): + if logger is None: + logger = logging + logging.info('Start training with %s', str(self.ctx)) + arg_shapes, out_shapes, aux_shapes = self.symbol.infer_shape(data=train_data.provide_data[0][1]) + arg_names = self.symbol.list_arguments() + if grad_req != 'null': + self.grad_params = {} + for name, shape in zip(arg_names, arg_shapes): + if not (name.endswith('data') or name.endswith('label')): + self.grad_params[name] = mx.nd.zeros(shape, self.ctx) + else: + self.grad_params = None + aux_names = self.symbol.list_auxiliary_states() + self.aux_params = {k: mx.nd.zeros(s) for k, s in zip(aux_names, aux_shapes)} + data_name = train_data.data_name + label_name = train_data.label_name + input_names = [data_name, label_name] + self.optimizer = opt.create(self.optimizer, rescale_grad=(1.0/train_data.get_batch_size()), **(self.kwargs)) + self.updater = get_updater(self.optimizer) + eval_metric = metric.create(eval_metric) + # begin training + for epoch in range(self.begin_epoch, self.num_epoch): + nbatch = 0 + train_data.reset() + eval_metric.reset() + for data in train_data: + nbatch += 1 + label_shape = data[label_name].shape + self.arg_params[data_name] = mx.nd.array(data[data_name], self.ctx) + self.arg_params[label_name] = mx.nd.array(data[label_name].reshape(label_shape[0], \ + label_shape[1]*label_shape[2]), self.ctx) + output_names = self.symbol.list_outputs() + self.exector = self.symbol.bind(self.ctx, self.arg_params, + args_grad=self.grad_params, + grad_req=grad_req, + aux_states=self.aux_params) + assert len(self.symbol.list_arguments()) == len(self.exector.grad_arrays) + update_dict = {name: nd for name, nd in zip(self.symbol.list_arguments(), \ + self.exector.grad_arrays) if nd is not None} + output_dict = {} + output_buff = {} + for key, arr in zip(self.symbol.list_outputs(), self.exector.outputs): + output_dict[key] = arr + output_buff[key] = mx.nd.empty(arr.shape, ctx=mx.cpu()) + self.exector.forward(is_train=True) + for key in output_dict: + output_dict[key].copyto(output_buff[key]) + self.exector.backward() + for key, arr in update_dict.items(): + if key != "bigscore_weight": + self.updater(key, arr, self.arg_params[key]) + pred_shape = self.exector.outputs[0].shape + label = mx.nd.array(data[label_name].reshape(label_shape[0], label_shape[1]*label_shape[2])) + pred = mx.nd.array(output_buff["softmax_output"].asnumpy().reshape(pred_shape[0], \ + pred_shape[1], pred_shape[2]*pred_shape[3])) + eval_metric.update([label], [pred]) + self.exector.outputs[0].wait_to_read() + batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, eval_metric=eval_metric) + batch_end_callback(batch_end_params) + if epoch_end_callback is not None: + epoch_end_callback(epoch, self.symbol, self.arg_params, self.aux_params) + name, value = eval_metric.get() + logger.info(" --->Epoch[%d] Train-%s=%f", epoch, name, value) + # evaluation + if eval_data: + logger.info(" in eval process...") + nbatch = 0 + eval_data.reset() + eval_metric.reset() + for data in eval_data: + nbatch += 1 + label_shape = data[label_name].shape + self.arg_params[data_name] = mx.nd.array(data[data_name], self.ctx) + self.arg_params[label_name] = mx.nd.array(data[label_name].reshape(label_shape[0], \ + label_shape[1]*label_shape[2]), self.ctx) + exector = self.symbol.bind(self.ctx, self.arg_params, + args_grad=self.grad_params, + grad_req=grad_req, + aux_states=self.aux_params) + cpu_output_array = mx.nd.zeros(exector.outputs[0].shape) + exector.forward(is_train=False) + exector.outputs[0].copyto(cpu_output_array) + pred_shape = cpu_output_array.shape + label = mx.nd.array(data[label_name].reshape(label_shape[0], \ + label_shape[1]*label_shape[2])) + pred = mx.nd.array(cpu_output_array.asnumpy().reshape(pred_shape[0], \ + pred_shape[1], pred_shape[2]*pred_shape[3])) + eval_metric.update([label], [pred]) + exector.outputs[0].wait_to_read() + name, value = eval_metric.get() + logger.info('batch[%d] Validation-%s=%f', nbatch, name, value) diff --git a/example/gluon/audio/urban_sounds/train.py b/example/gluon/audio/urban_sounds/train.py index 8a55c5b5bc67..c88f9fb55187 100644 --- a/example/gluon/audio/urban_sounds/train.py +++ b/example/gluon/audio/urban_sounds/train.py @@ -28,7 +28,7 @@ def evaluate_accuracy(data_iterator, net): """Function to evaluate accuracy of any data iterator passed to it as an argument""" - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() for data, label in data_iterator: output = net(data) predictions = nd.argmax(output, axis=1) diff --git a/example/gluon/dc_gan/dcgan.py b/example/gluon/dc_gan/dcgan.py index d7c36a0a3a67..6e03aae8bed6 100644 --- a/example/gluon/dc_gan/dcgan.py +++ b/example/gluon/dc_gan/dcgan.py @@ -259,7 +259,7 @@ def main(): real_label = mx.nd.ones((opt.batch_size,), ctx=ctx) fake_label = mx.nd.zeros((opt.batch_size,), ctx=ctx) - metric = mx.gluon.metric.Accuracy() + metric = mx.metric.Accuracy() print('Training... ') stamp = datetime.now().strftime('%Y_%m_%d-%H_%M') diff --git a/example/gluon/image_classification.py b/example/gluon/image_classification.py index 33583ff20175..7a845bf6f8eb 100644 --- a/example/gluon/image_classification.py +++ b/example/gluon/image_classification.py @@ -27,7 +27,7 @@ from mxnet.gluon.model_zoo import vision as models from mxnet import autograd as ag from mxnet.test_utils import get_mnist_iterator -from mxnet.gluon.metric import Accuracy, TopKAccuracy, CompositeEvalMetric +from mxnet.metric import Accuracy, TopKAccuracy, CompositeEvalMetric import numpy as np from data import (get_cifar10_iterator, get_imagenet_iterator, diff --git a/example/gluon/mnist/mnist.py b/example/gluon/mnist/mnist.py index 81259db8b939..5acaf143ca60 100644 --- a/example/gluon/mnist/mnist.py +++ b/example/gluon/mnist/mnist.py @@ -70,7 +70,7 @@ def transformer(data, label): # train def test(ctx): - metric = mx.gluon.metric.Accuracy() + metric = mx.metric.Accuracy() for data, label in val_data: data = data.as_in_context(ctx) label = label.as_in_context(ctx) @@ -86,7 +86,7 @@ def train(epochs, ctx): # Trainer is for updating parameters with gradient. trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': opt.lr, 'momentum': opt.momentum}) - metric = mx.gluon.metric.Accuracy() + metric = mx.metric.Accuracy() loss = gluon.loss.SoftmaxCrossEntropyLoss() for epoch in range(epochs): diff --git a/example/gluon/sn_gan/train.py b/example/gluon/sn_gan/train.py index fc4e87d632fe..46e44791cebd 100644 --- a/example/gluon/sn_gan/train.py +++ b/example/gluon/sn_gan/train.py @@ -102,7 +102,7 @@ def facc(label, pred): g_net.collect_params().zero_grad() d_net.collect_params().zero_grad() # define evaluation metric -metric = mx.gluon.metric.CustomMetric(facc) +metric = mx.metric.CustomMetric(facc) # initialize labels real_label = nd.ones(BATCH_SIZE, CTX) fake_label = nd.zeros(BATCH_SIZE, CTX) diff --git a/example/gluon/super_resolution/super_resolution.py b/example/gluon/super_resolution/super_resolution.py index 52bfc2241f82..4a3e8d92aa39 100644 --- a/example/gluon/super_resolution/super_resolution.py +++ b/example/gluon/super_resolution/super_resolution.py @@ -156,7 +156,7 @@ def hybrid_forward(self, F, x): return x net = SuperResolutionNet(upscale_factor) -metric = mx.gluon.metric.MSE() +metric = mx.metric.MSE() def test(ctx): val_data.reset() diff --git a/example/gluon/tree_lstm/main.py b/example/gluon/tree_lstm/main.py index 41e4f4f13ed8..53af3fa019e9 100644 --- a/example/gluon/tree_lstm/main.py +++ b/example/gluon/tree_lstm/main.py @@ -96,7 +96,7 @@ net = SimilarityTreeLSTM(sim_hidden_size, rnn_hidden_size, vocab.size, vocab.embed.shape[1], num_classes) # use pearson correlation and mean-square error for evaluation -metric = mx.gluon.metric.create(['pearsonr', 'mse']) +metric = mx.metric.create(['pearsonr', 'mse']) def to_target(x): target = np.zeros((1, num_classes)) diff --git a/example/image-classification/common/fit.py b/example/image-classification/common/fit.py new file mode 100644 index 000000000000..38ca296cf986 --- /dev/null +++ b/example/image-classification/common/fit.py @@ -0,0 +1,340 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" example train fit utility """ +import logging +import os +import time +import re +import math +import mxnet as mx + +def get_epoch_size(args, kv): + return math.ceil(int(args.num_examples / kv.num_workers) / args.batch_size) + +def _get_lr_scheduler(args, kv): + if 'lr_factor' not in args or args.lr_factor >= 1: + return (args.lr, None) + epoch_size = get_epoch_size(args, kv) + begin_epoch = args.load_epoch if args.load_epoch else 0 + if 'pow' in args.lr_step_epochs: + lr = args.lr + max_up = args.num_epochs * epoch_size + pwr = float(re.sub('pow[- ]*', '', args.lr_step_epochs)) + poly_sched = mx.lr_scheduler.PolyScheduler(max_up, lr, pwr) + return (lr, poly_sched) + step_epochs = [int(l) for l in args.lr_step_epochs.split(',')] + lr = args.lr + for s in step_epochs: + if begin_epoch >= s: + lr *= args.lr_factor + if lr != args.lr: + logging.info('Adjust learning rate to %e for epoch %d', + lr, begin_epoch) + + steps = [epoch_size * (x - begin_epoch) + for x in step_epochs if x - begin_epoch > 0] + if steps: + return (lr, mx.lr_scheduler.MultiFactorScheduler(step=steps, factor=args.lr_factor, + base_lr=args.lr)) + else: + return (lr, None) + +def _load_model(args, rank=0): + if 'load_epoch' not in args or args.load_epoch is None: + return (None, None, None) + assert args.model_prefix is not None + model_prefix = args.model_prefix + if rank > 0 and os.path.exists("%s-%d-symbol.json" % (model_prefix, rank)): + model_prefix += "-%d" % (rank) + sym, arg_params, aux_params = mx.model.load_checkpoint( + model_prefix, args.load_epoch) + logging.info('Loaded model %s_%04d.params', model_prefix, args.load_epoch) + return (sym, arg_params, aux_params) + + +def _save_model(args, rank=0): + if args.model_prefix is None: + return None + return mx.callback.do_checkpoint(args.model_prefix if rank == 0 else "%s-%d" % ( + args.model_prefix, rank), period=args.save_period) + + +def add_fit_args(parser): + """ + parser : argparse.ArgumentParser + return a parser added with args required by fit + """ + train = parser.add_argument_group('Training', 'model training') + train.add_argument('--network', type=str, + help='the neural network to use') + train.add_argument('--num-layers', type=int, + help='number of layers in the neural network, \ + required by some networks such as resnet') + train.add_argument('--gpus', type=str, + help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu') + train.add_argument('--kv-store', type=str, default='device', + help='key-value store type') + train.add_argument('--num-epochs', type=int, default=100, + help='max num of epochs') + train.add_argument('--lr', type=float, default=0.1, + help='initial learning rate') + train.add_argument('--lr-factor', type=float, default=0.1, + help='the ratio to reduce lr on each step') + train.add_argument('--lr-step-epochs', type=str, + help='the epochs to reduce the lr, e.g. 30,60') + train.add_argument('--initializer', type=str, default='default', + help='the initializer type') + train.add_argument('--optimizer', type=str, default='sgd', + help='the optimizer type') + train.add_argument('--mom', type=float, default=0.9, + help='momentum for sgd') + train.add_argument('--wd', type=float, default=0.0001, + help='weight decay for sgd') + train.add_argument('--batch-size', type=int, default=128, + help='the batch size') + train.add_argument('--disp-batches', type=int, default=20, + help='show progress for every n batches') + train.add_argument('--model-prefix', type=str, + help='model prefix') + train.add_argument('--save-period', type=int, default=1, help='params saving period') + parser.add_argument('--monitor', dest='monitor', type=int, default=0, + help='log network parameters every N iters if larger than 0') + train.add_argument('--load-epoch', type=int, + help='load the model on an epoch using the model-load-prefix') + train.add_argument('--top-k', type=int, default=0, + help='report the top-k accuracy. 0 means no report.') + train.add_argument('--loss', type=str, default='', + help='show the cross-entropy or nll loss. ce strands for cross-entropy, nll-loss stands for likelihood loss') + train.add_argument('--test-io', type=int, default=0, + help='1 means test reading speed without training') + train.add_argument('--dtype', type=str, default='float32', + help='precision: float32 or float16') + train.add_argument('--gc-type', type=str, default='none', + help='type of gradient compression to use, \ + takes `2bit` or `none` for now') + train.add_argument('--gc-threshold', type=float, default=0.5, + help='threshold for 2bit gradient compression') + # additional parameters for large batch sgd + train.add_argument('--macrobatch-size', type=int, default=0, + help='distributed effective batch size') + train.add_argument('--warmup-epochs', type=int, default=5, + help='the epochs to ramp-up lr to scaled large-batch value') + train.add_argument('--warmup-strategy', type=str, default='linear', + help='the ramping-up strategy for large batch sgd') + train.add_argument('--profile-worker-suffix', type=str, default='', + help='profile workers actions into this file. During distributed training\ + filename saved will be rank1_ followed by this suffix') + train.add_argument('--profile-server-suffix', type=str, default='', + help='profile server actions into a file with name like rank1_ followed by this suffix \ + during distributed training') + train.add_argument('--use-imagenet-data-augmentation', type=int, default=0, + help='enable data augmentation of ImageNet data, default disabled') + return train + + +def fit(args, network, data_loader, **kwargs): + """ + train a model + args : argparse returns + network : the symbol definition of the nerual network + data_loader : function that returns the train and val data iterators + """ + # kvstore + kv = mx.kvstore.create(args.kv_store) + if args.gc_type != 'none': + kv.set_gradient_compression({'type': args.gc_type, + 'threshold': args.gc_threshold}) + if args.profile_server_suffix: + mx.profiler.set_config(filename=args.profile_server_suffix, profile_all=True, profile_process='server') + mx.profiler.set_state(state='run', profile_process='server') + + if args.profile_worker_suffix: + if kv.num_workers > 1: + filename = 'rank' + str(kv.rank) + '_' + args.profile_worker_suffix + else: + filename = args.profile_worker_suffix + mx.profiler.set_config(filename=filename, profile_all=True, profile_process='worker') + mx.profiler.set_state(state='run', profile_process='worker') + + # logging + head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + logging.info('start with arguments %s', args) + + epoch_size = get_epoch_size(args, kv) + + # data iterators + (train, val) = data_loader(args, kv) + if 'dist' in args.kv_store and not 'async' in args.kv_store: + logging.info('Resizing training data to %d batches per machine', epoch_size) + # resize train iter to ensure each machine has same number of batches per epoch + # if not, dist_sync can hang at the end with one machine waiting for other machines + train = mx.io.ResizeIter(train, epoch_size) + + if args.test_io: + tic = time.time() + for i, batch in enumerate(train): + if isinstance(batch, list): + for b in batch: + for j in b.data: + j.wait_to_read() + else: + for j in batch.data: + j.wait_to_read() + if (i + 1) % args.disp_batches == 0: + logging.info('Batch [%d]\tSpeed: %.2f samples/sec', i, + args.disp_batches * args.batch_size / (time.time() - tic)) + tic = time.time() + return + + # load model + if 'arg_params' in kwargs and 'aux_params' in kwargs: + arg_params = kwargs['arg_params'] + aux_params = kwargs['aux_params'] + else: + sym, arg_params, aux_params = _load_model(args, kv.rank) + if sym is not None: + assert sym.tojson() == network.tojson() + + # save model + checkpoint = _save_model(args, kv.rank) + + # devices for training + devs = mx.cpu() if args.gpus is None or args.gpus == "" else [ + mx.gpu(int(i)) for i in args.gpus.split(',')] + + # learning rate + lr, lr_scheduler = _get_lr_scheduler(args, kv) + + # create model + model = mx.mod.Module( + context=devs, + symbol=network + ) + + lr_scheduler = lr_scheduler + optimizer_params = { + 'learning_rate': lr, + 'wd': args.wd, + 'lr_scheduler': lr_scheduler, + 'multi_precision': True} + + # Only a limited number of optimizers have 'momentum' property + has_momentum = {'sgd', 'dcasgd', 'nag', 'signum'} + if args.optimizer in has_momentum: + optimizer_params['momentum'] = args.mom + + monitor = mx.mon.Monitor( + args.monitor, pattern=".*") if args.monitor > 0 else None + + # A limited number of optimizers have a warmup period + has_warmup = {'lbnag'} + if args.optimizer in has_warmup: + nworkers = kv.num_workers + if epoch_size < 1: + epoch_size = 1 + macrobatch_size = args.macrobatch_size + if macrobatch_size < args.batch_size * nworkers: + macrobatch_size = args.batch_size * nworkers + #batch_scale = round(float(macrobatch_size) / args.batch_size / nworkers +0.4999) + batch_scale = math.ceil( + float(macrobatch_size) / args.batch_size / nworkers) + optimizer_params['updates_per_epoch'] = epoch_size + optimizer_params['begin_epoch'] = args.load_epoch if args.load_epoch else 0 + optimizer_params['batch_scale'] = batch_scale + optimizer_params['warmup_strategy'] = args.warmup_strategy + optimizer_params['warmup_epochs'] = args.warmup_epochs + optimizer_params['num_epochs'] = args.num_epochs + + if args.initializer == 'default': + if args.network == 'alexnet': + # AlexNet will not converge using Xavier + initializer = mx.init.Normal() + # VGG will not trend to converge using Xavier-Gaussian + elif args.network and 'vgg' in args.network: + initializer = mx.init.Xavier() + else: + initializer = mx.init.Xavier( + rnd_type='gaussian', factor_type="in", magnitude=2) + # initializer = mx.init.Xavier(factor_type="in", magnitude=2.34), + elif args.initializer == 'xavier': + initializer = mx.init.Xavier() + elif args.initializer == 'msra': + initializer = mx.init.MSRAPrelu() + elif args.initializer == 'orthogonal': + initializer = mx.init.Orthogonal() + elif args.initializer == 'normal': + initializer = mx.init.Normal() + elif args.initializer == 'uniform': + initializer = mx.init.Uniform() + elif args.initializer == 'one': + initializer = mx.init.One() + elif args.initializer == 'zero': + initializer = mx.init.Zero() + + # evaluation metrices + eval_metrics = ['accuracy'] + if args.top_k > 0: + eval_metrics.append(mx.metric.create( + 'top_k_accuracy', top_k=args.top_k)) + + supported_loss = ['ce', 'nll_loss'] + if len(args.loss) > 0: + # ce or nll loss is only applicable to softmax output + loss_type_list = args.loss.split(',') + if 'softmax_output' in network.list_outputs(): + for loss_type in loss_type_list: + loss_type = loss_type.strip() + if loss_type == 'nll': + loss_type = 'nll_loss' + if loss_type not in supported_loss: + logging.warning(loss_type + ' is not an valid loss type, only cross-entropy or ' \ + 'negative likelihood loss is supported!') + else: + eval_metrics.append(mx.metric.create(loss_type)) + else: + logging.warning("The output is not softmax_output, loss argument will be skipped!") + + # callbacks that run after each batch + batch_end_callbacks = [mx.callback.Speedometer( + args.batch_size, args.disp_batches)] + if 'batch_end_callback' in kwargs: + cbs = kwargs['batch_end_callback'] + batch_end_callbacks += cbs if isinstance(cbs, list) else [cbs] + + # run + model.fit(train, + begin_epoch=args.load_epoch if args.load_epoch else 0, + num_epoch=args.num_epochs, + eval_data=val, + eval_metric=eval_metrics, + kvstore=kv, + optimizer=args.optimizer, + optimizer_params=optimizer_params, + initializer=initializer, + arg_params=arg_params, + aux_params=aux_params, + batch_end_callback=batch_end_callbacks, + epoch_end_callback=checkpoint, + allow_missing=True, + monitor=monitor) + + if args.profile_server_suffix: + mx.profiler.set_state(state='run', profile_process='server') + if args.profile_worker_suffix: + mx.profiler.set_state(state='run', profile_process='worker') diff --git a/example/image-classification/score.py b/example/image-classification/score.py new file mode 100644 index 000000000000..f40e649f1f42 --- /dev/null +++ b/example/image-classification/score.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +from common import modelzoo, find_mxnet +import mxnet as mx +import time +import os +import logging + +def score(model, data_val, metrics, gpus, batch_size, rgb_mean=None, mean_img=None, + image_shape='3,224,224', data_nthreads=4, label_name='softmax_label', max_num_examples=None): + # create data iterator + data_shape = tuple([int(i) for i in image_shape.split(',')]) + if mean_img is not None: + mean_args = {'mean_img':mean_img} + elif rgb_mean is not None: + rgb_mean = [float(i) for i in rgb_mean.split(',')] + mean_args = {'mean_r':rgb_mean[0], 'mean_g':rgb_mean[1], + 'mean_b':rgb_mean[2]} + + data = mx.io.ImageRecordIter( + path_imgrec = data_val, + label_width = 1, + preprocess_threads = data_nthreads, + batch_size = batch_size, + data_shape = data_shape, + label_name = label_name, + rand_crop = False, + rand_mirror = False, + **mean_args) + + if isinstance(model, str): + # download model + dir_path = os.path.dirname(os.path.realpath(__file__)) + (prefix, epoch) = modelzoo.download_model( + model, os.path.join(dir_path, 'model')) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + elif isinstance(model, tuple) or isinstance(model, list): + assert len(model) == 3 + (sym, arg_params, aux_params) = model + else: + raise TypeError('model type [%s] is not supported' % str(type(model))) + + # create module + if gpus == '': + devs = mx.cpu() + else: + devs = [mx.gpu(int(i)) for i in gpus.split(',')] + + mod = mx.mod.Module(symbol=sym, context=devs, label_names=[label_name,]) + mod.bind(for_training=False, + data_shapes=data.provide_data, + label_shapes=data.provide_label) + mod.set_params(arg_params, aux_params) + if not isinstance(metrics, list): + metrics = [metrics,] + tic = time.time() + num = 0 + for batch in data: + mod.forward(batch, is_train=False) + for m in metrics: + mod.update_metric(m, batch.label) + num += batch_size + if max_num_examples is not None and num > max_num_examples: + break + return (num / (time.time() - tic), ) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='score a model on a dataset') + parser.add_argument('--model', type=str, required=True, + help = 'the model name.') + parser.add_argument('--gpus', type=str, default='0') + parser.add_argument('--batch-size', type=int, default=64) + parser.add_argument('--rgb-mean', type=str, default='0,0,0') + parser.add_argument('--data-val', type=str, required=True) + parser.add_argument('--image-shape', type=str, default='3,224,224') + parser.add_argument('--data-nthreads', type=int, default=4, + help='number of threads for data decoding') + args = parser.parse_args() + + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + metrics = [mx.metric.create('acc'), + mx.metric.create('top_k_accuracy', top_k = 5)] + + (speed,) = score(metrics = metrics, **vars(args)) + logging.info('Finished with %f images per second', speed) + + for m in metrics: + logging.info(m.get()) diff --git a/example/image-classification/test_score.py b/example/image-classification/test_score.py new file mode 100644 index 000000000000..58c5c66a7f1f --- /dev/null +++ b/example/image-classification/test_score.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +test pretrained models +""" +from __future__ import print_function +import mxnet as mx +from common import find_mxnet, modelzoo +from score import score +import pytest + +@pytest.fixture(scope="session") +def imagenet_val_5k_settings(): + mx.test_utils.download( + 'http://data.mxnet.io/data/val-5k-256.rec', 'data/val-5k-256.rec') + num_gpus = mx.context.num_gpus() + assert num_gpus > 0 + gpus = ','.join(map(str, range(num_gpus))) + batch_size = 16 * num_gpus + kwargs = {'gpus':gpus, 'batch_size':batch_size, 'max_num_examples':500} + return 'data/val-5k-256.rec', kwargs + +def test_imagenet1k_resnet(imagenet_val_5k_settings): + imagenet_val_5k, kwargs = imagenet_val_5k_settings + models = ['imagenet1k-resnet-50', 'imagenet1k-resnet-152'] + accs = [.77, .78] + for (m, g) in zip(models, accs): + acc = mx.metric.create('acc') + (speed,) = score(model=m, data_val=imagenet_val_5k, + rgb_mean='0,0,0', metrics=acc, **kwargs) + r = acc.get()[1] + print('Tested %s, acc = %f, speed = %f img/sec' % (m, r, speed)) + assert r > g and r < g + .1 + +def test_imagenet1k_inception_bn(imagenet_val_5k_settings): + imagenet_val_5k, kwargs = imagenet_val_5k_settings + acc = mx.metric.create('acc') + m = 'imagenet1k-inception-bn' + g = 0.75 + (speed,) = score(model=m, + data_val=imagenet_val_5k, + rgb_mean='123.68,116.779,103.939', metrics=acc, **kwargs) + r = acc.get()[1] + print('Tested %s acc = %f, speed = %f img/sec' % (m, r, speed)) + assert r > g and r < g + .1 + diff --git a/example/kaggle-ndsb2/Train.py b/example/kaggle-ndsb2/Train.py new file mode 100644 index 000000000000..51e308a2e21c --- /dev/null +++ b/example/kaggle-ndsb2/Train.py @@ -0,0 +1,234 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Training script, this is converted from a ipython notebook +""" + +import os +import csv +import sys +import numpy as np +import mxnet as mx +import logging + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +# In[2]: + +def get_lenet(): + """ A lenet style net, takes difference of each frame as input. + """ + source = mx.sym.Variable("data") + source = (source - 128) * (1.0/128) + frames = mx.sym.SliceChannel(source, num_outputs=30) + diffs = [frames[i+1] - frames[i] for i in range(29)] + source = mx.sym.Concat(*diffs) + net = mx.sym.Convolution(source, kernel=(5, 5), num_filter=40) + net = mx.sym.BatchNorm(net, fix_gamma=True) + net = mx.sym.Activation(net, act_type="relu") + net = mx.sym.Pooling(net, pool_type="max", kernel=(2,2), stride=(2,2)) + net = mx.sym.Convolution(net, kernel=(3, 3), num_filter=40) + net = mx.sym.BatchNorm(net, fix_gamma=True) + net = mx.sym.Activation(net, act_type="relu") + net = mx.sym.Pooling(net, pool_type="max", kernel=(2,2), stride=(2,2)) + # first fullc + flatten = mx.symbol.Flatten(net) + flatten = mx.symbol.Dropout(flatten) + fc1 = mx.symbol.FullyConnected(data=flatten, num_hidden=600) + # Name the final layer as softmax so it auto matches the naming of data iterator + # Otherwise we can also change the provide_data in the data iter + return mx.symbol.LogisticRegressionOutput(data=fc1, name='softmax') + +def CRPS(label, pred): + """ Custom evaluation metric on CRPS. + """ + for i in range(pred.shape[0]): + for j in range(pred.shape[1] - 1): + if pred[i, j] > pred[i, j + 1]: + pred[i, j + 1] = pred[i, j] + return np.sum(np.square(label - pred)) / label.size + + +# In[3]: + +def encode_label(label_data): + """Run encoding to encode the label into the CDF target. + """ + systole = label_data[:, 1] + diastole = label_data[:, 2] + systole_encode = np.array([ + (x < np.arange(600)) for x in systole + ], dtype=np.uint8) + diastole_encode = np.array([ + (x < np.arange(600)) for x in diastole + ], dtype=np.uint8) + return systole_encode, diastole_encode + +def encode_csv(label_csv, systole_csv, diastole_csv): + systole_encode, diastole_encode = encode_label(np.loadtxt(label_csv, delimiter=",")) + np.savetxt(systole_csv, systole_encode, delimiter=",", fmt="%g") + np.savetxt(diastole_csv, diastole_encode, delimiter=",", fmt="%g") + +# Write encoded label into the target csv +# We use CSV so that not all data need to sit into memory +# You can also use inmemory numpy array if your machine is large enough +encode_csv("./train-label.csv", "./train-systole.csv", "./train-diastole.csv") + + +# # Training the systole net + +# In[4]: + +network = get_lenet() +batch_size = 32 +devs = [mx.gpu(0)] +data_train = mx.io.CSVIter(data_csv="./train-64x64-data.csv", data_shape=(30, 64, 64), + label_csv="./train-systole.csv", label_shape=(600,), + batch_size=batch_size) + +data_validate = mx.io.CSVIter(data_csv="./validate-64x64-data.csv", data_shape=(30, 64, 64), + batch_size=1) + +systole_model = mx.model.FeedForward(ctx=devs, + symbol = network, + num_epoch = 65, + learning_rate = 0.001, + wd = 0.00001, + momentum = 0.9) + +systole_model.fit(X=data_train, eval_metric = mx.metric.np(CRPS)) + + +# # Predict systole + +# In[5]: + +systole_prob = systole_model.predict(data_validate) + + +# # Training the diastole net + +# In[6]: + +network = get_lenet() +batch_size = 32 +devs = [mx.gpu(0)] +data_train = mx.io.CSVIter(data_csv="./train-64x64-data.csv", data_shape=(30, 64, 64), + label_csv="./train-diastole.csv", label_shape=(600,), + batch_size=batch_size) + +diastole_model = mx.model.FeedForward(ctx=devs, + symbol = network, + num_epoch = 65, + learning_rate = 0.001, + wd = 0.00001, + momentum = 0.9) + +diastole_model.fit(X=data_train, eval_metric = mx.metric.np(CRPS)) + + +# # Predict diastole + +# In[7]: + +diastole_prob = diastole_model.predict(data_validate) + + +# # Generate Submission + +# In[8]: + +def accumulate_result(validate_lst, prob): + sum_result = {} + cnt_result = {} + size = prob.shape[0] + fi = csv.reader(open(validate_lst)) + for i in range(size): + line = fi.__next__() # Python2: line = fi.next() + idx = int(line[0]) + if idx not in cnt_result: + cnt_result[idx] = 0. + sum_result[idx] = np.zeros((1, prob.shape[1])) + cnt_result[idx] += 1 + sum_result[idx] += prob[i, :] + for i in cnt_result.keys(): + sum_result[i][:] /= cnt_result[i] + return sum_result + + +# In[9]: + +systole_result = accumulate_result("./validate-label.csv", systole_prob) +diastole_result = accumulate_result("./validate-label.csv", diastole_prob) + + +# In[10]: + +# we have 2 person missing due to frame selection, use udibr's hist result instead +def doHist(data): + h = np.zeros(600) + for j in np.ceil(data).astype(int): + h[j:] += 1 + h /= len(data) + return h +train_csv = np.genfromtxt("./train-label.csv", delimiter=',') +hSystole = doHist(train_csv[:, 1]) +hDiastole = doHist(train_csv[:, 2]) + + +# In[11]: + +def submission_helper(pred): + p = np.zeros(600) + pred.resize(p.shape) + p[0] = pred[0] + for j in range(1, 600): + a = p[j - 1] + b = pred[j] + if b < a: + p[j] = a + else: + p[j] = b + return p + + + +# In[12]: + +fi = csv.reader(open("data/sample_submission_validate.csv")) +f = open("submission.csv", "w") +fo = csv.writer(f, lineterminator='\n') +fo.writerow(fi.__next__()) # Python2: fo.writerow(fi.next()) +for line in fi: + idx = line[0] + key, target = idx.split('_') + key = int(key) + out = [idx] + if key in systole_result: + if target == 'Diastole': + out.extend(list(submission_helper(diastole_result[key]))) + else: + out.extend(list(submission_helper(systole_result[key]))) + else: + print("Miss: %s" % idx) + if target == 'Diastole': + out.extend(hDiastole) + else: + out.extend(hSystole) + fo.writerow(out) +f.close() diff --git a/example/model-parallel/matrix_factorization/train.py b/example/model-parallel/matrix_factorization/train.py new file mode 100644 index 000000000000..591dab3a6534 --- /dev/null +++ b/example/model-parallel/matrix_factorization/train.py @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import logging +import time +import mxnet as mx +import numpy as np +from get_data import get_movielens_iter, get_movielens_data +from model import matrix_fact_model_parallel_net + + +logging.basicConfig(level=logging.DEBUG) + +parser = argparse.ArgumentParser(description="Run model parallel version of matrix factorization", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-epoch', type=int, default=3, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=256, + help='number of examples per batch') +parser.add_argument('--print-every', type=int, default=100, + help='logging interval') +parser.add_argument('--factor-size', type=int, default=128, + help="the factor size of the embedding operation") +parser.add_argument('--num-gpus', type=int, default=2, + help="number of gpus to use") + +MOVIELENS = { + 'dataset': 'ml-10m', + 'train': './ml-10M100K/r1.train', + 'val': './ml-10M100K/r1.test', + 'max_user': 71569, + 'max_movie': 65135, +} + +if __name__ == '__main__': + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_epoch = args.num_epoch + batch_size = args.batch_size + optimizer = 'sgd' + factor_size = args.factor_size + print_every = args.print_every + num_gpus = args.num_gpus + + momentum = 0.9 + learning_rate = 0.1 + + # prepare dataset and iterators + max_user = MOVIELENS['max_user'] + max_movies = MOVIELENS['max_movie'] + get_movielens_data(MOVIELENS['dataset']) + train_iter = get_movielens_iter(MOVIELENS['train'], batch_size) + val_iter = get_movielens_iter(MOVIELENS['val'], batch_size) + + # construct the model + net = matrix_fact_model_parallel_net(factor_size, factor_size, max_user, max_movies) + + # construct the module + # map the ctx_group attribute to the context assignment + group2ctxs={'dev1':[mx.cpu()]*num_gpus, 'dev2':[mx.gpu(i) for i in range(num_gpus)]} + + # Creating a module by passing group2ctxs attribute which maps + # the ctx_group attribute to the context assignment + mod = mx.module.Module(symbol=net, context=[mx.cpu()]*num_gpus, data_names=['user', 'item'], + label_names=['score'], group2ctxs=group2ctxs) + + # the initializer used to initialize the parameters + initializer = mx.init.Xavier(factor_type="in", magnitude=2.34) + + # the parameters for the optimizer constructor + optimizer_params = { + 'learning_rate': learning_rate, + 'wd': 1e-4, + 'momentum': momentum, + 'rescale_grad': 1.0/batch_size} + + # use MSE as the metric + metric = mx.metric.create(['MSE']) + + speedometer = mx.callback.Speedometer(batch_size, print_every) + + # start training + mod.fit(train_iter, + val_iter, + eval_metric = metric, + num_epoch = num_epoch, + optimizer = optimizer, + optimizer_params = optimizer_params, + initializer = initializer, + batch_end_callback = speedometer) diff --git a/example/module/mnist_mlp.py b/example/module/mnist_mlp.py new file mode 100644 index 000000000000..7d63a584aec9 --- /dev/null +++ b/example/module/mnist_mlp.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import os, sys +from utils import get_data +import mxnet as mx +import numpy as np +import logging + +data = mx.symbol.Variable('data') +fc1 = mx.symbol.FullyConnected(data, name='fc1', num_hidden=128) +act1 = mx.symbol.Activation(fc1, name='relu1', act_type="relu") +fc2 = mx.symbol.FullyConnected(act1, name = 'fc2', num_hidden = 64) +act2 = mx.symbol.Activation(fc2, name='relu2', act_type="relu") +fc3 = mx.symbol.FullyConnected(act2, name='fc3', num_hidden=10) +softmax = mx.symbol.SoftmaxOutput(fc3, name = 'softmax') + +n_epoch = 2 +batch_size = 100 + +basedir = os.path.dirname(__file__) +get_data.get_mnist(os.path.join(basedir, "data")) + +train_dataiter = mx.io.MNISTIter( + image=os.path.join(basedir, "data", "train-images-idx3-ubyte"), + label=os.path.join(basedir, "data", "train-labels-idx1-ubyte"), + data_shape=(784,), + batch_size=batch_size, shuffle=True, flat=True, silent=False, seed=10) +val_dataiter = mx.io.MNISTIter( + image=os.path.join(basedir, "data", "t10k-images-idx3-ubyte"), + label=os.path.join(basedir, "data", "t10k-labels-idx1-ubyte"), + data_shape=(784,), + batch_size=batch_size, shuffle=True, flat=True, silent=False) + +################################################################################ +# Intermediate-level API +################################################################################ +mod = mx.mod.Module(softmax) +mod.bind(data_shapes=train_dataiter.provide_data, label_shapes=train_dataiter.provide_label) +mod.init_params() + +mod.init_optimizer(optimizer_params={'learning_rate':0.01, 'momentum': 0.9}) +metric = mx.metric.create('acc') + +for i_epoch in range(n_epoch): + for i_iter, batch in enumerate(train_dataiter): + mod.forward(batch) + mod.update_metric(metric, batch.label) + + mod.backward() + mod.update() + + for name, val in metric.get_name_value(): + print('epoch %03d: %s=%f' % (i_epoch, name, val)) + metric.reset() + train_dataiter.reset() + + +################################################################################ +# High-level API +################################################################################ +logging.basicConfig(level=logging.DEBUG) +train_dataiter.reset() +mod = mx.mod.Module(softmax) +mod.fit(train_dataiter, eval_data=val_dataiter, + optimizer_params={'learning_rate':0.01, 'momentum': 0.9}, num_epoch=n_epoch) + +# prediction iterator API +for preds, i_batch, batch in mod.iter_predict(val_dataiter): + pred_label = preds[0].asnumpy().argmax(axis=1) + label = batch.label[0].asnumpy().astype('int32') + if i_batch % 20 == 0: + print('batch %03d acc: %.3f' % (i_batch, (label == pred_label).sum() / float(len(pred_label)))) + +# a dummy call just to test if the API works for merge_batches=True +preds = mod.predict(val_dataiter) + +# perform prediction and calculate accuracy manually +preds = mod.predict(val_dataiter, merge_batches=False) +val_dataiter.reset() +acc_sum = 0.0; acc_cnt = 0 +for i, batch in enumerate(val_dataiter): + pred_label = preds[i][0].asnumpy().argmax(axis=1) + label = batch.label[0].asnumpy().astype('int32') + acc_sum += (label == pred_label).sum() + acc_cnt += len(pred_label) +print('validation Accuracy: %.3f' % (acc_sum / acc_cnt)) + +# evaluate on validation set with a evaluation metric +mod.score(val_dataiter, metric) +for name, val in metric.get_name_value(): + print('%s=%f' % (name, val)) + diff --git a/example/multi-task/multi-task-learning.ipynb b/example/multi-task/multi-task-learning.ipynb index e615559441f6..048d6d9862b8 100644 --- a/example/multi-task/multi-task-learning.ipynb +++ b/example/multi-task/multi-task-learning.ipynb @@ -267,8 +267,8 @@ "outputs": [], "source": [ "def evaluate_accuracy(net, data_iterator):\n", - " acc_digits = mx.gluon.metric.Accuracy(name='digits')\n", - " acc_odd_even = mx.gluon.metric.Accuracy(name='odd_even')\n", + " acc_digits = mx.metric.Accuracy(name='digits')\n", + " acc_odd_even = mx.metric.Accuracy(name='odd_even')\n", " \n", " for i, (data, label_digit, label_odd_even) in enumerate(data_iterator):\n", " data = data.as_in_context(ctx)\n", @@ -335,8 +335,8 @@ "source": [ "for e in range(epochs):\n", " # Accuracies for each task\n", - " acc_digits = mx.gluon.metric.Accuracy(name='digits')\n", - " acc_odd_even = mx.gluon.metric.Accuracy(name='odd_even')\n", + " acc_digits = mx.metric.Accuracy(name='digits')\n", + " acc_odd_even = mx.metric.Accuracy(name='odd_even')\n", " # Accumulative losses\n", " l_digits_ = 0.\n", " l_odd_even_ = 0. \n", diff --git a/example/multivariate_time_series/src/metrics.py b/example/multivariate_time_series/src/metrics.py new file mode 100644 index 000000000000..4818591068f8 --- /dev/null +++ b/example/multivariate_time_series/src/metrics.py @@ -0,0 +1,55 @@ +# !/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- + +import numpy as np +import mxnet as mx + +def rse(label, pred): + """computes the root relative squared error (condensed using standard deviation formula)""" + numerator = np.sqrt(np.mean(np.square(label - pred), axis = None)) + denominator = np.std(label, axis = None) + return numerator / denominator + +def rae(label, pred): + """computes the relative absolute error (condensed using standard deviation formula)""" + numerator = np.mean(np.abs(label - pred), axis=None) + denominator = np.mean(np.abs(label - np.mean(label, axis=None)), axis=None) + return numerator / denominator + +def corr(label, pred): + """computes the empirical correlation coefficient""" + numerator1 = label - np.mean(label, axis=0) + numerator2 = pred - np.mean(pred, axis = 0) + numerator = np.mean(numerator1 * numerator2, axis=0) + denominator = np.std(label, axis=0) * np.std(pred, axis=0) + return np.mean(numerator / denominator) + +def get_custom_metrics(): + """ + :return: mxnet metric object + """ + _rse = mx.metric.create(rse) + _rae = mx.metric.create(rae) + _corr = mx.metric.create(corr) + return mx.metric.create([_rae, _rse, _corr]) + +def evaluate(pred, label): + return {"RAE":rae(label, pred), "RSE":rse(label,pred),"CORR": corr(label,pred)} \ No newline at end of file diff --git a/example/named_entity_recognition/src/metrics.py b/example/named_entity_recognition/src/metrics.py new file mode 100644 index 000000000000..a1d270af6863 --- /dev/null +++ b/example/named_entity_recognition/src/metrics.py @@ -0,0 +1,87 @@ +# !/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# -*- coding: utf-8 -*- + +import logging +import mxnet as mx +import numpy as np +import pickle + +def load_obj(name): + with open(name + '.pkl', 'rb') as f: + return pickle.load(f) + +tag_dict = load_obj("../data/tag_to_index") +not_entity_index = tag_dict["O"] + +def classifer_metrics(label, pred): + """ + computes f1, precision and recall on the entity class + """ + prediction = np.argmax(pred, axis=1) + label = label.astype(int) + + pred_is_entity = prediction != not_entity_index + label_is_entity = label != not_entity_index + + corr_pred = (prediction == label) == (pred_is_entity == True) + + #how many entities are there? + # better to cast to float for safer further ratio computations + num_entities = float(np.sum(label_is_entity)) + entity_preds = float(np.sum(pred_is_entity)) + #how many times did we correctly predict an entity? + correct_entitites = float(np.sum(corr_pred[pred_is_entity])) + + #precision: when we predict entity, how often are we right? + if entity_preds == 0: + precision = np.nan + else: + precision = correct_entitites / entity_preds + + #recall: of the things that were an entity, how many did we catch? + recall = correct_entitites / num_entities + if num_entities == 0: + recall = np.nan + # To prevent dozens of warning: RuntimeWarning: divide by zero encountered in long_scalars + if precision + recall == 0: + f1 = 0 + else: + f1 = 2 * precision * recall / (precision + recall) + + logging.debug("Metrics results: precision=%f recall=%f f1=%f", precision, recall, f1) + return precision, recall, f1 + +def entity_precision(label, pred): + return classifer_metrics(label, pred)[0] + +def entity_recall(label, pred): + return classifer_metrics(label, pred)[1] + +def entity_f1(label, pred): + return classifer_metrics(label, pred)[2] + +def composite_classifier_metrics(): + metric1 = mx.metric.CustomMetric(feval=entity_precision, name='entity precision') + metric2 = mx.metric.CustomMetric(feval=entity_recall, name='entity recall') + metric3 = mx.metric.CustomMetric(feval=entity_f1, name='entity f1 score') + metric4 = mx.metric.Accuracy() + + return mx.metric.CompositeEvalMetric([metric4, metric1, metric2, metric3]) diff --git a/example/nce-loss/nce.py b/example/nce-loss/nce.py new file mode 100644 index 000000000000..e59220a026a8 --- /dev/null +++ b/example/nce-loss/nce.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=missing-docstring +from __future__ import print_function + +from operator import itemgetter + +import mxnet as mx +import numpy as np + + +def nce_loss(data, label, label_weight, embed_weight, vocab_size, num_hidden): + label_embed = mx.sym.Embedding(data=label, input_dim=vocab_size, + weight=embed_weight, + output_dim=num_hidden, name='label_embed') + data = mx.sym.Reshape(data=data, shape=(-1, 1, num_hidden)) + pred = mx.sym.broadcast_mul(data, label_embed) + pred = mx.sym.sum(data=pred, axis=2) + return mx.sym.LogisticRegressionOutput(data=pred, + label=label_weight) + + +def nce_loss_subwords( + data, label, label_mask, label_weight, embed_weight, vocab_size, num_hidden): + """NCE-Loss layer under subword-units input. + """ + # get subword-units embedding. + label_units_embed = mx.sym.Embedding(data=label, + input_dim=vocab_size, + weight=embed_weight, + output_dim=num_hidden) + # get valid subword-units embedding with the help of label_mask + # it's achieved by multiplying zeros to useless units in order to handle variable-length input. + label_units_embed = mx.sym.broadcast_mul(lhs=label_units_embed, + rhs=label_mask, + name='label_units_embed') + # sum over them to get label word embedding. + label_embed = mx.sym.sum(label_units_embed, axis=2, name='label_embed') + + # by boardcast_mul and sum you can get prediction scores in all label_embed inputs, + # which is easy to feed into LogisticRegressionOutput and make your code more concise. + data = mx.sym.Reshape(data=data, shape=(-1, 1, num_hidden)) + pred = mx.sym.broadcast_mul(data, label_embed) + pred = mx.sym.sum(data=pred, axis=2) + + return mx.sym.LogisticRegressionOutput(data=pred, + label=label_weight) + + +class NceAccuracy(mx.metric.EvalMetric): + def __init__(self): + super(NceAccuracy, self).__init__('nce-accuracy') + + def update(self, labels, preds): + label_weight = labels[1].asnumpy() + preds = preds[0].asnumpy() + for i in range(preds.shape[0]): + if np.argmax(label_weight[i]) == np.argmax(preds[i]): + self.sum_metric += 1 + self.num_inst += 1 + + +class NceAuc(mx.metric.EvalMetric): + def __init__(self): + super(NceAuc, self).__init__('nce-auc') + + def update(self, labels, preds): + label_weight = labels[1].asnumpy() + preds = preds[0].asnumpy() + tmp = [] + for i in range(preds.shape[0]): + for j in range(preds.shape[1]): + tmp.append((label_weight[i][j], preds[i][j])) + tmp = sorted(tmp, key=itemgetter(1), reverse=True) + m = 0.0 + n = 0.0 + z = 0.0 + k = 0 + for a, _ in tmp: + if a > 0.5: + m += 1.0 + z += len(tmp) - k + else: + n += 1.0 + k += 1 + z -= m * (m + 1.0) / 2.0 + z /= m + z /= n + self.sum_metric += z + self.num_inst += 1 + + +class NceLSTMAuc(mx.metric.EvalMetric): + def __init__(self): + super(NceLSTMAuc, self).__init__('nce-lstm-auc') + + def update(self, labels, preds): + preds = np.array([x.asnumpy() for x in preds]) + preds = preds.reshape((preds.shape[0] * preds.shape[1], preds.shape[2])) + label_weight = labels[1].asnumpy() + label_weight = label_weight.transpose((1, 0, 2)) + label_weight = label_weight.reshape((preds.shape[0], preds.shape[1])) + + tmp = [] + for i in range(preds.shape[0]): + for j in range(preds.shape[1]): + tmp.append((label_weight[i][j], preds[i][j])) + tmp = sorted(tmp, key=itemgetter(1), reverse=True) + m = 0.0 + n = 0.0 + z = 0.0 + k = 0 + for a, _ in tmp: + if a > 0.5: + m += 1.0 + z += len(tmp) - k + else: + n += 1.0 + k += 1 + z -= m * (m + 1.0) / 2.0 + z /= m + z /= n + self.sum_metric += z + self.num_inst += 1 diff --git a/example/neural_collaborative_filtering/train.py b/example/neural_collaborative_filtering/train.py new file mode 100644 index 000000000000..c68f271a6f0d --- /dev/null +++ b/example/neural_collaborative_filtering/train.py @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +import os +import time +import argparse +import logging +import math +import random +import numpy as np +import mxnet as mx +from mxnet import gluon +from core.model import get_model +from core.dataset import NCFTrainData, NCFTestData +from core.evaluate import * + + +logging.basicConfig(level=logging.DEBUG) + +parser = argparse.ArgumentParser(description="Run matrix factorization with embedding", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--path', nargs='?', default='./data/', + help='Input data path.') +parser.add_argument('--dataset', nargs='?', default='ml-20m', + help='The dataset name.') +parser.add_argument('--batch-size', type=int, default=2048, + help='number of training examples per batch') +parser.add_argument('--eval-batch-size', type=int, default=1000, + help='number of evaluate examples per batch') +parser.add_argument('--model-type', type=str, default='neumf', choices=['neumf', 'gmf', 'mlp'], + help="mdoel type") +parser.add_argument('--num-negative', type=int, default=4, + help="number of negative samples per positive sample while training.") +parser.add_argument('--layers', default='[256, 256, 128, 64]', + help="list of number hiddens of fc layers in mlp model.") +parser.add_argument('--factor-size-gmf', type=int, default=64, + help="outdim of gmf embedding layers.") +parser.add_argument('--num-hidden', type=int, default=1, + help="num-hidden of neumf fc layer") +parser.add_argument('--log-interval', type=int, default=100, + help='logging interval') +parser.add_argument('--learning-rate', type=float, default=0.0005, + help='learning rate for optimizer') +parser.add_argument('--beta1', '-b1', type=float, default=0.9, + help='beta1 for Adam') +parser.add_argument('--beta2', '-b2', type=float, default=0.999, + help='beta1 for Adam') +parser.add_argument('--eps', type=float, default=1e-8, + help='eps for Adam') +parser.add_argument('--topk', type=int, default=10, + help="topk for accuracy evaluation.") +parser.add_argument('--gpu', type=int, default=None, + help="list of gpus to run, e.g. 0 or 0,2. empty means using cpu().") +parser.add_argument('--workers', type=int, default=8, help='thread number for dataloader.') +parser.add_argument('--epoch', type=int, default=14, help='training epoch') +parser.add_argument('--seed', type=int, default=3, help='random seed to use. Default=3.') +parser.add_argument('--deploy', action='store_true', help="whether to load static graph for deployment") + + +def cross_entropy(label, pred, eps=1e-12): + ce = 0 + for l, p in zip(label, pred): + ce += -( l*np.log(p+eps) + (1-l)*np.log(1-p+eps)) + return ce + +if __name__ == '__main__': + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + + mx.random.seed(args.seed) + np.random.seed(args.seed) + batch_size = args.batch_size + eval_batch_size = args.eval_batch_size + model_type = args.model_type + model_layers = eval(args.layers) + factor_size_gmf = args.factor_size_gmf + factor_size_mlp = int(model_layers[0]/2) + num_hidden = args.num_hidden + learning_rate=args.learning_rate + beta1=args.beta1 + beta2=args.beta2 + eps=args.eps + ctx = mx.cpu() if args.gpu is None else mx.gpu(args.gpu) + topK = args.topk + num_negatives = args.num_negative + num_workers = args.workers + epoch = args.epoch + log_interval = args.log_interval + + # prepare dataset + logging.info('Prepare Dataset') + train_dataset = NCFTrainData((args.path + args.dataset + '/train-ratings.csv'), num_negatives) + test_data = NCFTestData(args.path + args.dataset) + train_dataloader = mx.gluon.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, last_batch='rollover') + logging.info('Prepare Dataset completed') + # construct the model + net = get_model(model_type, factor_size_mlp, factor_size_gmf, + model_layers, num_hidden, train_dataset.nb_users, train_dataset.nb_items) + + # initialize the module + mod = mx.module.Module(net, context=ctx, data_names=['user', 'item'], label_names=['softmax_label']) + provide_data = [mx.io.DataDesc(name='item', shape=((batch_size,))), + mx.io.DataDesc(name='user', shape=((batch_size,)))] + provide_label = [mx.io.DataDesc(name='softmax_label', shape=((batch_size,)))] + mod.bind(for_training=True, data_shapes=provide_data, label_shapes=provide_label) + mod.init_params() + mod.init_optimizer(optimizer='adam', optimizer_params=[('learning_rate', learning_rate), ('beta1',beta1), ('beta2',beta2), ('epsilon',eps)]) + + metric = mx.metric.create(cross_entropy) + speedometer = mx.callback.Speedometer(batch_size, log_interval) + best_hr, best_ndcg, best_iter = -1, -1, -1 + logging.info('Training started ...') + for epoch in range(epoch): + metric.reset() + for nbatch, seqs in enumerate(train_dataloader): + user_id, item_id, labels = seqs + batch = mx.io.DataBatch(data = [item_id.astype('int32').as_in_context(ctx), + user_id.astype('int32').as_in_context(ctx)], + label = [labels.as_in_context(ctx)]) + mod.forward(batch) + mod.backward() + mod.update() + predicts=mod.get_outputs()[0] + metric.update(labels = labels, preds = predicts) + speedometer_param = mx.model.BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=metric, locals=locals()) + speedometer(speedometer_param) + + # save model + dir_path = os.path.dirname(os.path.realpath(__file__)) + model_path = os.path.join(dir_path, 'model', args.dataset) + if not os.path.exists(model_path): + os.makedirs(model_path) + mod.save_checkpoint(os.path.join(model_path, model_type), epoch) + # compute hit ratio + (hits, ndcgs) = evaluate_model(mod, test_data.testRatings, test_data.testNegatives, topK, eval_batch_size, ctx, logging) + hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() + logging.info('Iteration %d: HR = %.4f, NDCG = %.4f' % (epoch, hr, ndcg)) + # best hit ratio + if hr > best_hr: + best_hr, best_ndcg, best_iter = hr, ndcg, epoch + + logging.info("End. Best Iteration %d: HR = %.4f, NDCG = %.4f. " % (best_iter, best_hr, best_ndcg)) + logging.info('Training completed.') + diff --git a/example/quantization/imagenet_inference.py b/example/quantization/imagenet_inference.py new file mode 100644 index 000000000000..4d690d37d00c --- /dev/null +++ b/example/quantization/imagenet_inference.py @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import logging +import os +import time +import numpy as np +import mxnet as mx +from mxnet import nd +from mxnet.contrib.quantization import * +from mxnet.contrib import amp + + +def download_dataset(dataset_url, dataset_dir, logger=None): + if logger is not None: + logger.info('Downloading dataset for inference from %s to %s' % (dataset_url, dataset_dir)) + mx.test_utils.download(dataset_url, dataset_dir) + + +def load_model(symbol_file, param_file, logger=None): + cur_path = os.path.dirname(os.path.realpath(__file__)) + symbol_file_path = os.path.join(cur_path, symbol_file) + if logger is not None: + logger.info('Loading symbol from file %s' % symbol_file_path) + symbol = mx.sym.load(symbol_file_path) + + param_file_path = os.path.join(cur_path, param_file) + if logger is not None: + logger.info('Loading params from file %s' % param_file_path) + save_dict = nd.load(param_file_path) + arg_params = {} + aux_params = {} + for k, v in save_dict.items(): + tp, name = k.split(':', 1) + if tp == 'arg': + arg_params[name] = v + if tp == 'aux': + aux_params[name] = v + return symbol, arg_params, aux_params + + +def advance_data_iter(data_iter, n): + assert n >= 0 + if n == 0: + return data_iter + has_next_batch = True + while has_next_batch: + try: + data_iter.next() + n -= 1 + if n == 0: + return data_iter + except StopIteration: + has_next_batch = False + + +def score(sym, arg_params, aux_params, data, devs, label_name, max_num_examples, logger=None): + metrics = [mx.metric.create('acc'), + mx.metric.create('top_k_accuracy', top_k=5)] + if not isinstance(metrics, list): + metrics = [metrics, ] + mod = mx.mod.Module(symbol=sym, context=devs, label_names=[label_name, ]) + mod.bind(for_training=False, + data_shapes=data.provide_data, + label_shapes=data.provide_label) + mod.set_params(arg_params, aux_params) + + tic = time.time() + num = 0 + for batch in data: + mod.forward(batch, is_train=False) + for m in metrics: + mod.update_metric(m, batch.label) + num += batch_size + if max_num_examples is not None and num >= max_num_examples: + break + + speed = num / (time.time() - tic) + + if logger is not None: + logger.info('Finished inference with %d images' % num) + logger.info('Finished with %f images per second', speed) + logger.warn('Note: GPU performance is expected to be slower than CPU. Please refer quantization/README.md for details') + for m in metrics: + logger.info(m.get()) + + +def low_precison_convert(model_name, low_precision, sym, arg_params, aux_params, excluded_sym_names=[]): + if low_precision == 'bfloat16': + if model_name.find('imagenet1k-resnet-152') != -1: + excluded_sym_names += ['conv0'] + elif model_name.find('imagenet1k-inception-bn') != -1: + excluded_sym_names += ['conv_1'] + elif model_name.find('resnet') != -1 and model_name.find('v1') != -1: + excluded_sym_names += ['resnetv10_conv0_fwd'] + elif model_name.find('resnet') != -1 and model_name.find('v2') != -1: + excluded_sym_names += ['resnetv20_conv0_fwd'] + elif model_name.find('vgg') != -1: + excluded_sym_names += ['vgg0_conv0_fwd'] + elif model_name.find('squeezenet1') != -1: + excluded_sym_names += ['squeezenet0_conv0_fwd'] + elif model_name.find('mobilenet') != -1 and model_name.find('v2') == -1: + excluded_sym_names += ['mobilenet0_conv0_fwd'] + elif model_name.find('mobilenet') != -1 and model_name.find('v2') != -1: + excluded_sym_names += ['mobilenetv20_conv0_fwd'] + elif model_name.find('inceptionv3') != -1: + excluded_sym_names += ['inception30_conv0_fwd'] + return amp.convert_model(sym, + arg_params, + aux_params, + target_dtype=low_precision, + excluded_sym_names=excluded_sym_names, + cast_optional_params=True) + +def benchmark_score(symbol_file, ctx, batch_size, num_batches, data_layer_type, low_precision, logger=None): + # get mod + cur_path = os.path.dirname(os.path.realpath(__file__)) + symbol_file_path = os.path.join(cur_path, symbol_file) + if logger is not None: + logger.info('Loading symbol from file %s' % symbol_file_path) + sym = mx.sym.load(symbol_file_path) + mod = mx.mod.Module(symbol=sym, context=ctx) + if data_layer_type == "int8": + dshape = mx.io.DataDesc(name='data', shape=( + batch_size,) + data_shape, dtype=np.int8) + elif data_layer_type == 'uint8': + dshape = mx.io.DataDesc(name='data', shape=( + batch_size,) + data_shape, dtype=np.uint8) + else: # float32 + dshape = mx.io.DataDesc(name='data', shape=( + batch_size,) + data_shape, dtype=np.float32) + mod.bind(for_training=False, + inputs_need_grad=False, + data_shapes=[dshape]) + mod.init_params(initializer=mx.init.Xavier(magnitude=2.)) + + if low_precision: + arg_params, aux_params = mod.get_params() + sym, arg_params, aux_params = low_precison_convert(symbol_file, + low_precision, + sym, arg_params, + aux_params) + mod = mx.mod.Module(symbol=sym, context=ctx) + mod.bind(for_training=False, + inputs_need_grad=False, + data_shapes=[dshape], + label_shapes=[['softmax_label', (batch_size,)]]) + mod.set_params(arg_params, aux_params) + + # get data + if data_layer_type == "float32": + data = [mx.random.uniform(-1.0, 1.0, shape=shape, ctx=ctx, dtype=data_layer_type) + for _, shape in mod.data_shapes] + else: + data = [mx.nd.full(shape=shape, val=127, ctx=ctx, dtype=data_layer_type) + for _, shape in mod.data_shapes] + batch = mx.io.DataBatch(data, []) # empty label + + # run + dry_run = 5 # use 5 iterations to warm up + for i in range(dry_run+num_batches): + if i == dry_run: + tic = time.time() + mod.forward(batch, is_train=False) + for output in mod.get_outputs(): + output.wait_to_read() + + # return num images per second + return num_batches*batch_size/(time.time() - tic) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Score a model on a dataset') + parser.add_argument('--ctx', type=str, default='gpu') + parser.add_argument('--benchmark', type=bool, default=False, help='dummy data benchmark') + parser.add_argument('--symbol-file', type=str, required=True, help='symbol file path') + parser.add_argument('--param-file', type=str, required=False, help='param file path') + parser.add_argument('--batch-size', type=int, default=32) + parser.add_argument('--label-name', type=str, default='softmax_label') + parser.add_argument('--dataset', type=str, required=False, help='dataset path') + parser.add_argument('--rgb-mean', type=str, default='0,0,0') + parser.add_argument('--rgb-std', type=str, default='1,1,1') + parser.add_argument('--image-shape', type=str, default='3,224,224') + parser.add_argument('--data-nthreads', type=int, default=60, help='number of threads for data decoding') + parser.add_argument('--num-skipped-batches', type=int, default=0, help='skip the number of batches for inference') + parser.add_argument('--num-inference-batches', type=int, required=True, help='number of images used for inference') + parser.add_argument('--shuffle-dataset', action='store_true', default=True, + help='shuffle the calibration dataset') + parser.add_argument('--shuffle-chunk-seed', type=int, default=3982304, + help='shuffling chunk seed, see' + ' https://mxnet.apache.org/api/python/io/io.html?highlight=imager#mxnet.io.ImageRecordIter' + ' for more details') + parser.add_argument('--shuffle-seed', type=int, default=48564309, + help='shuffling seed, see' + ' https://mxnet.apache.org/api/python/io/io.html?highlight=imager#mxnet.io.ImageRecordIter' + ' for more details') + parser.add_argument('--data-layer-type', type=str, default='float32', + choices=['float32', 'int8', 'uint8'], + help='data type for data layer') + parser.add_argument('--low-precision', type=str, default='', + choices=['', 'float16', 'bfloat16'], + help='enable low precision') + + args = parser.parse_args() + + if args.ctx == 'gpu': + ctx = mx.gpu(0) + elif args.ctx == 'cpu': + ctx = mx.cpu(0) + else: + raise ValueError('ctx %s is not supported in this script' % args.ctx) + + logging.basicConfig() + logger = logging.getLogger('logger') + logger.setLevel(logging.INFO) + + symbol_file = args.symbol_file + param_file = args.param_file + data_nthreads = args.data_nthreads + + batch_size = args.batch_size + logger.info('batch size = %d for inference' % batch_size) + + rgb_mean = args.rgb_mean + logger.info('rgb_mean = %s' % rgb_mean) + rgb_mean = [float(i) for i in rgb_mean.split(',')] + mean_args = {'mean_r': rgb_mean[0], 'mean_g': rgb_mean[1], 'mean_b': rgb_mean[2]} + rgb_std = args.rgb_std + logger.info('rgb_std = %s' % rgb_std) + rgb_std = [float(i) for i in rgb_std.split(',')] + std_args = {'std_r': rgb_std[0], 'std_g': rgb_std[1], 'std_b': rgb_std[2]} + combine_mean_std = {} + combine_mean_std.update(mean_args) + combine_mean_std.update(std_args) + + label_name = args.label_name + logger.info('label_name = %s' % label_name) + + image_shape = args.image_shape + data_shape = tuple([int(i) for i in image_shape.split(',')]) + logger.info('Input data shape = %s' % str(data_shape)) + + data_layer_type = args.data_layer_type + + if args.low_precision: + if args.ctx == 'gpu': + assert args.low_precision == 'float16', "Not supported low-precision options for GPU." + elif args.ctx == 'cpu': + assert args.low_precision == 'bfloat16', "Not supported low-precision options for CPU." + + if args.benchmark == False: + dataset = args.dataset + download_dataset('http://data.mxnet.io/data/val_256_q90.rec', dataset) + logger.info('Dataset for inference: %s' % dataset) + + # creating data iterator + data = mx.io.ImageRecordIter( + path_imgrec=dataset, + label_width=1, + preprocess_threads=data_nthreads, + batch_size=batch_size, + data_shape=data_shape, + label_name=label_name, + rand_crop=False, + rand_mirror=False, + shuffle=args.shuffle_dataset, + shuffle_chunk_seed=args.shuffle_chunk_seed, + seed=args.shuffle_seed, + dtype=data_layer_type, + ctx=args.ctx, + **combine_mean_std) + + # loading model + sym, arg_params, aux_params = load_model(symbol_file, param_file, logger) + + if args.low_precision: + sym, arg_params, aux_params = low_precison_convert(symbol_file, + args.low_precision, + sym, arg_params, + aux_params) + # make sure that fp32 inference works on the same images as calibrated quantized model + logger.info('Skipping the first %d batches' % args.num_skipped_batches) + data = advance_data_iter(data, args.num_skipped_batches) + + num_inference_images = args.num_inference_batches * batch_size + logger.info('Running model %s for inference' % symbol_file) + score(sym, arg_params, aux_params, data, [ctx], label_name, + max_num_examples=num_inference_images, logger=logger) + else: + logger.info('Running model %s for inference' % symbol_file) + speed = benchmark_score(symbol_file, ctx, batch_size, + args.num_inference_batches, data_layer_type, args.low_precision, logger) + logger.info('batch size %2d, image/sec: %f', batch_size, speed) diff --git a/example/rcnn/symnet/metric.py b/example/rcnn/symnet/metric.py new file mode 100644 index 000000000000..fa8d7919e919 --- /dev/null +++ b/example/rcnn/symnet/metric.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np + + +def get_names(): + pred = ['rpn_cls_prob', 'rpn_bbox_loss', 'rcnn_cls_prob', 'rcnn_bbox_loss', 'rcnn_label'] + label = ['rpn_label', 'rpn_bbox_target', 'rpn_bbox_weight'] + return pred, label + + +class RPNAccMetric(mx.metric.EvalMetric): + def __init__(self): + super(RPNAccMetric, self).__init__('RPNAcc') + self.pred, self.label = get_names() + + def update(self, labels, preds): + pred = preds[self.pred.index('rpn_cls_prob')] + label = labels[self.label.index('rpn_label')] + + # pred (b, c, p) or (b, c, h, w) + pred_label = mx.ndarray.argmax_channel(pred).asnumpy().astype('int32') + pred_label = pred_label.reshape((pred_label.shape[0], -1)) + # label (b, p) + label = label.asnumpy().astype('int32') + + # filter with keep_inds + keep_inds = np.where(label != -1) + pred_label = pred_label[keep_inds] + label = label[keep_inds] + + self.sum_metric += np.sum(pred_label.flat == label.flat) + self.num_inst += len(pred_label.flat) + + +class RCNNAccMetric(mx.metric.EvalMetric): + def __init__(self): + super(RCNNAccMetric, self).__init__('RCNNAcc') + self.pred, self.label = get_names() + + def update(self, labels, preds): + pred = preds[self.pred.index('rcnn_cls_prob')] + label = preds[self.pred.index('rcnn_label')] + + last_dim = pred.shape[-1] + pred_label = pred.asnumpy().reshape(-1, last_dim).argmax(axis=1).astype('int32') + label = label.asnumpy().reshape(-1,).astype('int32') + + self.sum_metric += np.sum(pred_label.flat == label.flat) + self.num_inst += len(pred_label.flat) + + +class RPNLogLossMetric(mx.metric.EvalMetric): + def __init__(self): + super(RPNLogLossMetric, self).__init__('RPNLogLoss') + self.pred, self.label = get_names() + + def update(self, labels, preds): + pred = preds[self.pred.index('rpn_cls_prob')] + label = labels[self.label.index('rpn_label')] + + # label (b, p) + label = label.asnumpy().astype('int32').reshape((-1)) + # pred (b, c, p) or (b, c, h, w) --> (b, p, c) --> (b*p, c) + pred = pred.asnumpy().reshape((pred.shape[0], pred.shape[1], -1)).transpose((0, 2, 1)) + pred = pred.reshape((label.shape[0], -1)) + + # filter with keep_inds + keep_inds = np.where(label != -1)[0] + label = label[keep_inds] + cls = pred[keep_inds, label] + + cls += 1e-14 + cls_loss = -1 * np.log(cls) + cls_loss = np.sum(cls_loss) + self.sum_metric += cls_loss + self.num_inst += label.shape[0] + + +class RCNNLogLossMetric(mx.metric.EvalMetric): + def __init__(self): + super(RCNNLogLossMetric, self).__init__('RCNNLogLoss') + self.pred, self.label = get_names() + + def update(self, labels, preds): + pred = preds[self.pred.index('rcnn_cls_prob')] + label = preds[self.pred.index('rcnn_label')] + + last_dim = pred.shape[-1] + pred = pred.asnumpy().reshape(-1, last_dim) + label = label.asnumpy().reshape(-1,).astype('int32') + cls = pred[np.arange(label.shape[0]), label] + + cls += 1e-14 + cls_loss = -1 * np.log(cls) + cls_loss = np.sum(cls_loss) + self.sum_metric += cls_loss + self.num_inst += label.shape[0] + + +class RPNL1LossMetric(mx.metric.EvalMetric): + def __init__(self): + super(RPNL1LossMetric, self).__init__('RPNL1Loss') + self.pred, self.label = get_names() + + def update(self, labels, preds): + bbox_loss = preds[self.pred.index('rpn_bbox_loss')].asnumpy() + bbox_weight = labels[self.label.index('rpn_bbox_weight')].asnumpy() + + # calculate num_inst (average on those fg anchors) + num_inst = np.sum(bbox_weight > 0) / 4 + + self.sum_metric += np.sum(bbox_loss) + self.num_inst += num_inst + + +class RCNNL1LossMetric(mx.metric.EvalMetric): + def __init__(self): + super(RCNNL1LossMetric, self).__init__('RCNNL1Loss') + self.pred, self.label = get_names() + + def update(self, labels, preds): + bbox_loss = preds[self.pred.index('rcnn_bbox_loss')].asnumpy() + label = preds[self.pred.index('rcnn_label')].asnumpy() + + # calculate num_inst + keep_inds = np.where(label != 0)[0] + num_inst = len(keep_inds) + + self.sum_metric += np.sum(bbox_loss) + self.num_inst += num_inst diff --git a/example/rcnn/train.py b/example/rcnn/train.py new file mode 100644 index 000000000000..7b1f2f7f31a5 --- /dev/null +++ b/example/rcnn/train.py @@ -0,0 +1,303 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import ast +import pprint + +import mxnet as mx +from mxnet.module import Module + +from symdata.loader import AnchorGenerator, AnchorSampler, AnchorLoader +from symnet.logger import logger +from symnet.model import load_param, infer_data_shape, check_shape, initialize_frcnn, get_fixed_params +from symnet.metric import RPNAccMetric, RPNLogLossMetric, RPNL1LossMetric, RCNNAccMetric, RCNNLogLossMetric, RCNNL1LossMetric + + +def train_net(sym, roidb, args): + # print config + logger.info('called with args\n{}'.format(pprint.pformat(vars(args)))) + + # setup multi-gpu + ctx = [mx.cpu()] if not args.gpus else [mx.gpu(int(i)) for i in args.gpus.split(',')] + batch_size = args.rcnn_batch_size * len(ctx) + + # load training data + feat_sym = sym.get_internals()['rpn_cls_score_output'] + ag = AnchorGenerator(feat_stride=args.rpn_feat_stride, + anchor_scales=args.rpn_anchor_scales, anchor_ratios=args.rpn_anchor_ratios) + asp = AnchorSampler(allowed_border=args.rpn_allowed_border, batch_rois=args.rpn_batch_rois, + fg_fraction=args.rpn_fg_fraction, fg_overlap=args.rpn_fg_overlap, + bg_overlap=args.rpn_bg_overlap) + train_data = AnchorLoader(roidb, batch_size, args.img_short_side, args.img_long_side, + args.img_pixel_means, args.img_pixel_stds, feat_sym, ag, asp, shuffle=True) + + # produce shape max possible + _, out_shape, _ = feat_sym.infer_shape(data=(1, 3, args.img_long_side, args.img_long_side)) + feat_height, feat_width = out_shape[0][-2:] + rpn_num_anchors = len(args.rpn_anchor_scales) * len(args.rpn_anchor_ratios) + data_names = ['data', 'im_info', 'gt_boxes'] + label_names = ['label', 'bbox_target', 'bbox_weight'] + data_shapes = [('data', (batch_size, 3, args.img_long_side, args.img_long_side)), + ('im_info', (batch_size, 3)), + ('gt_boxes', (batch_size, 100, 5))] + label_shapes = [('label', (batch_size, 1, rpn_num_anchors * feat_height, feat_width)), + ('bbox_target', (batch_size, 4 * rpn_num_anchors, feat_height, feat_width)), + ('bbox_weight', (batch_size, 4 * rpn_num_anchors, feat_height, feat_width))] + + # print shapes + data_shape_dict, out_shape_dict = infer_data_shape(sym, data_shapes + label_shapes) + logger.info('max input shape\n%s' % pprint.pformat(data_shape_dict)) + logger.info('max output shape\n%s' % pprint.pformat(out_shape_dict)) + + # load and initialize params + if args.resume: + arg_params, aux_params = load_param(args.resume) + else: + arg_params, aux_params = load_param(args.pretrained) + arg_params, aux_params = initialize_frcnn(sym, data_shapes, arg_params, aux_params) + + # check parameter shapes + check_shape(sym, data_shapes + label_shapes, arg_params, aux_params) + + # check fixed params + fixed_param_names = get_fixed_params(sym, args.net_fixed_params) + logger.info('locking params\n%s' % pprint.pformat(fixed_param_names)) + + # metric + rpn_eval_metric = RPNAccMetric() + rpn_cls_metric = RPNLogLossMetric() + rpn_bbox_metric = RPNL1LossMetric() + eval_metric = RCNNAccMetric() + cls_metric = RCNNLogLossMetric() + bbox_metric = RCNNL1LossMetric() + eval_metrics = mx.metric.CompositeEvalMetric() + for child_metric in [rpn_eval_metric, rpn_cls_metric, rpn_bbox_metric, eval_metric, cls_metric, bbox_metric]: + eval_metrics.add(child_metric) + + # callback + batch_end_callback = mx.callback.Speedometer(batch_size, frequent=args.log_interval, auto_reset=False) + epoch_end_callback = mx.callback.do_checkpoint(args.save_prefix) + + # learning schedule + base_lr = args.lr + lr_factor = 0.1 + lr_epoch = [int(epoch) for epoch in args.lr_decay_epoch.split(',')] + lr_epoch_diff = [epoch - args.start_epoch for epoch in lr_epoch if epoch > args.start_epoch] + lr = base_lr * (lr_factor ** (len(lr_epoch) - len(lr_epoch_diff))) + lr_iters = [int(epoch * len(roidb) / batch_size) for epoch in lr_epoch_diff] + logger.info('lr %f lr_epoch_diff %s lr_iters %s' % (lr, lr_epoch_diff, lr_iters)) + lr_scheduler = mx.lr_scheduler.MultiFactorScheduler(lr_iters, lr_factor) + # optimizer + optimizer_params = {'momentum': 0.9, + 'wd': 0.0005, + 'learning_rate': lr, + 'lr_scheduler': lr_scheduler, + 'rescale_grad': (1.0 / batch_size), + 'clip_gradient': 5} + + # train + mod = Module(sym, data_names=data_names, label_names=label_names, + logger=logger, context=ctx, work_load_list=None, + fixed_param_names=fixed_param_names) + mod.fit(train_data, eval_metric=eval_metrics, epoch_end_callback=epoch_end_callback, + batch_end_callback=batch_end_callback, kvstore='device', + optimizer='sgd', optimizer_params=optimizer_params, + arg_params=arg_params, aux_params=aux_params, begin_epoch=args.start_epoch, num_epoch=args.epochs) + + +def parse_args(): + parser = argparse.ArgumentParser(description='Train Faster R-CNN network', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--network', type=str, default='vgg16', help='base network') + parser.add_argument('--pretrained', type=str, default='', help='path to pretrained model') + parser.add_argument('--dataset', type=str, default='voc', help='training dataset') + parser.add_argument('--imageset', type=str, default='', help='imageset splits') + parser.add_argument('--gpus', type=str, help='GPU devices, eg: "0,1,2,3" , not set to use CPU') + parser.add_argument('--epochs', type=int, default=10, help='training epochs') + parser.add_argument('--lr', type=float, default=0.001, help='base learning rate') + parser.add_argument('--lr-decay-epoch', type=str, default='7', help='epoch to decay lr') + parser.add_argument('--resume', type=str, default='', help='path to last saved model') + parser.add_argument('--start-epoch', type=int, default=0, help='start epoch for resuming') + parser.add_argument('--log-interval', type=int, default=100, help='logging mini batch interval') + parser.add_argument('--save-prefix', type=str, default='', help='saving params prefix') + # faster rcnn params + parser.add_argument('--img-short-side', type=int, default=600) + parser.add_argument('--img-long-side', type=int, default=1000) + parser.add_argument('--img-pixel-means', type=str, default='(0.0, 0.0, 0.0)') + parser.add_argument('--img-pixel-stds', type=str, default='(1.0, 1.0, 1.0)') + parser.add_argument('--net-fixed-params', type=str, default='["conv0", "stage1", "gamma", "beta"]') + parser.add_argument('--rpn-feat-stride', type=int, default=16) + parser.add_argument('--rpn-anchor-scales', type=str, default='(8, 16, 32)') + parser.add_argument('--rpn-anchor-ratios', type=str, default='(0.5, 1, 2)') + parser.add_argument('--rpn-pre-nms-topk', type=int, default=12000) + parser.add_argument('--rpn-post-nms-topk', type=int, default=2000) + parser.add_argument('--rpn-nms-thresh', type=float, default=0.7) + parser.add_argument('--rpn-min-size', type=int, default=16) + parser.add_argument('--rpn-batch-rois', type=int, default=256) + parser.add_argument('--rpn-allowed-border', type=int, default=0) + parser.add_argument('--rpn-fg-fraction', type=float, default=0.5) + parser.add_argument('--rpn-fg-overlap', type=float, default=0.7) + parser.add_argument('--rpn-bg-overlap', type=float, default=0.3) + parser.add_argument('--rcnn-num-classes', type=int, default=21) + parser.add_argument('--rcnn-feat-stride', type=int, default=16) + parser.add_argument('--rcnn-pooled-size', type=str, default='(14, 14)') + parser.add_argument('--rcnn-batch-size', type=int, default=1) + parser.add_argument('--rcnn-batch-rois', type=int, default=128) + parser.add_argument('--rcnn-fg-fraction', type=float, default=0.25) + parser.add_argument('--rcnn-fg-overlap', type=float, default=0.5) + parser.add_argument('--rcnn-bbox-stds', type=str, default='(0.1, 0.1, 0.2, 0.2)') + args = parser.parse_args() + args.img_pixel_means = ast.literal_eval(args.img_pixel_means) + args.img_pixel_stds = ast.literal_eval(args.img_pixel_stds) + args.net_fixed_params = ast.literal_eval(args.net_fixed_params) + args.rpn_anchor_scales = ast.literal_eval(args.rpn_anchor_scales) + args.rpn_anchor_ratios = ast.literal_eval(args.rpn_anchor_ratios) + args.rcnn_pooled_size = ast.literal_eval(args.rcnn_pooled_size) + args.rcnn_bbox_stds = ast.literal_eval(args.rcnn_bbox_stds) + return args + + +def get_voc(args): + from symimdb.pascal_voc import PascalVOC + if not args.imageset: + args.imageset = '2007_trainval' + args.rcnn_num_classes = len(PascalVOC.classes) + + isets = args.imageset.split('+') + roidb = [] + for iset in isets: + imdb = PascalVOC(iset, 'data', 'data/VOCdevkit') + imdb.append_flipped_images() + roidb.extend(imdb.roidb) + return roidb + + +def get_coco(args): + from symimdb.coco import coco + if not args.imageset: + args.imageset = 'train2017' + args.rcnn_num_classes = len(coco.classes) + + isets = args.imageset.split('+') + roidb = [] + for iset in isets: + imdb = coco(iset, 'data', 'data/coco') + imdb.filter_roidb() + imdb.append_flipped_images() + roidb.extend(imdb.roidb) + return roidb + + +def get_vgg16_train(args): + from symnet.symbol_vgg import get_vgg_train + if not args.pretrained: + args.pretrained = 'model/vgg16-0000.params' + if not args.save_prefix: + args.save_prefix = 'model/vgg16' + args.img_pixel_means = (123.68, 116.779, 103.939) + args.img_pixel_stds = (1.0, 1.0, 1.0) + args.net_fixed_params = ['conv1', 'conv2'] + args.rpn_feat_stride = 16 + args.rcnn_feat_stride = 16 + args.rcnn_pooled_size = (7, 7) + return get_vgg_train(anchor_scales=args.rpn_anchor_scales, anchor_ratios=args.rpn_anchor_ratios, + rpn_feature_stride=args.rpn_feat_stride, rpn_pre_topk=args.rpn_pre_nms_topk, + rpn_post_topk=args.rpn_post_nms_topk, rpn_nms_thresh=args.rpn_nms_thresh, + rpn_min_size=args.rpn_min_size, rpn_batch_rois=args.rpn_batch_rois, + num_classes=args.rcnn_num_classes, rcnn_feature_stride=args.rcnn_feat_stride, + rcnn_pooled_size=args.rcnn_pooled_size, rcnn_batch_size=args.rcnn_batch_size, + rcnn_batch_rois=args.rcnn_batch_rois, rcnn_fg_fraction=args.rcnn_fg_fraction, + rcnn_fg_overlap=args.rcnn_fg_overlap, rcnn_bbox_stds=args.rcnn_bbox_stds) + + +def get_resnet50_train(args): + from symnet.symbol_resnet import get_resnet_train + if not args.pretrained: + args.pretrained = 'model/resnet-50-0000.params' + if not args.save_prefix: + args.save_prefix = 'model/resnet50' + args.img_pixel_means = (0.0, 0.0, 0.0) + args.img_pixel_stds = (1.0, 1.0, 1.0) + args.net_fixed_params = ['conv0', 'stage1', 'gamma', 'beta'] + args.rpn_feat_stride = 16 + args.rcnn_feat_stride = 16 + args.rcnn_pooled_size = (14, 14) + return get_resnet_train(anchor_scales=args.rpn_anchor_scales, anchor_ratios=args.rpn_anchor_ratios, + rpn_feature_stride=args.rpn_feat_stride, rpn_pre_topk=args.rpn_pre_nms_topk, + rpn_post_topk=args.rpn_post_nms_topk, rpn_nms_thresh=args.rpn_nms_thresh, + rpn_min_size=args.rpn_min_size, rpn_batch_rois=args.rpn_batch_rois, + num_classes=args.rcnn_num_classes, rcnn_feature_stride=args.rcnn_feat_stride, + rcnn_pooled_size=args.rcnn_pooled_size, rcnn_batch_size=args.rcnn_batch_size, + rcnn_batch_rois=args.rcnn_batch_rois, rcnn_fg_fraction=args.rcnn_fg_fraction, + rcnn_fg_overlap=args.rcnn_fg_overlap, rcnn_bbox_stds=args.rcnn_bbox_stds, + units=(3, 4, 6, 3), filter_list=(256, 512, 1024, 2048)) + + +def get_resnet101_train(args): + from symnet.symbol_resnet import get_resnet_train + if not args.pretrained: + args.pretrained = 'model/resnet-101-0000.params' + if not args.save_prefix: + args.save_prefix = 'model/resnet101' + args.img_pixel_means = (0.0, 0.0, 0.0) + args.img_pixel_stds = (1.0, 1.0, 1.0) + args.net_fixed_params = ['conv0', 'stage1', 'gamma', 'beta'] + args.rpn_feat_stride = 16 + args.rcnn_feat_stride = 16 + args.rcnn_pooled_size = (14, 14) + return get_resnet_train(anchor_scales=args.rpn_anchor_scales, anchor_ratios=args.rpn_anchor_ratios, + rpn_feature_stride=args.rpn_feat_stride, rpn_pre_topk=args.rpn_pre_nms_topk, + rpn_post_topk=args.rpn_post_nms_topk, rpn_nms_thresh=args.rpn_nms_thresh, + rpn_min_size=args.rpn_min_size, rpn_batch_rois=args.rpn_batch_rois, + num_classes=args.rcnn_num_classes, rcnn_feature_stride=args.rcnn_feat_stride, + rcnn_pooled_size=args.rcnn_pooled_size, rcnn_batch_size=args.rcnn_batch_size, + rcnn_batch_rois=args.rcnn_batch_rois, rcnn_fg_fraction=args.rcnn_fg_fraction, + rcnn_fg_overlap=args.rcnn_fg_overlap, rcnn_bbox_stds=args.rcnn_bbox_stds, + units=(3, 4, 23, 3), filter_list=(256, 512, 1024, 2048)) + + +def get_dataset(dataset, args): + datasets = { + 'voc': get_voc, + 'coco': get_coco + } + if dataset not in datasets: + raise ValueError("dataset {} not supported".format(dataset)) + return datasets[dataset](args) + + +def get_network(network, args): + networks = { + 'vgg16': get_vgg16_train, + 'resnet50': get_resnet50_train, + 'resnet101': get_resnet101_train + } + if network not in networks: + raise ValueError("network {} not supported".format(network)) + return networks[network](args) + + +def main(): + args = parse_args() + roidb = get_dataset(args.dataset, args) + sym = get_network(args.network, args) + train_net(sym, roidb, args) + + +if __name__ == '__main__': + main() diff --git a/example/rnn/bucketing/cudnn_rnn_bucketing.py b/example/rnn/bucketing/cudnn_rnn_bucketing.py new file mode 100644 index 000000000000..38275ae3dfb8 --- /dev/null +++ b/example/rnn/bucketing/cudnn_rnn_bucketing.py @@ -0,0 +1,272 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +import mxnet as mx +import argparse +from mxnet.contrib.amp import amp + +parser = argparse.ArgumentParser(description="Train RNN on Sherlock Holmes", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--test', default=False, action='store_true', + help='whether to do testing instead of training') +parser.add_argument('--model-prefix', type=str, default=None, + help='path to save/load model') +parser.add_argument('--load-epoch', type=int, default=0, + help='load from epoch') +parser.add_argument('--num-layers', type=int, default=2, + help='number of stacked RNN layers') +parser.add_argument('--num-hidden', type=int, default=200, + help='hidden layer size') +parser.add_argument('--num-embed', type=int, default=200, + help='embedding layer size') +parser.add_argument('--bidirectional', action='store_true', + help='uses bidirectional layers if specified') +parser.add_argument('--gpus', type=str, + help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu. ' \ + 'Increase batch size when using multiple gpus for best performance.') +parser.add_argument('--kv-store', type=str, default='device', + help='key-value store type') +parser.add_argument('--num-epochs', type=int, default=25, + help='max num of epochs') +parser.add_argument('--lr', type=float, default=0.01, + help='initial learning rate') +parser.add_argument('--optimizer', type=str, default='sgd', + help='the optimizer type') +parser.add_argument('--mom', type=float, default=0.0, + help='momentum for sgd') +parser.add_argument('--wd', type=float, default=0.00001, + help='weight decay for sgd') +parser.add_argument('--batch-size', type=int, default=32, + help='the batch size.') +parser.add_argument('--disp-batches', type=int, default=50, + help='show progress for every n batches') +# When training a deep, complex model *on multiple GPUs* it's recommended to +# stack fused RNN cells (one layer per cell) together instead of one with all +# layers. The reason is that fused RNN cells don't set gradients to be ready +# until the computation for the entire layer is completed. Breaking a +# multi-layer fused RNN cell into several one-layer ones allows gradients to be +# processed ealier. This reduces communication overhead, especially with +# multiple GPUs. +parser.add_argument('--stack-rnn', default=False, + help='stack fused RNN cells to reduce communication overhead') +parser.add_argument('--dropout', type=float, default='0.0', + help='dropout probability (1.0 - keep probability)') +parser.add_argument('--rnntype', type=str, default='lstm', + help='rnn type: gru, lstm, rnn_tanh and rnn_relu are supported') +parser.add_argument('--dtype', type=str, default='float32', + help='if float16 is provided AMP convert model' + 'is used to convert model to mixed precision model' + 'before running inference') + +#buckets = [32] +buckets = [10, 20, 30, 40, 50, 60] + +start_label = 1 +invalid_label = 0 + +def tokenize_text(fname, vocab=None, invalid_label=-1, start_label=0): + lines = open(fname).readlines() + lines = [filter(None, i.split(' ')) for i in lines] + sentences, vocab = mx.rnn.encode_sentences(lines, vocab=vocab, invalid_label=invalid_label, start_label=start_label) + return sentences, vocab + +def get_data(layout): + train_sent, vocab = tokenize_text("./data/sherlockholmes.train.txt", start_label=start_label, + invalid_label=invalid_label) + val_sent, _ = tokenize_text("./data/sherlockholmes.test.txt", vocab=vocab, start_label=start_label, + invalid_label=invalid_label) + + data_train = mx.rnn.BucketSentenceIter(train_sent, args.batch_size, buckets=buckets, + invalid_label=invalid_label, layout=layout) + data_val = mx.rnn.BucketSentenceIter(val_sent, args.batch_size, buckets=buckets, + invalid_label=invalid_label, layout=layout) + return data_train, data_val, vocab + + +def train(args): + data_train, data_val, vocab = get_data('TN') + if args.stack_rnn: + cell = mx.rnn.SequentialRNNCell() + for i in range(args.num_layers): + cell.add(mx.rnn.FusedRNNCell(args.num_hidden, num_layers=1, + mode=args.rnntype, prefix='%s_l%d'%(args.rnntype,i), + bidirectional=args.bidirectional)) + if args.dropout > 0 and i < args.num_layers - 1 and args.rnntype == 'lstm': + cell.add(mx.rnn.DropoutCell(args.dropout, prefix='%s_d%d'%(args.rnntype,i))) + else: + cell = mx.rnn.FusedRNNCell(args.num_hidden, num_layers=args.num_layers, dropout=args.dropout, + mode=args.rnntype, bidirectional=args.bidirectional) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=len(vocab), output_dim=args.num_embed,name='embed') + + output, _ = cell.unroll(seq_len, inputs=embed, merge_outputs=True, layout='TNC') + + pred = mx.sym.Reshape(output, + shape=(-1, args.num_hidden*(1+args.bidirectional))) + pred = mx.sym.FullyConnected(data=pred, num_hidden=len(vocab), name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + + return pred, ('data',), ('softmax_label',) + + if args.gpus: + contexts = [mx.gpu(int(i)) for i in args.gpus.split(',')] + else: + contexts = mx.cpu(0) + + model = mx.mod.BucketingModule( + sym_gen = sym_gen, + default_bucket_key = data_train.default_bucket_key, + context = contexts) + + if args.load_epoch: + _, arg_params, aux_params = mx.rnn.load_rnn_checkpoint( + cell, args.model_prefix, args.load_epoch) + else: + arg_params = None + aux_params = None + + opt_params = { + 'learning_rate': args.lr, + 'wd': args.wd + } + + if args.optimizer not in ['adadelta', 'adagrad', 'adam', 'rmsprop']: + opt_params['momentum'] = args.mom + + model.fit( + train_data = data_train, + eval_data = data_val, + eval_metric = mx.metric.Perplexity(invalid_label), + kvstore = args.kv_store, + optimizer = args.optimizer, + optimizer_params = opt_params, + initializer = mx.init.Xavier(factor_type="in", magnitude=2.34), + arg_params = arg_params, + aux_params = aux_params, + begin_epoch = args.load_epoch, + num_epoch = args.num_epochs, + batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches, auto_reset=False), + epoch_end_callback = mx.rnn.do_rnn_checkpoint(cell, args.model_prefix, 1) + if args.model_prefix else None) + +def test(args): + assert args.model_prefix, "Must specifiy path to load from" + _, data_val, vocab = get_data('NT') + + if not args.stack_rnn: + stack = mx.rnn.FusedRNNCell(args.num_hidden, num_layers=args.num_layers, + mode=args.rnntype, bidirectional=args.bidirectional).unfuse() + else: + stack = mx.rnn.SequentialRNNCell() + for i in range(args.num_layers): + if args.rnntype == 'lstm': + cell = mx.rnn.LSTMCell(num_hidden=args.num_hidden, prefix='%s_%dl0_'%(args.rnntype,i)) + if args.bidirectional: + cell = mx.rnn.BidirectionalCell( + cell, + mx.rnn.LSTMCell(num_hidden=args.num_hidden, prefix='%s_%dr0_'%(args.rnntype,i)), + output_prefix='bi_%s_%d'%(args.rnntype,i)) + elif args.rnntype == 'gru': + cell = mx.rnn.GRUCell(num_hidden=args.num_hidden, prefix='%s_%dl0_'%(args.rnntype,i)) + if args.bidirectional: + cell = mx.rnn.BidirectionalCell( + cell, + mx.rnn.GRUCell(num_hidden=args.num_hidden, prefix='%s_%dr0_'%(args.rnntype,i)), + output_prefix='bi_%s_%d'%(args.rnntype,i)) + elif args.rnntype == 'rnn_tanh': + cell = mx.rnn.RNNCell(num_hidden=args.num_hidden, activation='tanh', prefix='%s_%dl0_'%(args.rnntype,i)) + if args.bidirectional: + cell = mx.rnn.BidirectionalCell( + cell, + mx.rnn.RNNCell(num_hidden=args.num_hidden, activation='tanh', prefix='%s_%dr0_'%(args.rnntype,i)), + output_prefix='bi_%s_%d'%(args.rnntype,i)) + elif args.rnntype == 'rnn_relu': + cell = mx.rnn.RNNCell(num_hidden=args.num_hidden, activation='relu', prefix='%s_%dl0_'%(args.rnntype,i)) + if args.bidirectional: + cell = mx.rnn.BidirectionalCell( + cell, + mx.rnn.RNNCell(num_hidden=args.num_hidden, activation='relu', prefix='%s_%dr0_'%(args.rnntype,i)), + output_prefix='bi_%s_%d'%(args.rnntype,i)) + + stack.add(cell) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=len(vocab), + output_dim=args.num_embed, name='embed') + + stack.reset() + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, + shape=(-1, args.num_hidden*(1+args.bidirectional))) + pred = mx.sym.FullyConnected(data=pred, num_hidden=len(vocab), name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + + return pred, ('data',), ('softmax_label',) + + if args.gpus: + contexts = [mx.gpu(int(i)) for i in args.gpus.split(',')] + else: + contexts = mx.cpu(0) + + model = mx.mod.BucketingModule( + sym_gen = sym_gen, + default_bucket_key = data_val.default_bucket_key, + context = contexts) + model.bind(data_val.provide_data, data_val.provide_label, for_training=False) + + _, arg_params, aux_params = mx.rnn.load_rnn_checkpoint(stack, args.model_prefix, args.load_epoch) + model.set_params(arg_params, aux_params) + + if args.dtype == "float32": + model.set_params(arg_params, aux_params) + model.score(data_val, mx.metric.Perplexity(invalid_label), + batch_end_callback=mx.callback.Speedometer(args.batch_size, 5)) + else: + assert args.dtype == "float16", "Only float32 and float16 are supported currently" + model = amp.convert_bucketing_module(model, target_dtype="float16") + model.bind(data_val.provide_data, data_val.provide_label, + for_training=False) + model.score(data_val, mx.metric.Perplexity(invalid_label), + batch_end_callback=mx.callback.Speedometer(args.batch_size, 5)) + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + args = parser.parse_args() + + if args.num_layers >= 4 and len(args.gpus.split(',')) >= 4 and not args.stack_rnn: + print('WARNING: stack-rnn is recommended to train complex model on multiple GPUs') + + if args.test: + # Demonstrates how to load a model trained with CuDNN RNN and predict + # with non-fused MXNet symbol + test(args) + else: + train(args) diff --git a/example/rnn/bucketing/lstm_bucketing.py b/example/rnn/bucketing/lstm_bucketing.py new file mode 100644 index 000000000000..7f150104f458 --- /dev/null +++ b/example/rnn/bucketing/lstm_bucketing.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import numpy as np +import mxnet as mx +import argparse +import os + +parser = argparse.ArgumentParser(description="Train RNN on Sherlock Holmes", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-layers', type=int, default=2, + help='number of stacked RNN layers') +parser.add_argument('--num-hidden', type=int, default=200, + help='hidden layer size') +parser.add_argument('--num-embed', type=int, default=200, + help='embedding layer size') +parser.add_argument('--gpus', type=str, + help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu. ' \ + 'Increase batch size when using multiple gpus for best performance.') +parser.add_argument('--kv-store', type=str, default='device', + help='key-value store type') +parser.add_argument('--num-epochs', type=int, default=25, + help='max num of epochs') +parser.add_argument('--lr', type=float, default=0.01, + help='initial learning rate') +parser.add_argument('--optimizer', type=str, default='sgd', + help='the optimizer type') +parser.add_argument('--mom', type=float, default=0.0, + help='momentum for sgd') +parser.add_argument('--wd', type=float, default=0.00001, + help='weight decay for sgd') +parser.add_argument('--batch-size', type=int, default=32, + help='the batch size.') +parser.add_argument('--disp-batches', type=int, default=50, + help='show progress for every n batches') + +def tokenize_text(fname, vocab=None, invalid_label=-1, start_label=0): + if not os.path.isfile(fname): + raise IOError("Please use get_sherlockholmes_data.sh to download requied file (data/sherlockholmes.train.txt)") + lines = open(fname).readlines() + lines = [filter(None, i.split(' ')) for i in lines] + sentences, vocab = mx.rnn.encode_sentences(lines, vocab=vocab, invalid_label=invalid_label, + start_label=start_label) + return sentences, vocab + + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + args = parser.parse_args() + + #buckets = [] + buckets = [10, 20, 30, 40, 50, 60] + + start_label = 1 + invalid_label = 0 + + train_sent, vocab = tokenize_text("./data/sherlockholmes.train.txt", start_label=start_label, + invalid_label=invalid_label) + val_sent, _ = tokenize_text("./data/sherlockholmes.test.txt", vocab=vocab, start_label=start_label, + invalid_label=invalid_label) + + data_train = mx.rnn.BucketSentenceIter(train_sent, args.batch_size, buckets=buckets, + invalid_label=invalid_label) + data_val = mx.rnn.BucketSentenceIter(val_sent, args.batch_size, buckets=buckets, + invalid_label=invalid_label) + + stack = mx.rnn.SequentialRNNCell() + for i in range(args.num_layers): + stack.add(mx.rnn.LSTMCell(num_hidden=args.num_hidden, prefix='lstm_l%d_'%i)) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=len(vocab), + output_dim=args.num_embed, name='embed') + + stack.reset() + outputs = stack.unroll(seq_len, inputs=embed, merge_outputs=True)[0] + + pred = mx.sym.Reshape(outputs, shape=(-1, args.num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=len(vocab), name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + + return pred, ('data',), ('softmax_label',) + + if args.gpus: + contexts = [mx.gpu(int(i)) for i in args.gpus.split(',')] + else: + contexts = mx.cpu(0) + + model = mx.mod.BucketingModule( + sym_gen = sym_gen, + default_bucket_key = data_train.default_bucket_key, + context = contexts) + + model.fit( + train_data = data_train, + eval_data = data_val, + eval_metric = mx.metric.Perplexity(invalid_label), + kvstore = args.kv_store, + optimizer = args.optimizer, + optimizer_params = { 'learning_rate': args.lr, + 'momentum': args.mom, + 'wd': args.wd }, + initializer = mx.init.Xavier(factor_type="in", magnitude=2.34), + num_epoch = args.num_epochs, + batch_end_callback = mx.callback.Speedometer(args.batch_size, args.disp_batches, auto_reset=False)) diff --git a/example/rnn/old/char-rnn.ipynb b/example/rnn/old/char-rnn.ipynb new file mode 100644 index 000000000000..1ec56cd9aa8c --- /dev/null +++ b/example/rnn/old/char-rnn.ipynb @@ -0,0 +1,549 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import mxnet as mx\n", + "import numpy as np\n", + "import random\n", + "import bisect" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# set up logging\n", + "import logging\n", + "reload(logging)\n", + "logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.DEBUG, datefmt='%I:%M:%S')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# A Glance of LSTM structure and embedding layer\n", + "\n", + "We will build a LSTM network to learn from char only. At each time, input is a char. We will see this LSTM is able to learn words and grammers from sequence of chars.\n", + "\n", + "The following figure is showing an unrolled LSTM network, and how we generate embedding of a char. The one-hot to embedding operation is a special case of fully connected network.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "from lstm import lstm_unroll, lstm_inference_symbol\n", + "from bucket_io import BucketSentenceIter\n", + "from rnn_model import LSTMInferenceModel" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Read from doc\n", + "def read_content(path):\n", + " with open(path) as ins:\n", + " content = ins.read()\n", + " return content\n", + "\n", + "# Build a vocabulary of what char we have in the content\n", + "def build_vocab(path):\n", + " content = read_content(path)\n", + " content = list(content)\n", + " idx = 1 # 0 is left for zero-padding\n", + " the_vocab = {}\n", + " for word in content:\n", + " if len(word) == 0:\n", + " continue\n", + " if not word in the_vocab:\n", + " the_vocab[word] = idx\n", + " idx += 1\n", + " return the_vocab\n", + "\n", + "# We will assign each char with a special numerical id\n", + "def text2id(sentence, the_vocab):\n", + " words = list(sentence)\n", + " words = [the_vocab[w] for w in words if len(w) > 0]\n", + " return words" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Evaluation \n", + "def Perplexity(label, pred):\n", + " label = label.T.reshape((-1,))\n", + " loss = 0.\n", + " for i in range(pred.shape[0]):\n", + " loss += -np.log(max(1e-10, pred[i][int(label[i])]))\n", + " return np.exp(loss / label.size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get Data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "data_url = \"http://data.mxnet.io/mxnet/data/char_lstm.zip\"\n", + "os.system(\"wget %s\" % data_url)\n", + "os.system(\"unzip -o char_lstm.zip\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample training data:\n", + "```\n", + "all to Renewal Keynote Address Call to Renewal Pt 1Call to Renewal Part 2 TOPIC: Our Past, Our Future & Vision for America June\n", + "28, 2006 Call to Renewal' Keynote Address Complete Text Good morning. I appreciate the opportunity to speak here at the Call to R\n", + "enewal's Building a Covenant for a New America conference. I've had the opportunity to take a look at your Covenant for a New Ame\n", + "rica. It is filled with outstanding policies and prescriptions for much of what ails this country. So I'd like to congratulate yo\n", + "u all on the thoughtful presentations you've given so far about poverty and justice in America, and for putting fire under the fe\n", + "et of the political leadership here in Washington.But today I'd like to talk about the connection between religion and politics a\n", + "nd perhaps offer some thoughts about how we can sort through some of the often bitter arguments that we've been seeing over the l\n", + "ast several years.I do so because, as you all know, we can affirm the importance of poverty in the Bible; and we can raise up and\n", + " pass out this Covenant for a New America. We can talk to the press, and we can discuss the religious call to address poverty and\n", + " environmental stewardship all we want, but it won't have an impact unless we tackle head-on the mutual suspicion that sometimes\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LSTM Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# The batch size for training\n", + "batch_size = 32\n", + "# We can support various length input\n", + "# For this problem, we cut each input sentence to length of 129\n", + "# So we only need fix length bucket\n", + "buckets = [129]\n", + "# hidden unit in LSTM cell\n", + "num_hidden = 512\n", + "# embedding dimension, which is, map a char to a 256 dim vector\n", + "num_embed = 256\n", + "# number of lstm layer\n", + "num_lstm_layer = 3" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# we will show a quick demo in 2 epoch\n", + "# and we will see result by training 75 epoch\n", + "num_epoch = 2\n", + "# learning rate \n", + "learning_rate = 0.01\n", + "# we will use pure sgd without momentum\n", + "momentum = 0.0" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# we can select multi-gpu for training\n", + "# for this demo we only use one\n", + "devs = [mx.context.gpu(i) for i in range(1)]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# build char vocabluary from input\n", + "vocab = build_vocab(\"./obama.txt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# generate symbol for a length\n", + "def sym_gen(seq_len):\n", + " return lstm_unroll(num_lstm_layer, seq_len, len(vocab) + 1,\n", + " num_hidden=num_hidden, num_embed=num_embed,\n", + " num_label=len(vocab) + 1, dropout=0.2)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# initalize states for LSTM\n", + "init_c = [('l%d_init_c'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)]\n", + "init_h = [('l%d_init_h'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)]\n", + "init_states = init_c + init_h" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Summary of dataset ==================\n", + "bucket of len 129 : 8290 samples\n" + ] + } + ], + "source": [ + "# we can build an iterator for text\n", + "data_train = BucketSentenceIter(\"./obama.txt\", vocab, buckets, batch_size,\n", + " init_states, seperate_char='\\n',\n", + " text2id=text2id, read_content=read_content)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# the network symbol\n", + "symbol = sym_gen(buckets[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train model" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Train a LSTM network as simple as feedforward network\n", + "model = mx.model.FeedForward(ctx=devs,\n", + " symbol=symbol,\n", + " num_epoch=num_epoch,\n", + " learning_rate=learning_rate,\n", + " momentum=momentum,\n", + " wd=0.0001,\n", + " initializer=mx.init.Xavier(factor_type=\"in\", magnitude=2.34))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "05:01:35 INFO:Start training with [gpu(0)]\n" + ] + } + ], + "source": [ + "# Fit it\n", + "model.fit(X=data_train,\n", + " eval_metric = mx.metric.np(Perplexity),\n", + " batch_end_callback=mx.callback.Speedometer(batch_size, 50),\n", + " epoch_end_callback=mx.callback.do_checkpoint(\"obama\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Inference from model" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# helper strcuture for prediction\n", + "def MakeRevertVocab(vocab):\n", + " dic = {}\n", + " for k, v in vocab.items():\n", + " dic[v] = k\n", + " return dic" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# make input from char\n", + "def MakeInput(char, vocab, arr):\n", + " idx = vocab[char]\n", + " tmp = np.zeros((1,))\n", + " tmp[0] = idx\n", + " arr[:] = tmp" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# helper function for random sample \n", + "def _cdf(weights):\n", + " total = sum(weights)\n", + " result = []\n", + " cumsum = 0\n", + " for w in weights:\n", + " cumsum += w\n", + " result.append(cumsum / total)\n", + " return result\n", + "\n", + "def _choice(population, weights):\n", + " assert len(population) == len(weights)\n", + " cdf_vals = _cdf(weights)\n", + " x = random.random()\n", + " idx = bisect.bisect(cdf_vals, x)\n", + " return population[idx]\n", + "\n", + "# we can use random output or fixed output by choosing largest probability\n", + "def MakeOutput(prob, vocab, sample=False, temperature=1.):\n", + " if sample == False:\n", + " idx = np.argmax(prob, axis=1)[0]\n", + " else:\n", + " fix_dict = [\"\"] + [vocab[i] for i in range(1, len(vocab) + 1)]\n", + " scale_prob = np.clip(prob, 1e-6, 1 - 1e-6)\n", + " rescale = np.exp(np.log(scale_prob) / temperature)\n", + " rescale[:] /= rescale.sum()\n", + " return _choice(fix_dict, rescale[0, :])\n", + " try:\n", + " char = vocab[idx]\n", + " except:\n", + " char = ''\n", + " return char" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# load from check-point\n", + "_, arg_params, __ = mx.model.load_checkpoint(\"obama\", 75)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# build an inference model\n", + "model = LSTMInferenceModel(num_lstm_layer, len(vocab) + 1,\n", + " num_hidden=num_hidden, num_embed=num_embed,\n", + " num_label=len(vocab) + 1, arg_params=arg_params, ctx=mx.gpu(), dropout=0.2)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# generate a sequence of 1200 chars\n", + "\n", + "seq_length = 1200\n", + "input_ndarray = mx.nd.zeros((1,))\n", + "revert_vocab = MakeRevertVocab(vocab)\n", + "# Feel free to change the starter sentence\n", + "output ='The joke'\n", + "random_sample = True\n", + "new_sentence = True\n", + "\n", + "ignore_length = len(output)\n", + "\n", + "for i in range(seq_length):\n", + " if i <= ignore_length - 1:\n", + " MakeInput(output[i], vocab, input_ndarray)\n", + " else:\n", + " MakeInput(output[-1], vocab, input_ndarray)\n", + " prob = model.forward(input_ndarray, new_sentence)\n", + " new_sentence = False\n", + " next_char = MakeOutput(prob, revert_vocab, random_sample)\n", + " if next_char == '':\n", + " new_sentence = True\n", + " if i >= ignore_length - 1:\n", + " output += next_char\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The joke learning to be struggle for our daughter. We are the ones who can't pay their relationship. The Judiciary Commencement ce designed to deficit to the party of almost unemployment instead, just to look at home, little proof for America, Carguin are showing struggle against our pride. That if you came from tharger by a party that would increase the pervasive sense of new global warming against the challenge of governments - to get a corporation.As a highealth care, your own retirement security information about his family decided to get a job or aspect what will allow cannot simply by sagging high school system and stin twenty-five years. But led my faith designed to leave all their buddets and responsibility. But I sund this dangerous weapons, explain withdrawal oful -clears axdication in Iraq.What is the time for American policy became their efforts, and given them that a man doesn't make sure that that my own, you'll be faced with you. Four years, reforms illness all that kind of choose to understand is a broadeary. You instills in search of a reducithis recision, of us, with public services from using that barealies, but that must continue to limb line, they know th\n" + ] + } + ], + "source": [ + "# Let's see what we can learned from char in Obama's speech.\n", + "print(output)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/example/rnn/old/gru_bucketing.py b/example/rnn/old/gru_bucketing.py new file mode 100644 index 000000000000..b9f651a90dc0 --- /dev/null +++ b/example/rnn/old/gru_bucketing.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme +# pylint: disable=superfluous-parens, no-member, invalid-name +import sys +sys.path.insert(0, "../../python") +import numpy as np +import mxnet as mx + +from gru import gru_unroll +from bucket_io import BucketSentenceIter, default_build_vocab, DummyIter + +def Perplexity(label, pred): + label = label.T.reshape((-1,)) + loss = 0. + for i in range(pred.shape[0]): + loss += -np.log(max(1e-10, pred[i][int(label[i])])) + return np.exp(loss / label.size) + +if __name__ == '__main__': + batch_size = 32 + #buckets = [10, 20, 30, 40, 50, 60] + #buckets = [32] + buckets = [] + num_hidden = 200 + num_embed = 200 + num_lstm_layer = 2 + + num_epoch = 25 + learning_rate = 0.01 + momentum = 0.0 + + # dummy data is used to test speed without IO + dummy_data = False + + #contexts = [mx.context.gpu(i) for i in range(1)] + contexts = mx.context.cpu() + + vocab = default_build_vocab("./data/sherlockholmes.train.txt") + + def sym_gen(seq_len): + return gru_unroll(num_lstm_layer, seq_len, len(vocab), + num_hidden=num_hidden, num_embed=num_embed, + num_label=len(vocab)) + + init_h = [('l%d_init_h'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)] + + data_train = BucketSentenceIter("./data/sherlockholmes.train.txt", vocab, + buckets, batch_size, init_h) + data_val = BucketSentenceIter("./data/sherlockholmes.valid.txt", vocab, + buckets, batch_size, init_h) + + if dummy_data: + data_train = DummyIter(data_train) + data_val = DummyIter(data_val) + + if len(buckets) == 1: + # only 1 bucket, disable bucketing + symbol = sym_gen(buckets[0]) + else: + symbol = sym_gen + + model = mx.model.FeedForward(ctx=contexts, + symbol=symbol, + num_epoch=num_epoch, + learning_rate=learning_rate, + momentum=momentum, + wd=0.00001, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) + + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + model.fit(X=data_train, eval_data=data_val, + eval_metric = mx.metric.np(Perplexity), + batch_end_callback=mx.callback.Speedometer(batch_size, 50),) + diff --git a/example/rnn/old/lstm_bucketing.py b/example/rnn/old/lstm_bucketing.py new file mode 100644 index 000000000000..0fe4116250a2 --- /dev/null +++ b/example/rnn/old/lstm_bucketing.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=C0111,too-many-arguments,too-many-instance-attributes,too-many-locals,redefined-outer-name,fixme +# pylint: disable=superfluous-parens, no-member, invalid-name +import sys +sys.path.insert(0, "../../python") +import numpy as np +import mxnet as mx + +from lstm import lstm_unroll +from bucket_io import BucketSentenceIter, default_build_vocab, DummyIter + +def Perplexity(label, pred): + label = label.T.reshape((-1,)) + loss = 0. + for i in range(pred.shape[0]): + loss += -np.log(max(1e-10, pred[i][int(label[i])])) + return np.exp(loss / label.size) + +if __name__ == '__main__': + N = 8 + batch_size = 32*N + #buckets = [10, 20, 30, 40, 50, 60] + buckets = [32] + #buckets = [] + num_hidden = 200 + num_embed = 200 + num_lstm_layer = 2 + + num_epoch = 25 + learning_rate = 0.01 + momentum = 0.0 + + # dummy data is used to test speed without IO + dummy_data = False + + contexts = [mx.context.gpu(i) for i in range(N)] + + vocab = default_build_vocab("./data/sherlockholmes.train.txt") + + def sym_gen(seq_len): + return lstm_unroll(num_lstm_layer, seq_len, len(vocab), + num_hidden=num_hidden, num_embed=num_embed, + num_label=len(vocab)) + + init_c = [('l%d_init_c'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)] + init_h = [('l%d_init_h'%l, (batch_size, num_hidden)) for l in range(num_lstm_layer)] + init_states = init_c + init_h + + data_train = BucketSentenceIter("./data/sherlockholmes.train.txt", vocab, + buckets, batch_size, init_states) + data_val = BucketSentenceIter("./data/sherlockholmes.valid.txt", vocab, + buckets, batch_size, init_states) + + if dummy_data: + data_train = DummyIter(data_train) + data_val = DummyIter(data_val) + + if len(buckets) == 1: + # only 1 bucket, disable bucketing + symbol = sym_gen(buckets[0]) + else: + symbol = sym_gen + + model = mx.model.FeedForward(ctx=contexts, + symbol=symbol, + num_epoch=num_epoch, + learning_rate=learning_rate, + momentum=momentum, + wd=0.00001, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) + + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + model.fit(X=data_train, eval_data=data_val, kvstore='device', + eval_metric = mx.metric.np(Perplexity), + batch_end_callback=mx.callback.Speedometer(batch_size, 50),) + diff --git a/example/rnn/old/rnn_cell_demo.py b/example/rnn/old/rnn_cell_demo.py new file mode 100644 index 000000000000..c5772fa3a5b7 --- /dev/null +++ b/example/rnn/old/rnn_cell_demo.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A simple demo of new RNN cell with sherlockholmes language model.""" + +import os + +import numpy as np +import mxnet as mx + +from bucket_io import BucketSentenceIter, default_build_vocab + + +data_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'data')) + + +def Perplexity(label, pred): + # TODO(tofix): we make a transpose of label here, because when + # using the RNN cell, we called swap axis to the data. + label = label.T.reshape((-1,)) + loss = 0. + for i in range(pred.shape[0]): + loss += -np.log(max(1e-10, pred[i][int(label[i])])) + return np.exp(loss / label.size) + + +if __name__ == '__main__': + batch_size = 128 + buckets = [10, 20, 30, 40, 50, 60] + num_hidden = 200 + num_embed = 200 + num_lstm_layer = 2 + + num_epoch = 2 + learning_rate = 0.01 + momentum = 0.0 + + contexts = [mx.context.gpu(i) for i in range(4)] + vocab = default_build_vocab(os.path.join(data_dir, 'sherlockholmes.train.txt')) + + init_h = [('LSTM_init_h', (batch_size, num_lstm_layer, num_hidden))] + init_c = [('LSTM_init_c', (batch_size, num_lstm_layer, num_hidden))] + init_states = init_c + init_h + + data_train = BucketSentenceIter(os.path.join(data_dir, 'sherlockholmes.train.txt'), + vocab, buckets, batch_size, init_states) + data_val = BucketSentenceIter(os.path.join(data_dir, 'sherlockholmes.valid.txt'), + vocab, buckets, batch_size, init_states) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=len(vocab), + output_dim=num_embed, name='embed') + + # TODO(tofix) + # The inputs and labels from IO are all in batch-major. + # We need to transform them into time-major to use RNN cells. + embed_tm = mx.sym.SwapAxis(embed, dim1=0, dim2=1) + label_tm = mx.sym.SwapAxis(label, dim1=0, dim2=1) + + # TODO(tofix) + # Create transformed RNN initial states. Normally we do + # no need to do this. But the RNN symbol expects the state + # to be time-major shape layout, while the current mxnet + # IO and high-level training logic assume everything from + # the data iter have batch_size as the first dimension. + # So until we have extended our IO and training logic to + # support this more general case, this dummy axis swap is + # needed. + rnn_h_init = mx.sym.SwapAxis(mx.sym.Variable('LSTM_init_h'), + dim1=0, dim2=1) + rnn_c_init = mx.sym.SwapAxis(mx.sym.Variable('LSTM_init_c'), + dim1=0, dim2=1) + + # TODO(tofix) + # currently all the LSTM parameters are concatenated as + # a huge vector, and named '_parameters'. By default + # mxnet initializer does not know how to initilize this + # guy because its name does not ends with _weight or _bias + # or anything familiar. Here we just use a temp workaround + # to create a variable and name it as LSTM_bias to get + # this demo running. Note by default bias is initialized + # as zeros, so this is not a good scheme. But calling it + # LSTM_weight is not good, as this is 1D vector, while + # the initialization scheme of a weight parameter needs + # at least two dimensions. + rnn_params = mx.sym.Variable('LSTM_bias') + + # RNN cell takes input of shape (time, batch, feature) + rnn = mx.sym.RNN(data=embed_tm, state_size=num_hidden, + num_layers=num_lstm_layer, mode='lstm', + name='LSTM', + # The following params can be omitted + # provided we do not need to apply the + # workarounds mentioned above + state=rnn_h_init, + state_cell=rnn_c_init, + parameters=rnn_params) + + # the RNN cell output is of shape (time, batch, dim) + # if we need the states and cell states in the last time + # step (e.g. when building encoder-decoder models), we + # can set state_outputs=True, and the RNN cell will have + # extra outputs: rnn['LSTM_output'], rnn['LSTM_state'] + # and for LSTM, also rnn['LSTM_state_cell'] + + # now we collapse the time and batch dimension to do the + # final linear logistic regression prediction + hidden = mx.sym.Reshape(data=rnn, shape=(-1, num_hidden)) + label_cl = mx.sym.Reshape(data=label_tm, shape=(-1,)) + + pred = mx.sym.FullyConnected(data=hidden, num_hidden=len(vocab), + name='pred') + sm = mx.sym.SoftmaxOutput(data=pred, label=label_cl, name='softmax') + + data_names = ['data', 'LSTM_init_h', 'LSTM_init_c'] + label_names = ['softmax_label'] + + return (sm, data_names, label_names) + + if len(buckets) == 1: + mod = mx.mod.Module(*sym_gen(buckets[0]), context=contexts) + else: + mod = mx.mod.BucketingModule(sym_gen, default_bucket_key=data_train.default_bucket_key, + context=contexts) + + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + + mod.fit(data_train, eval_data=data_val, num_epoch=num_epoch, + eval_metric=mx.metric.np(Perplexity), + batch_end_callback=mx.callback.Speedometer(batch_size, 50), + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + optimizer='sgd', + optimizer_params={'learning_rate': learning_rate, + 'momentum': momentum, 'wd': 0.00001}) diff --git a/example/sparse/factorization_machine/metric.py b/example/sparse/factorization_machine/metric.py new file mode 100644 index 000000000000..a8c52c781c0f --- /dev/null +++ b/example/sparse/factorization_machine/metric.py @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np +from operator import itemgetter + +@mx.metric.register +@mx.metric.alias('log_loss') +class LogLossMetric(mx.metric.EvalMetric): + """Computes the negative log-likelihood loss. + + The negative log-likelihoodd loss over a batch of sample size :math:`N` is given by + + .. math:: + -\\sum_{n=1}^{N}\\sum_{k=1}^{K}t_{nk}\\log (y_{nk}), + + where :math:`K` is the number of classes, :math:`y_{nk}` is the prediceted probability for + :math:`k`-th class for :math:`n`-th sample. :math:`t_{nk}=1` if and only if sample + :math:`n` belongs to class :math:`k`. + + Parameters + ---------- + eps : float + Negative log-likelihood loss is undefined for predicted value is 0, + so predicted values are added with the small constant. + name : str + Name of this metric instance for display. + output_names : list of str, or None + Name of predictions that should be used when updating with update_dict. + By default include all predictions. + label_names : list of str, or None + Name of labels that should be used when updating with update_dict. + By default include all labels. + + Examples + -------- + >>> predicts = [mx.nd.array([[0.3], [0], [0.4]])] + >>> labels = [mx.nd.array([0, 1, 1])] + >>> log_loss= mx.metric.NegativeLogLikelihood() + >>> log_loss.update(labels, predicts) + >>> print(log_loss.get()) + ('log-loss', 0.57159948348999023) + """ + def __init__(self, eps=1e-12, name='log-loss', + output_names=None, label_names=None): + super(LogLossMetric, self).__init__( + name, eps=eps, + output_names=output_names, label_names=label_names) + self.eps = eps + + def update(self, labels, preds): + """Updates the internal evaluation result. + + Parameters + ---------- + labels : list of `NDArray` + The labels of the data. + + preds : list of `NDArray` + Predicted values. + """ + mx.metric.check_label_shapes(labels, preds) + + for label, pred in zip(labels, preds): + label = label.asnumpy() + pred = pred.asnumpy() + pred = np.column_stack((1 - pred, pred)) + + label = label.ravel() + num_examples = pred.shape[0] + assert label.shape[0] == num_examples, (label.shape[0], num_examples) + prob = pred[np.arange(num_examples, dtype=np.int64), np.int64(label)] + self.sum_metric += (-np.log(prob + self.eps)).sum() + self.num_inst += num_examples + +@mx.metric.register +@mx.metric.alias('auc') +class AUCMetric(mx.metric.EvalMetric): + def __init__(self, eps=1e-12): + super(AUCMetric, self).__init__( + 'auc') + self.eps = eps + + def update(self, labels, preds): + mx.metric.check_label_shapes(labels, preds) + label_weight = labels[0].asnumpy() + preds = preds[0].asnumpy() + tmp = [] + for i in range(preds.shape[0]): + tmp.append((label_weight[i], preds[i])) + tmp = sorted(tmp, key=itemgetter(1), reverse=True) + label_sum = label_weight.sum() + if label_sum == 0 or label_sum == label_weight.size: + raise Exception("AUC with one class is undefined") + + label_one_num = np.count_nonzero(label_weight) + label_zero_num = len(label_weight) - label_one_num + total_area = label_zero_num * label_one_num + height = 0 + width = 0 + area = 0 + for a, _ in tmp: + if a == 1.0: + height += 1.0 + else: + width += 1.0 + area += height + + self.sum_metric += area / total_area + self.num_inst += 1 diff --git a/example/sparse/factorization_machine/train.py b/example/sparse/factorization_machine/train.py new file mode 100644 index 000000000000..b30f9cc81acf --- /dev/null +++ b/example/sparse/factorization_machine/train.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from metric import * +from mxnet.test_utils import * +from model import factorization_machine_model +import argparse, os + +parser = argparse.ArgumentParser(description="Run factorization machine with criteo dataset", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--data-train', type=str, default=None, + help='training dataset in LibSVM format.') +parser.add_argument('--data-test', type=str, default=None, + help='test dataset in LibSVM format.') +parser.add_argument('--num-epoch', type=int, default=1, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=1000, + help='number of examples per batch') +parser.add_argument('--input-size', type=int, default=1000000, + help='number of features in the input') +parser.add_argument('--factor-size', type=int, default=16, + help='number of latent variables') +parser.add_argument('--factor-lr', type=float, default=0.0001, + help='learning rate for factor terms') +parser.add_argument('--linear-lr', type=float, default=0.001, + help='learning rate for linear terms') +parser.add_argument('--bias-lr', type=float, default=0.1, + help='learning rate for bias terms') +parser.add_argument('--factor-wd', type=float, default=0.00001, + help='weight decay rate for factor terms') +parser.add_argument('--linear-wd', type=float, default=0.001, + help='weight decay rate for linear terms') +parser.add_argument('--bias-wd', type=float, default=0.01, + help='weight decay rate for bias terms') +parser.add_argument('--factor-sigma', type=float, default=0.001, + help='standard deviation for initialization of factor terms') +parser.add_argument('--linear-sigma', type=float, default=0.01, + help='standard deviation for initialization of linear terms') +parser.add_argument('--bias-sigma', type=float, default=0.01, + help='standard deviation for initialization of bias terms') +parser.add_argument('--log-interval', type=int, default=100, + help='number of batches between logging messages') +parser.add_argument('--kvstore', type=str, default='local', + help='what kvstore to use', choices=["dist_async", "local"]) + + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_epoch = args.num_epoch + batch_size = args.batch_size + kvstore = args.kvstore + factor_size = args.factor_size + num_features = args.input_size + log_interval = args.log_interval + assert(args.data_train is not None and args.data_test is not None), \ + "dataset for training or test is missing" + + def batch_row_ids(data_batch): + """ Generate row ids based on the current mini-batch """ + idx = data_batch.data[0].indices + return {'w': idx, 'v': idx} + + def all_row_ids(data_batch): + """ Generate row ids for all rows """ + all_rows = mx.nd.arange(0, num_features, dtype='int64') + return {'w': all_rows, 'v': all_rows} + + # create kvstore + kv = mx.kvstore.create(kvstore) + # data iterator + train_data = mx.io.LibSVMIter(data_libsvm=args.data_train, data_shape=(num_features,), + batch_size=batch_size) + eval_data = mx.io.LibSVMIter(data_libsvm=args.data_test, data_shape=(num_features,), + batch_size=batch_size) + # model + lr_config = {'v': args.factor_lr, 'w': args.linear_lr, 'w0': args.bias_lr} + wd_config = {'v': args.factor_wd, 'w': args.linear_wd, 'w0': args.bias_wd} + init_config = {'v': mx.initializer.Normal(args.factor_sigma), + 'w': mx.initializer.Normal(args.linear_sigma), + 'w0': mx.initializer.Normal(args.bias_sigma)} + model = factorization_machine_model(factor_size, num_features, lr_config, wd_config, init_config) + + # module + mod = mx.mod.Module(symbol=model) + mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) + mod.init_params() + optimizer_params=(('learning_rate', 1), ('wd', 1), ('beta1', 0.9), + ('beta2', 0.999), ('epsilon', 1e-8)) + mod.init_optimizer(optimizer='adam', kvstore=kv, optimizer_params=optimizer_params) + + # metrics + metric = mx.metric.create(['log_loss', 'auc']) + speedometer = mx.callback.Speedometer(batch_size, log_interval) + + logging.info('Training started ...') + train_iter = iter(train_data) + eval_iter = iter(eval_data) + for epoch in range(num_epoch): + nbatch = 0 + metric.reset() + for batch in train_iter: + try: + nbatch += 1 + # manually pull sparse weights from kvstore so that _square_sum + # only computes the rows necessary + mod.prepare(batch, sparse_row_id_fn=batch_row_ids) + mod.forward_backward(batch) + # update all parameters (including the weight parameter) + mod.update() + # update training metric + mod.update_metric(metric, batch.label) + speedometer_param = mx.model.BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=metric, locals=locals()) + speedometer(speedometer_param) + except: + continue + + # pull all updated rows before validation + mod.prepare(None, all_row_ids) + # evaluate metric on validation dataset + score = mod.score(eval_iter, ['log_loss']) + logging.info("epoch %d, eval log loss = %s" % (epoch, score[0][1])) + # reset the iterator for next pass of data + train_iter.reset() + eval_iter.reset() + logging.info('Training completed.') diff --git a/example/sparse/linear_classification/train.py b/example/sparse/linear_classification/train.py new file mode 100644 index 000000000000..0a8acfd87bef --- /dev/null +++ b/example/sparse/linear_classification/train.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet.test_utils import * +from data import get_avazu_data +from linear_model import * +import argparse +import os + +parser = argparse.ArgumentParser(description="Run sparse linear classification " \ + "with distributed kvstore", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-epoch', type=int, default=5, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=8192, + help='number of examples per batch') +parser.add_argument('--kvstore', type=str, default=None, + help='what kvstore to use', + choices=["dist_sync", "dist_async", "local"]) +parser.add_argument('--optimizer', type=str, default='sgd', + help='what optimizer to use', + choices=["adagrad", "sgd", "adam"]) + +AVAZU = { + 'train': 'avazu-app', + 'test': 'avazu-app.t', + 'url': "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/", + # 1000000 + 1 since LibSVMIter uses zero-based indexing + 'num_features': 1000001, +} + +def batch_row_ids(data_batch): + """ Generate row ids based on the current mini-batch """ + return {'weight': data_batch.data[0].indices} + +def all_row_ids(data_batch): + """ Generate row ids for all rows """ + all_rows = mx.nd.arange(0, AVAZU['num_features'], dtype='int64') + return {'weight': all_rows} + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_epoch = args.num_epoch + kvstore = args.kvstore + batch_size = args.batch_size + optimizer = args.optimizer + + # create kvstore + kv = mx.kvstore.create(kvstore) if kvstore else None + rank = kv.rank if kv else 0 + num_worker = kv.num_workers if kv else 1 + + # dataset + num_features = AVAZU['num_features'] + data_dir = os.path.join(os.getcwd(), 'data') + train_data = os.path.join(data_dir, AVAZU['train']) + val_data = os.path.join(data_dir, AVAZU['test']) + get_avazu_data(data_dir, AVAZU['train'], AVAZU['url']) + get_avazu_data(data_dir, AVAZU['test'], AVAZU['url']) + + # data iterator + train_data = mx.io.LibSVMIter(data_libsvm=train_data, data_shape=(num_features,), + batch_size=batch_size, num_parts=num_worker, + part_index=rank) + eval_data = mx.io.LibSVMIter(data_libsvm=val_data, data_shape=(num_features,), + batch_size=batch_size) + + # model + # The positive class weight, says how much more we should upweight the importance of + # positive instances in the objective function. + # This is used to combat the extreme class imbalance. + positive_class_weight = 2 + model = linear_model(num_features, positive_class_weight) + + # module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['softmax_label']) + mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) + mod.init_params() + optim = mx.optimizer.create(optimizer, learning_rate=0.01, rescale_grad=1.0/batch_size/num_worker) + mod.init_optimizer(optimizer=optim, kvstore=kv) + # use accuracy as the metric + metric = mx.metric.create(['nll_loss']) + + # get the sparse weight parameter + speedometer = mx.callback.Speedometer(batch_size, 100) + + logging.info('Training started ...') + for epoch in range(num_epoch): + nbatch = 0 + metric.reset() + for batch in train_data: + nbatch += 1 + # for distributed training, we need to manually pull sparse weights from kvstore + mod.prepare(batch, sparse_row_id_fn=batch_row_ids) + mod.forward_backward(batch) + # update all parameters (including the weight parameter) + mod.update() + # update training metric + mod.update_metric(metric, batch.label) + speedometer_param = mx.model.BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=metric, locals=locals()) + speedometer(speedometer_param) + + # prepare the module weight with all row ids for inference. Alternatively, one could call + # score = mod.score(val_iter, ['MSE'], sparse_row_id_fn=batch_row_ids) + # to fetch the weight per mini-batch + mod.prepare(None, all_row_ids) + # evaluate metric on validation dataset + score = mod.score(eval_data, ['nll_loss']) + logging.info('epoch %d, eval nll = %s ' % (epoch, score[0][1])) + + # prepare the module weight with all row ids before making a checkpoint. + mod.prepare(None, all_row_ids) + mod.save_checkpoint("checkpoint", epoch) + # reset the iterator for next pass of data + train_data.reset() + eval_data.reset() + logging.info('Training completed.') diff --git a/example/sparse/matrix_factorization/train.py b/example/sparse/matrix_factorization/train.py new file mode 100644 index 000000000000..44bab2c416ba --- /dev/null +++ b/example/sparse/matrix_factorization/train.py @@ -0,0 +1,132 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import logging +import mxnet as mx +import numpy as np +from data import get_movielens_iter, get_movielens_data +from model import matrix_fact_net +import os + +logging.basicConfig(level=logging.DEBUG) + +parser = argparse.ArgumentParser(description="Run matrix factorization with sparse embedding", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-epoch', type=int, default=3, + help='number of epochs to train') +parser.add_argument('--seed', type=int, default=1, + help='random seed') +parser.add_argument('--batch-size', type=int, default=128, + help='number of examples per batch') +parser.add_argument('--log-interval', type=int, default=100, + help='logging interval') +parser.add_argument('--factor-size', type=int, default=128, + help="the factor size of the embedding operation") +parser.add_argument('--gpus', type=str, + help="list of gpus to run, e.g. 0 or 0,2. empty means using cpu().") +parser.add_argument('--dense', action='store_true', help="whether to use dense embedding") + +MOVIELENS = { + 'dataset': 'ml-10m', + 'train': './data/ml-10M100K/r1.train', + 'val': './data/ml-10M100K/r1.test', + 'max_user': 71569, + 'max_movie': 65135, +} + +def batch_row_ids(data_batch): + """ Generate row ids based on the current mini-batch """ + item = data_batch.data[0] + user = data_batch.data[1] + return {'user_weight': user.astype(np.int64), + 'item_weight': item.astype(np.int64)} + +def all_row_ids(data_batch): + """ Generate row ids for all rows """ + all_users = mx.nd.arange(0, MOVIELENS['max_user'], dtype='int64') + all_movies = mx.nd.arange(0, MOVIELENS['max_movie'], dtype='int64') + return {'user_weight': all_users, 'item_weight': all_movies} + +if __name__ == '__main__': + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_epoch = args.num_epoch + batch_size = args.batch_size + optimizer = 'sgd' + factor_size = args.factor_size + log_interval = args.log_interval + + momentum = 0.9 + ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] if args.gpus else [mx.cpu()] + learning_rate = 0.1 + mx.random.seed(args.seed) + np.random.seed(args.seed) + + # prepare dataset and iterators + max_user = MOVIELENS['max_user'] + max_movies = MOVIELENS['max_movie'] + data_dir = os.path.join(os.getcwd(), 'data') + get_movielens_data(data_dir, MOVIELENS['dataset']) + train_iter = get_movielens_iter(MOVIELENS['train'], batch_size) + val_iter = get_movielens_iter(MOVIELENS['val'], batch_size) + + # construct the model + net = matrix_fact_net(factor_size, factor_size, max_user, max_movies, dense=args.dense) + + # initialize the module + mod = mx.module.Module(net, context=ctx, data_names=['user', 'item'], + label_names=['score']) + mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) + mod.init_params(initializer=mx.init.Xavier(factor_type="in", magnitude=2.34)) + optim = mx.optimizer.create(optimizer, learning_rate=learning_rate, + rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=optim, kvstore='device') + # use MSE as the metric + metric = mx.metric.create(['MSE']) + speedometer = mx.callback.Speedometer(batch_size, log_interval) + logging.info('Training started ...') + for epoch in range(num_epoch): + nbatch = 0 + metric.reset() + for batch in train_iter: + nbatch += 1 + mod.prepare(batch, sparse_row_id_fn=batch_row_ids) + mod.forward_backward(batch) + # update all parameters + mod.update() + # update training metric + mod.update_metric(metric, batch.label) + speedometer_param = mx.model.BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=metric, locals=locals()) + speedometer(speedometer_param) + + # prepare the module weight with all row ids for inference. Alternatively, one could call + # score = mod.score(val_iter, ['MSE'], sparse_row_id_fn=batch_row_ids) + # to fetch the weight per mini-batch + mod.prepare(None, sparse_row_id_fn=all_row_ids) + # evaluate metric on validation dataset + score = mod.score(val_iter, ['MSE']) + logging.info('epoch %d, eval MSE = %s ' % (epoch, score[0][1])) + # reset the iterator for next pass of data + train_iter.reset() + val_iter.reset() + logging.info('Training completed.') diff --git a/example/sparse/wide_deep/inference.py b/example/sparse/wide_deep/inference.py new file mode 100644 index 000000000000..e14396e50c15 --- /dev/null +++ b/example/sparse/wide_deep/inference.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet.test_utils import * +from config import * +from data import get_uci_adult +from model import wide_deep_model +import argparse +import os +import time + +parser = argparse.ArgumentParser(description="Run sparse wide and deep inference", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-infer-batch', type=int, default=100, + help='number of batches to inference') +parser.add_argument('--load-epoch', type=int, default=0, + help='loading the params of the corresponding training epoch.') +parser.add_argument('--batch-size', type=int, default=100, + help='number of examples per batch') +parser.add_argument('--benchmark', action='store_true', default=False, + help='run the script for benchmark mode, not set for accuracy test.') +parser.add_argument('--verbose', action='store_true', default=False, + help='accurcy for each batch will be logged if set') +parser.add_argument('--gpu', action='store_true', default=False, + help='Inference on GPU with CUDA') +parser.add_argument('--model-prefix', type=str, default='checkpoint', + help='the model prefix') + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_iters = args.num_infer_batch + batch_size = args.batch_size + benchmark = args.benchmark + verbose = args.verbose + model_prefix = args.model_prefix + load_epoch = args.load_epoch + ctx = mx.gpu(0) if args.gpu else mx.cpu() + # dataset + data_dir = os.path.join(os.getcwd(), 'data') + val_data = os.path.join(data_dir, ADULT['test']) + val_csr, val_dns, val_label = get_uci_adult(data_dir, ADULT['test'], ADULT['url']) + # load parameters and symbol + sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, load_epoch) + # data iterator + eval_data = mx.io.NDArrayIter({'csr_data': val_csr, 'dns_data': val_dns}, + {'softmax_label': val_label}, batch_size, + shuffle=True, last_batch_handle='discard') + # module + mod = mx.mod.Module(symbol=sym, context=ctx, data_names=['csr_data', 'dns_data'], + label_names=['softmax_label']) + mod.bind(data_shapes=eval_data.provide_data, label_shapes=eval_data.provide_label) + # get the sparse weight parameter + mod.set_params(arg_params=arg_params, aux_params=aux_params) + + data_iter = iter(eval_data) + nbatch = 0 + if benchmark: + logging.info('Inference benchmark started ...') + tic = time.time() + for i in range(num_iters): + try: + batch = data_iter.next() + except StopIteration: + data_iter.reset() + else: + mod.forward(batch, is_train=False) + for output in mod.get_outputs(): + output.wait_to_read() + nbatch += 1 + score = (nbatch*batch_size)/(time.time() - tic) + logging.info('batch size %d, process %s samples/s' % (batch_size, score)) + else: + logging.info('Inference started ...') + # use accuracy as the metric + metric = mx.metric.create(['acc']) + accuracy_avg = 0.0 + for batch in data_iter: + nbatch += 1 + metric.reset() + mod.forward(batch, is_train=False) + mod.update_metric(metric, batch.label) + accuracy_avg += metric.get()[1][0] + if args.verbose: + logging.info('batch %d, accuracy = %s' % (nbatch, metric.get())) + logging.info('averged accuracy on eval set is %.5f' % (accuracy_avg/nbatch)) diff --git a/example/sparse/wide_deep/train.py b/example/sparse/wide_deep/train.py new file mode 100644 index 000000000000..eea70301660d --- /dev/null +++ b/example/sparse/wide_deep/train.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet.test_utils import * +from config import * +from data import get_uci_adult +from model import wide_deep_model +import argparse +import os + + +parser = argparse.ArgumentParser(description="Run sparse wide and deep classification ", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--num-epoch', type=int, default=10, + help='number of epochs to train') +parser.add_argument('--batch-size', type=int, default=100, + help='number of examples per batch') +parser.add_argument('--lr', type=float, default=0.001, + help='learning rate') +parser.add_argument('--gpu', action='store_true', default=False, + help='Train on GPU with CUDA') +parser.add_argument('--optimizer', type=str, default='adam', + help='what optimizer to use', + choices=["ftrl", "sgd", "adam"]) +parser.add_argument('--log-interval', type=int, default=100, + help='number of batches to wait before logging training status') + + +if __name__ == '__main__': + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + # arg parser + args = parser.parse_args() + logging.info(args) + num_epoch = args.num_epoch + batch_size = args.batch_size + optimizer = args.optimizer + log_interval = args.log_interval + lr = args.lr + ctx = mx.gpu(0) if args.gpu else mx.cpu() + + # dataset + data_dir = os.path.join(os.getcwd(), 'data') + train_data = os.path.join(data_dir, ADULT['train']) + val_data = os.path.join(data_dir, ADULT['test']) + train_csr, train_dns, train_label = get_uci_adult(data_dir, ADULT['train'], ADULT['url']) + val_csr, val_dns, val_label = get_uci_adult(data_dir, ADULT['test'], ADULT['url']) + + model = wide_deep_model(ADULT['num_linear_features'], ADULT['num_embed_features'], + ADULT['num_cont_features'], ADULT['embed_input_dims'], + ADULT['hidden_units']) + + # data iterator + train_data = mx.io.NDArrayIter({'csr_data': train_csr, 'dns_data': train_dns}, + {'softmax_label': train_label}, batch_size, + shuffle=True, last_batch_handle='discard') + eval_data = mx.io.NDArrayIter({'csr_data': val_csr, 'dns_data': val_dns}, + {'softmax_label': val_label}, batch_size, + shuffle=True, last_batch_handle='discard') + + # module + mod = mx.mod.Module(symbol=model, context=ctx, data_names=['csr_data', 'dns_data'], + label_names=['softmax_label']) + mod.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label) + mod.init_params() + optim = mx.optimizer.create(optimizer, learning_rate=lr, rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=optim) + # use accuracy as the metric + metric = mx.metric.create(['acc']) + # get the sparse weight parameter + speedometer = mx.callback.Speedometer(batch_size, log_interval) + + logging.info('Training started ...') + + data_iter = iter(train_data) + for epoch in range(num_epoch): + nbatch = 0 + metric.reset() + for batch in data_iter: + nbatch += 1 + mod.forward_backward(batch) + # update all parameters (including the weight parameter) + mod.update() + # update training metric + mod.update_metric(metric, batch.label) + speedometer_param = mx.model.BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=metric, locals=locals()) + speedometer(speedometer_param) + # evaluate metric on validation dataset + score = mod.score(eval_data, ['acc']) + logging.info('epoch %d, accuracy = %s' % (epoch, score[0][1])) + + mod.save_checkpoint("checkpoint", epoch, save_optimizer_states=True) + # reset the iterator for next pass of data + data_iter.reset() + + logging.info('Training completed.') diff --git a/example/speech_recognition/stt_metric.py b/example/speech_recognition/stt_metric.py new file mode 100644 index 000000000000..26609627ea58 --- /dev/null +++ b/example/speech_recognition/stt_metric.py @@ -0,0 +1,252 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np + +from label_util import LabelUtil +from log_util import LogUtil + + +def check_label_shapes(labels, preds, shape=0): + """Check to see if the two arrays are the same size.""" + + if shape == 0: + label_shape, pred_shape = len(labels), len(preds) + else: + label_shape, pred_shape = labels.shape, preds.shape + + if label_shape != pred_shape: + raise ValueError("Shape of labels {} does not match shape of " + "predictions {}".format(label_shape, pred_shape)) + + +class STTMetric(mx.metric.EvalMetric): + def __init__(self, batch_size, num_gpu, is_epoch_end=False, is_logging=True): + super(STTMetric, self).__init__('STTMetric') + + self.batch_size = batch_size + self.num_gpu = num_gpu + self.total_n_label = 0 + self.total_l_dist = 0 + self.is_epoch_end = is_epoch_end + self.total_ctc_loss = 0. + self.batch_loss = 0. + self.is_logging = is_logging + + def update(self, labels, preds): + check_label_shapes(labels, preds) + if self.is_logging: + log = LogUtil.getInstance().getlogger() + labelUtil = LabelUtil.getInstance() + self.batch_loss = 0. + + for label, pred in zip(labels, preds): + label = label.asnumpy() + pred = pred.asnumpy() + + seq_length = len(pred) / int(int(self.batch_size) / int(self.num_gpu)) + + for i in range(int(int(self.batch_size) / int(self.num_gpu))): + l = remove_blank(label[i]) + p = [] + for k in range(int(seq_length)): + p.append(np.argmax(pred[k * int(int(self.batch_size) / int(self.num_gpu)) + i])) + p = pred_best(p) + + l_distance = levenshtein_distance(l, p) + self.total_n_label += len(l) + self.total_l_dist += l_distance + this_cer = float(l_distance) / float(len(l)) + if self.is_logging: + log.info("label: %s " % (labelUtil.convert_num_to_word(l))) + log.info("pred : %s , cer: %f (distance: %d/ label length: %d)" % ( + labelUtil.convert_num_to_word(p), this_cer, l_distance, len(l))) + self.num_inst += 1 + self.sum_metric += this_cer + if self.is_epoch_end: + loss = ctc_loss(l, pred, i, int(seq_length), int(self.batch_size), int(self.num_gpu)) + self.batch_loss += loss + if self.is_logging: + log.info("loss: %f " % loss) + self.total_ctc_loss += self.batch_loss + + def get_batch_loss(self): + return self.batch_loss + + def get_name_value(self): + try: + total_cer = float(self.total_l_dist) / float(self.total_n_label) + except ZeroDivisionError: + total_cer = float('inf') + + return total_cer, self.total_n_label, self.total_l_dist, self.total_ctc_loss + + def reset(self): + self.total_n_label = 0 + self.total_l_dist = 0 + self.num_inst = 0 + self.sum_metric = 0.0 + self.total_ctc_loss = 0.0 + + +def pred_best(p): + ret = [] + p1 = [0] + p + for i in range(len(p)): + c1 = p1[i] + c2 = p1[i + 1] + if c2 == 0 or c2 == c1: + continue + ret.append(c2) + return ret + + +def remove_blank(l): + ret = [] + for i in range(l.size): + if l[i] == 0: + break + ret.append(l[i]) + return ret + + +def remove_space(l): + labelUtil = LabelUtil.getInstance() + ret = [] + for i in range(len(l)): + if l[i] != labelUtil.get_space_index(): + ret.append(l[i]) + return ret + + +def ctc_loss(label, prob, remainder, seq_length, batch_size, num_gpu=1, big_num=1e10): + label_ = [0, 0] + prob[prob < 1 / big_num] = 1 / big_num + log_prob = np.log(prob) + + l = len(label) + for i in range(l): + label_.append(int(label[i])) + label_.append(0) + + l_ = 2 * l + 1 + a = np.full((seq_length, l_ + 1), -big_num) + a[0][1] = log_prob[remainder][0] + a[0][2] = log_prob[remainder][label_[2]] + for i in range(1, seq_length): + row = i * int(batch_size / num_gpu) + remainder + a[i][1] = a[i - 1][1] + log_prob[row][0] + a[i][2] = np.logaddexp(a[i - 1][2], a[i - 1][1]) + log_prob[row][label_[2]] + for j in range(3, l_ + 1): + a[i][j] = np.logaddexp(a[i - 1][j], a[i - 1][j - 1]) + if label_[j] != 0 and label_[j] != label_[j - 2]: + a[i][j] = np.logaddexp(a[i][j], a[i - 1][j - 2]) + a[i][j] += log_prob[row][label_[j]] + + return -np.logaddexp(a[seq_length - 1][l_], a[seq_length - 1][l_ - 1]) + + +# label is done with remove_blank +# pred is got from pred_best +def levenshtein_distance(label, pred): + n_label = len(label) + 1 + n_pred = len(pred) + 1 + if (label == pred): + return 0 + if (len(label) == 0): + return len(pred) + if (len(pred) == 0): + return len(label) + + v0 = [i for i in range(n_label)] + v1 = [0 for i in range(n_label)] + + for i in range(len(pred)): + v1[0] = i + 1 + + for j in range(len(label)): + cost = 0 if label[j] == pred[i] else 1 + v1[j + 1] = min(v1[j] + 1, v0[j + 1] + 1, v0[j] + cost) + + for j in range(n_label): + v0[j] = v1[j] + + return v1[len(label)] + + +def char_match_1way(char_label, char_pred, criteria, n_whole_label): + n_label = len(char_label) + n_pred = len(char_pred) + + pred_pos = 0 + accuracy = 0. + next_accu = 0. + n_matched = 0. + next_n_matched = 0. + + for i_index in range(n_label): + tail_label = n_label - 1 - i_index + c_label = char_label[i_index] + + for j_index in range(pred_pos, n_pred): + tail_pred = n_pred - 1 - j_index + c_pred = char_pred[j_index] + + if tail_label < tail_pred * criteria or tail_pred < tail_label * criteria: + break + if c_label == c_pred: + n_matched += 1.0 + pred_pos = j_index + 1 + break + + accuracy = n_matched / n_whole_label + + if n_label > 0.7 * n_whole_label: + next_label = char_label[1:] + next_accu, next_n_matched = char_match_1way(next_label, char_pred, criteria, n_whole_label) + + if next_accu > accuracy: + accuracy = next_accu + n_matched = next_n_matched + return accuracy, n_matched + + +def char_match_2way(label, pred): + criterias = [0.98, 0.96, 0.93, 0.9, 0.85, 0.8, 0.7] + r_pred = pred[::-1] + r_label = label[::-1] + n_whole_label = len(remove_space(label)) + + val1_max = 0. + val2_max = 0. + val1_max_matched = 0. + val2_max_matched = 0. + for criteria in criterias: + val1, val1_matched = char_match_1way(label, pred, criteria, n_whole_label) + val2, val2_matched = char_match_1way(r_label, r_pred, criteria, n_whole_label) + + if val1 > val1_max: + val1_max = val1 + val1_max_matched = val1_matched + if val2 > val2_max: + val2_max = val2 + val2_max_matched = val2_matched + + val = val1_max if val1_max > val2_max else val2_max + val_matched = val1_max_matched if val1_max > val2_max else val2_max_matched + return val, val_matched, n_whole_label diff --git a/example/ssd/evaluate/eval_metric.py b/example/ssd/evaluate/eval_metric.py new file mode 100644 index 000000000000..1deb381fb859 --- /dev/null +++ b/example/ssd/evaluate/eval_metric.py @@ -0,0 +1,295 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np + +class MApMetric(mx.metric.EvalMetric): + """ + Calculate mean AP for object detection task + + Parameters: + --------- + ovp_thresh : float + overlap threshold for TP + use_difficult : boolean + use difficult ground-truths if applicable, otherwise just ignore + class_names : list of str + optional, if provided, will print out AP for each class + pred_idx : int + prediction index in network output list + """ + def __init__(self, ovp_thresh=0.5, use_difficult=False, class_names=None, pred_idx=0): + super(MApMetric, self).__init__('mAP') + if class_names is None: + self.num = None + else: + assert isinstance(class_names, (list, tuple)) + for name in class_names: + assert isinstance(name, str), "must provide names as str" + num = len(class_names) + self.name = class_names + ['mAP'] + self.num = num + 1 + self.reset() + self.ovp_thresh = ovp_thresh + self.use_difficult = use_difficult + self.class_names = class_names + self.pred_idx = int(pred_idx) + + def reset(self): + """Clear the internal statistics to initial state.""" + if getattr(self, 'num', None) is None: + self.num_inst = 0 + self.sum_metric = 0.0 + else: + self.num_inst = [0] * self.num + self.sum_metric = [0.0] * self.num + self.records = dict() + self.counts = dict() + + def get(self): + """Get the current evaluation result. + + Returns + ------- + name : str + Name of the metric. + value : float + Value of the evaluation. + """ + self._update() # update metric at this time + if self.num is None: + if self.num_inst == 0: + return (self.name, float('nan')) + else: + return (self.name, self.sum_metric / self.num_inst) + else: + names = ['%s'%(self.name[i]) for i in range(self.num)] + values = [x / y if y != 0 else float('nan') \ + for x, y in zip(self.sum_metric, self.num_inst)] + return (names, values) + + def update(self, labels, preds): + """ + Update internal records. This function now only update internal buffer, + sum_metric and num_inst are updated in _update() function instead when + get() is called to return results. + + Params: + ---------- + labels: mx.nd.array (n * 6) or (n * 5), difficult column is optional + 2-d array of ground-truths, n objects(id-xmin-ymin-xmax-ymax-[difficult]) + preds: mx.nd.array (m * 6) + 2-d array of detections, m objects(id-score-xmin-ymin-xmax-ymax) + """ + def iou(x, ys): + """ + Calculate intersection-over-union overlap + Params: + ---------- + x : numpy.array + single box [xmin, ymin ,xmax, ymax] + ys : numpy.array + multiple box [[xmin, ymin, xmax, ymax], [...], ] + Returns: + ----------- + numpy.array + [iou1, iou2, ...], size == ys.shape[0] + """ + ixmin = np.maximum(ys[:, 0], x[0]) + iymin = np.maximum(ys[:, 1], x[1]) + ixmax = np.minimum(ys[:, 2], x[2]) + iymax = np.minimum(ys[:, 3], x[3]) + iw = np.maximum(ixmax - ixmin, 0.) + ih = np.maximum(iymax - iymin, 0.) + inters = iw * ih + uni = (x[2] - x[0]) * (x[3] - x[1]) + (ys[:, 2] - ys[:, 0]) * \ + (ys[:, 3] - ys[:, 1]) - inters + ious = inters / uni + ious[uni < 1e-12] = 0 # in case bad boxes + return ious + + # independant execution for each image + for i in range(labels[0].shape[0]): + # get as numpy arrays + label = labels[0][i].asnumpy() + if np.sum(label[:, 0] >= 0) < 1: + continue + pred = preds[self.pred_idx][i].asnumpy() + # calculate for each class + while (pred.shape[0] > 0): + cid = int(pred[0, 0]) + indices = np.where(pred[:, 0].astype(int) == cid)[0] + if cid < 0: + pred = np.delete(pred, indices, axis=0) + continue + dets = pred[indices] + pred = np.delete(pred, indices, axis=0) + # sort by score, desceding + dets = dets[dets[:,1].argsort()[::-1]] + records = np.hstack((dets[:, 1][:, np.newaxis], np.zeros((dets.shape[0], 1)))) + # ground-truths + label_indices = np.where(label[:, 0].astype(int) == cid)[0] + gts = label[label_indices, :] + label = np.delete(label, label_indices, axis=0) + if gts.size > 0: + found = [False] * gts.shape[0] + for j in range(dets.shape[0]): + # compute overlaps + ious = iou(dets[j, 2:], gts[:, 1:5]) + ovargmax = np.argmax(ious) + ovmax = ious[ovargmax] + if ovmax > self.ovp_thresh: + if (not self.use_difficult and + gts.shape[1] >= 6 and + gts[ovargmax, 5] > 0): + pass + else: + if not found[ovargmax]: + records[j, -1] = 1 # tp + found[ovargmax] = True + else: + # duplicate + records[j, -1] = 2 # fp + else: + records[j, -1] = 2 # fp + else: + # no gt, mark all fp + records[:, -1] = 2 + + # ground truth count + if (not self.use_difficult and gts.shape[1] >= 6): + gt_count = np.sum(gts[:, 5] < 1) + else: + gt_count = gts.shape[0] + + # now we push records to buffer + # first column: score, second column: tp/fp + # 0: not set(matched to difficult or something), 1: tp, 2: fp + records = records[np.where(records[:, -1] > 0)[0], :] + if records.size > 0: + self._insert(cid, records, gt_count) + + # add missing class if not present in prediction + while (label.shape[0] > 0): + cid = int(label[0, 0]) + label_indices = np.where(label[:, 0].astype(int) == cid)[0] + label = np.delete(label, label_indices, axis=0) + if cid < 0: + continue + gt_count = label_indices.size + self._insert(cid, np.array([[0, 0]]), gt_count) + + def _update(self): + """ update num_inst and sum_metric """ + aps = [] + for k, v in self.records.items(): + recall, prec = self._recall_prec(v, self.counts[k]) + ap = self._average_precision(recall, prec) + aps.append(ap) + if self.num is not None and k < (self.num - 1): + self.sum_metric[k] = ap + self.num_inst[k] = 1 + if self.num is None: + self.num_inst = 1 + self.sum_metric = np.mean(aps) + else: + self.num_inst[-1] = 1 + self.sum_metric[-1] = np.mean(aps) + + def _recall_prec(self, record, count): + """ get recall and precision from internal records """ + record = np.delete(record, np.where(record[:, 1].astype(int) == 0)[0], axis=0) + sorted_records = record[record[:,0].argsort()[::-1]] + tp = np.cumsum(sorted_records[:, 1].astype(int) == 1) + fp = np.cumsum(sorted_records[:, 1].astype(int) == 2) + if count <= 0: + recall = tp * 0.0 + else: + recall = tp / float(count) + prec = tp.astype(float) / (tp + fp) + return recall, prec + + def _average_precision(self, rec, prec): + """ + calculate average precision + + Params: + ---------- + rec : numpy.array + cumulated recall + prec : numpy.array + cumulated precision + Returns: + ---------- + ap as float + """ + # append sentinel values at both ends + mrec = np.concatenate(([0.], rec, [1.])) + mpre = np.concatenate(([0.], prec, [0.])) + + # compute precision integration ladder + for i in range(mpre.size - 1, 0, -1): + mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) + + # look for recall value changes + i = np.where(mrec[1:] != mrec[:-1])[0] + + # sum (\delta recall) * prec + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) + return ap + + def _insert(self, key, records, count): + """ Insert records according to key """ + if key not in self.records: + assert key not in self.counts + self.records[key] = records + self.counts[key] = count + else: + self.records[key] = np.vstack((self.records[key], records)) + assert key in self.counts + self.counts[key] += count + + +class VOC07MApMetric(MApMetric): + """ Mean average precision metric for PASCAL V0C 07 dataset """ + def __init__(self, *args, **kwargs): + super(VOC07MApMetric, self).__init__(*args, **kwargs) + + def _average_precision(self, rec, prec): + """ + calculate average precision, override the default one, + special 11-point metric + + Params: + ---------- + rec : numpy.array + cumulated recall + prec : numpy.array + cumulated precision + Returns: + ---------- + ap as float + """ + ap = 0. + for t in np.arange(0., 1.1, 0.1): + if np.sum(rec >= t) == 0: + p = 0 + else: + p = np.max(prec[rec >= t]) + ap += p / 11. + return ap diff --git a/example/ssd/train/metric.py b/example/ssd/train/metric.py new file mode 100644 index 000000000000..eeb9796bf4a8 --- /dev/null +++ b/example/ssd/train/metric.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np + + +class MultiBoxMetric(mx.metric.EvalMetric): + """Calculate metrics for Multibox training """ + def __init__(self, eps=1e-8): + super(MultiBoxMetric, self).__init__('MultiBox') + self.eps = eps + self.num = 2 + self.name = ['CrossEntropy', 'SmoothL1'] + self.reset() + + def reset(self): + """ + override reset behavior + """ + if getattr(self, 'num', None) is None: + self.num_inst = 0 + self.sum_metric = 0.0 + else: + self.num_inst = [0] * self.num + self.sum_metric = [0.0] * self.num + + def reset_local(self): + """ + override reset behavior + """ + if getattr(self, 'num', None) is None: + self.num_inst = 0 + self.sum_metric = 0.0 + else: + self.num_inst = [0] * self.num + self.sum_metric = [0.0] * self.num + + def update(self, labels, preds): + """ + Implementation of updating metrics + """ + # get generated multi label from network + cls_prob = preds[0].asnumpy() + loc_loss = preds[1].asnumpy() + cls_label = preds[2].asnumpy() + valid_count = np.sum(cls_label >= 0) + # overall accuracy & object accuracy + label = cls_label.flatten() + mask = np.where(label >= 0)[0] + indices = np.int64(label[mask]) + prob = cls_prob.transpose((0, 2, 1)).reshape((-1, cls_prob.shape[1])) + prob = prob[mask, indices] + self.sum_metric[0] += (-np.log(prob + self.eps)).sum() + self.num_inst[0] += valid_count + # smoothl1loss + self.sum_metric[1] += np.sum(loc_loss) + self.num_inst[1] += valid_count + + def get(self): + """Get the current evaluation result. + Override the default behavior + + Returns + ------- + name : str + Name of the metric. + value : float + Value of the evaluation. + """ + if self.num is None: + if self.num_inst == 0: + return (self.name, float('nan')) + else: + return (self.name, self.sum_metric / self.num_inst) + else: + names = ['%s'%(self.name[i]) for i in range(self.num)] + values = [x / y if y != 0 else float('nan') \ + for x, y in zip(self.sum_metric, self.num_inst)] + return (names, values) diff --git a/example/svm_mnist/svm_mnist.py b/example/svm_mnist/svm_mnist.py new file mode 100644 index 000000000000..e166cb6ac707 --- /dev/null +++ b/example/svm_mnist/svm_mnist.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +############################################################# +## Please read the README.md document for better reference ## +############################################################# +from __future__ import print_function + +import logging +import random + +import mxnet as mx +import numpy as np +from sklearn.datasets import fetch_mldata +from sklearn.decomposition import PCA + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +np.random.seed(1234) # set seed for deterministic ordering +mx.random.seed(1234) +random.seed(1234) + +# Network declaration as symbols. The following pattern was based +# on the article, but feel free to play with the number of nodes +# and with the activation function +data = mx.symbol.Variable('data') +fc1 = mx.symbol.FullyConnected(data = data, name='fc1', num_hidden=512) +act1 = mx.symbol.Activation(data = fc1, name='relu1', act_type="relu") +fc2 = mx.symbol.FullyConnected(data = act1, name = 'fc2', num_hidden = 512) +act2 = mx.symbol.Activation(data = fc2, name='relu2', act_type="relu") +fc3 = mx.symbol.FullyConnected(data = act2, name='fc3', num_hidden=10) + +# Here we add the ultimate layer based on L2-SVM objective +mlp_svm_l2 = mx.symbol.SVMOutput(data=fc3, name='svm_l2') + +# With L1-SVM objective +mlp_svm_l1 = mx.symbol.SVMOutput(data=fc3, name='svm_l1', use_linear=True) + +# Compare with softmax cross entropy loss +mlp_softmax = mx.symbol.SoftmaxOutput(data=fc3, name='softmax') + +print("Preparing data...") +mnist_data = mx.test_utils.get_mnist() +X = np.concatenate([mnist_data['train_data'], mnist_data['test_data']]) +Y = np.concatenate([mnist_data['train_label'], mnist_data['test_label']]) +X = X.reshape((X.shape[0], -1)).astype(np.float32) * 255 + +# Now we fetch MNIST dataset, add some noise, as the article suggests, +# permutate and assign the examples to be used on our network +mnist_pca = PCA(n_components=70).fit_transform(X) +noise = np.random.normal(size=mnist_pca.shape) +mnist_pca += noise +p = np.random.permutation(mnist_pca.shape[0]) +X = mnist_pca[p] / 255. +Y = Y[p] +X_show = X[p] + +# This is just to normalize the input and separate train set and test set +X_train = X[:60000] +X_test = X[60000:] +X_show = X_show[60000:] +Y_train = Y[:60000] +Y_test = Y[60000:] +print("Data prepared.") +# Article's suggestion on batch size +batch_size = 200 + +ctx = mx.gpu() if mx.context.num_gpus() > 0 else mx.cpu() + +results = {} +for output in [mlp_svm_l2, mlp_svm_l1, mlp_softmax]: + + print("\nTesting with %s \n" % output.name) + + label = output.name + "_label" + + train_iter = mx.io.NDArrayIter(X_train, Y_train, batch_size=batch_size, label_name=label) + test_iter = mx.io.NDArrayIter(X_test, Y_test, batch_size=batch_size, label_name=label) + + # Here we instatiate and fit the model for our data + # The article actually suggests using 400 epochs, + # But I reduced to 10, for convenience + + mod = mx.mod.Module( + context = ctx, + symbol = output, # Use the network we just defined + label_names = [label], + ) + mod.fit( + train_data=train_iter, + eval_data=test_iter, # Testing data set. MXNet computes scores on test set every epoch + batch_end_callback = mx.callback.Speedometer(batch_size, 200), # Logging module to print out progress + num_epoch = 10, # Train for 10 epochs + optimizer_params = { + 'learning_rate': 0.1, # Learning rate + 'momentum': 0.9, # Momentum for SGD with momentum + 'wd': 0.00001, # Weight decay for regularization + }) + results[output.name] = mod.score(test_iter, mx.metric.Accuracy())[0][1]*100 + print('Accuracy for %s:'%output.name, mod.score(test_iter, mx.metric.Accuracy())[0][1]*100, '%\n') + +for key, value in results.items(): + print(key, value, "%s") + +#svm_l2 97.85 %s +#svm_l1 98.15 %s +#softmax 97.69 %s diff --git a/example/svrg_module/api_usage_example/example_api_train.py b/example/svrg_module/api_usage_example/example_api_train.py new file mode 100644 index 000000000000..f6cd1b2e592c --- /dev/null +++ b/example/svrg_module/api_usage_example/example_api_train.py @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import mxnet as mx +import numpy as np +from mxnet.contrib.svrg_optimization.svrg_module import SVRGModule + + +def test_svrg_intermediate_level_api(args): + """Demonstrates intermediate level SVRGModule API where the training process + need to be explicitly defined. KVstore is not explicitly created. + + Parameters + ---------- + args: args + Command line arguments + """ + num_epoch = args.epochs + batch_size = args.batch_size + update_freq = args.update_freq + + di, mod = create_network(batch_size, update_freq) + + mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label) + mod.init_params(initializer=mx.init.Uniform(0.01), allow_missing=False, force_init=False, allow_extra=False) + kv = mx.kv.create("local") + mod.init_optimizer(kvstore=kv, optimizer='sgd', optimizer_params=(('learning_rate', 0.025),)) + metrics = mx.metric.create("mse") + for e in range(num_epoch): + metrics.reset() + if e % mod.update_freq == 0: + mod.update_full_grads(di) + di.reset() + for batch in di: + mod.forward_backward(data_batch=batch) + mod.update() + mod.update_metric(metrics, batch.label) + mod.logger.info('Epoch[%d] Train cost=%f', e, metrics.get()[1]) + + +def test_svrg_high_level_api(args): + """Demonstrates suggested usage of high level SVRGModule API. KVStore is explicitly created. + + Parameters + ---------- + args: args + Command line arguments + """ + num_epoch = args.epochs + batch_size = args.batch_size + update_freq = args.update_freq + + di, mod = create_network(batch_size, update_freq) + mod.fit(di, eval_metric='mse', optimizer='sgd', optimizer_params=(('learning_rate', 0.025),), num_epoch=num_epoch, + kvstore='local') + + +def create_network(batch_size, update_freq): + """Create a linear regression network for performing SVRG optimization. + Parameters + ---------- + batch_size: int + Size of data split + update_freq: int + Update Frequency for calculating full gradients + + Returns + ---------- + di: mx.io.NDArrayIter + Data iterator + update_freq: SVRGModule + An instance of SVRGModule for performing SVRG optimization + """ + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + + train_data = np.random.randint(1, 5, [1000, 2]) + weights = np.array([1.0, 2.0]) + train_label = train_data.dot(weights) + + di = mx.io.NDArrayIter(train_data, train_label, batch_size=batch_size, shuffle=True, label_name='lin_reg_label') + X = mx.sym.Variable('data') + Y = mx.symbol.Variable('lin_reg_label') + fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden=1) + lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro") + + mod = SVRGModule( + symbol=lro, + data_names=['data'], + label_names=['lin_reg_label'], update_freq=update_freq, logger=logging + ) + + return di, mod + +# run as a script +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument('-e', dest='epochs', default=100, type=int) + parser.add_argument('-bs', dest='batch_size', default=32, type=int) + parser.add_argument('-f', dest="update_freq", default=2, type=int) + args = parser.parse_args() + + print("========================== Intermediate Level API ==========================") + test_svrg_intermediate_level_api(args) + print("========================== High Level API ==========================") + test_svrg_high_level_api(args) diff --git a/example/svrg_module/api_usage_example/example_inference.py b/example/svrg_module/api_usage_example/example_inference.py new file mode 100644 index 000000000000..312f9796074d --- /dev/null +++ b/example/svrg_module/api_usage_example/example_inference.py @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import mxnet as mx +import numpy as np +import logging +from mxnet.contrib.svrg_optimization.svrg_module import SVRGModule + + +def test_svrg_inference(args): + epoch = args.epochs + batch_size = args.batch_size + update_freq = args.update_freq + + train_iter, val_iter, mod = create_network(batch_size, update_freq) + mod.fit(train_iter, eval_data=val_iter, eval_metric='mse', optimizer='sgd', + optimizer_params=(('learning_rate', 0.025),), + num_epoch=epoch) + + +def get_validation_score(args): + epoch = args.epochs + batch_size = args.batch_size + update_freq = args.update_freq + + train_iter, val_iter, mod = create_network(batch_size, update_freq) + mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) + mod.init_params(initializer=mx.init.Uniform(0.01), allow_missing=False, force_init=False, allow_extra=False) + mod.init_optimizer(kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.025),)) + metrics = mx.metric.create("mse") + for e in range(epoch): + metrics.reset() + if e % mod.update_freq == 0: + mod.update_full_grads(train_iter) + train_iter.reset() + for batch in train_iter: + mod.forward_backward(data_batch=batch) + mod.update() + mod.update_metric(metrics, batch.label) + + y = mod.predict(val_iter) + + # test-train data split, 20% test data out of 1000 data samples + assert y.shape == (200, 1) + score = mod.score(val_iter, ['mse']) + print("Training Loss on Validation Set is {}".format(score[0][1])) + + +def create_network(batch_size, update_freq): + """Create a linear regression network for performing SVRG optimization. + :return: an instance of mx.io.NDArrayIter + :return: an instance of mx.mod.svrgmodule for performing SVRG optimization + """ + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.INFO, format=head) + data = np.random.randint(1, 5, [1000, 2]) + + #Test_Train data split + n_train = int(data.shape[0] * 0.8) + weights = np.array([1.0, 2.0]) + label = data.dot(weights) + + di = mx.io.NDArrayIter(data[:n_train, :], label[:n_train], batch_size=batch_size, shuffle=True, label_name='lin_reg_label') + val_iter = mx.io.NDArrayIter(data[n_train:, :], label[n_train:], batch_size=batch_size) + + X = mx.sym.Variable('data') + Y = mx.symbol.Variable('lin_reg_label') + fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden=1) + lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro") + + mod = SVRGModule( + symbol=lro, + data_names=['data'], + label_names=['lin_reg_label'], update_freq=update_freq, logger=logging) + + return di, val_iter, mod + + +# run as a script +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-e', dest='epochs', default=100, type=int) + parser.add_argument('-bs', dest='batch_size', default=32, type=int) + parser.add_argument('-f', dest="update_freq", default=2, type=int) + args = parser.parse_args() + + print("========================== SVRG Module Inference ==========================") + test_svrg_inference(args) + print("========================SVRG Module Score ============================") + get_validation_score(args) diff --git a/example/svrg_module/benchmarks/svrg_benchmark.ipynb b/example/svrg_module/benchmarks/svrg_benchmark.ipynb new file mode 100644 index 000000000000..54ae81281db3 --- /dev/null +++ b/example/svrg_module/benchmarks/svrg_benchmark.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Linear Regression Using SVRGModule on YearPredictionMSD Dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this notebook, a linear regression model will be fit on YearPredictionMSD dataset, which contains predictions of the release year of a song based on its audio features. The dataset has 90 features and over 400,000 samples. The dataset is downsampled to 5,000 in this experiment." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import sys\n", + "import tempfile\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.patches as mpatches\n", + "import mxnet as mx\n", + "from mxnet.contrib.svrg_optimization.svrg_module import SVRGModule\n", + "import numpy as np\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "from sklearn.datasets import load_svmlight_file\n", + "\n", + "sys.path.insert(0, \"../linear_regression\")\n", + "from data_reader import get_year_prediction_data\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Read Data\n", + "The first step is to get the training features and labels and normalize the data. In this example, we will use 5000 data samples. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting data...\n", + "Reading data from disk...\n" + ] + } + ], + "source": [ + "feature_dim, train_features, train_labels = get_year_prediction_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "train_features = train_features[-5000:]\n", + "train_labels = train_labels[-5000:]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create Linear Regression Network" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def create_lin_reg_network(batch_size=100):\n", + " train_iter = mx.io.NDArrayIter(train_features, train_labels, batch_size=batch_size, shuffle=True,\n", + " data_name='data', label_name='label')\n", + " data = mx.sym.Variable(\"data\")\n", + " label = mx.sym.Variable(\"label\")\n", + " weight = mx.sym.Variable(\"fc_weight\", shape=(1, 90))\n", + " net = mx.sym.dot(data, weight.transpose())\n", + " bias = mx.sym.Variable(\"fc_bias\", shape=(1,), wd_mult=0.0, lr_mult=10.0)\n", + " net = mx.sym.broadcast_plus(net, bias)\n", + " net = mx.sym.LinearRegressionOutput(data=net, label=label)\n", + " \n", + " return train_iter, net" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SVRGModule with SVRG Optimization\n", + "In this example, we will use intermediate level API for SVRGModule and the dump mse per epoch to JSON file for plotting graphs." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def train_svrg_lin_reg(num_epoch=100, batch_size=100, update_freq=2, output='svrg_lr.json', \n", + " optimizer_params=None):\n", + "\n", + " di, net = create_lin_reg_network(batch_size=batch_size)\n", + " \n", + " #Create a SVRGModule\n", + " mod = SVRGModule(symbol=net, context=mx.cpu(0), data_names=['data'], label_names=['label'], update_freq=update_freq)\n", + " mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label)\n", + " mod.init_params(initializer=mx.init.Zero(), allow_missing=False, force_init=False, allow_extra=False)\n", + " mod.init_optimizer(kvstore='local', optimizer='sgd', optimizer_params=optimizer_params)\n", + " metrics = mx.metric.create(\"mse\")\n", + " \n", + " results = {}\n", + " for e in range(num_epoch):\n", + " results[e] = {}\n", + " metrics.reset()\n", + " if e % mod.update_freq == 0:\n", + " mod.update_full_grads(di)\n", + " di.reset()\n", + " for batch in di:\n", + " mod.forward_backward(data_batch=batch)\n", + " mod.update()\n", + " mod.update_metric(metrics, batch.label)\n", + " results[e][\"mse\"] = metrics.get()[1]\n", + " \n", + " f = open(output, 'w+')\n", + " f.write(json.dumps(results, indent=4, sort_keys=True))\n", + " f.close()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Module with SGD Optimization " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def train_sgd_lin_reg(num_epoch=100, batch_size=100, update_freq=2, output='sgd_lr.json', \n", + " optimizer_params=None):\n", + " \n", + " di, net = create_lin_reg_network(batch_size=batch_size)\n", + " \n", + " #Create a standard module\n", + " mod = mx.mod.Module(symbol=net, context=mx.cpu(0), data_names=['data'], label_names=['label'])\n", + " mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label)\n", + " mod.init_params(initializer=mx.init.Zero(), allow_missing=False, force_init=False, allow_extra=False)\n", + " mod.init_optimizer(kvstore='local', optimizer='sgd', optimizer_params=optimizer_params)\n", + " metrics = mx.metric.create(\"mse\")\n", + " \n", + " results = {}\n", + " for e in range(num_epoch):\n", + " results[e] = {}\n", + " metrics.reset()\n", + " di.reset()\n", + " for batch in di:\n", + " mod.forward_backward(data_batch=batch)\n", + " mod.update()\n", + " mod.update_metric(metrics, batch.label)\n", + " results[e][\"mse\"] = metrics.get()[1]\n", + " f = open(output, 'w+')\n", + " f.write(json.dumps(results, indent=4, sort_keys=True))\n", + " f.close()\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training Loss over 100 Epochs Using lr_scheduler\n", + "When a large learning rate is used with SGD, training loss will drop fast but will oscillates above the minimum and never converges. With a small learning rate, it will eventually reach the minimum after many iterations. A common practice is to use learning rate scheduling by starting with a large learning rate and gradually decreasing it. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "train_svrg_lin_reg(optimizer_params={'lr_scheduler': mx.lr_scheduler.FactorScheduler(step=10, factor=0.99)})\n", + "train_sgd_lin_reg(optimizer_params={'lr_scheduler': mx.lr_scheduler.FactorScheduler(step=10, factor=0.99)})" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5,0,'Epochs')" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABKIAAALMCAYAAADXShqaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xmc5XV5J/rPc051V3UViMriCC2CI0bFVlTQGBTnxklsgdF4nUxkJIm5JjF3NDrEEMxczeJkMnOzR6Nm4oxLFhK8xsyYCOISjctoEBAXFkcgCA2YNK0gWxVdVd/7xzldnC6ququ7TlV1db/fr9d58dvPU6f56/N6nu+vWmsBAAAAgJXWWesCAAAAADg0CKIAAAAAWBWCKAAAAABWhSAKAAAAgFUhiAIAAABgVQiiAAAAAFgVgigAYN2oqm5V3VNVxw/zWg5sVfWTVfWpta4DAFg+QRQAsGL6QdCuz2xV3T+w/4p9fV5rbaa1dlhr7eZhXruvqurXquq9w37uEr+7U1UXVNX1/d/zm1X1n6pq4yp9/7/s/1veM+9z2mp8PwCwvo2sdQEAwMGrtXbYru2quinJT7bWPr7Y9VU10lqbXo3a1rG3J/n+JK9IckWSJyZ5b5InJfk/h/lFe/j3uLm1dsIwvwsAODToiAIA1ky/s+iiqvrzqro7yblV9Zyq+kJV3VlVt1fVW6tqQ//6kapqVXVCf/9P++cvqaq7q+rzVXXivl7bP/+iqvrfVXVXVb2tqj5XVa/cj7/p5Kr6u379X62qswbOnV1V1/a/f1tVndc/fkxVXdy/59tV9elFnv3EJK9Ock5r7e9ba9Otta8l+ddJzq6qM6rq9Kq6tao6A/f9cFVd2d/uVNV/qKobquqOqvqLqnpE/9zj+7/ZT1TVzUk+uh9//2f7HVqX93/Lv9r1/P75l1bV1f2/9W+r6nsGzj22qv5HVW3v1/b7uz+6frd/341V9YMDJ15VVTf1f9cbq+rl+1o3ALA6BFEAwFp7aZILkxyR5KIk00len+SoJKcn2Zpe+LKYf5vkzUkemeTmJP9xX6+tqmOSvD/J+f3v/Yckz9rXP6Q/Hvc3ST6c5Ogk5yW5qKoe37/kPUle1Vo7PMlTk/xd//j5SW7s3/PPkrxpka/4l0n+obV25eDB1tpNSb6Y5AeS/K8kO5M8f97ffWF/+7wkZyU5I8nmJPckeeu87zkjvU6rs7J/fqz/OTZJJfndJKmqJyX5kyQ/m97f+vEkH6qqDVU1kt7vdn2SE5I8Jr1/k12+L8lXkxzZf95/7z/zYUl+J8kP9H/X05N8ZT/rBgBWmCAKAFhrn22t/XVrbba1dn9r7YsD3T43Jvmj7B6qzPeB1trlrbWdSf4sySn7ce3ZSa5qrf3P/rnfTXLHfvwtpyfZmOQ3W2s7+2OIlyTZ1aGzM8mTq+rw1tq3BwKlnemFNse31h5orS3YEZVeSHb7IuduT3JUa60l+Ysk5yRJVT08yQv7x5LkZ5L8h9bara21ySS/muSHBzuokvxya+2+1tr9i3zX8f3OpMHP6MD597XWrmmt3Zvkl5K8vKqq/zt8qLX2t/3f+b+kF0A+O8lz+n/fBa21e/v/L3xu4Jk3tNbe3VqbSfK+JJur6qj+uZbkKVU11lq7vbV2zSJ1AwBrTBAFAKy1WwZ3quqJVfXhqvpWVX03yVvSCygW862B7fuSHLbYhXu49tjBOvphzrYl1D7fsemtn9QGjn0zyXH97ZcmeXGSm6vqU1X17P7x/9K/7hP9kbnzF3n+HUkevci5R+fB8OzCJC/rjzS+LMnft9Z2/T3HJ/nrXQFSel1GSXLMwLN2+zdZwM2ttYfP+0wtcv83k4ym14V2bH8/SdJam03vdz4uvQ6om/pB00Lm/9slyWGtte+mF7q9Jsm3qupvquoJe6kfAFgjgigAYK21efv/NcnXkjy+tfaw9DpqaoVruD29MbUkvcWI8mB4tC9uS/KY/v27HJ/k1iTpd3q9OL3Q52/S71JqrX23tXZefwHwH0pyQVUt1AX2iSQnVtUzBg/218E6rX8+rbWvpBfcvDC7j+UlveDnB+aFSGOttbmgZ16Qtj8eM7B9fJKpJN9O7/d57EDdnfR+91vTC68eW1Xdff2y1tolrbV/mV4Yd316/w8BAAcgQRQAcKA5PMldSe7trym0p/WhhuVvkjyjqv5Vf62i16e3htGedKtqbOAzmt76TNNJ3tBf9+j7k5yZ3jpRm6rq31bVw/pjaXcnmU2S/vf+836AdVeSmV3nBrXWrk3y35L8eVU9q6q6VfWUJB9Icklr7VMDl1+Y3npQz+mf3+UPk/x6VR3f/+5jqurF+/BbLcWP9TvbJtIb/Xt/P9x6f5IXV9W/6HdrnZ/e7/D3ST6fZEe/tvH+73X63r6oqh7d//3GkzyQ5N4s8NsBAAcGQRQAcKB5Q5IfTy+g+K/pLWC+olpr/5jkR9Jb9HpHkn+e5EvpdfIs5twk9w98vt4fT/tXSV6S3pjcW5P829baN/r3/HiSb/ZHDl/Vf0aSfE+Sv01v4fDPJfn91tpnFvne/zu9NZL+PL3Q5ZIkH0vyb+Zdd2GS70/ysdbadwaO/06Sj6Q3Bnh3euHZaXv4OxdyfFXdM+/zQwPn/yTJn6bXadZN8u+TpLV2df83eGeS7ektRP/i/npa0+mt1fWk9Lqjbk7vbYB7000v0Lo9vX+770tvTA8AOADV8juvAQAOLv3xsNuS/Os9BEIsoKo+m+S/tdbeu9a1AAAHHh1RAABJqmprVT28P2L35vTeZHfZGpcFAHBQEUQBAPQ8N8mN6Y2MvTDJS+e9CQ4AgGUymgcAAADAqtARBQAAAMCqGFnrAlbbUUcd1U444YS1LgMAAADgoHHFFVfc0Vo7em/XHXJB1AknnJDLL798rcsAAAAAOGhU1TeXcp3RPAAAAABWhSAKAAAAgFUhiAIAAABgVRxya0QBAAAAh5adO3dm27ZtmZycXOtS1r2xsbFs3rw5GzZs2K/7BVEAAADAQW3btm05/PDDc8IJJ6Sq1rqcdau1lh07dmTbtm058cQT9+sZRvMAAACAg9rk5GSOPPJIIdQyVVWOPPLIZXWWCaIAAACAg54QajiW+zsKogAAAABYFdaIAgAAAA4pWz77tWzfOT205x29YSRffe5Thva8g5mOKAAAAOCQMswQaiWeN2hmZmbFnr0WBFEAAAAAK+zee+/NWWedlac97Wl5ylOekve973354R/+4bnzn/rUp3L22WcnSQ477LC84Q1vyNOe9rR8/vOfz8UXX5wnPvGJeeYzn5nXve51c9ct5Fd+5Vfy4z/+43ne856Xxz72sfngBz+YX/iFX8iWLVuydevW7Ny5M0nyxje+MU9+8pPz1Kc+NT//8z+fJNm+fXte9rKX5bTTTstpp52Wz33uc0P/HYzmAQAAAKywj3zkIzn22GPz4Q9/OEly11135c1vfnPuvffeTExM5KKLLsrLX/7yJL3Q6tnPfnZ++7d/O5OTkznppJPy6U9/OieeeGLOOeecvX7XDTfckE9+8pO55ppr8pznPCd/+Zd/md/4jd/IS1/60nz4wx/O8573vPzVX/1VrrvuulRV7rzzziTJ61//+px33nl57nOfm5tvvjkvfOELc+211w71d9ARBQAAALDCtmzZko997GO54IIL8pnPfCZHHHFEtm7dmr/+67/O9PR0PvzhD+clL3lJkqTb7eZlL3tZkuS6667L4x73uJx44olJsqQg6kUvelE2bNiQLVu2ZGZmJlu3bp2r4aabbsoRRxyRsbGxvOpVr8oHP/jBjI+PJ0k+/vGP57WvfW1OOeWUvPjFL853v/vd3HPPPUP9HXREAQAAAKywJzzhCbnyyitz8cUX501velNe8IIX5OUvf3n+4A/+II985CNz6qmn5vDDD0+SjI2Npdvt7vd3jY6OJkk6nU42bNiQqprbn56ezsjISC677LJ84hOfyAc+8IH8wR/8Qf72b/82s7Oz+cIXvpCxsbHl/8GL0BEFAAAAsMJuu+22jI+P59xzz83555+fK6+8Ms9//vNz5ZVX5l3vetfcWN583/M935Mbb7wxN910U5LkoosuWnYt99xzT+66666ceeaZ+d3f/d18+ctfTpL84A/+YN72trfNXXfVVVct+7vm0xEFAAAAHFKO3jAy1DfdHb1h7/HKV7/61Zx//vlzXUrvfOc70+12c/bZZ+e9731v3ve+9y1436ZNm/KOd7wjW7duzcTERE477bRl13v33XfnJS95SSYnJ9Nay+/8zu8kSd761rfmNa95TZ761Kdmeno6Z5xxRv7wD/9w2d83qFprQ33gge7UU09tl19++VqXAQAAAKySa6+9Nk960pPWuoz9ds899+Swww5Lay2vec1rctJJJ+W8885bs3oW+j2r6orW2ql7u9doHgAAAMAB7F3veldOOeWUnHzyybnrrrvy6le/eq1L2m9G8wAAAAAOYOedd95DOqDe85735Pd///d3O3b66afn7W9/+2qWts8EUQAAAADrzE/8xE/kJ37iJ9a6jH1mNA8AAACAVSGIAgAAAGBVCKIAAAAAWBWCKAAAAABWhcXKAQAAgEPKZz777DzwwB1De97GjUflec/9+6E972CmIwoAAAA4pAwzhFqJ5813wgkn5I47VvY7VosgCgAAAIBVccgGUTMzU7nllvfmsi++JJ/93Om58spX5B//8W/SWlvr0gAAAICDzL333puzzjorT3va0/KUpzwlF110US6++OI88YlPzDOf+cy87nWvy9lnn50k2bFjR37wB38wJ598cn7yJ39yj1nFTTfdlCc+8Yl55StfmSc84Ql5xStekY9//OM5/fTTc9JJJ+Wyyy5Lkvzd3/1dTjnllJxyyil5+tOfnrvvvjtJ8pu/+Zs57bTT8tSnPjW//Mu/vOK/wyEZRM3MTOaqL78y//sb/zF33/21TE19K9+58wv52tWvz9e//mZhFAAAADBUH/nIR3Lsscfmy1/+cr72ta9l69atefWrX51LLrkkV1xxRbZv3z537a/+6q/muc99bq6++uq89KUvzc0337zHZ19//fV5wxvekOuuuy7XXXddLrzwwnz2s5/Nb/3Wb+XXf/3XkyS/9Vu/lbe//e256qqr8pnPfCabNm3KRz/60XzjG9/IZZddlquuuipXXHFFPv3pT6/o73BIBlE33/Lfc+edly147tbb/jw7dnxqdQsCAAAADmpbtmzJxz72sVxwwQX5zGc+k3/4h3/I4x73uJx44olJknPOOWfu2k9/+tM599xzkyRnnXVWHvGIR+zx2SeeeGK2bNmSTqeTk08+OS94wQtSVdmyZUtuuummJMnpp5+en/u5n8tb3/rW3HnnnRkZGclHP/rRfPSjH83Tn/70POMZz8h1112Xb3zjGyvzA/Qdkm/Nu/22D+zx/G23fyBHHfV/rFI1AAAAwMHuCU94Qq688spcfPHFedOb3pQXvOAFQ3v26Ojo3Han05nb73Q6mZ6eTpK88Y1vzFlnnZWLL744p59+ei699NK01vKLv/iLefWrXz20WvbmkOyImpy6fY/npyZvW6VKAAAAgNW2ceNRq/682267LePj4zn33HNz/vnn53Of+1xuvPHGuY6liy66aO7aM844IxdeeGGS5JJLLsl3vvOdZdd4ww03ZMuWLbngggty2mmn5brrrssLX/jCvPvd784999yTJLn11lvzT//0T8v+rj05JDuixkYfnfsnF5+vHB07dhWrAQAAAFbT857796v+nV/96ldz/vnnp9PpZMOGDXnnO9+Z22+/PVu3bs3ExEROO+20uWt/+Zd/Oeecc05OPvnkfN/3fV+OP/74ZX//7/3e7+WTn/zk3Pjei170ooyOjubaa6/Nc57znCTJYYcdlj/90z/NMcccs+zvW0wdagtzn3rqqe0DH/i/csONv73oNU972n/PUUf+i9UrCgAAAFgx1157bZ70pCetdRkPcc899+Swww5Lay2vec1rctJJJ+W8885b67L2aqHfs6quaK2durd7D8nRvMc85lV5+MOfveC54457RY585PNXuSIAAADgUPOud70rp5xySk4++eTcddddq7pW01o5JEfzut3RPP2U9+TW296fG274jczM3Jeqbk4++fdzzNFbU1VrXSIAAABwkDvvvPOW3AG1Y8eOBRc4/8QnPpEjjzxy2KWtmEMyiEqSTmc0j9n8o7lj+8fz7e98NknlUce8aK3LAgAAAFZAa21dN54ceeSRueqqq9a6jCx3iadDcjRvULe7KUnS2nRmZx9Y42oAAACAYRsbG8uOHTuWHaIc6lpr2bFjR8bGxvb7GYdsR9Qu3e743PbMzP3pdDauYTUAAADAsG3evDnbtm3L9u3b17qUdW9sbCybN2/e7/sFUbsFUfdmw4Yj1rAaAAAAYNg2bNiQE088ca3LIEbzHtIRBQAAAMDKEETN64gCAAAAYGUIovqLlSc6ogAAAABW0iEfRHV264i6bw0rAQAAADi4HdJB1M33T+UT39k5t//dB+5ew2oAAAAADm6HbBD1tm/+Y773C9fmg9un5o790tdvyCXb71zDqgAAAAAOXodkEHXJ9jvzn268PbNJpjI6d7za/fnpq7+Z6++bXLviAAAAAA5Sh2QQ9Ye3bJ/bnszY3PZYJrOztbxn2x1rURYAAADAQe2QDKK+9N0HFyWfGgiiRtMb0/vS3RYtBwAAABi2QzKI2tR98M+e3C2I6o3kjXcOyZ8FAAAAYEUdkonLWUcfMbc9uEbUWD+IOnPgPAAAAADDcUgGUa9/7KPy8JFukoeO5j1xYiw/8uhHrlVpAAAAAAetQzKIeuym0fzPZ5yU5z3isN06ok4cnckHTnl8JrrdNawOAAAA4OB0SAZRSfI9E2P5/055fJ5xxMOyMyNJkpPHk6M2jqxxZQAAAAAHp0M2iNrlERtG5sbzZma8LQ8AAABgpaxaEFVVW6vq61V1fVW9cYHzx1fVJ6vqS1X1lao6s3/8FVV11cBntqpO6Z/7VP+Zu84ds691jXc7c2/O2zktiAIAAABYKasyh1ZV3SRvT/IDSbYl+WJVfai1ds3AZW9K8v7W2jur6slJLk5yQmvtz5L8Wf85W5L8j9baVQP3vaK1dvn+1jbR7cytEzWtIwoAAABgxaxWR9SzklzfWruxtfZAkr9I8pJ517QkD+tvH5HktgWec07/3qEZ7Iiamb1/mI8GAAAAYMBqBVHHJbllYH9b/9igX0lyblVtS68b6mcXeM6PJPnzecfe0x/Le3NV1UJfXlU/XVWXV9Xl27dv3+3cRLc7t0bUrI4oAAAAgBVzIC1Wfk6S97bWNic5M8mfVNVcfVX17CT3tda+NnDPK1prW5I8r//50YUe3Fr7o9baqa21U48++ujdzo0PjOZl9v60NjvEPwkAAACAXVYriLo1yWMG9jf3jw16VZL3J0lr7fNJxpIcNXD+5ZnXDdVau7X/37uTXJjeCOA+6QVRY3P7MzPG8wAAAABWwmoFUV9MclJVnVhVG9MLlT4075qbk7wgSarqSekFUdv7+50k/yYD60NV1UhVHdXf3pDk7CRfyz4aXCMqsU4UAAAAwEpZlbfmtdamq+q1SS5N0k3y7tba1VX1liSXt9Y+lOQNSd5VVeelt3D5K1trrf+IM5Lc0lq7ceCxo0ku7YdQ3SQfT/Kufa1tvDMwmhfrRAEAAACslFUJopKktXZxeouQDx77pYHta5Kcvsi9n0ryvfOO3Zvkmcuta2LeaN60IAoAAABgRRxIi5WviflrROmIAgAAAFgZgqhuN5MDo3kWKwcAAABYGYd8EDV/NG9m5t41rAYAAADg4HXIB1HzR/N0RAEAAACsjEM+iJrodjK5WxBljSgAAACAlXDIB1G9jqjBNaIEUQAAAAAr4ZAPojZWZWfpiAIAAABYaYd8EFVV6XQ3ze0LogAAAABWxiEfRCVJpzM+t22xcgAAAICVIYhKMtIdDKJ0RAEAAACsBEFUkq4gCgAAAGDFCaKSjHY3ZTaVJJmZFUQBAAAArARBVJJNI91MZTRJMjMtiAIAAABYCYKoJOPdTqYyliTZaTQPAAAAYEUIopJMdDtzHVHTgigAAACAFSGISjLe7Way3xE1M3P/GlcDAAAAcHASRGVXR9SuIEpHFAAAAMBKEEQlGe88OJrXvDUPAAAAYEUIotJbrHzXaF7adGZnH1jbggAAAAAOQoKo7D6al1gnCgAAAGAlCKLS64jaPYgyngcAAAAwbIKo7OqIGp3bF0QBAAAADJ8gKsl4t5tJQRQAAADAihJEZVdH1Ka5fWtEAQAAAAyfICq71oga7Ii6dw2rAQAAADg4CaKyQBA1qyMKAAAAYNgEUemN5k0OjuZNWyMKAAAAYNgEUUnGO/M7ogRRAAAAAMMmiEqy6SFrRBnNAwAAABg2QVSSTlVaZ3xu32LlAAAAAMMniOrrdAfWiNIRBQAAADB0gqi+bndibntmxhpRAAAAAMMmiOrr7tYRJYgCAAAAGDZBVN/G7uAaUYIoAAAAgGETRPWNjWzMzowkSWatEQUAAAAwdIKovoluJ1MZS5Ls9NY8AAAAgKETRPWNd7uZymiSZHraaB4AAADAsAmi+sY7nUz2O6KmrREFAAAAMHSCqL7B0byZWWtEAQAAAAybIKpvvPtgR9SsjigAAACAoRNE9Y13O3NrRGX2/rQ2u7YFAQAAABxkBFF9uwVRSWZnJ9ewGgAAAICDjyCqrzeat2luf8Z4HgAAAMBQCaL6Jrrd3TqiBFEAAAAAwyWI6ps/mjcz4815AAAAAMMkiOqb6HYyZTQPAAAAYMUIovp6a0QZzQMAAABYKYKovl5H1NjcviAKAAAAYLgEUX3jHUEUAAAAwEoSRPU9dLFyQRQAAADAMAmi+nprRA12RHlrHgAAAMAwCaL6NnY6manBIOreNawGAAAA4OAjiBrQ6W6a29YRBQAAADBcgqgB1Rmf256ZtUYUAAAAwDAJogaMjAwEUdOCKAAAAIBhEkQNGOkOdkQZzQMAAAAYJkHUgA2dscymklisHAAAAGDYBFEDJkY25IGMJkmmjeYBAAAADJUgasB4t5PJjCVJpr01DwAAAGCoBFEDJrqdTO3qiDKaBwAAADBUgqgB4wNB1IyOKAAAAIChEkQN6I3mbUqSzMxYIwoAAABgmARRAwZH89qsIAoAAABgmARRAwZH89KmMzu7c20LAgAAADiICKIGTHS7c6N5iXWiAAAAAIZJEDVgt46oJDPenAcAAAAwNIKoAeOdTqYyNrevIwoAAABgeFYtiKqqrVX19aq6vqreuMD546vqk1X1par6SlWd2T9+QlXdX1VX9T9/OHDPM6vqq/1nvrWqajk1TnQ7mdQRBQAAALAiViWIqqpukrcneVGSJyc5p6qePO+yNyV5f2vt6UlenuQdA+duaK2d0v/8zMDxdyb5qSQn9T9bl1NnbzRPRxQAAADASlitjqhnJbm+tXZja+2BJH+R5CXzrmlJHtbfPiLJbXt6YFU9OsnDWmtfaK21JH+c5IeWU+RDgqjZ+5bzOAAAAAAGrFYQdVySWwb2t/WPDfqVJOdW1bYkFyf52YFzJ/ZH9v6uqp438Mxte3nmPnnoYuWCKAAAAIBhOZAWKz8nyXtba5uTnJnkT6qqk+T2JMf3R/Z+LsmFVfWwPTznIarqp6vq8qq6fPv27YteN9HtZnK30TxBFAAAAMCwrFYQdWuSxwzsb+4fG/SqJO9Pktba55OMJTmqtTbVWtvRP35FkhuSPKF//+a9PDP9+/6otXZqa+3Uo48+etEirREFAAAAsHJWK4j6YpKTqurEqtqY3mLkH5p3zc1JXpAkVfWk9IKo7VV1dH+x81TV49JblPzG1trtSb5bVd/bf1vejyX5n8spclOnjOYBAAAArJCR1fiS1tp0Vb02yaVJukne3Vq7uqrekuTy1tqHkrwhybuq6rz0Fi5/ZWutVdUZSd5SVTuTzCb5mdbat/uP/ndJ3ptkU5JL+p/9VlWp7ngy09sXRAEAAAAMz6oEUUnSWrs4vUXIB4/90sD2NUlOX+C+v0zyl4s88/IkTxlmnZ3OJkEUAAAAwAo4kBYrPyB0u+Nz24IoAAAAgOERRM0zGETNWqwcAAAAYGgEUfNsGAiipmfuXcNKAAAAAA4ugqh5No2MZrq/dJaOKAAAAIDhEUTNM97tZDJjSZKd1ogCAAAAGBpB1DwT3U6mMpokmZ42mgcAAAAwLIKoecYHgyijeQAAAABDI4iaZ1O3k8lsSpLMzBrNAwAAABgWQdQ8g6N5s9aIAgAAABgaQdQ8450HFyvP7GRam13bggAAAAAOEoKoeSZGunMdUUnL7OzkmtYDAAAAcLAQRM0z3ulkaldHVJIZ43kAAAAAQyGImmeiOzCal2TGm/MAAAAAhkIQNc/4wGLliY4oAAAAgGERRM3TC6KM5gEAAAAMmyBqnoeO5gmiAAAAAIZBEDWP0TwAAACAlSGImmei281UNs3tW6wcAAAAYDgEUfPoiAIAAABYGYKoecY7nUwKogAAAACGThA1z0inMtsZHM0TRAEAAAAMgyBqAR1BFAAAAMDQCaIW0OmMz23PzAqiAAAAAIZBELWA7shAEOWteQAAAABDIYhawIbuxNy20TwAAACA4RBELWBDdyyzqSSCKAAAAIBhEUQtYHxkJA9kNIkgCgAAAGBYBFELmOh2MtUPonZOC6IAAAAAhkEQtYDxbieTGUuSTOuIAgAAABgKQdQCBjuiBFEAAAAAwyGIWsB4p5OpfkfUrCAKAAAAYCgEUQsYHM1rs/evcTUAAAAABwdB1AImut250by0nZmd3bm2BQEAAAAcBARRCxjvPjialyQzM7qiAAAAAJZLELWAiYHRvCSZmbVOFAAAAMByCaIWMD7w1rwkmZkWRAEAAAAslyBqAeM6ogAAAACGThC1AGtEAQAAAAyfIGoBDxnNm7l3DasBAAAAODgIohYw0e3qiAIAAAAYMkHUAsY789aI0hEFAAAAsGyCqAVMPGQ0T0cUAAAAwHIJohYw2qk8MNARNTvjrXkAAAAAyyWIWkBVpbrjc/vTgigAAADGmaJ+AAAgAElEQVSAZRNELaLT2TS3PWs0DwAAAGDZBFGL6A50RM3oiAIAAABYNkHUIjaMCKIAAAAAhkkQtYgRHVEAAAAAQyWIWsSmkdFMZySJxcoBAAAAhkEQtYjxbidTGU0iiAIAAAAYBkHUIjZ1OpnMWJJkpyAKAAAAYNkEUYuYGOiImpkWRAEAAAAslyBqEePdTiazKUkyMyuIAgAAAFguQdQiJrrduY6o2Zn717gaAAAAgPVPELWIwcXKM3t/WmtrWxAAAADAOieIWkRvNG+sv9cyOzu5pvUAAAAArHeCqEX0Fisfm9uf8eY8AAAAgGURRC1it9G8CKIAAAAAlksQtYiJ3UbzBFEAAAAAyyWIWsR4x2geAAAAwDAJohYxMdI1mgcAAAAwRIKoRYx35o/m3b+G1QAAAACsf4KoRYx7ax4AAADAUAmiFjEhiAIAAAAYKkHUIjZ1O5kcXCNq1mgeAAAAwHIIohbRrUo6m+b2Z6bvXcNqAAAAANY/QdSeDAZROqIAAAAAlmXVgqiq2lpVX6+q66vqjQucP76qPllVX6qqr1TVmf3jP1BVV1TVV/v//f6Bez7Vf+ZV/c8xw6y52x2f27ZGFAAAAMDyjKzGl1RVN8nbk/xAkm1JvlhVH2qtXTNw2ZuSvL+19s6qenKSi5OckOSOJP+qtXZbVT0lyaVJjhu47xWttctXou4RQRQAAADA0KxWR9SzklzfWruxtfZAkr9I8pJ517QkD+tvH5HktiRprX2ptXZb//jVSTZV1WhWwUh3Ym5bEAUAAACwPKsVRB2X5JaB/W3ZvaspSX4lyblVtS29bqifXeA5L0tyZWttauDYe/pjeW+uqhpizdnYHZvbFkQBAAAALM+BtFj5OUne21rbnOTMJH9SVXP1VdXJSf7fJK8euOcVrbUtSZ7X//zoQg+uqp+uqsur6vLt27cvuaDxkZFMphdGTQuiAAAAAJZltYKoW5M8ZmB/c//YoFcleX+StNY+n2QsyVFJUlWbk/xVkh9rrd2w64bW2q39/96d5ML0RgAforX2R621U1trpx599NFLLnq828lUelOA09OCKAAAAIDlWK0g6otJTqqqE6tqY5KXJ/nQvGtuTvKCJKmqJ6UXRG2vqocn+XCSN7bWPrfr4qoaqapdQdWGJGcn+dowi54YDKJ0RAEAAAAsy6oEUa216SSvTe+Nd9em93a8q6vqLVX14v5lb0jyU1X15SR/nuSVrbXWv+/xSX6pvxbUVVV1TJLRJJdW1VeSXJVeh9W7hln3eLdjNA8AAABgSEZW64taaxentwj54LFfGti+JsnpC9z3a0l+bZHHPnOYNc430e1mqh9Ezc7cv5JfBQAAAHDQO5AWKz/gDK4R1WZ1RAEAAAAshyBqDwZH89J2ZnZ2em0LAgAAAFjHBFF70OuIGpvbn501ngcAAACwvwRRezDeGeiISjI9c+8aVgMAAACwvgmi9mBiYI2oxILlAAAAAMshiNqD+aN5OqIAAAAA9p8gag8mut3dRvNmdEQBAAAA7DdB1B6MP2Q07741rAYAAABgfRNE7cFDR/MEUQAAAAD7SxC1BxPd3d+apyMKAAAAYP8JovZg/mieNaIAAAAA9p8gag82VmW6Bhcr99Y8AAAAgP0liNqDqkp1x+f2dUQBAAAA7D9B1F50OoNBlDWiAAAAAPaXIGovuoIoAAAAgKEQRO1Fd2TT3LbRPAAAAID9J4jai427rRFlsXIAAACA/SWI2ovRkdFMZyRJMm00DwAAAGC/CaL2YqLbyVRGkyTTRvMAAAAA9psgai/GB4KondNG8wAAAAD2lyBqL8a73UxmLIm35gEAAAAshyBqL3qjef0gatZoHgAAAMD+EkTtxXinM9cRNasjCgAAAGC/CaL2YnCx8szen9ba2hYEAAAAsE4JovZifGA0L2mZnZ1c03oAAAAA1itB1F5MdB8czUssWA4AAACwvwRRezE+OJqXZGbGguUAAAAA+0MQtRfj3e7AaF4yM3PvGlYDAAAAsH4JovbioaN5OqIAAAAA9ocgai8eOppnjSgAAACA/SGI2guLlQMAAAAMhyBqL3odUYIoAAAAgOUSRO3FeGfeaN6sNaIAAAAA9ocgai82Gc0DAAAAGApB1F50qpLOprl9QRQAAADA/hFELUG3Mz63LYgCAAAA2D+CqCWoriAKAAAAYLkEUUuwoTs4mmexcgAAAID9IYhagu7IxNz2zMy9a1gJAAAAwPoliFqCjd3Bt+bpiAIAAADYH4KoJZgYGclkRpMk09aIAgAAANgvgqglGO92M5VeV9TOaaN5AAAAAPtDELUEE91OpuY6oozmAQAAAOwPQdQSjHc6cx1RFisHAAAA2D+CqCUY73Yy2Q+iZnVEAQAAAOwXQdQSjA+M5rVZQRQAAADA/hBELUEviOp1RKU9kNnZ6bUtCAAAAGAdEkQtwcTAaF6SzOqKAgAAANhngqglGO9250bzkmRm5r41rAYAAABgfRJELcH8jihBFAAAAMC+E0QtwW5rREUQBQAAALA/BFFLMPjWvCSZmbFGFAAAAMC+EkQtwUNH8+5dw2oAAAAA1idB1BKMd+aP5umIAgAAANhXgqglmHjIaJ6OKAAAAIB9JYhagvFud95ono4oAAAAgH0liFqCDZ3KTG2a2/fWPAAAAIB9J4haouoKogAAAACWQxC1RJ3O+Ny2IAoAAABg3wmilmhksCNq1hpRAAAAAPtKELVEnZGBjqhpHVEAAAAA+0oQtUQbuwNB1KwgCgAAAGBfCaKWaGNVptNNktx997W5775/WOOKAAAAANYXQdQS3H77B3PfnZ/KVMaSJPdNbsvnv/ADuemmd6xxZQAAAADrhyBqL+6+++pcc+0FGWv3ZSqjSZKWStJyw42/ne3bP7a2BQIAAACsE4Kovbhl2x8nmc1opjLZ74jqBVE9N9/ynjWqDAAAAGB9EUTtxd13X50kGc3k3GhepT3kPAAAAAB7Jojai27/bXmjmcxdOSJJ0sns3PmR7sSa1AUAAACw3qxaEFVVW6vq61V1fVW9cYHzx1fVJ6vqS1X1lao6c+DcL/bv+3pVvXCpzxyGY455UZJkLFPZkaN63zt4/lFnLnAXAAAAAPOtShBVVd0kb0/yoiRPTnJOVT153mVvSvL+1trTk7w8yTv69z65v39ykq1J3lFV3SU+c9mOffS/ycTEEzKaybkgapeNGx+Vxx7/U8P+SgAAAICD0n4FUVX1vKo6fR9ueVaS61trN7bWHkjyF0leMu+aluRh/e0jktzW335Jkr9orU211v4hyfX95y3lmcs2MjKRZzz9z/LPjnjqbkHU4YdvyanPvCijo48a9lcCAAAAHJSWFERV1aeq6rn97Z9P8sEkH6iqC5b4PccluWVgf1v/2KBfSXJuVW1LcnGSn93LvUt55q76f7qqLq+qy7dv377Ekh+0ceMj84THnrtbEHXM0VuzadNj9vlZAAAAAIeqpXZEbUnyhf72q5P8iyTPTvLvhljLOUne21rbnOTMJH9SVUMZHWyt/VFr7dTW2qlHH330fj1jvNPJHQNB1OTU7cMoDQAAAOCQMbLE6zpJZqvqcUlGWmtXJ0lVPXKJ99+aZLB9aHP/2KBXpbcGVFprn6+qsSRH7eXevT1zaMa7nXwnR87tT03etoerAQAAAJhvqR1H/yvJ7yX5jSR/lST9UGrHEu//YpKTqurEqtqY3uLjH5p3zc1JXtB/9pOSjCXZ3r/u5VU1WlUnJjkpyWVLfObQjHc7ma4N+U4enkRHFAAAAMC+WmoQ9cokk0m+nuSX+seenORtS7m5tTad5LVJLk1ybXpvx7u6qt5SVS/uX/aGJD9VVV9O8udJXtl6rk7y/iTXJPlIkte01mYWe+YS/559Nt7t/VTf7o/nTeqIAgAAANgn1Vpb6xpW1amnntouv/zyfb7vjgem85TPfS2vb7+ZZ/WXy3r+GV/JyMjEsEsEAAAAWFeq6orW2ql7u26pb817fVWd0t9+VlXdWFXfqKpnLbfQ9WJXR9Tgm/OmjOcBAAAALNlSR/PekOSm/vZ/SfL2JL+V5K0rUNMBaVOnUsnub84zngcAAACwZEsNoh7eWruzqg5LckqS32ut/dckT1y50g4cN943lZd+6fq07N4R9b/vvGnNagIAAABYb0aWeN22qnp2kpOTfKa1NlNVhyeZWbnSDgw7HpjOy750fW5/YGdvfyCIev/N1+ToR0/m8eNja1UeAAAAwLqx1I6oX0jy10nekuTX+sfOTvLFlSjqQPLeW++YC6GS3YOoh7Xt+YNv/tNalAUAAACw7iypI6q19jdJjpl3+K/6n4Pax3d8d7f97+aITGckI5nOI7MjH5h3HgAAAICFLXU0L1X1uCQvT3JckluTXNRau2GlCjtQTLe2236rTna0I/Oo/GOOyvaHnAcAAABgYUsazauqM5N8Ob2Fyu9L8rQkX6qqs1awtgPCsx8+8ZBju8bzHpkd+d4jxle7JAAAAIB1aakdUf85yQ+11j6x60BVfX+S30vy4ZUo7EDxquOOzp/dtiP3zz7Y+bQriNqYnfmZRy+5qQwAAADgkLbUxcqPT/Kpecc+3T9+UDtxfDR/vOVxOXLDg4HT4ILlJ49aIwoAAABgKZYaRH05yb+fd+x1Sb4y3HIOTM975OG54jlPzmsf01uvfTCImpy6ba3KAgAAAFhXlhpE/bskr6mqW6rqc1V1c5LX9o8fEsa6nbzsnz0iybwgalIQBQAAALAUS1rgqLV2TVV9T5LTkxyb5LYk/6u19sBKFneg2Ty2McnuQdTU5O1rVQ4AAADAurLklbZbazszsE5UVW2sqhtba49bicIORIePdHN4t5Md04OjeYIoAAAAgKVY6mjeQirJCUOqY904bmxj7q/x3J/xJMmkjigAAACAJVlOEJUkbShVrCPHjfbG8+7oj+dNWSMKAAAAYEmWG0Qdco4b25BkIIh64J8yO7tzLUsCAAAAWBf2uEZUVb17f+89WO3qiPr23ILlLVNT/5hNmzavXVEAAAAA68DewqRb93L+14dVyHpx7LyOqCSZnLxNEAUAAACwF3sMolprb16tQtaLBzuijpw7NuXNeQAAAAB7ZY2offTgGlFHzx2btGA5AAAAwF4JovbRo0c3pJLsGOiImtQRBQAAALBXgqh9tLHTyTEbR/KdHJmWSqIjCgAAAGApBFH74djRjZmuDbkrD09ijSgAAACApdjbW/OSJFX1Y4ucmkqyLcllrbWdQ6vqAHfc2IZ86e7em/Menu/oiAIAAABYgiUFUUl+OslpSXakFzwdl+SoJF9KckKSB6rqh1prV65EkQea48Z6b87bkaPy+Hwj09PfzfT0PRkZOWyNKwMAAAA4cC11NO/KJG9srR3bWntWa+24JBck+fskxyZ5d5K3rVCNB5zjRntvzrNgOQAAAMDSLTWI+tEkb5137G1Jfqy1NpvkPyc5eZiFHcge7Ig6eu7YlPE8AAAAgD1aahD1T0leNO/Y1iTb+9tjSWaGVdSB7tjRXhB1R46aOzY5qSMKAAAAYE+WukbUv09yUVV9KcktSR6T5OlJfqR//nuTvGP45R2YNo/1RvO+PRhETemIAgAAANiTJQVRrbVLqurxSc5Kb02ov03yw621f+qfvzTJpStW5QHmyA0jGe1U7pgZ7IgSRAEAAADsyVI7otIPnd6zgrWsG52qPHp0Q75538MynZGMZDpTRvMAAAAA9mhJa0RV1WOr6o+r6itVdePgZ6ULPFAdN7oxrTrZ0R/PM5oHAAAAsGdL7Yi6ML21of6fJPetXDnrx7H9daJ25Mg8Kt/K1NS30tpsqpa6/jsAAADAoWWpQdSWJGe01g6ZN+Ptzea5N+cdnSSZnX0gD+z8dkY3HrWn2wAAAAAOWUtt3/lskqeuZCHrzXFjvSBqx8Cb86YsWA4AAACwqKV2RH0jyaVV9YEk3xo80Vp7y9CrWgeOHX1wNG+Xycnb87CHyesAAAAAFrLUIOqRSS5Ncnj/s0sbekXrxIMdUUfPHbNgOQAAAMDilhREtdZ+dKULWW+OW6Ajamry9rUqBwAAAOCAt2gQVVWbW2vb+tvHL3Zda+3mlSjsQHfYSDdHjHTz7Z0PrhE1aY0oAAAAgEXtqSPq2jw4hndTemN4Ne+alqQ7/LLWh+NGN+Sa6fHc3yayKfdmckpHFAAAAMBi9vTWvCMGtjck2dj/7+Bn48qVduA7tr9O1Pb+eJ6OKAAAAIDFLRpEtdZmB7ZnFvusTpkHpgfXieotWP7AA9szO/vAWpYEAAAAcMBa0mLlVfXYJP8xySlJDhs811p73ArUtS48+Oa8XQuWt0xN/WM2bXrM2hUFAAAAcIBaUhCV5MIktyT5f5Lct3LlrC/zO6KS3nieIAoAAADgoZYaRG1JcsahPoo330M7omLBcgAAAIBF7Gmx8kGfTfLUlSxkPTp2riPqqLljUxYsBwAAAFjQUjuivpHk0qr6QJJvDZ5orb1l6FWtE48e3ZjK7kGUjigAAACAhS01iHpkkkuTHN7/7NKGXtE6sqFTedTGDdk+9ci0VCotkzqiAAAAABa0pCCqtfajK13IenXc2IZ864ENuas9Ig/PtwVRAAAAAItYNIiqqs2ttW397eMXu661dvNKFLZeHDe2MVd8977ckSPz8Hw7U0bzAAAAABa0p46oa/PgGN5N6Y3h1bxrWpLu8MtaP3YtWH5Hjsrj841MT9+d6em7MzJy+F7uBAAAADi07OmteUcMbG9IsrH/38HPxpUrbX3YPNb7Cb49uGD5pK4oAAAAgPkWDaJaa7MD2zOLfVanzAPXYEfULpNT1okCAAAAmG9Ji5VXVTfJq5M8P8lRGRjRa619/8qUtj4c1++I2jEQRE3piAIAAAB4iD2N5g36nSSvS3JZkmcn+XCSzUk+u0J1rRvHjT40iPLmPAAAAICHWmoQ9a+TbG2t/XaSmf5/X5LkjBWrbJ04ckM3Y53Kjhw9d8xoHgAAAMBDLTWIGk/yzf72fVW1qf3/7N13lORVnf//161cnXOenGdggrSDIIigAmICAwLm7H4VFRe/YljToqu7urhr5ofCV0UMrO7iqiAGUIJAT07MTE/sNNO5q7uru7rC/f3R1TXV09VhpLuqw/Nxzhw+de+tT78/zAzn8Dr3vj/WHpD0vJkpa+4wxqjK61FAeQpruF8UzcoBAAAAAADGmmoQ9ayk2vj1NkmfMcbcJomtP4o3LDcmcTyPHlEAAAAAAABjTTWIukXSyFv0/lHSRZLeIOn9M1HUXHOmYXmxJGkwdEpJLx0EAAAAAACApvDWvPgb81ZL+pkkWWsPSnrxzJY1t1T7ho/kjeyIsnZIQ0Md8npLJ/oaAAAAAADAgjLpjihrbVTSN6y1oTTUMyelenNeKMTxPAAAAAAAgGRTPZr3G2PMNTNayRx29o4oSRocpH0WAAAAAABAskmP5sU5JP3SGPOYpAZJdmTCWvvOmShsLqmK74hqJ4gCAAAAAAAY11SDqMOS/m0mC5nLqr3DO6I6k4MojuYBAAAAAACMMmEQZYy50Vp7n7X2n9JV0FyU7XKq0OVUR5gdUQAAAAAAAOOZrEfU99JSxTxQ5XNr0PgVVLYkmpUDAAAAAACcbbIgykzXDzLGXG2MOWiMqTfG3JZi/g5jzM74r0PGmO74+OVJ4zuNMYPGmGvjc/cYY44lzW2ernrP1cib89rix/PYEQUAAAAAADDaZD2inMaYyzVBIGWt/dNkP8QY45T0LUkvk9Qo6RljzAPW2v1J97klaf3NkrbEx/8saXN8vEhSvaTfJ93+Y9ba+yerYaZV+YaDqA6VaIlOaGioTbFYSA6HN8OVAQAAAAAAzA6TBVFeSd/X+EGUlbR8Cj9nq6R6a+1RSTLG/FTSayTtH2f9jZI+m2L89ZJ+Z60NTuFnptVIw/KOpIblodBp+f2LM1USAAAAAADArDLZ0bx+a+1ya+2ycX5NJYSSpGpJDUmfG+NjYxhjlkhaJinVTqsbJN131tgXjTG740f7Um4/Msa81xhTZ4ypa2trm2LJ56YmaUfUCI7nAQAAAAAAnDFZEJUJN0i631obTR40xlRKOl/SQ0nDn5C0VtLzJRVJ+niqG1pr77TW1lpra0tLS2ek6KoUO6IGB2lYDgAAAAAAMCJdzcqbJC1K+lwTH0sl1a4nSbpe0q+steGRAWttix0WknS3ho8AZkR1qh1RIXZEAQAAAAAAjJgwiLLW5k7Tz3lG0ipjzDJjjEfDYdMDZy8yxqyVVCjpyRT3uFFnBVTxXVIyxhhJ10raO031nrMKj1sOndUjih1RAAAAAAAACZM1K58W1tqIMeaDGj5W55T0A2vtPmPMFyTVWWtHQqkbJP3UWmuTv2+MWarhHVWPnnXre40xpRreubVT0vtn7ikm5nIYVXjdOj1YJCsjI8uOKAAAAAAAgCRpCaIkyVr7W0m/PWvsM2d9/tw43z2uFM3NrbVXTF+Fz12116PmUFjdtkiF6qBZOQAAAAAAQJLZ2Kx8zqryDTcsb1exJJqVAwAAAAAAJCOImkbV3tENy6PRPkUivZksCQAAAAAAYNYgiJpGIzuiRr05j+N5AAAAAAAAkgiiplXNWTuiJIIoAAAAAACAEQRR06g61Y6oEH2iAAAAAAAAJIKoaVUV3xHVzo4oAAAAAACAMQiiplGR2ymfw4zaEXXy5J06cOATGhhoymBlAAAAAAAAmUcQNY2MMSo1AfUqT0MaPqZnbVTNLT9X3bbXamDgZIYrBAAAAAAAyByCqGnU07NT+ZGjkjHqTNoVJUlDQ+2qP/JvGaoMAAAAAAAg8wiiptGpU/+tYrVLGt0nakRb2+8VifSnuywAAAAAAIBZgSBqGg2FO5OCqNIx89ZGFIn2prssAAAAAACAWYEgahplZy1PBFEntGzMvMuVJ4+7KN1lAQAAAAAAzAoEUdOoqup6lZguSdJRrUw573B40l0WAAAAAADArEAQNY18vipduPydkqQTWqKYTGKuoOBCLV/2kUyVBgAAAAAAkHEEUdNsY83VkqSw8apDFZIkh8OnzZvukdPpz2RpAAAAAAAAGUUQNc2ynA4VuZ2SpGb3RklSLDaowcHGTJYFAAAAAACQcQRRM6DaO9wH6kBsRWIs0Ls7U+UAAAAAAADMCgRRM6DK55Yk7YmeeXNeb2BPpsoBAAAAAACYFQiiZsDIjqgGLZIxw9eBXoIoAAAAAACwsBFEzYBq33D4FDVuyb9aktTbu0+xWCSTZQEAAAAAAGQUQdQMqPa6E9f93nWShhuWB4NHMlUSAAAAAABAxhFEzYCRHVGS1O5clbgOBGhYDgAAAAAAFi6CqBnQF4kmrn/WU5m4pk8UAAAAAABYyAiiptl9LR26affRxOdd4XINyitJ6u7ZlamyAAAAAAAAMo4gahqdGAjpYwcbZJPGrHHquJZLknr7nlUsFspMcQAAAAAAABlGEDWN7mvpVMSOHT+qFZIkhyLqCjyb5qoAAAAAAABmB4KoaXR0IPVup2Nambg+1c3xPAAAAAAAsDARRE2jUrcr5fiR+I4oSYoG96WrHAAAAAAAgFmFIGoavaGiKOV4qyrUryxJ0kDf3nSWBAAAAAAAMGsQRE2jzXlZem9N6ZhxaxxqMqskSf39hxWNDqS7NAAAAAAAgIwjiJpmn19Zpf9ct1gbc/2JMZ/D6KKq50uSrI2qt29/psoDAAAAAADIGIKoaWaM0fUVRfp97Rq9rrxQkjQYs/LknJ9Y0xvYk6nyAAAAAAAAMoYgagZtzs1KXB+1ZxqWBwiiAAAAAADAAkQQNYM2550JonYM5sntHm5mHugliAIAAAAAAAsPQdQM2pDjl9MMX+/sHVBe3vDxvGDwqCKR3gxWBgAAAAAAkH4EUTMoy+nQ2myfJGl3b1A5iT5RVr29+zJXGAAAAAAAQAYQRM2wkT5RvdGY+r1rE+OBwO5MlQQAAAAAAJARBFEzLLlP1OHkhuX0iQIAAAAAAAsMQdQMS35z3o6BLHm9FZJ4cx4AAAAAAFh4CKJm2Npsv3yO4Y7lO3uDys09T5I0ONigcLgrk6UBAAAAAACkFUHUDHM7jDbk+CVJe/sGlJ17fmKOXVEAAAAAAGAhIYhKgy3xPlGhmFW3e01inD5RAAAAAABgISGISoPkPlHPxpYnrnvZEQUAAAAAABYQgqg0SH5z3vagWz7fIknsiAIAAAAAAAsLQVQaLPd7lesc/le9MxBUXt5wn6hQ6JRCodZMlgYAAAAAAJA2BFFp4DBGm+LH8w4GB+XLOS8xx64oAAAAAACwUBBEpcnI8byoldqcqxLj9IkCAAAAAAALBUFUmiQ3LN8XXZa4DvTuzkQ5AAAAAAAAaUcQlSbJDct3BB3Kyhp+e14gsEfW2kyVBQAAAAAAkDYEUWlS7XWrxO2SFG9YnrtRkhQOd2pwsDmTpQEAAAAAAKQFQVSaGGMSu6KODoTkyl6fmOulYTkAAAAAAFgACKLSaEtSn6gW5+rEdSBAnygAAAAAADD/EUSlUXKfqN3hxTLGKUkKsCMKAAAAAAAsAARRaZT85rwd/TFlZ6+SNHw0z9pYpsoCAAAAAABIC4KoNCr2uLTI55Ek7ewNKjf3fElSJNKrgYETmSwNAAAAAABgxhFEpdnIrqiWUFjyn2lYHghwPA8AAAAAAMxvBFFpltwnqsGxKnFNnygAAAAAADDfEUSl2eZcf+J6V7hKxrglSb3siAIAAAAAAPMcQVSabcrNkolf7+iLKCdnrSSpt2+frI1mrjAAAAAAAIAZRhCVZjkup1ZmeSVJOwNB5cUblkejQfX3H8lkaQAAAAAAADOKICoDRvpEdUWiCnnXJMbrj/ybegK7ZK3NVGkAAAAAAAAzhiAqA0benCdJf238a+K6o+NPqqt7rfbs+QdFo4OZKA0AAAAAAGDGEERlwJakN+dtD/0RqHQAACAASURBVFeOmW9rf1iH67+YzpIAAAAAAABmHEFUBmzI8ctlho/fHdHqlGtamu9XONyVzrIAAAAAAABmFEFUBngdDq1090mSjmuZUnWEitkh9fYeSG9hAAAAAAAAM4ggKkPWe4OSpJDxq0PFKdc4nb50lgQAAAAAADCjCKIyZGvJ0sT1AW0YM+/1lCs3d2MaKwIAAAAAAJhZaQuijDFXG2MOGmPqjTG3pZi/wxizM/7rkDGmO2kumjT3QNL4MmPMU/F7/swY40nX8zxXF5YsTlzXa82Y+RUrbpXD4UpnSQAAAAAAADMqLUGUMcYp6VuSXi5pvaQbjTHrk9dYa2+x1m621m6W9A1Jv0yaHhiZs9a+Omn8K5LusNaulNQl6V0z+iDTaFWWT36HkSQdNaODqJKSl6qy8rWZKAsAAAAAAGDGpGtH1FZJ9dbao9baIUk/lfSaCdbfKOm+iW5ojDGSrpB0f3zo/0m6dhpqTQuXw2hjbpYkqcEs03mbfiKHwytJCgaPyNpULcwBAAAAAADmrnQFUdWSGpI+N8bHxjDGLJG0TNKfkoZ9xpg6Y8zfjDEjYVOxpG5rbWQK93xv/Pt1bW1tz+U5ptXmeBAVtlKT+zyVFF8hSQoGj6mv/2AmSwMAAAAAAJh2s7FZ+Q2S7rfWRpPGllhrayXdJOnrxpgV53JDa+2d1tpaa21taWnpdNb6nGzOy0pc7wwEVVZ2deJza+vvMlESAAAAAADAjElXENUkaVHS55r4WCo36Kxjedbapvg/j0p6RNIWSR2SCowxIx29J7rnrLQlOYjqDaq4+PLE8bzW1t9xPA8AAAAAAMwr6QqinpG0Kv6WO4+Gw6YHzl5kjFkrqVDSk0ljhcYYb/y6RNILJe23wynNnyW9Pr70bZL+Z0afYpot8XlU6HJKknYGBuRyZau4+DJJw32i+vsPZ7I8AAAAAACAaZWWICrex+mDkh6SdEDSz621+4wxXzDGJL8F7wZJP7WjtwKtk1RnjNml4eDpy9ba/fG5j0v6qDGmXsM9o74/088ynYwx2hTvE3U4OKi+SFRlpS9PzHM8DwAAAAAAzCeuyZdMD2vtbyX99qyxz5z1+XMpvveEpPPHuedRDb+Rb87anJelR7p6ZSXt7h3Q1pIr5HB4FIsNqbXtd1q+/MOZLhEAAAAAAGBazMZm5QuKy5jE9Y27jugjhzrkzb9EktTff1j9/fWZKg0AAAAAAGBaEURl0I+bO/TV46cSn0PW6v7TXfpmz5kNYKc5ngcAAAAAAOYJgqgM6QxH9KnDjSnnnoxdoEj81GQbQRQAAAAAAJgnCKIy5Net3QrFbMq5AZOtXdosSerrP6j+/qPpLA0AAAAAAGBGEERlSOtQeML5p3XRmbVt7IoCAAAAAABzH0FUhiz3eyec367nS2b4eF4rx/MAAAAAAMA8QBCVIdeUFqjI7Rx3fmtRhYqLht+e19d3QMHgsXSVBgAAAAAAMCMIojLE73Tozg1L5XeYMXNlHpe+unaRyspenhhrbX0wneUBAAAAAABMO4KoDLqkMFePbF2r9y8q1cqsM0f1rinJV43Po9KSl8pwPA8AAAAAAMwTBFEZtsTv1edWVuuPz1+jHOfwb8cjXb2y1srtLlBR4cWSpN6+fQoGT2SyVAAAAAAAgOeEIGqW8DocuqI4T5J0fGBIB4ODkqSysmsSa1rbOJ4HAAAAAADmLoKoWeSqeBAlSb9vD0iSSktfKmOGm5pzPA8AAAAAAMxlBFGzyEuK8+SM9y5/sL1HkuR2F6pw5Hhe7x4NDDRkqjwAAAAAAIDnhCBqFilwu3RRfo4kaXsgqNOhsCSprPTqxBqO5wEAAAAAgLmKIGqWuaokP3H9cMfI8bwrOZ4HAAAAAADmPIKoWebKkjN9okaO53k8RSoouFCSFAjs0sBAU0ZqAwAAAAAAeC4IomaZJX6v1mX7JEl/7epVfzQqSSore3liTRvH8wAAAAAAwBxEEDULXR0/nheKWT3a2StJKiu9UiO/Xac5ngcAAAAAAOYggqhZKLlP1EPtw32iPJ4S5ec/T5IUCOzQM3Wv09Fj31BoqD0jNQIAAAAAAJwrgqhZaGOuXxUetyTp4Y4eRa1VMHhc/f2HEmsCgZ06duzreuqpqxTo3ZupUgEAAAAAAKaMIGoWchiTaFreGY7qmZ5+7dt/qyKRwJi14XC39u69WdZG010mAAAAAADAOSGImqWSj+f9uvmoAoEd464dGDipzs7H01EWAAAAAADA340gapa6pDBH2c7h356Hu4ZkJ1kfDB6b+aIAAAAAAACeA4KoWcrrcOjyolxJ0skhl1pUPeF6t6coHWUBAAAAAAD83QiiZrHk43m7XC8Zd53LlavSkvHnAQAAAAAAZgOCqFnspcV5cprh673eV8gYV8p1a1Z/Xk5nVhorAwAAAAAAOHcEUbNYodulrfnZkqRdQZeWnP8zlZS8VJI7aZVRfn5tRuoDAAAAAAA4FwRRs9zV8eN5VtJTQ4u0aeP3dMXlB7Rm9RfiK6yam+/LWH0AAAAAAABTRRA1yyX3iXqoo0eSZIxRRcVr5HTmSJKamn+uWGwoI/UBAAAAAABMFUHULLfU79WabJ8k6S+dvQpGY5IklytHlRXXSZLC4Q61tj2UsRoBAAAAAACmgiBqDhg5njcQs/pLZ29ivLr6psR1U+O9aa8LAAAAAADgXBBEzQFXFeclrkeO50lSTs5qFRRslSR19zyjvr6Daa8NAAAAAABgqgii5oDNeVkq87gkSb9vDyhqbWKupvpNievGpp+kvTYAAAAAAICpIoiaAxzG6Mri4eN5HeGItgeCibnS0ivl8ZRIkk6d+pUikb6M1AgAAAAAADAZgqg54qqSM8fzHmw/czzP4fCoqvJ6SVI02q9Tpx9Ie20AAAAAAABTQRA1R1xamCu/Y/i36/dJQZQkVVffqJHfyqbGH8smHd0DAAAAAACYLQii5gif06HLinIlSYeDIb1n7zH98nSXQrGYfL4qlZRcIUnq6z+onp5tmSwVAAAAAAAgJYKoOaJ9KKK9fWd6Q/26rUf/Z/8JveSZg2oYHDqrafm9mSgRAAAAAABgQgRRc8RHnj2pxsHwmPH6YEjv2XtchYUvlN+/WJLU2vo7DQ21p7tEAAAAAACACRFEzQFHgyH9oSMw7vzO3qDqAgOqrr5JkmRtWM3N96erPAAAAAAAgCkhiJoD9vYNTLpmT9+AqipfL4fDI0lqav6JrI3OdGkAAAAAAABTRhA1B+Q4J/9tynE65XYXqrzslZKkwcEmdXQ8OtOlAQAAAAAATBlB1BxwcUGOitzOcee9DqMrS/IkSdU0LQcAAAAAALMUQdQc4HM69E8rqsadv3VphQrdLklSXt4m5eZukCR1dDyqgYGTaakRAAAAAABgMgRRc8SNlcW6c8NSrc7yjZl7WXw3lCQZY1RT/eb4J6umpvvSVCEAAAAAAMDECKLmkFeXFejRrWtUd9F6/fPKMzuk/r+GtlHrystfKZczV5J0suFubd/xVh05+u8aHGxOa70AAAAAAADJCKLmGGOManwevb26VJVetyTp/tNdah+KJNZYG5XD6Ytfh9XV9biOH/+Wnvzblero+GtG6gYAAAAAACCImqPcDqN3VpdIkkIxqx81tyfmDtd/WUNDbWO+E4sNaM/eDyocDqStTgAAAAAAgBEEUXPYm6uK5XcYSdLdTe0aisUUifTq1KlfjfudaLRPp07/d7pKBAAAAAAASCCImsMK3S69oaJIktQ6FNH/tHZrcLBZsVhowu/199enozwAAAAAAIBRCKLmuPfUlCau72xok9OVN8HqYW53wUyWBAAAAAAAkBJB1By3KtunK4qG35C3p29AOwdzVVBw4YTfqSh/dTpKAwAAAAAAGIUgah5436KyxPWdDW1averTcjqzU64tLr5C2dkr01UaAAAAAABAAkHUPPCiwhytyfZJkh5s71Gna4VqL7hfZaUvlzGuUWv7+w9P2kMKAAAAAABgJhBEzQPGGL033ivKSrqrsU05Oat1/vnf1Isv26cXX7ZX5fHjeIODDWpo/GEGqwUAAAAAAAsVQdQ88dryQhW5nZKkn7R0KhCJSpIcDpecTr9WLL9VDodHknT8+LcUDndlrFYAAAAAALAwEUTNE36nQ2+rKpEk9Udjuq+lY/S8v1qLFr1TkhSJ9OrosW+kvUYAAAAAALCwEUTNI2+vLpHbGEnSXY3tilo7an7pkvfL7S6SJDU13atg8FjaawQAAAAAAAsXQdQ8Uu5169ryAklSw+CQHmzvGTXvcuVq+bKPSJKsjai+/itprxEAAAAAACxcBFHzzEjTckm6s6FtzHxV1RuVlbVSktTW/rC6up5KW20AAAAAAGBhI4iaZ87PzdJFBdmSpKd6+rUzEBw173C4tGrlbYnPh+u/JGtjaa0RAAAAAAAsTARR81DyrqivHGvRE1196gpHEmPFxS9WYeHFkqTe3r06dfqBtNcIAAAAAAAWHoKoeejKknyVul2SpD939uq1O+u1+fF9+tjBBgWjMRljtGrlJyUNNzY/cuSrikYHM1gxAAAAAABYCAii5qGH2wNqS9oBJUkha/Wj5g69Z+9xWWuVm7tOlZWvG54Ltaih4QeZKBUAAAAAACwgBFHzjLVWXzraMu78HzsDeqqnX5K0fPktcjj8kqTjJ76r0FB7WmoEAAAAAAALE0HUPHN8YEiHghMfs3uwvUeS5PNWaMnid0uSotF+bdt2vXbseKsOHvqc+voOznitAAAAAABgYSGImmdCU3gDXihmE9c1Ne+Qw+GVJA0MnFBn1+NqbPyRnnr6FWpsvHfG6gQAAAAAAAtP2oIoY8zVxpiDxph6Y8xtKebvMMbsjP86ZIzpjo9vNsY8aYzZZ4zZbYx5Y9J37jHGHEv63uZ0Pc9stczvVZHbOeGarfnZietTp/5LsVgoxSqrg4c+q96+Z6e5QgAAAAAAsFClJYgyxjglfUvSyyWtl3SjMWZ98hpr7S3W2s3W2s2SviHpl/GpoKS3Wms3SLpa0teNMQVJX/3YyPestTtn/GFmOa/DoffWlI47X+Fx6ZrS/MTnxqYfT3A3q6amn0xjdQAAAAAAYCFL146orZLqrbVHrbVDkn4q6TUTrL9R0n2SZK09ZK09HL9ultQqafykBfrQknK9vbok5VyxxyW3MZKkWCyigYETE96rv79+2usDAAAAAAALU7qCqGpJDUmfG+NjYxhjlkhaJulPKea2SvJIOpI0/MX4kb07jDHece75XmNMnTGmrq2t7e99hjnDYYy+vLpGT1y4Tp9aXqmPLC7T6qzhfzX7+gb1s1OdkiRjnHI6cya8l8ddNOP1AgAAAACAhWE2Niu/QdL91tpo8qAxplLSjyS9w9pER+5PSFor6fmSiiR9PNUNrbV3WmtrrbW1paULZzPV8iyvbl5SrttWVOk7G5YmfrNvP9KinnBExhhVVEy0MU2qqHj1zBcKAAAAAAAWhHQFUU2SFiV9romPpXKD4sfyRhhj8iT9RtKnrLV/Gxm31rbYYSFJd2v4CCBS2JDjTxzX6whH9K/HTkmSli29WT5fzTjfcsjnWzTOHAAAAAAAwLlJVxD1jKRVxphlxhiPhsOmB85eZIxZK6lQ0pNJYx5Jv5L0Q2vt/Wetr4z/00i6VtLeGXuCeeDjyypU7HZJku5uatf+vgF5vaWqveAXqq66UU7n8Nv0XM7c+Ddi2rvvw4pGgxmqGAAAAAAAzCdpCaKstRFJH5T0kKQDkn5urd1njPmCMSb57NcNkn5qrbVJY9dLepGktxtjdsZ/bY7P3WuM2SNpj6QSSbfP+MPMYflulz61olKSFJP0iUONstbK6y3T2rW367IX7dSLL9unSy+tU2HhxZKkYPCIDh76QgarBgAAAAAA84UZnfnMf7W1tbauri7TZWRMzFq9cvthbQ8M73L61rrFel3F2IbkoVCrnnr6FQqHhxubb1h/B/2iAAAAAABASsaYbdba2snWzcZm5ZhBDmP0pVU1MvHPnz/SrN5IdMw6r7dMG9Z/LfH52YOfVjB4PD1FAgAAAACAeYkgagHanJelt1QVS5JahyL62vFTKdcVF79ISxa/T5IUjfZr774PKRYLpa1OAAAAAAAwvxBELVC3La9UocspSbqrsU0H+wdTrlu+/Bbl5W2RJPX27lN9/b+mrUYAAAAAADC/EEQtUEVulz6xfLhxecRKn4o3Lj+bw+HWeRu+LpcrT5LU0HiP2tr+kNZaAQAAAADA/ECz8gUsaq1eXndIu/sGJEnXlxeqyufRiiyvXllaIL/zTE7Z2vqg9uz9gCTJ5cpTdfVNikQC8norVFlxnXy+qow8AwAAAAAAyLypNisniFrgtvX06xXbD48ZL3a7dNd5S3VRQU5i7NmDn1FT070p7uLU6tWf1qKat85gpQAAAAAAYLbirXmYkpZQOOV4Rziit+w+qpbQUGKsvOyV49wlqkOHPq/OzidmoEIAAAAAADBfEEQtcN9uaB13ri8a0w+bOhKfm5p/MuG9Ghrvma6yAAAAAADAPEQQtYBFrdX2QHDCNc/09CeuA4HdE66dbB4AAAAAACxsBFELmJHkNmbCNV7HmT8iTmf2hGudTv90lAUAAAAAAOYpgqgFzGGMrizJm3DN1aVn5stKr5pwbVHRi6alLgAAAAAAMD8RRC1wH11aIb8j9a4oI2l1li/xuabmLfL7F497r66uJxUOd093iQAAAAAAYJ4giFrgNuT49YvNK3V+zpljdSOxlJX0f/afUMdQRJLkdufrguf9VGVl10hyJlY7nTmSpGDwiHbuepcikTN9pQAAAAAAAEYYa22ma0ir2tpaW1dXl+kyZh1rrQ4GB9U5FNUSv1ufONSk33cEJEmXFubovo0r5EraORUOd2kwdFpeT6kkq23bb1AweEySVFh4sTZvuksOhzcTjwIAAAAAANLMGLPNWls72Tp2REGSZIzR2my/Li7MUbXPq2+uX6Ll/uEg6a9dffrysZZR693uQuXmrJXHUyyPp0RbNv9QXm+FJKmr6wnt3XeLYrFI2p8DAAAAAADMXgRRSCnP5dT3z1uqLOfwH5FvnmzV/7aO3//J56vSls0/lNtdJElqa3tIzx78lKyNpaVeAAAAAAAw+7kyXQBmr3U5ft2xdpHet++EJOnDz57Uqmyf1mT7Uq7Pzl6hzZvv1vbtb1I02qeWlvvlMB75/IsV7D8st7tA5eWvUl7e+el8DAAAAAAAMEvQIwqT+nx9k77T0CZJWuH36ne1q5Xnco67vqvrae3c9XbFYqGU89VVN2rNmi/IGDbkAQAAAAAwH9AjCtPmU8ur9MKC4TfjHRkI6eb9J/SXzoB+0tKhRzoDisRGh5mFhVu1Zs3t496vqfk+NTb+cEZrBgAAAAAAsw9H8zApl8PoexuW6qq6g2oKhfVQR0APxd+oJ0nVXrf+Y91iXVKYmxgbHGiY8J4nG+5RTc3bZIyZcB0AAAAAAJg/2BGFKSnxuHT7quqUc02hsN68+6gO9g8mxnr79k14v8HBBkUivdNaIwAAAAAAmN0IojBlj3f3jTs3GLP6zsnWxGenM3uSuznkcHinqTIAAAAAADAXEERhyh7tnHgH01+6zsyXl718wrVud76i0f5pqQsAAAAAAMwNBFGYESUlL1FR4QvHnQ+Hu1S37XXq7z+SxqoAAAAAAEAmEURhyl6U1Iw8lYvycxLXxji1ceOdWrzoXXI6z4zn59fK5xvuNTUwcFJ1216vzs7HZ6ZgAAAAAAAwqxhrbaZrSKva2lpbV1eX6TLmpBMDIb3kmYPqi8ZSzm/JzdIvt6yU3zk634xGBxUKnZLLlSuPp1jhcLf27PmAurr/JkkyxqU1qz+v6uobFI2GNDBwXA6HV37/Et6qBwAAAADAHGCM2WatrZ10HUEUzkVdT78+sP+ETgwOJcackqLx65cW5+kH5y2VxzHxZrtYLKyDBz+j5pafJ8by8raov/+ootEeSVJ29mqtXPExlZRcMd2PAQAAAAAAphFB1DgIop67qLV6oqtPDaEhVXrcqvZ69Lpd9WobikiSXlVaoO+sXyKXY+LdTNZanWz4vurrvyxpvD+HRhvP/65KS186vQ8BAAAAAACmzVSDKHpE4Zw5jdGlRbm6qbJYlxfnaXWOTz/ftEIFLqck6ddt3frowZOKTRJyGmO0ZPG7tWrVP02wyqr+yJdlberjgAAAAAAAYO4giMK0WJfj132bVign3h/q56e69MnDTZrKjrtoJDDhfDB4TH39h6alTgAAAAAAkDmuTBeA+WNLXpZ+vHG5btx1RAMxq3ua2hWzVsVupw4FQypwOXVdeaFeWJAzqgl5NBqc9N6xKawBAAAAAACzG0EUptULCnJ0z/nL9ZbdRzVkrX7Y3DFq/t6WTl1bVqBvrjvTQyo3b+Mkd3XI662YoYoBAAAAAEC6cDQP0+6yolx9aVX1uPP/3dqt7za0Jj6XlrxUft/iCe4Y0/Ydb1Ff38FprBIAAAAAAKQbQRRmREMoPOH83U3tif5RDodbmzbdJZ9vbHjlcHglSQMDx/VM3WvV0vKr6S8WAAAAAACkBUfzMCP29w1MON8UCqsnElWBe/iPYHb2Cl30gofV2vqgenp2yOH0qrT0Svl9S7R//z+qs+sxxWKD2n/gVvUEtmvVyk+pp2ebOjsflyQVFb1QhYUXyRiyVQAAAAAAZiszlbeazSe1tbW2rq4u02XMex/cf0L3n+4ad94l6chlG+V1TB4cWRvV0WP/qePHv5kYczqzxjQ5z89/njZtvFNud+HfXTcAAAAAADh3xpht1traydaxfQQz4tryicMgl8PoQN/glO5ljFMrlt+iTRvvksuVLyn1m/Z6erZr3/5/PPdiAQAAAABAWhBEYUZcUZSrq0vyxp0fjFm9Zsdh/aylc8r3LCm5XOed940J13R0PKq+/sNTvicAAAAAAEgfgijMCIcxunPDUt26tEKlnuE+UC4jXVWcpxcV5kiSQjGrDz97Up861KhwbGpHRIdCpyZdE+jZ9fcXDgAAAAAAZgzNyjFjPA6Hbl1WoVuWlqszHFG206ksp0Mxa/X1E6f1b8dOyUr6flO79vUN6J9XVevXrd3aFgjK6zB6eWm+Xl9eJL/zTF7qcPgm/8HGzNxDAQAAAACAvxvNypExv2/v0Qf2n1BvNCZJMpLO/tO4LtunX2xeqZL4rqpwOKDHHr9Isdj4/aX8/sVat/YrKizcOkOVAwAAAACAZDQrx6x3ZUm+HqxdrZV+r6SxIZQkHegf1CcPNyY+u915Wrb0gxPed2DgpLbvuFHPPvtpRSK9kqRIpFetrQ/p1KkHFAwen65HAAAAAAAA54CjecioFVk+vbOmRJ883DTumt+0dqttVVilHrckacmS98vpzNLxE9/V0FCrJMnjKVV19U3q7d2n9vY/SJKamu9Te8efVFT4QrW2PaRotD9xz9KSl2ndun+V2z1+Q3UAAAAAADC9CKKQcY2D4Qnno5KODwwlgihjjBYtepuqq29Sf/8hSVJ29io5HB5Za9Xa9jsdPPg5hcMdCoVOq+XUL8fcs639YYX3vF/P23KvDD2lAAAAAABIC47mIeOKPZPnoX7H2D+qDodbubkblJu7QQ6HR9JwSFVedo0uesHvVVFx3YT37O5+St3dT/19RQMAAAAAgHNGEIWMu66sQM5J1rxj7zH9tbN3yvd0uwu0aNHbJ13X0fnYlO8JAAAAAACeG4IoZFyVz6PblldOuKZhcEhv2HVEHzvYoN5IVJLUPDikn5/q1M9PdappcGjsl6bwRsi+vgOKxVJ8FwAAAAAATDtjp/A/6/NJbW2traury3QZSOGB1m5952SrdvQG5XUYXV2Sr7dVFevupg79uq07sa7K69babJ8e7exVND7mkHRDZZH+ZXWNvPFjfLFYSI89fonC4c4Jf67XW6klS96nqso3KBrt14mTd+rUqQcUiXQrO3uVaqrfosrK19NLCgAAAACAcRhjtllrayddRxCF2SZqrRzSqODnf1u7dduhRrWHIxN+902VRfra2sWJzydO3qX6+n+Z0s91u4tlbVSRSPeYueqqG7VmzT8TRgEAAAAAkMJUgyiO5mHWcRozJvB5ZVmB/nLhWl1Tkj/hd+9rGX1Mb/Gid2nZ0g/JYTyj1hUVXaoLLrg/vtNpuFl6ONyRMoSSpKbm+9Td/czf8zgAAAAAACBu8teVAbNEkdulV5Tm67ftPeOuiUl6vLtP11cUSRreVbV8+YdVU/MWdXT+RbHooPLyNys3Z60kqSB/i5Yt/ZBOnPiumpp/MuHPbzn1SxUWbp225wEAAAAAYKEhiMK80zYUHjPm8RSpsuLalOv9/mqtXPl/Jw2iuruf0cBAo/z+GkmStVGdOvWAmlt+oVCoRT5ftaoqr1d5+StlDJsNAQAAAAA4G0EU5pSLC3PklBJNylP50tEWNQ6G9dGl5Sr1uCVJPeGIHu3q02AspuflZWlllm/Ud5zObLndxQqHO8a978DAcT3x5GUqKNiqiopr1d72J7V3/CFp/qS6up5UR8cjWr/+q4RRAAAAAACchSAKc0ql16M3VRXrh83jB0ZRK93d1K6fnerUP9SUSkb69sk2DcRiiTUvK87TN9YtVoF7+K+AMQ5VV71Rx098e9IaurufVnf30+POnzr9PyoufrEqKl59Dk8GAAAAAMD8x5YNzDm3r6rWW6uK5Uwac2r4jXm3r6xSqWc4XApGY/raidP62vHTo0IoSXq4I6C37zmm5LdGLl36ARUWvCDlz1yx/GNau/ZLKsh//pRqbGr+6Tk9EwAAAAAAC4FJ/h/xhaC2ttbW1dVlugxMg5bQkJ7o6pM0fGSv0jv8Zrz+SFTfa2zTt062qj8am+gWun/zCl1SmJv4HIuFdfr0Azp1+tcKh7uUk71a1dVvUn7+5sSaovDb3wAAIABJREFUYPCEnn7mVYpG+8e9r9tdqksveWLM8byhoU4NDJyUx1Mkv3/xOT8zAAAAAACzkTFmm7W2dtJ1BFGYrx7tDOiNu45OuOZDi8v0yRVV53zvZ565ToHe3ROuycpapurqN6my4nWSojp46Atqbf2trI1IkvLytmjN6s8oL2/jOf98AAAAAABmk6kGURzNw7xV6J68Bdpfunp1YiA0aqw7HNFPWjr07ZOt+mNHQNEUYW1l1RsmvXcweEyHD9+uxx6/WE8++TKdPv1AIoSSpEBgh7Ztv0l9fQen8DQAAAAAAMx9NCvHvLU226cSt0vt4ci4a3b2Duiivx3Qy0vz9d6aUu3pDer2oy0ajJ0Jn5b5PfrBecu0LsefGKuqfIPa2/+ojo5HxtwzP79WXk+p2toflrURxWIDisUGUv78WGxAx45/U+ef941R4339h9XY+CMFArvldGaprPRKVVa+QS5X9jn+WwAAAAAAYPbgaB7mte81tOqz9c0p51xGikzxj3+p26W/Xrg28ZY9abifVHPzz9Tc8gsNDrbI76tWVdX1qqx8gxwOl0KhVjU1/0wnTnxHsVhogrs7deHWXys7e5WMcai19UHt3ffhUbunJCk7e5Wet+VeeTzFUysaAAAAAIA0oUfUOAiiFhZrre44cVr/ceK0Qkm7nK4oytW/rVmkP3UGdGdDmw4HJwqKhn1uRZXev7jsnGvYvv3N6up+ctJ1TmeOcnPXq7t7m6RoyjXl5a/WeRvuOOcaAAAAAACYSQRR4yCIWpi6whE90tmrgVhMF+Rla022LzFnrdXv2nv0zr3HJ7zHS4pyde+mFaPGotbqia4+NYWGVOPz6OKCHDmMGbXmyNF/1/Hj35qW5zDGpUsveUpud8G03A8AAAAAgOkw1SCKHlFYEArdLl1XXphyzhijy4vyJr3Ho129+siBk3p9RaEuLsjR0z39uvnASTUMDiXWLPV79K11S3RB/pleTtVVN+rkyR+M2yeqqvIGyVgFArvijcvHD4etjSgQ2K3i4hclxmKxIbW2PaSuridl5FBR8aUqKX6JHA7+egMAAAAAZhd2RAFxr99Rr8e6+6a0ttzjUsdQRKnaoOc4HfrD89doqd+bGOvsfFx79t6sSKQnaaXR8mUf1rJlNydGjhz5dx0/MdnuKaOiwheqsvJ1ys09T7v3/IOCwfpRK3Jzz9PmTT9I2U/KWqtQqEWx2JB8vhoCKwAAAADAc8bRvHEQRGE8T3b36fU76lN2Zyp0OeVzONQyFJ7Svd5ZXaIvra4ZNRaJ9Ol0628U7D8it7tQ5eWvkt8/es3AQKOeePJySbEpVu0Yd21x8WXavOkHo8ba2/+sI0e/pr6+A5Ikj6dMixe/S4sXvVPGOKb4MwEAAAAAGG2qQRT/5wnEXVSQo7vPX6Yan3vU+IsLc/XnrWtVd/F63b95ha6vKJQZ5x4j/tgRGDP2dK/0ha4L9b6u1+pj3S/Tf3f7FY6NDoL9/hqtWHFrynu6XPlauvRm5eVtShodP7Dq6HhUvb2HEp9b2x7Srt3vSYRQkjQ01Kr6+n/R4fp/meSJAAAAAAB47tgRBZwlaq2e6elXVziiNdl+Lc/yjlmz5fG9ahlKdTDvjA05Pl1TUqBrSvP1UFuPvnz81Jg1lxbm6EfnL5fPOToTbmt7WCcb7lYgsEtOZ5bKSq/SkiXvT+yg6u+v16FD/6zOrscmeRojn69KPt8i9fbuUTTaP+66iy/6s/z+RaNGA4HdOn36fxWOBJSbs1YVFdfJ7c6f5GcCAAAAABaaWXc0zxhztaT/kOSUdJe19stnzd8h6fL4xyxJZdbagvjc2yR9Oj53u7X2/8XHL5B0jyS/pN9K+rCd5IEIojAdbnn2pO5r6ZyWe926tEK3Lqs45++1tPxK+w+k3j3191i8+N1aueLjMsYha6M68Oyn1NLyi1FrnM4cbTz/Oyoqunjafi4AAAAAYO6bVUGUMcYp6ZCkl0lqlPSMpButtfvHWX+zpC3W2ncaY4ok1Umq1fDrxLZJusBa22WMeVrShyQ9peEg6j+ttb+bqBaCKEyHQ/2DurLuoAZjY//+eIxRbX626nr6NTSFv1+VXre2X7Rexpw58DcUi+m3bT3a0RtUlsOhV5YVaEOOf9T3wuFuPfb4xYrFQinva4xHBQVbNTjYqMHBBlmbqvvVaC5XgfLzt8jaqDo7/5JyjdOZrYsv+pM8npLEmLUxtbU/rJaWXyoUOi2/f5Gqq25QYeHFo54LAAAAADA/TTWIStfrsrZKqrfWHpUkY8xPJb1GUsogStKNkj4bv75K0sPW2s74dx+WdLUx5hFJedbav8XHfyjpWkkTBlHAdFid7dOPNy7Xhw+cVFPoTAPzRT6PvrlusS4syFFvJKo/dgT06cNNag+Pf4yvJRTWnY1turokX0v8Xh3sH9Sbdx9Vw+BQYs0dJ07rDRWFumPNYrkcw8GO212g5cs+rPoj/5ryvmvWfE7VVW+UJIVCrXrs8UuklK3Yz4hEutXR8ecJ10Sj/Wpu+S8tXfI+SZK1Ue3bf6tOn34gsaa3d49aW38b32V1G2EUAAAAAEBS+oKoakkNSZ8bJV2YaqExZomkZZL+NMF3q+O/GlOMp7rneyW9V5IWL1587tUDKVxSmKunXrBef+nqVUsorGqfW5cW5soZD11yXU5dW16ohzsC+q/TXRPe67P1zfpsfbNW+L06PRRWX3RsE/JfnOpSjdejjy+vTIwtWfI+ud2FOnb82xocHP5rkpW1QsuXfUjl5a9MrPN6y1RZeZ1aWu5P+fPd7mKVFF+unsBOBYP1kz770aNfV0fHo8rOXqlIuEenW/835bqTJ+9SUeHFKi6+bMxcf3+9gsFj8nhKlJe3ibf2AQAAAMACkK4g6lzcIOl+O5VzRFNkrb1T0p3S8NG86bov4HIYXVGcN+Ga6yuKJg2iRhwZSH3MbsTdTe360JJy+ZOam9e5rtR/uM5TqxpkZbTIuUT/6KpU+VnfXb3qMxocbFJX15Ojxn3eKm3efI+ys1dIkgYGGvXEk2ODo2TWDqm7+yl1dz816TM1Nd03KogaGGjQ/gP/V93dTyfGsrKWae2aL6qwMGU+rXC4W/399XI6s5WTs5YdVgAAAAAwR6UriGqSlPw6rpr4WCo3SPrAWd998VnffSQ+XjPFewIZ86LCHL2pskj3pmhuvibbp++tX6K6QFB/6OjRHzoCikwQlXZHovr04UZdVZKv5+Vl69dt3frEofjGQDMcPbX3Degtu4/q2+uX6LrywsR3Xa5sbdn8Q3V2PaH2tj8oFgupoKBWZWWvkNPpS6zz+2tUXHyZOjoeHbcOn69aoVCrrA2Pu2ZEe8eftWfPB5Wfv0XZ2at04MAnFRpqGbUmGDymnbveodoL7ldu7vrEeDQa1KHDt+tUy68Us8NHFbOylmnVyk+qpOSKSX82AAAAAGB2SVezcpeGm5W/RMNh0TOSbrLW7jtr3VpJD0paNvL2u3iz8m2Snhdftl3Dzco7UzQr/4a19rcT1UKzcmRCzFr9/FSn7mnq0OHgoIrcLr2+vFDvX1SqfPeZPPhz9U36bkPblO9rNNzBP5USt0vbL14vj+PM7qn2oYjubmrT79sDCsWstuZn692LSrQ2e3Qj9N7efarb9kbFYgNj7ltcfJk2bbxL1sY0ONigbdtv0tBQ65RrnkhZ2TU6/7xvSBpugL5z5zvU2fVYipUObd70AxUXXzpq1FqrnsB2dXc9LWOcKi5+sXJyVk9LbQAAAACA8c2qt+ZJkjHmGklfl+SU9ANr7ReNMV+QVGetfSC+5nOSfNba28767jslfTL+8YvW2rvj47WS7pHk13CT8pvtJA9EEIXZbFtPv16x/fC03e8rq2v0pspiuRxGxwdCum57vVqGRu9ichuj75+3VFeW5I8aDwT2qL7+y+rq/pskyeXKVVXVG7V82UfldHoT644f/7aOHP3auDW4XQUKR7qnXHNOzrr4G/msOjtThVDDcnPP09bn/0/i89BQp/bs/eCY44Ll5a/SurVfGVUzAAAAAGB6zbogarYgiMJsZq3Vu/cd12/aesbMOSR9b/0SFXlc2h4I6penu3Sgf3DSe/odDp2X41fj4NCYEGpEntOh7RdvUI7LmRjrCkd0Z0Ob/nD6mMKRXlXkLNI7FlXrqrMCq2g0qO073qxAYNeY+xYXX66N539X4XCHenp26OChL2ho6PSkNU/V4sXvVX7eJvmzlunQwc+pu+fplOuqq2/S2jX/PGosHO5RU/NP1dH+Z1kbUUHBVtXUvFk+X9W01QcAAAAACwVB1DgIojDbhWIxfeloi37c3KH++NvzVmV59U8rqkbtWvpbd5+u3TH5G+6m6vZVVXp3TZkkqW0orNdsr9fRFM3Tb1lSPurNfZIUifTr5Mm7dKL5vxQNt8rtrdGymhtVU/MWORyexLojR76q4ye+M24NTme2nM4sDQ11SBr75sC/n1OXXvJEfKeVNDBwUtu3v0mDoeYxP3/zph+ooGDsfzvD4YA6Oh5RJNqn3NwNysvdSNN0AAAAAIgjiBoHQRTmiv5IVIeDIWU5HVqV5R0Telhr9dK6g9rXl3pX1Eq/Vy8szNHu3gHt7RtQeAp/18s8Lq3N9un0UFgH+8d/g98fn79GG3LO9JXaEQjqH589qf1JO7S25mfrjrWLtCLrTCP0wcEW/e3/Z++8w+Oozr59z2xfrbTqXbKaLclVsi3b2JQ49N5LCKGGkkAooQQSEggkJHQCARISQg8l9BZwaAaMe5ObJEuyeu/avjtzvj9WWu16d+W8+ZK8ecPc16VrtWd/e/bMmdkyv3me56w/BkVxxOhVorrqGVJTVyCEQnv70+xtvOuAY/57sVpLsCdVYbWW0N3zFi5X7BRIozGTFctXRxhobW1P0tT8AKo6tX1JSdXMm/twzAgqj6eLnp638PkGsFiLyM46CYPBHqXT0NDQ0NDQ0NDQ0ND4b0EzouKgGVEa/000u7yctb2RDk9kyt1Mq4lXqkrJMQXNlD3jblZuqv+nve4xaUncVpZHgdlIq8fL0ZsacCjREUxZRj0f1ZSTYTSE2kZGNrOx9irkwFSBc1WyMGvW7czIOyPUpihu1q47HK83dipf0YwrSUk9CJezmd6+dxkZiZ2W949QUnI9uTlnYDSm093zOnv2/CimzmotZemSd5DlqfpTra1P0Nh0L+ERXTqdlTmzHyQj44ioPoQQjI1txe1ux2jMICVlKZKki9JpaGhoaGhoaGhoaGj8J6MZUXHQjCiN/zacisJrPcN8OeJABlamJnFyZjJmnRyhO2FzA5vGXDH70EsSR6Ql0uTy0uzyovydr62XgjWoxmOYUJPcUJTNDcXZQNB0ubG+gxe7eqliM9l0M4adTSxlrj2Dl6pKSNBNmTAORz3btl+G19sR0Wd+3vnMmvVTJCm4jS5XK2vXHU78NQRlZNkUcxXA6ZAkAyAQIhBXU1F+F7m5ZyFJEv39f6N2xxVx+1q29K9YrcWhtnFHHbt3/RCHc8okNJvzqaz4Fampy6P6GHfU0dnxPOOOOgz6RLKyTiAr68SI6K39mfyM19IINTQ0NDQ0NDQ0NDT+lWhGVBw0I0rj68puh5vTtjYyEoi0mWTgN5WFnJmdCsBoIMCCNbvwqP+czwarTubYdDsFZiPOgMIfOgfiam8syub6CdMKYMuokwtq91LkX0sxzbixsIFlHJw9h99UFKKXp8yVPXU/pqvr5Zj9lpXdTGHBd/F6exgY+Jj6htv+Kds2iSTp0evtKIozIn1vf3JyzqKy4pdIkozX28f6Dcfh9w9H6WTZxOJFr5GYWBlq6+p+lT17bmH/2ll2+yKqFjyFXp8Q0T4w8AmtrU8wMroZWTaQnn44RUVXkmiriDk2IVRcrn0IoWC1liDL+v/BDGhoaGhoaGhoaGhofN3RjKg4aEaUxteZNreXx9v7+XBgFK8qWGpP4IqCDJYk2yJ0Nzd08PQ0htGVBZmMKwr73F6+HHbEjUP6n5Ksl3moYgZ5ZgN2vY6jN9UzHIgdbRUeaQXg8Ht5fvPtFLrexIQv2IaN/pQLuGjBtejkYPSUEIJ164/B5Ypd6F2SjBQWXITP14/TtY+xsa3/pK2b7F+PyZiJQMXr7YmrS08/gtmVd6PTJeD19rJ23eFxI7MKCi5m1syfhO63dzxHQ8PtUTpZtlBd/QzJ9kUR7d3dr9G87xE8nnYATMYsZsy4jPz8C2JGUqmqj5GRTSiqm0TbbMzmnCiNhoaGhoaGhoaGhsbXC82IioNmRGloHJgRf4AztjWx0xGdynZ7aS5XFGaG7l+0Yx9/HRiN25degsC/4GMmSS+zdmklaRP1p767cx/v9o9iFU6KaURBTxNl+CUT187I4uawlf4GhzayefsF6ERkQXaBREn5rymZqFV1INMKwG5fjE424w8MMz6+m/jpgf8/6GCahElZNlJaehNGQxqSJLNr1/UIYptWtoRylix5L2QwdXS8QH3Dz2Jqi4uupqTkmoi2ru5XaWy8B79/cPLVyco8joqKX6DXJ0ZohRAMDX0eLNzuH8JqLSEv71vYEmbG3RZFcePzDWE0pqDTWePqNDQ0NDQ0NDQ0NDT+s9CMqDhoRpSGxt+HU1F4vmuQN3tHGAsoVNrMXJyXwfKUyOipjaNOTt6yl1hxS2ZZ4uPF5dgNeto9Pn66tyNunap/FJtOJs2gp9Xji6uxSBLbD55Lkj5Yf+rnjZ2827aJk3iNaragQ6GOSt7hVLLTlvPnBaWh5/b2vsvOXZFmjAAkwJRQyfKaN5DloBm2e8+P6O5+Ne44kpKq0OtseLw9uFzN7J9m9+8iLW0lFkshep2NtvYn46YTSpKBg1eswWhMA6Cn9x127bo2pjY5eSkLq18IGVxCKOzafT29ve/s3yvl5XeQn3duRKvPN0RT07309L6NqnqQJSOZWcdTVvojTKaMqNdTVS99/atwOvdi0NvJzDxOi8zS0NDQ0NDQ0NDQ+F9EM6LioBlRGhr/fN7sHeaG+vaIlfPSDHoenz2DQ1OnomRWD41z9vamuP2ckZXCYnsCnR4f7/eP0uT2xtX+TzFJEpkmA8l6Hbsc7mktoL8umkl1UrDm0is9Q7yx+0nO4XlsOEKaWhawynojby1ZinEi7c/pauGL9adgEOMhs2ry1mcs5shlb4VqOe1t/DVtbX+IOwaLuRC7fSEBxcH4+M5p0/j+lZhMuSRYizEYUhgYXI2ijMfVzp/3e9LTv4kkybS1P8Xevb+Io5RYuuQ9bLZyAAIBB5s2n4HTuTdKabHMoGbx6xgMyaG20bHt7Kj9Hl5f+IqKOkqKf0BR0VVR6YRDQ1/R1v4ko6Nb0clm0jOOZEbhZVgseVGvJ4RgZGQDQ0NfApCauoLk5KXTFnv3+gZQFS8mU5ZWW0tDQ0NDQ0NDQ+Nri2ZExUEzojQ0/jU4Agrv9Y/S6/NTaDZyTLo9auU+IQQ/bezkjx3R9adWJNt4YX5J6DnrRhycsjV+Sly2Uc+SZBvdHj8NLg+jgb93rb8DIwHpRj0ZBj373F7cqkAWAbLpwoqbIdIYktIB+NXMPM7KScUqy7zYM8QDe9ZwPk8yh50ABNCxkWU8yyW8sriGqqRgupnT3cnn647FKJxRr6+gp6r6ZTJTqgAYG6tl46ZTQ6bW/qSkrKCw4CL8gVGGhr6kp+eNf9pc/CPodAkoiofp0gmTkqrIzj4Fvc7G0PCaacdcXPQDSkqCkVg+3xBr1x1JIDASUzu78l5yck4L3e/sfJG6+lujdAZDMgsXvhSRJuj3j1K743uMjKyP0CYnL2H+vMcjzDCA4eF1NDXdx+hEHTGTMYuCwosoLLgktKLjJEKoDA6upqf3bfz+YWwJs8jL+1bEKorhKIqLvr4P8Xq7MZvzycg4Ep3OEm+KUBQ3Xm8Per0dozE1rk5DQ0NDQ0NDQ0PjX4VmRMVBM6I0NP53EULwwcAoz3YN0uzykm7Uc1Z2KufkpGKS5Qjdd3e18F5/dP0pgyTxalUpSyeKrO9xuFm5sT7ua8rAoiQrIwGFbq8/InLrn4VBklARKBMfqQbhw4gXH0b8kgmAGruVy/IzSdbr2DLm5IXmr7iC31JAG/JEbalR7PyB73FMyUlcU5QFgE9VeWDNDdT434p63SFSSal8jiNygqvhKYqX1WsORgSGYo7TlnwIC+c9iN8/gsNRz46dV8Y0uCbbDPpkAoojbqH0fweSpCfBWopOn4DfN4zLvS+u1mTKZc7s+9HrbajCz6ZNZ0GcelnJyUtZtPDPofvbay9jYODjmNq0tJVULfhj6P7g0Jds335JzHnJz/sO5eW3h+6rqp+du66mv3/Vfkodsyt/HWGcAfT1f8iePT8iEJiKPtPrk5kz+z7S01dGaAMBB41N99LT/TqKGkx7TUs9lLKZP45Zi8vr7aOz62XGx2qRdRYyM44mI+PouJFcLlcrwyPrkJBJTV2B2ZwbU6ehoaGhoaGhoaGhGVFx0IwoDY3/O/hUlXv29fBs5wBjE+bRoiQrPy3NZdl+K/1NVzT9qsJMbi0NnkA7Awrz1+zEqcb/7FtiT8ClqPR4/Qz4/w0GjBDo8SOj4sMEkoRVljk2w45NJ9Pj9fPhwChlNFDFFgpoRUJlLxWsZiUFtlzeWTQTiyzhE4Jz1vyFS/x3YiGy2Hw3OXyZ8gC/q14Sant7/cUkOFfHNKPGTfM4eXkwUsnnG2LdhmMI+GMbXAB2+0L0+iQCASejo5v536qB9T8hKakag8GOUBWGhr+YVltWdgtWywwk2UR9/c9CqwzGYtnSVSQkBGuNtbQ8RlPz/XGUOg5a9mEoMmpsfCebNp0e0+CSJANLat4KpTSqqp8tW789MdeR6PV2aha/FhFxNTT0FbU7LkdRIuu02e2LqVrwJHr91HtKUdzsqbtlvxpfMrm5Z1I+63Zk2RjRh9PZSEvL4wwMfoyq+rDbFzGj8FLS0g6NGpsQCn39H9Lb+x6BwCg2WyX5eedOEx3mZXDwU7zeXiyWQlJTD5k2BTIQcOL19mI0pmAwpMTVaWhoaGhoaGho/HPRjKg4aEaUhsb/PTyKSqvHR6JOJtdsjKlxBhSuq2/n7b6plC29BBfnZXBbWS66sBo/D7T0cM++2DWXTs5M5vdzioBgVNYh6/fQ6I5fCP2QFBvJej1jAYU1I+P/khUC/150EpgkGZeqohMB7AxjZxQDfsZJpI9sFEnPFQUZ5JgM6CWJexqbOUZ9hVIaMePBhAcTXgZI53fS9XyyfAWZpuCcP7rpfirGHov52t3kc/yKD0gxBdPHvtpxC+7+V6J0k4ZXQdF1pCbNIaA4aG39Aw7HrrjbJeusGPR2FMURESX0n4wkGTEY7MiyGa+3e9qIMputguTkGiTJwODg59Ou0picfBCFhRchyyZGhjfS0vrbuNrsrFOYMydogAUC46z56lACgbGY2tzcc6is+GXo/s6d19Db925M7f4RX2NjtWzZeh6KEp1mWl5+Z0RhekXxUFt7OUPDX0boJEnP7Mp7yc4+KaK9v/8j9tTdjN8/HGozmXKYO+chkpMjf+P4/WM0Nt1NT8+bEwX4JdLTv8nMsltimlwjI5to73iG8fFd6HU2MjOPJT//vKjVHyf77u55jZGRDUiSnrS0w8jKPBGdzhRzjpzORgYGP0MIhZTkGpKSquPWGVNVP6OjW1AUJzZbxQGjzvz+Efz+YYzGzFDNuekQQkxb40xDQ0NDQ0ND45+FZkTFQTOiNDT+u2l2eVk36sAgSRyWkkimyRClEUJwX0sPj7X141aDUTs64KycVO6amY8lrLbVm73DXLG7NeZrzbGZ+WBROQY5eJL3s72dPNHRH3dsl+SlMyfRwqg/uCLhP7MY+78KGUjQyVh1Mn2+AHYxRCa9EaaVDyO1VHFQWhbLk21YdDKvd7Uxz/En0hjAhBcTXox4MeGjjnLySn/JhfmZmGSJ91o+xbLv0rhjGCl4iNNnnghAU8drtDTcFFerGIuYlX8miuJgZHRLVL2nSSa/+XSyCVX9z98P/yh6fTI6nRlV9eP3D06jlMnP+zY6vQ0l4KCj87lptDoWLHgSszkbWTJTu+MKnM662L1KRlasWBOqW9XYdC+trb+LqZUkPQct+yRURH5srJZNm8+MaeLpdAksXfI+Fks+EIya2rzlbMbHd0RpDYZUaha/GVGcvqPzz9TX/zRKa7WWsGjhixiN6aE2h6OerdsuwOeLfG8nJMykuuq5iFUdVdXLnj0/pqf3zQhtcvIS5s19NKp+V0/vOzTu/VVY4X2JjIyjqaz4ZVRNMperlb2NdzEw8AmgIstmsrNPpqz0ZgyGpAitqnppa3+ars6XcHvaMBozyck5naIZl8c02kZHt9De8SwORx16fSJZmSeQm3tWzLpkPt8Q3d1/YWR0M5JkICP9cLKyjkeW45tyg4OfI1BJSVlGUuLcmDoIGpXDI+tQFBdJiXOxWArjaoN6N4HAOAZD6gEXCggEHPh8AxiNaTHnYH+EUKNqvWloaGhoaGhMj2ZExUEzojQ0NCYZ9QdYO+LELwQ19gSyY5hWAM91DfCLpu6IguiHpth4dPYMMoxTz+n3+Tl+817aPNERVMuTbby8oDRkWn08OMa3a5vjju2XM/M4PSuFcUXlxa5BHmjtjatN1MkclpqIU1Fpcnljvv5/KtLEn054SWJswrDyYcCHER8DZOA15HNEWhIWncxepwvDyHtk0IcBP0Z86PFjIICMwjrTt7i+fCkmWabHPUJP/fex4cRAAD0BDPgw4MeAj2a5iisPC9aIcnt6+XztNzDgjznOAHpmz/oZsqTD7x9hb/MDyNMUY7dYitHrrSiKB5cr/kqRXwcMhjSMxlQkSY/T2YAQ8ectMXEuyfbFyLKR/oFPpo0Oy0g/krz885AlA4NDn8c1uABycs6isuIuJEnC7e7gq7UH8P0DAAAgAElEQVTfJF4x/ezsU5gzOxhJJoTC2nVH4nbHNqPTUg+lquqp0P36+tvjmnjJyUtZWP1CKDqpv/8jandcHlOblFTN4kUvI0k6ADyeLjZuOhWfL3qhh8TEuSxa+DI6nRkAVQ1QW3spg0OfR2ltttksWvhiRBpme8ezNDT8PLpf2xyqq5+PMLnGxmrZtv0i/P7IxQJstkqqq57BaEwLtSmKlz17boqKrEtLO4y5c34TZQZ1dP6Zpqb7CASmUqwz0o+ksvJuDAZ7hNbtbqex6V76+z9EiAB6fTJ5uWdTXHx1aB4m8fkGaWz8Nb2976IKH5KkJzPzWGaW3YLJlBWhFUKls/NFOjqfw+nci16fRFbWSRQXfT9KOzkfbe1PBVfk1JnJSD+C/IILMYUZmZP4/SN0db0SsSJnbu7ZUYbj5DgGhz6nv38VquIlyV5FTvYpcQ00v3+U/v6/4Q8Mk5Awk7TUQ0LHzv5Mprp6vN2Yzbmkp62MSrWdGodg3LELl7MJgyGVlJRlyHLs70kI1qAbd+xGJ1ux26vi9gugqj4czgYkZBISZk1rJAoh8Hq7UVUvZnPBAU1HVfWhql50OpsWDaihoaHxb0YzouKgGVEaGhr/CC5F5YvhccYDCnMTLVQkxF7BrNfr5+593bzeO4xHFaQadJyXk8a1RdlYdZHF2G9r7IoZQXV8hp3fzy5CP2FaORWFQ9fX0emNbZI8XFnIWdnBSIt+n5+FX+3GH+ezvchi5LHKGbhUlRaXlxsaOqbd7kVJVvSSxGhAoc7pmVb7fxGdBGZZRgb8AQd2RvczrPzo8dNPFsUps8g1GdFJEjt7PmeGaJrQBnU6FHQE8GImIe9Kqux2DJLEJ/VPkB/Yih4F3YRuykDz4824ksMLlmAgwJf1j2B3/g09AWREVN0ut/UgqgqOQ1W97Ov+gIAj/veZIllISaxAVd14PN0RJ/hfTyRk2YAQIMR0Zq2EPakaSTYQCIzjcOyOqZpMM83KOhmjMRWhBujofIFYtdEmtUUzrgymCUo6mpvvxePpiqstK7uZlORlSJKeltbH6Ot7P+6IZ5b9hNzcs5AkPb29b7On7pa4q2wWF19DSfHVALhc+1i77qiYYwbIyzuPivKgSaWqPr5auxKvN3Zac0bGUcyf93jo/p49t9DV/UrEOCb/T0/7JgsW/CGk7e5+g917bog5D3b7IhYtfCkUnRQ05U6LilADSElZTtWCp0JGRSAwzsZNp8c0g83mAmoWvx6KUhNCsKfuZrq7X43Smkw5LF78KmZTdqitp+dtdu2+nv3nzmTKYtHClyKiuZzOJrZsPQ+fry9CazRmUF39XMTCAoriYnvt5QwPfxWhNRjSqKr6U1REWXvHczQ2/noiHTWIxTKD+fMeD9WTm2Rg8DN2774Rf1itP6MxnTmzHyA1dUWE1uVqZdfuHzI2ti1Mm0lF+R1kZBwZoQ0EHNQ33EZPzztMGrxGYwZlpTeSk3N6hFYIQVv7H2ltfSI0DpMpm6KiK8nL/VaUcTQw8ClNzffjcOwJjaGw8BIKCy6OilhzuVpobn6Qvv4PEcKP2ZxHfv75FBZcFGXMBQIO2tr+SFf3q/h8/ZjN+eTlnUNB/gVRBpoQKt09r9PZ+SIu1z6MxnRysk+loOACdDor+zMyupmO9mcZG9+JXp9AZsZx5Od/O6aR6HZ30NHxLCMjG5EkHWlp3yAv79yYq58qipuenjcn0n79JCcvJTfnzJja4Eqtn9Hb9x5KwEli4hxyc8+KaagCjI5uo7vnDXy+fqyWInJzz8JqLYqp9Xi66e55Hbe7HZMxg5yc06ap8eehr+/9ULRlZubxJCSUxNQKoTI8vJaRkU3IsoH09MOjjuFwHI6GoOEuJqItk+bH1fp8QwwOfkpAcZGUNJ+kxPlxTcpAYJz+/o/w+4exWotJSzs0rrErhMrI6GY87g5M5mxSkpfE1U6O2enci95gJyV56QGM3V7GxnYgyyaSk2uiTPbIMTsZG9sOQFLSgmnTtlU1wLhjF6rixWYrjzL6I7dP4Ha34PcPY7EUHXBVXo+3B6+3F7MpB5Mpc1qt3z+Gx9OBwZB8wJR0VfXidrej01kPqJ00rpFkTMasA5rRfv8oqvBjNKQdUKsobgKKE6Mh9YARs6oaQFGc6PW2aY+J4JhVAgEHOp31gEa7EAJVdSNJhmmPn6lx+JEk+YBjmBwHiL9LOzmWWHOmGVFx0IwoDQ2Nfwd+VeBQFJL0uoj6VOEIIfhkaJznuwZpdXvJMhk4OzuVEzOTo57T5PJw0Y4WGlxTJxsmSeLG4myumhH5w/L37X3c1hh9gmuUJF6uKuWgsELvl+1qiairFc5ByQm8UT11gnTq1r2sHYmuAzTJH+YUkWnU41JUfrK3g+ZpamtVJpgpsZpwKyqbRp2hYvQaU0hCRT9hXOlRkFHxyIlkmcwYJIlhnxtboC0UDRbUBdARQI9CjzSDJZmzMMgSXc4hzGOrJh4LoJ+IJps0xMaxs7DgFMxygHF3P86+F9Djn9AqoT51BJBQyUg/EqssoQaGGRn6BBk19CchkCdOziUEen0KRp0JIXxR5oEWq/C/gYzJmIEk6QgExgko09Vdk0m2L0KS9fh8w3FTMCdNo+zs0zAYklFVL52df0bEMFMntcXF10ycUMjsbfxVzIUQJrUzy24lJWUJkqSned/D9Pd/EFdbUf5LMjKOQADtbU/T2vZ4lHaSgoJLKCm+GknSMTKyiW3bL4xr4OXknMHsyrsB8PuH+WLNwQg1tjmfmnoI1VVPB8clBOs3nIjTuSem1ppQzrIl74V+zNfV3Upn14sxx2EwZrLioM9Ctcn6+j5kx87vx+xXb0hn+bJVoZPM8fE9bNh0KojoCxqSZGLpkrdJSCgDgifia9cfh8/bFWMcOhZWP09KypLQ9m3ddgHDw2tijnnOnIfIzjoxdL+p+QFaWh6NOeaZZT+hsPDi0P2+/g/ZseNKQET1XVBwMbNm/iR03+Xax4ZNZ6AEor/PsrNOYfbs+0JzHAg42bTlXJyOnVHalNTDqJr/ROhkUAjB7j0/oqfntShtQuI8Fle/EHHSHy/t12QpYcmiyLTf4ZGNbN12MUKNXEBCb8yiZuELEeaO19vLhi3fxrffqrGyPplF1c9EGJSK4mVr7eWM7r8Ah2ymat5jpKUdFmoSQlDfcAednc/uN2IdFeV3kJd3TkRrd/dr7K77MYSlTAskSouvo7j4ygjt6Nh2tmy/DNUfGcmZl3ch5bNujTiB9Xp72bztUtzOyHqR6ZknMm/23RGpv4rioXb3DQz1/zVCm5RyMFVzH4mI4hRC0Lzvt+xrfQwp7AKENXEhC+f/NsqY6+x6hbqGOyFsn+hN+VTPfzTKBB4bq2Xrzh8S8EztE50pnwVz7iMluSZC6/F0s23nD3GObQi1SYYM5sy6jaysYyO0iuJi556f0t/3NtKk0a2zM7PkWgoLzo/QCqHStO+3tLT9EUkN/j4TcgIzCi6mrOTqKLOku+ctdu+9B/zBCwpCMpKTczaVs26JSq8eGd1Mbd3t+J3BizECPWmZxzO3/PaodHCXq5XauttwjnwxoZVISjmUeRV3RqTFQ9CA2t3wC/r73gntE3NiDfMrfkZi4uwIraoGaNz3MK0dzyMrwYtpeuts5s26Oco8B+jseo265kfAF1xMRjKXMqf0WrKyjovSDg+vZ8fe+/A7tgTHbCygvPhy8nPPiTJXnM4mdjTcjWP4MyQUVH06xQXfoWTGFVHGkc83yO7GB+jvfRtZuFDlJHJzz6K85Ooog1BRvOzd9yjtnS8iK0Ookpn0zBOZXfbDKCNPCEF7xws0tP4JydeKQE9i2hHMLfthaHGccPr7P2Zn82Oozm0IZMz2Fcwruwa7vTpKOzZWy47Gh3GPfAEo6GzVzC35Phn7rdQM4HZ3srvpYYYG/oqkupAsFVQUXUxu9qmhedOMqDhoRpSGhsb/VVQhWDPsYJfDTZJexzEZdlINsa+cvNs3wqNtfWwdd6EDjkxP4rqibBYkRl69HQ8oXLxzH18MOyLaFyRaeH5+SUTq4Y5xF6dubcQRwzS6oiCD28umfmy81z/CJTtbYo7NppP5cmllKBXyz92D/LAu/gp0t5Xmck5OKh5V5S89Q9zVHDsiA6DUYuLSggx8qsrWMRdvxDHZIFj/anmyjYAQDPgDNLomakUJgQ4FgYSKDFpqx78MvfBNGGLBlExZ0pGsByN+FP8QFhyYJlI1g3+B0K0qmUg3J6ITfjzeXsxidMI4C0QYckHzTCLRlIxO+PH7h9ELZyiCTYcyEX2mhm51kgEJBUkEYOJxja83AjAaUpAkHYriRYlj4E0aJraECmSdGUVx43TWxzW4AJLtNRiN6cEVJQc+mjr5jEFa6jdITKwMRsp1voTwx69LmJZ+FOkTJ2ttXa/hdtTG1SYk1TAj70xApn/wc/r73o6rNVgrmF12PSDhcNTT1Hxv/O0zZLJw7sPIsg5/YIxt2y9DipMWKyQLNYteRK9LAATrtnwH/NFp6ZPvxprFb2Ax5yNJOjbvuB7nyKcxtRKwsPrFkHnW0Pxb2lsejKstr7iX/NzTgODJXO2Oy+JuX17hlVSU/RAInqCtWfuNuPvPnnEyi+c9AAQjDD/58hCkQHS6LYDBVs2hS6ai877cfD7e0TUxtRhyWLnis9AJ8Y6Gu+jreDLmmFXJwmErVofSaNu7XqOh7qaYWoHE0pq3SEycA8D4+C7Wbzwl7vbNnfs4WZlHAUGz4bOvViIrkd/BoejQ0lspnXFRsE0IVq8/GcW1K6Y2Lec7VFXeHmrfuPNGxvpejxltaUo+lIMXTqVLN7U9Q0vjHTH7lcyz+Mayd0Lz1j/wGbW1l8TcNlVn57CDVoWMRLe7gy/WH49OdURpFcnMiiVvh8wBRXHz6drjJ8yDyHkWyCxY8BQZaQeH5mLNlovwjn4Rc58Uz7qTkvypBUC2N9zNQMcTMcecnncpC8pvDt1v736bhj3XxdSaU45kRfVUevvY2A7WbT4bnfBGj8M6h5VLXg1FDnq9faxedxI6pT+knbxV9Nl8Y9nboeNNVb18uuFMCNvXIa1kZXnNa9hss0KPfbX9GtyD70ZpVWQWzH+KzPSDQ4/tbv493S33RGkBimbdRWn+2aHHuvs/Y+eOy5BRorYvNf97VM+aitB1OPayZuMZ6EX0vjamHM3BVY+GDBi/f4RP1p+G3hedzi+s81lZ81LoQoKqBvhs0/kIR3QtU78+h5XL3oxI816/66c4ev8cpQ1INpbXvEJiWPRgfevzdDTdFqVV0TN33hPkZEyZ0b2DX7F9+8XoYpSmyC+7k/LCqePN6dzHFxvPxKAOR2mTci6ipvJWQDOi4qIZURoaGl8nfKqKTpLiRmVB8IfPulEnnw6OoQAHJ9s4LDUROcZz9jjc3NfSw4cDowQEzLSauLwgk2/npEZdQfp9ex+/aOqOSBNMN+j549wiloVFZQVUwUU79/G3wegV3Q5OtvHCghJMcvCqnltROWpTPXtd0QXGdcDLVaUcnJIY2vbl6/fQ4Ymd0nhBbhp3lxcAwQi2Jet20x0n/THToOOLpZUoE2M4YfNeun2xtQC3luRQYDESUAV3NXeH0ioloSKjoKAPGVxL7QlU2iz4VJVVA6MM+BUQAjNu9ASCcUtSMCTfppPJNRkJCEGP149LVScip4JXpxV0mnn2r0AEo7yCxpUaMrD0YWbW1F8gLDpNmYgUm9wzk5rJ/yf7U5AntLpQdJkS8fypx8Lbw/+UsOdGP18Xdl8P6FCQJgw7OTSG8Mi2yf+ZuD/5+Nfrd6PGfxMSkiSjCmXaaEyBDoPehiTJ+AIOpBhRZFNaGas5HyTw+kdRp0mBFkCirRJJ0k2kJbXF1UkEU6x0ugRU1cvo6OZpzUxbYhUWcyYIQe/AJ9PWMDRaK0ixlYEk0dX/GTo1flSkZCokO3UJINE9tAG8sevlAQT0GRTnnICETP/oDtxh0T9RWimBshkXISHjdLfT1/tGXK2CjrKiK9HpzAQCDppbH0eaJtqysPAKzOZsECq7Gu/HIOJHcmfkfIvUpGAUzs7mJzD4418Us6QcTlH2UUhI7Ol4AzG+Nq5WTahhQfGFgERb/+eM9r4UV+s1FLOk8scAjDqaaGn+ddx97ZOSqJn3ILJswB8YY/vOq9HFMQYVdFTPexSDISVY73D7DzCp0YuWhCJPy+8mOTFoAn2x83bMnu1xx5xZeB1FWYcDsLnxCZThaYzr9DNZWBKMdGzs/pDB9ofian0JKzh07u1IksTQWB31u6+KOxcuQxGHL3oaSZKDKwNvPBUDsSPxvdhYedD76HQmhBB8uO40EpTozAEIJlsvrXkPy0QK4AdbrsLqjDaBQ2n0s39HXvpyJEni8z33ofY9E1ebXPhjqorPA2B3+xv0Nf8k7vaJtLP45rxganz/yE52bjszrtZpWcKJy54HJHy+ET5dsyLuXLh0uZxw6GokSUYIlXe+WElCIHaZDj8mvnnwVxiNwVqG76y/CKszuv7k5LgWLHqHdPtszYiKh2ZEaWhoaPz/41cFPqGSoJs+j7zP6+ft/hEGfQHKrCaOz0jGrIvOq/ergue6Bnihe5AOj58ck4Fv5aRyYV56yISapNfr59q6Nj4dmvrxnGcycNesfI5Oj6x1sMfh5tztzVGm0crURJ6cWxxRt+vTwTHO37Evqr6WQZJ4el4xh6dNhaK/3jvM9+Ospnhcup0/zZtKqZhu5cVUg44vl1aGItum0wI8NbeIYzOCPwg+GRzj3GkK3l+Yl8atJbn4hWDTqJPv7JhIHxACE55gvJAUjEorNBu5d1Y+CtDh8XHTRO0wq3CQQxcqMl3k4pWsyMAPi7IwyzIOReWR1l4k4WMGLWTRg0BiiDRaKcIjWTksJZEskx6/Kni3fxRJdZNDF3ZGEMi4sTBIOsOkkmM2kmU0oApBvcuDUNwkM4KECNkifow4sKGT9Vh1MgEhcCvqRG0DENpKZ/9ahAiZVDrUsOizcAMu/L6CHn/I4Jo0vKQoI02N0EyaevH7nboVYael+/8fTBAl1DYV/abGNOBim3Gx/qZ7njhAX1PpqxoaGhoaGv8ugr9wpaiWaGt3Ehlp4ntVxDC4h0khnaDB6Uo9jxOrfq4ZUfHQjCgNDQ2N/w6aXV4anB7sBh1L7Alxo75cisqbfcNsHnVilmWOzbCzIjn2ako7xl38tq2P1RMm1yEpiVw1IzMqpRHglZ4hftXcHYqiMkkS5+Sk8vOyvCiz7fG2Pn7V3I0v7Ds332zgT3OLmR/WtxCCHzV08GxX9FXLS/PTuaMsLzRuIQQ/2NPGq73RIdKlFhNvLZxJunGqzsl3d7XwXn/01XoJeGZeMUeFmXiX72rhrThpjRfmpfPrWfmh+zfWt/NcjPFCMGLusyUVoX1zZ1MXj7YFCzZLImgCBCbMML0EXy2tpNASDFufrHUmCZVZ1JHCMH1k0kzwSv4Tc4o4KTNoyk0ag8limIP5lCx6GcXOGg6lW8rngtxUbi7JRRGwadTBhTtbmCV2cxifkEc7LhLYzkI+5xuUJKbzQEUhiipo83i5YncbFWInB7GGdPoRSLRSxFcczKCuiHtm5WOUZcYDCrc0dFAhtlLNFoz4JowzA3XMZjM1XFqQTbbJgF8VPNLaQ6FSSzm7ISwxcJRkNrKUxanZVCRYUITgrb5hknx1FLEvzMoIxjPtpQKbNZ95Nisqgs2jLiTvPrInTMGphEOZYVIY1RUyL9GKCrS5vfi8PSQzHKX1Y2BAymKGxYIKjAUU3L4xrLgixjCpd2PBog+magSEwK/4kSdq+whk1L+z+OnXCUlMmVZSmEEV3hYZBTdZLy48Ak+diGpTIiIjpFDkmghrY7/HRcRelEL3I2u9TY5pUh/rNmj5qaHTlci+iNu/FPH8qedG9knYWCP7kcNeM7pO3fRjme6PA4xncn+Fjyt8rvavlTd9n/HGEWvOvl7nbRoaGv/Z7KGSSoI1ENstR3HhQY9rRlQ8NCNKQ0NDQ+OfRUAVbBt34VFV5tgspMSp2QXBFQ3f6x9lxB+gPMHMEWl2DHK0GSaE4LOhcV7sGaLb4yfPbODbOWkcnBJtnilC8FzXIE93DtDo8pBq0HN6Vgo/mJEVVT/Mq6r8sqmb57sGcanBE6Qyq4mfleZGmFAQNO9+WNfGm2FmlAycm5PGXbPyMIZFqbkUlct2tfDRfqmVxRYjLy4opcgyVQDVrah8p7aZL0ciay3ogN9UFnJG9tSKPF5V5bza5qj6ZQCnZ6XwSGVhKH00oAq+XdvM6uHoFJMZZiPvLpoZqnc2nSlnkCRerSplaVjq6FW7W0Nmn0U4UdHhnUiVvLk4m2uLplZSu72xk9+192MQXmbSgA6FJspwSTZWpiby4oKpgqKPtvVxZ1MXdjHMYjZgxk0LxexmHhkmI2uXVpKgD5o3r/YMcdWeNnJFB4fwGUmM0ks2n7OSMSmVv9WUM8cWXMnzy+FxztjWRLnYzUo+Iosehknlc1ayjYU8UFnIuTnBeh1NLg+HrK9joVjP0bxHIa24sLKOFbzHSZyeVxxKXx3xB6j+ajfVympO4E1m0IKKxA6qeJ0zsSVV88HiWaH9sXTtLmq8L3ECb2LBDYAHIx9xDO/J57Nh+Twsehkh4LQte8kff55jeRs9aug0vIkyfs+V/Km6hplWMwL4cX0biQOPchBrwqKfJPzoeInvcFzpyRyVbkcAT3b009D5FifwJhIipFWQWcWxZGSdwnfzMxDAx4NjrG55l+N4m8kaJBC0GZopZY/tQm4uLUQlGGn5etMqjub9kGby17QDGx/qzuWW8nnoJYkhf4AnGz7jSN4nWDtlKlpLReJjjuHc0kNINujxqSpPNG7gUPV95IkowPCorjpmMyv3OMom5uLl1lqq/JOF2yPtjlHseJJPoybFjhDwQW87xe730U/UvwuPFPNhpN1yFIdl5ICADcND2B0fYMK3n30S1O/VLeUb2cHC5g1ON+rIR1hxhnRAyK5plUpZmDkfvSTR5/MzOLSGJMaItMqC2iFSyU1djE2vw62oNA9uC5m/k3M3OR4PZnSJi8k0mlAR1A/Vky56oywfJval0zSXfGvwfd042kGq2hn22pGRdKP6GRQkpCGAducQSYG2iDkLH/OYlEquLQeBoN/rxuRr3W+sU2NxYyHFmocswXggAN72kCoyig8C6DGacjHKEl5VxeftCdUuJGwLQ2PXp2HR6RFCxeUbwoAvNG9TzwjeBuQkLDoDAnD7HejxE1ltaSpmQsGIWRf8LvEpPvR4w1473EgLzrNB1gECVVWQJ2rOTGlFxKvIyEiSIHguOmU4To5l/zjH8G3e39zbP/5Rjhrf1BFMVK9EtIWP4u8zB9Wo8YXv+f1jUPY3iWO1x2uL7i8yBjRyjiLbpn497N8WWxNr7JHbFdnf1GOxt3O6x6Zvj71dscYbfbzEen70vor3fyyinx97f4Vv199D9OvGntfI/mNdaIg1X9F9xn7u39d3+OMObCwjmKbakPRdvrf4Fs2IiodmRGloaGhofJ1xBBQaXB6sOplyq3na5Yr3ubysGXEgA4ekJlJgNsbUCSHYMOrkw4ExfEKlxp7Asen2CMNqkoAqeLd/hDf7hhkLqMy2mTk/N51ZCdFLU3tVlee7Bnm5Z4her59Cs4nzctM4MzslqoaZR1F5pK2X57oG6fMFsMoyp2encGNRNpmmyCWOfarKPft6eLZzILRi46IkKz8tzY2oXzY5hp/t7eTP3UOhtM0EncxVhZlcOyNyaeiAKvjx3g6e7xqMSLw6PDWJx2YXYg8zB1Uh+MneTp7qjCxWnGsy8Nz8kpCxNDm/dzR18Xh7ZGFqvQT3lRdwzoSxNMl9+3q4ryW6qP852ak8UFEQMXd/7Ojn1r2dUdq5NguvVZVGjPmVniGu2dOGAMzCjYIOv2QkSSfzWnUZ88Ki+1YNjHLxzn3oVDczaUBGpZGZuCQbv6ko5OycKdNx86iT07c1YlBGqGYzZjw0UUYTM7kgLz1khkEwEvLYzQ1k+ndxMKtJZJxucvmUI5hhL+KN6rLQcTfgC3D0pnp8ng4O5VPS6WdowpRTjYWsqplFjil4THsUleM2NzDuqONwVlFAGw5srOUQtkjLeKW6PHRsCCE4a3sT3UNbOJZ3KKMBPwY2sZQPOIEbZs3lu/kZoTFfX9fG5q4vOJnXmMMOAHYzj7c4jarcQ3mwojCkfbClh7eaP+ZMXqRi4irzIGm8z4l02M5gVU1FaP+92jPEk7tf50L+QDpTx9EOFvBn/TW8v2xFKCpy3YiD67d8wGU8Qj5T9UBamcEfuIpHFx/NwqTgik6tbi9nr1vF98V95IVpnSTwDJdweuV5ITPTGVA4eu1qLvLfRRH7QloVib9yItb86/jFrMLQvJ24fj0nuW6jlMaI4203c/gy+Q5eXlgVaru8djezB37M3Ik5m6SfDJ403M67K44KpW7fubcFS/tNVLM5QutHz++ka/nNsouZMWGKP9/ZT1P9TRxMdK2TdziFkxbcxjcnUrHXDI/z6tbbOIG3orS1LMBY8huuLAoen+0eHz//6ld8i/1XoIM+Mtmc8Qj3zQuem7kVlcu+fIzzlYeiopy8mHjF8gueWhZchUoIwWXrXuQM9+3oY6TGvCRfykOH3BhKlb9x28ccNnRNyAAO51MO54KlDzNz4vP2/oad5HdcEnH8TFJHJUVznuL4rOCx/JeuThx151HIVG2rScNmiFS6Cv7ENTPnAbB5ZIw1W77DHHZGaf3oWZ18P79aeAIQvEjz2Jff42A+jRoDwDumH/DAQVcjSRI+VfDjL27jGPWliH6DtxKrpaO5ccWDJOiDx8VP1j/B4e4Hw7TSxDgk9jKLQxY9zyxb8HPr1zs/YvHgDaHIwiktDJCBZdYzHJcdXJTl6eZtZHdcgZnJlVslIrAAACAASURBVDOnPlO9GNiX8zCXli1FAJ/2dTNWfz4pDIf6DR/7+sSbuGHBGQA0u9ys2XIhM2iNsnokJNYYTuemmutBAldA5ekNP2Se2LiffRf8v1ZaxPk1D2DV6xBC8ODGe1gSeC9qfgUSvWQzf8GTlFqDnwGP1L7AQucfQ32Fj9mNGUp+xzczg+/rZ/Z+TungHaHjeH97qCXzTr5VvBSAD7qbsLRdjXHCpNw/Haw24RIum3sWAsHu0VE66i7HzmhoDqbmArbpj+Sy6uuQJIlBn59Pt/2AfNojtJM0SZWcUn03Fp2MIuC5zbdTKbaEbd8UI6RQMedxCqzB4+JP23/PXN9fY2r9GGDG/SzPCM7Fn+vfYeb4kxH7I5zOtJs4uXgZCHivfRuZfXeGzVUk9Qnn8u2K0wHYNNSNb9/1GELzFkmd/hC+Pf8HAHR7PezadW3M4w2gTSrl2AW/wCxLeFTBX7ffSqFoChvHlH6MJMorHyLHbEEgeGnHI8wKrAvbPkhmmFy68GNAV/kuR+aUaUZUPDQjSkNDQ0ND478XIQQORcWqk6ct0g/Bk8E2jw+bTiYvjsk2Sb/Pz+ZRF3pZYpk9AZs+fqpZu8fHp4Nj+IVgebKNyjBTaX8anB7e7R/BEVCZl2jhuAx7VF20SWrHXbzaM0y/z0+RxcS3clJDqYz7s2nUyfNdg7S4vWSbDJydnco3UhNjGo8bRhz8qXMgtCLnKZkpnJubGrMG3NoRB79r72PDiBODLHFMup3vFWRSbI0ex/oRBw+19rJ6aBwVWGZP4JoZWaxMS4rSbhtzcfe+7lDttzyTgcsKMrg0PyPKdKx3eri9sTOktcgyZ2an8NPSXBL32y+tbi+3NHTwSVhNuUNSbPxqVj5l1kjzs9/n54b6dlYNjIV+6BdZjNw1Mz9kTEwyFlC4rq4tIrIuQSdz3YwsrizMjFyaXlW5paGDl7qHQrXMJEnmrOxU7i7Pj9jfqhDc3tjFHzv6SRCjGPAzTArV9kT+NLc4tNroJA+19HBfcydl1JGAg04KEKZCnppbzEJ75FLhL3QNcnN9O4WigTQGGSCDDqmM+yoLOSssGhGCRuIVO5spUneSQyfjJLGNRVxSmM/PSnMjtm/zqJPztjeRE9hOKXvxYmYTS1iQVsyf5hZjCUtVbnJ5OHvrXlK8m5jDTgQStVThsyzkL9Vl5Ia9Dwd8Ac7a2oDs3MhCNmLATwMV7NEfwjMLKlkUtn0uReW87Y24R1ZzEGuw4qKNGXzGkdw6e3HE9ilCcNWuFlr7PuJQPgml/X7KkRxW+M2o7ftFYydftH3I4awimy5GSeZLDkOXegJPz58ZYbg/0znA8/XvcgzvUsQ+PJhZx3IarKfz/MJFESvR/m1glF/veJ9jxRtUsAsFPdtYyKf603l04eERZnTtuIubtrzLUcorzGMbMoJGZvK+dCqXzT2b4ydqBwJ0enxcuuljDvU9yyI2IiPoJ4MPOY75xd/luuLckHY8oHDB5nXMdz7JMtZgIICTBD7lcNwZl/HY3IqIyNPLtm8nb/hRlvNFcHVTZDZTw+aEK3h60YqIz8Wf1DVA1/0czOqQ8dBCMe8aLuGhxaeFjEGA37d1Ud94L0fwIaaJqKthUnhTOocrq6/goLALBB/0jfDmzvs5njewTphtXkz8jWOoKf8R5+ZlhbTbx138ZtMjnCJeIIGpouWbqcGTexs/q6iMmLfrNr7Eqf7HyaQv1L6H2Wy138wTC5eHvlOcAYXvbXyT49z3ksHUBYIhUnnHdB0PLzmL5AkTXxWCq7auZunIbREmsAcTb8gXcfOSH0R8Ft1Rt530rh9TTl2oTUViFcdz7Pyf8830qX39VFsrA403snA/A3Yri0gpu4dLCotCbR8PDLG69kccxicR2nYKaM66hzvmLAm17R538YdNd3CceDXCKB0jkVW223hk8SnoJyK6+31+bl33KKcGHscwsXAKgA8Dbxh+wK+XXR6K0PaqKldveIkTXXdFGaXvS2dwac3tVEwc90IIbtz2EcuGbwmZKpNsYBk18x7miIypizAPNGwlo+NqcoksQt5CMWrRb7m0pCLU9pfOFgbrv8dMGiK0I9jZnn4fd8z/Zqht3fAwq7dexWLWRWh9GPir9Sc8sOS80Fy0uDw8seFmDlffYX/e01/ETw+6ORQ57wgo3Lr2QY73/y7KjP5COoJzah6kYsIkVYXgho1/5kjHnaH30iS7mUPlvD9wRMbUcX/njo+Z1389iURGifeQjafod1xSMi/U9mRzLeaW75FN5MUrBwlsS7+X2+YfHWr7W28nTbu+GzVvAXSsst7EPUsvQSdJmhEVD82I0tDQ0NDQ0ND49xFQg7819TFSUfdnPKDgUVXSDPqYK3eG0+/zM+xXyDMZQmmM8ejy+Oj0+sk2GeJG9k3S7vGx1+kh2aCjKtE67Tha3F62jrkwyRKHpCRGGWHhdHh8fDmRProiJX6EIQRPij8cGMWjChYlWVliT4gbvdjt9fF23wgjfoXyBDPHTmNm9nn9vNo7TLfXR57JyOnZKRHmSDiDvgB/6Rli72Tab3YKFQmxTdVRf4C/9A6zdcyFVSdzfIadQ1Nir77qCCj8pXeYL4fHkZFYmZbIqZkpEYbVJJ6JGn+rBsbwqoIl9gTOzU2NOWa/Kni7b5g3+0YYDyhU2ixckJcWc8yqEHw4MMpLPUP0eP3MsJg4LyeNQ2KkQENwMYtnuwZpdntJN+g5KzuV07JSYqZXbxx18seOfnaMu7HpZE7MTOaCvHSSYhwbux1uft/ez1cjDgySxBFpSVxakBHz2Gh1e3msrY9PBoZQhcLilFS+V5BJVVJ0DcN+n5/H2vp4v7cXv+KmxJbJpYWZUQt6QPA993h7H2909+L0jZBhSee8/GzOz02LMvN9qsofOwZ4pbOdcU8vZmMaJ+UU8v3CzKhjXwjBC91DPNe+j3HnPiSdjYMy53BtcTb5Mbbv3b4R/tC6j9Hx3Qj0lKRXc21xfswajWuGx3lkXwv9I1uQESQlzefyotKoNPPJOb6/eR/dA19hxEvAXMEZMxZwQW5a1L5u9/j4VVMHDX3rMIsxXPoCjsxfyHUzsqJqP474A9zd3MHO7s9IVPtwyGnMylrJj0oLoo5Pr6pyf3MX6zo/J1lpwYUNe+pKfjRzZlQ0sCoET7T18UH7l6T4duHDiJJ4CFeVzQ+tChzOqz1DvNS8hmRP8Px2xLyYs0tWcOZ+5jLA6qFxfrd3HYnOLzDgpU9fycEFR3FVUU7Uvq4dd3Ff/WbMY38jgXH6pBkUZh/PrTNLoj5v2z0+flG/A//ge6QwyBAZWNKP59ZZcyLMZQh+VtzR0MBg39tkig4cJOFMPIobKpZERNVC8Hj7dWMzjV1vk6/uxYuJbvNhXDRzJUeGma8QPN4eb+tkfcubFCrbEUi06RdzeMlJnJ+XFbWvX+3u453Gtyjyr8OAn1a5ktK807mhdGbUd9Xng2M8Vf8e+Z5PsOKiQyrCmnEqP62ojnpf73G4eXD3KtIdfyWFIfrJwmE/mVvmHBr1vu7x+vnFri8xj7wZMrm7rEdxZeVxURcSHAGFO+s24+l7jRk04cZCk/FQzph1KsdlpkdoA6rg3sbddHa+TKnYiYKevbpF1BSdwyWFhRFzIYTgybZWNu17kVnqpmBJAWk2WbnncNPMeVGfce/09PFOw0uUBz7HgptWSlDTzuTW2ctDEdSaERUHzYjS0NDQ0NDQ0NDQ0ND496AIgQzTpoJP4lNVdJJ0wIhWCBo8qiCmiRlL61UFiTr5gOPwKCrjikKKXn9AA92rqoz4Fex6XcxVgcPxq4I+n59EvS6mMRmOIgSdHh8WnRzXLJ5ECEG7xwdAvtk4rXkuhKDL68ejqhSYjTFT6MPp9voY8SsUmI3TRgJD0ADt9frJNhlDacHxGPEHaPf4SDHoYxqT4TgVhUaXF6ssU2Y1Tbv/vKpKndODDFQkWGKaxZMoQrDb4canCioSzNNe0BBCUOf0MBZQKLOaSTvA9jW7vPR4/RRajAfcvk6Pj31uLxlGA+UxyhSEM+ALUOd0Y9PpmJ9omXZfjwcUto+70EsS1UnWuBcoIHjMbx13EVAF8xMtEWn5+6MIwZYxF+MBhdk2S1SkrmZE/b/27jzakqq64/h3yySgDDIokzYRULIUjcEpCxDBGEOMoIKR5YSASXQZMA5EY4KtxCkYNGqQpaioOBE1qHFgkOAUAwoyNDII2sggKoIMIQ7IyR+n3ut6VXvvqifdt6ub32etu/re93593qnat845Ve/e+wK6ECUiIiIiIiIisnKNvRA1fPlYRERERERERERkJdCFKBERERERERERmQldiBIRERERERERkZnQhSgREREREREREZkJXYgSEREREREREZGZ0IUoERERERERERGZCV2IEhERERERERGRmdCFKBERERERERERmQldiBIRERERERERkZnQhSgREREREREREZkJXYgSEREREREREZGZ0IUoERERERERERGZCV2IEhERERERERGRmdCFKBERERERERERmQldiBIRERERERERkZnQhSgREREREREREZkJXYgSEREREREREZGZ0IUoERERERERERGZCV2IEhERERERERGRmdCFKBERERERERERmQldiBIRERERERERkZnQhSgREREREREREZkJXYgSEREREREREZGZ0IUoERERERERERGZCV2IEhERERERERGRmdCFKBERERERERERmQldiBIRERERERERkZmwUsrq7sNMmdnPgKs7X94SuHERzSwmvzZnp9KPNS07lX5MITuVfkwhO5V+TCE7lX6sadmp9GMK2an0YwrZqfRjCtmp9GNNy06lH1PITqUfU8hOpR9TyE6lH2tadir9mEJ2Kv1YGdkHlVK2GvzfpZR7/A34zqrKr83ZqfRjTctOpR9TyE6lH1PITqUfU8hOpR9rWnYq/ZhCdir9mEJ2Kv2YQnYq/VjTslPpxxSyU+nHFLJT6ccUslPpx5qWnUo/ppCdSj9W5fZ1b3prnoiIiIiIiIiIzIQuRImIiIiIiIiIyEzoQlT13lWYX5uzU+nHmpadSj+mkJ1KP6aQnUo/ppCdSj/WtOxU+jGF7FT6MYXsVPoxhexU+rGmZafSjylkp9KPKWSn0o8pZKfSjzUtO5V+TCE7lX6syu1b4B73YeUiIiIiIiIiIrJ66BVRIiIiIiIiIiIyE7oQJSIiIiIiIiIis3F3/uTe2nADngJcDlwJvHog+wHgp8CygdwOwH8B3wMuAY5MsvcGzgUubLKvH9HndYDvAv85IrscuBi4gIE/sQhsBnwKuAy4FHh8kHtI097c7VbgZUm7f9ts2zLg48C9k+yRTe4Sr02vBsD9gDOA7zf/bp5kD2ravgvYfaDdY5t9cRHwH8BmSfaYJncBcDqw7dBzBngFUIAtB/qxFLiutb/3y9oG/qbp9yXAPyftfrLV5nLggiT7SOB/5p5HwGOS7COAbzXPu88Dm2THhVe/JNurX5KN6hflezWMsl4Nk3Z79cva7dYvabdXvyTbq1+SjernjlXAjsA51DH0k8D6SfalTW7+eZ9kP0odm5dRn2frJdn3N1+7iDqG3SfKtvbzO4HbB/pwEvDD1n5+5EDegDcCV1DH0COS7Ndb7V4PnJpk9wXOb7LfAHZKsvs02WXAh4B1o7nDq12S7dUuyfZql2R7tRsz37Xrl7Tt1i/I9mqXZHu1S7K92iXZrHbL6czlxHOfl43mPi8bjZ1e1p37onw0/wVtL8Wf+9x28ec+r91o7vOy0dznZaOxs7e2imqX5KP6edmofl42WruE60Gndl67Ue3cdr3aJW1H9fOyUf28bK9+BOtdr35J1lu3RNmodlHeW7eka3QWrluidnv1y9rt1i9p11u3RFlv3RJl3WOv6VvvHIRg7guy7twXZN25L8i6c5+Xjea9oN2T8NctXjab97y8O/cFWXfuC7Lu3IdzTkg873nZaNz0su6xl+SjsTM8j6U/dnrtLsUfO9128ec9r91o3PSy0bjpZcNjb8xtdHBtvFEXgVcBv0c9eboQ+P0kvxfwKIYvRG0DPKq5f1/qAe62Sx0E5gaf9agD4+MG2n858DHGX4jacijXZD8EHN7cX799EA7swxuABwXf3446IG7YPD4FOCTIPqx5gm8ErAucSWvRHtWAOum9urn/auCtSXZX6kR2NgsHJS/7ZFYMhm8daLc96R0BnJA9Z6gXAU4DrmbhxOa1vRR45ZjnI/DEZr9t0DzeesxzF/gX4Oik3dOBP23u7wecnWS/DTyhuX8ocEx2XHj1S7K9+iXZqH5RvlfDKOvVMGm3V78k26tf1odu/ZJ2e/VLslH93LGKekw/u/n6CcCLk+wfAEtojUtJdr/me0ZdrGTttmt3HPV5FI6twO7AR1hxISpq9yTgQOd4ifIvBD4M3KtVv8ExHvg08Pyk3SuAXZuvv6Tpl5f9I+AaYJfm628ADmv9nAVzh1e7JNurXZLt1S7J9mqXte3VL2nbrV+Q7dUu60O3dkm7vdp5Weor1LPaefs9mvu8bDT3edlo7PSy7twX5b2xM2l7Kf7c52Wjuc/tQ3fsTNqN5j4vG42dvbVVVLskH9XPy0b187LR2sVdDwa189qNaudl3dpl/Qjq57Ud1c/LuvVr/az59W5WPyfr1i7IurVL8uHx181G9QvadesXZMP6eX3wahe069YuyEbHnnsOgr9uibLeuiXKeuuWKOutW8JzJvrrlqjdk+jMe0nWnfeyfrTanFu3RG176xYveyjO3EdwToh/zhBlvXOGKBuNm1HeO2cIz2PpnzNE7S6lf84QZb1zhjHn0nPnDFG73jlDlE3HzaHbPf2teY8Briyl/KCU8mvgE8D+UbiU8jXgpqFGSyk/LqWc39y/jXqVebsgW0optzcP12tuJWrbzLYH/gw4cagfi2Fmm1IvKry/6devSym/GPFf9wWuKqVcnWTWBTY0s3WpT+Drg9yuwDmllDtKKXcCXwWe0Q4ENdifuqig+feAKFtKubSUcnn3BwfZ05t+QL0yvH2SvbX1cGOaGibPmbcDR9Gp9djnWJJ9MfCWUsqvmsxPh9o1MwOeRZ00o2yh/nYQYFOaGgbZXYCvNffPAJ7ZZKPjole/KOvVL8lG9YvyvRoOHMsLarjI4z7K9uo31G67fkm2V78kG9UvGqv2of42D1bUz82WUr5bSlne2RdR9ovN9wr1lT/bJ9lbW/tiwxXN9rNmtg71N19HDfWBQJJ/MfCGUspdTe6nQ22b2SbNPjw1yXr187K/BX5dSrmi+fp8/bpzR7OverXzss229GqXZHu1S7K92mVte/WLspEg26vdULvt2iVZd+x0slsQ1C7hzn0eb+xMsu7YGWTduW+AO//dTe7cl+nOfQG3foHe2JmsrdzaRXmvfkm2V78k26vfwHpwQe0Ws3ZMsm7thtpu1y/J9uqXZN25r6W93h069uazI469dnbMsdfODx1/3TV6duyNWc972aFjr9ducuy1s0PHXjub1a57DvJjgrnPyV4fzX1B1p37gmw09/Wy0bznZZ1+Zll33htquzv3Bdmoft3s/+LPfdE5oXfsudng2Iuy0bEX5b1jLzuP7R57g+e8Q33GP/bSdjvHXpT1ahdlh8bN1D39QtR21Kuwc64lOHH8XZnZEurV9HOSzDpmdgH1LU5nlFLCLPAO6hP5rpFdKMDpZnaemf1lktsR+BnwQTP7rpmdaGYbj2j/2SSLuFLKdcDbgB9RB/9bSimnB/FlwJ5mtoWZbcSKtzANuX8p5cfN/RuA+4/4P4t1KPClLGBmbzSza4DnUK80R7n9getKKRcu4ue/1MwuMrMPmNnmSW4X6j48x8y+amaPHtH2nsBPSinfTzIvA45ttu9twGuS7CWsuKB7EE4NO8dFWr8xx9CIrFu/bj6rYTs7VEOnH2H9Otm0fsH2ufXrZNP6dbJh/bpjFfUVpb9oTd7zY+hixrUsa2brAc8DvpxlzeyD1OfPQ4F3JdmXAp9rPeeG+vDGpnZvN7MNBvIPBv7CzL5jZl8ys51H7IsDgK+0FqVe9nDgi2Z2bbMv3hLU41xgXTPbvWn7wFb9unPHFgS1c7KZMNutXZT1apfk3fol/fDq52Xd2mXbR6d2QdatnZO9kbh24M/l0dg5dt4fk22PnW42GTd7+WTsjPrhjZ1eNho7s+3rjp1eNho7vaw3dkZrq6h2i1mLjcnO1S/MOvVzs0Htsj50axdlo9oNbV+7flHWq1+UHVq7tNe7Q+vOdG08MhutOxfkk+NvQXZo3eL0I1t3trND605v+6J1Zzs7tO5sZ93aeecgwHk4c99izleGsu25L8t2574k25v3BvqwYN5Lsu68N2JfzM99SbY39wX1OAV/7ovOCb1jbzHnj2Oy7WMvzDvHnpsNjr2sH91jL8p6x97Q9rWPvSjrHXtRdvCcL1UW8fKpte1GfbKf2Hr8PODdA/9nCQNvzWtl70Md8J4xMr8Z9TNbHhZ8/6nA8c39vRn31rztyoqX610I7BXkdgfuBB7bPP5XBl5eR305843UQSHKbA6cBWxF/Y39qcBzk/xhzT77GvAe4B1DNaBOKO3v3zxUL5yXSCfZ11LfL2xjngfUA/b1XpZ69f8cYNPm8XL6L4/ubt/9qS9Bvhf1vdwfSLLLqCdyRn3F3w/n+p1s33uAVwz04Z3AM5v7zwLOTLIPpb6s8zzgdcDPs+NioH7uMRTUL8r26jd0fDo1nM8O1dDZvqx+3WxWv2j7vPp1283q182m9Wsyc2PVHtRXlc59fYfucwxnXOvus4Hs+/DHAS+7DnA88MIguxf1swrmXn59e9Yu9e2LBmxA/c3b0QP52+dq0TxXvj6iz1+aq03S7mdYMTa/ita85WQfT/0ch3OBf6K+x783d1BfGt6rnZft/Kz52o3IztduRHZB7YI+b+vVL2rbq1+S7dVuRJ/na5e026tdku3VrvWzenM5wdjpZVuZs1n41q4su2DszLLBuOn12R07g6w7dgZZd+wc2L4FY2fQrjt2Btne2Emwtkpql67FWPgWk6HsfP2Gsu36Bdljvdol29erXZKNaje0ffP1S9ru1S/JhnMfnfVuVD8vGx17A9lo3RKuu+kff/NZhtct3e3L1i3dbLZuibbPW7d0283WLd2sWzuCcxD8uS89X2HheDWUbc99Q9n5uS/IPh9/3ou2zZv3oqy7ZhnR5/bcF7XtzX1R1p37cM4JicfO8PyR/ryXZb1zvvTclNax52TfSzzvedsXzXteNho7s+3rznteu9G852UHzxmy2+jg2nhrnvindZ5Irxn4P0sYcSGqOcBOA16+yD4dTfC+bODN1Cv3y6lXge8ATl5E20uTth8ALG893hP4wkB7+wOnD2QOAt7fevx8mgX4iP6+CXjJUA2oHwy4TXN/G+DyoXox8kIU9X3O3wI2Gvs8AB7Y6d98Fng49dULy5vbndTfDDxgZNvdbe8+/jLwxNbjq4Ctku1bF/gJ9W1P2c+5hRULCwNuHdnfXYBzs+Miqp+XjeoXZZP6pcdnu4bdbFbDEe0uidrN6pdsX69+Qbtu/Ub0d0H9Ot87mrq4uJEVC6QFY2on+8rW4+UEn9vSzlIntVNpPr9gqN3ma3vhf5bP0U17N7RqdxetBelAu3t77bbz1A+M3LG1n28Z2L4tqSer7h9waO3jqzrPze+N7POTqb9t9OaOj3q1C7Int9qcr12W7dZuqN1u7YL8zV79Rra9N/Vilpv1ajewfQtqF2S/4NVuZH+fDJwSPC+WUp9v4dzXzbYen43zOTXdLMHYGbXb2r5oLlgK/CMD81/S9hKv7da+COe+YPvcuc9pN5z7Bvq7C/Wkyl1bRbWL8l79smy3fkPttusXZL8S1O7hI9pdkrT7hah2A9u3oH5J2736jdwX3bXLgvVuVD8vmx17XrZbu6F8dPy1swysPQfaXRK12zzO1p3e9kXrzm672boz6+987fDPQd6DP/el5yssnPvCLP25b/A8iGbuC7I/xJ/3xrS7d9Lu8QRrloHt68590T725r4xfXbnPppzQsbNewvOH8nnvfksA/Oe17Z37HWyRzJu3vPaXZK0+xLGzXvt7Rua9+baHTPvef0Nzxmi2z39rXnfBnY2sx3NbH3qyzw/d3cbNTOjvvf80lLKcQPZrcxss+b+hsAfUweGnlLKa0op25dSljR9PauU8tyk7Y3N7L5z96kH97Kg7RuAa8zsIc2X9qUOGpmDGX7p8Y+Ax5nZRs1+2Zf6WTRRn7du/n0g9er8xwbah1qzFzT3XwB8dsT/GWRmT6G+deJppZQ7BrI7tx7uT1zDi0spW5dSljR1vJb6gdE3JG1v03r4dIIaNk6lfngdZrYLK35rFHkScFkp5dokA/X9wU9o7u9D/WsVUX/nangv4B+oH+CXHRe9+i3yGHKzUf2SfK+GXjaqIXVw99rt1S/Zvqh+0b5YUL+k3V79kv0Q1c8bqy6lvhLnwOa/z9Vv9LgWZc3scOBPgINL8/kFQfZyM9uptf1Pa/6/lz2vlPKAVu3uKKXslPRhm1a7B9Ace8n2zdev2d9XDOyLA6kXXn45sI83bZ4PzH0t6fNc/TYA/o764bXe3PEcr3aLmWeirFc7Lws8z6td0vbmXv2SfvTql2xfr3YD+2JB7YLt29+rXdLfXu2ax9Fc7o2do+f9KOuNnUnWnfuC/LeDsfO2oG1v7Iy2zxs770j2RXfsjNr1xs5oX/TGzmRt5a5bFrMWi7Je/ZJsr35B9vxg7XJx0G6vdsm2ufPewL5YUL8k26tfsi/cua/RXe9m684xa2M3O2Ld2c1na8/57Ii1Z7fdbN3Z3b5s3enti2jd2c1m685uf6Paeecg38OZ+4JsdL7iZr25L8l6c5+XPc6b95J2vXVLtG29eS/rc/O9BXNfso97ApFKmgAABgdJREFUc1/S52ju884J3WMvyLq8bHbsBflo7utmPxQde0G77rEXbJ977CX7onfsBVn32Av6m42bw8ZesVpbb9T3OF5BvYr42oHsx6nvaf1N80Q6LMjtQf38gLk/6zj/5xed7G7UP998UfNk6731I/h/ezPw1jzqXwO8kBV/3nto+x5J/TONFzVP7s2T7MbUK+Kbjujr66kH6DLqX3zYIMl+nTqAXQjsO6YG1M86+UpzoJwJ3C/JPr25/yvqhYPTkuyV1M8Qm6vhCUn20832XUT985XbRdnO9ixn4cujvbY/Qv2zmBdRB+Btkuz61N/uL6P+KdR9sn5Q/4rFX4/Yx3tQX3Z5IfUlpn+YZI+kHlNXUD8PZe6quntcePVLsr36JdmoflG+V8Mo69UwabdXvyTbq1/Wh279knZ79UuyUf3csYo6zpzb7O9/p74cPMoe0dTvTupEd2KSvZM6Ls/17WgvS3358jebfbyM+kqfTaJ2O7W7fWDbzmq1ezIr/kpdlN+M+lv4i6m/VXtE1g/qb+meMjQfUJ/3Fzf1O7vZ51H2WOqC73I6fzq4O3d4tUuyvdol2V7tvGxUu7HzHf5bK9v9cOsXZHu1y/rQrV3Sbq92SdatHcFcjj92Rllv7IyyvbEzyUZz3+D6gxVjZ9S2N3ZGWW/sDPtAf+yM2vXGzigbjZ29tZVXu1ZfvHy0dvGy0dznZaP6petBFr46xGs3Wrd4WXfdkvWjW7+k7Wjt4mWj+vXWu1H9gmxUOy/r1i7JR/VL1+id+nntRvXzstG60+1DUDuv3ah2XtatXfO93jkIwdwXZN25L8i6c1+Qdec+LxvNe0G70brFy4bzXtQPnLkvaNud+4JsNPf1zgmJjz0vGx17XjY79rx8dOwNnccuZ8Wx57UbHXteNjr23D7gH3teu9Gx52XDY2/MbW6QFRERERERERERWaXu6W/NExERERERERGRGdGFKBERERERERERmQldiBIRERERERERkZnQhSgREREREREREZkJXYgSEREREREREZGZ0IUoERERkTWUmRUz22l190NERERkLF2IEhEREVlJzGy5mf2fmd3eur17dfdLREREZCrWXd0dEBEREVnL/Hkp5czV3QkRERGRKdIrokRERERWMTM7xMy+aWbvNrNbzOwyM9u39f1tzexzZnaTmV1pZi9qfW8dM/t7M7vKzG4zs/PMbIdW808ys++b2S/M7N/MzJr/t5OZfbX5eTea2SdnuMkiIiIiLr0iSkRERGQ2Hgt8CtgSeAbwGTPbsZRyE/AJYBmwLfBQ4Awzu6qUchbwcuBgYD/gCmA34I5Wu08FHg1sApwHfB74MnAMcDrwRGB9YPdVvYEiIiIiQ6yUsrr7ICIiIrJWMLPl1AtNd7a+/CrgN8CbgO1Ks/gys3OBdwFnA8uBzUoptzXfezOwTSnlEDO7HDiqlPJZ5+cVYM9Syjeax6cA55dS3mJmHwZ+CbyhlHLtKthcERERkUXTW/NEREREVq4DSimbtW7va75+XVn4G8Crqa+A2ha4ae4iVOt72zX3dwCuSn7eDa37dwD3ae4fBRhwrpldYmaH/o7bIyIiIrLS6EKUiIiIyGxsN/f5TY0HAtc3t/uZ2X0737uuuX8N8ODF/rBSyg2llBeVUrYF/go43sx2+t26LiIiIrJy6EKUiIiIyGxsDRxhZuuZ2UHArsAXSynXAP8NvNnM7m1muwGHASc3/+9E4Bgz29mq3cxsi6EfZmYHmdn2zcObgQLctbI3SkRERGQx9GHlIiIiIivX583st63HZwCfBc4BdgZuBH4CHFhK+XmTORg4gfrqqJuB15VSzmy+dxywAfWDx7cELgOePqIfjwbeYWabNj/vyFLKD+7OhomIiIjcXfqwchEREZFVzMwOAQ4vpeyxuvsiIiIisjrprXkiIiIiIiIiIjITuhAlIiIiIiIiIiIzobfmiYiIiIiIiIjITOgVUSIiIiIiIiIiMhO6ECUiIiIiIiIiIjOhC1EiIiIiIiIiIjITuhAlIiIiIiIiIiIzoQtRIiIiIiIiIiIyE/8PNRi49G6D8RwAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# plot graph\n", + "#Plot training loss over Epochs:\n", + "color = sns.color_palette()\n", + "#Draw Weight Variance Ratio\n", + "dataplot3 = {\"svrg_mse\": [], \"sgd_mse\": []}\n", + "with open('sgd_lr.json') as sgd_data, open('svrg_lr.json') as svrg_data:\n", + " sgd = json.load(sgd_data)\n", + " svrg = json.load(svrg_data)\n", + " for epoch in range(100):\n", + " dataplot3[\"svrg_mse\"].append(svrg[str(epoch)][\"mse\"])\n", + " dataplot3[\"sgd_mse\"].append(sgd[str(epoch)][\"mse\"])\n", + "\n", + "x3 = list(range(100))\n", + "plt.figure(figsize=(20, 12))\n", + "plt.title(\"Training Loss Over Epochs\")\n", + "sns.pointplot(x3, dataplot3['svrg_mse'], color=color[9])\n", + "sns.pointplot(x3, dataplot3['sgd_mse'], color=color[8])\n", + "color_patch1 = mpatches.Patch(color=color[9], label=\"svrg_mse\")\n", + "color_patch2 = mpatches.Patch(color=color[8], label=\"sgd_mse\")\n", + "plt.legend(handles=[color_patch1, color_patch2])\n", + "plt.ylabel('Training Loss', fontsize=12)\n", + "plt.xlabel('Epochs', fontsize=12)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Training Loss Comparison with SGD with fixed learning rates\n", + "Choosing learning rate (0.0025, 0.001, 0.005) for SGD and a relatively large learning rate 0.025 for SVRG, we can see SVRG smoothly goes down faster than SGD. Learning rate for SVRG does not need to decay to zero, which means we can start with a larger learning rate." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "train_svrg_lin_reg(output=\"svrg_0.025.json\", optimizer_params=(('learning_rate', 0.025),))\n", + "train_sgd_lin_reg(output=\"sgd_0.001.json\", optimizer_params=((\"learning_rate\", 0.001),))\n", + "train_sgd_lin_reg(output=\"sgd_0.0025.json\", optimizer_params=((\"learning_rate\", 0.0025),))\n", + "train_sgd_lin_reg(output=\"sgd_0.005.json\", optimizer_params=((\"learning_rate\", 0.005),))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5,0,'Epochs')" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAABJwAAALMCAYAAAChcKgRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xt4VeWd//33nSMkAYQAVgEFDMcQjAp4QO2MTgUPE4dqK1RaHemMbe3j/NCh6EyrdQ7q0zoddWRsa2u105+FeZx2cCogSlXAjkWhsQqkgkCRQwsECJCEkMN6/shmEyAJIedkv1/Xlcu117r3vb5ry1+f676/K0RRhCRJkiRJktRakjq6AEmSJEmSJHUvBk6SJEmSJElqVQZOkiRJkiRJalUGTpIkSZIkSWpVBk6SJEmSJElqVQZOkiRJkiRJalUGTpIkqdMJISSHEA6FEM5pzbHq3EIIXwwhvNHRdUiSpJYzcJIkSS0WC3yO/tWEEMrrfL71dOeLoqg6iqKsKIq2tubY0xVC+KcQwnOtPW8T750UQpgbQtgY+z1/H0L45xBCWjvd/89i/y8PnfA3sT3uL0mSuraUji5AkiR1fVEUZR09DiFsAb4YRdFrDY0PIaREUVTVHrV1YfOAq4BbgdXAaOA5YAzw6da8USP/P7ZGUTS0Ne8lSZISgyucJElSm4utFFoQQvhpCOEgMDOEcGkI4e0Qwv4Qws4QwpMhhNTY+JQQQhRCGBr7/JPY9cUhhIMhhP8NIQw73bGx69eGED4MIZSEEP4thPBWCOH2ZjxTbgjhzVj974cQrq9z7YYQwvrY/beFEGbHzg8MISyKfWdvCGF5A3OPBu4EZkRR9OsoiqqiKPoAuBm4IYRwZQhhcghhewghqc73PhNCWBM7Tgoh/F0I4aMQwp4QwvwQQt/YtZzYb/aXIYStwNJmPP/K2Iqrd2O/5c+Pzh+7Pi2EsDb2rL8MIYyqc+3cEMJ/hxB2x2p74vipw7/GvrcphHBNnQuzQghbYr/rphDC9NOtW5IktQ8DJ0mS1F6mAS8AfYAFQBXwN0B/YDIwldqQpSGfA74B9AO2Av94umNDCAOB/wTmxO67GZh0ug8S29b2C+BlYAAwG1gQQsiJDfkRMCuKol7AeODN2Pk5wKbYdz4BfL2BW/wZsDmKojV1T0ZRtAV4B/gU8CugEvjkCc/9Qux4NnA9cCUwGDgEPHnCfa6kduXU9TTPF2J/ZwMB+FeAEMIY4D+A/4faZ30NeCmEkBpCSKH2d9sIDAWGUPv/5KjLgPeB7Nh8P4zN2Rv4DvCp2O86GfhtM+uWJEltzMBJkiS1l5VRFP1PFEU1URSVR1H0Tp3VO5uA73N8eHKiF6MoejeKokrg/wL5zRh7A1AYRdHC2LV/BfY041kmA2nAt6MoqoxtH1wMHF1xUwmMDSH0iqJob53gqJLacOacKIqORFFU7wonasOwnQ1c2wn0j6IoAuYDMwBCCGcAU2LnAL4E/F0URdujKDoMPAR8pu6KKODBKIrKoigqb+Be58RWGtX9S69z/fkoitZFUVQKPABMDyGE2O/wUhRFv4z9zo9SGzReDFwae765URSVxv4tvFVnzo+iKHo2iqJq4HlgcAihf+xaBIwLIfSIomhnFEXrGqhbkiR1MAMnSZLUXj6u+yGEMDqE8HII4Q8hhAPAP1AbRDTkD3WOy4CshgY2MvbsunXEQpttTaj9RGdT298oqnPu98Cg2PE0oADYGkJ4I4Rwcez8o7Fxy2Jb3eY0MP8e4KwGrp3FsZDsBeCm2FbEm4BfR1F09HnOAf7naFBE7aohgIF15jru/0k9tkZRdMYJfxUNfP/3QDq1q8rOjn0GIIqiGmp/50HUrmjaEguU6nPi/zuArCiKDlAbrt0F/CGE8IsQwshT1C9JkjqIgZMkSWov0Qmfvwd8AOREUdSb2hUyoY1r2Ent9jKgtlkQx0Ki07EDGBL7/lHnANsBYiu3CqgNd35BbNVRFEUHoiiaHWvE/RfA3BBCfau6lgHDQggX1j0Z61M1MXadKIp+S21AM4Xjt9NBbcDzqRPCoh5RFMUDnRMCs+YYUuf4HKAC2Evt73NunbqTqP3dt1MbUp0bQkg+3ZtFUbQ4iqI/ozZ020jtvyFJktQJGThJkqSO0gsoAUpjPX8a69/UWn4BXBhC+PNYL6G/obbHUGOSQwg96vylU9s/qQq4N9aX6CrgOmr7OPUMIXwuhNA7tp3sIFADELvvebGgqgSoPnqtriiK1gM/AH4aQpgUQkgOIYwDXgQWR1H0Rp3hL1Dbr+nS2PWjvgs8HEI4J3bvgSGEgtP4rZriC7GVapnUbtn7z1iI9Z9AQQjhT2Krr+ZQ+zv8GvhfoDhWW0bs95p8qhuFEM6K/X4ZwBGglHp+O0mS1DkYOEmSpI5yL3AbtUHE96htJN6moij6I3ALtc2ni4HzgN9QuzKnITOB8jp/v4ttK/tz4EZqt7c9CXwuiqINse/cBvw+tlVwVmwOgFHAL6lt4P0W8EQURSsauO+Xqe1h9FNqw5XFwKvAZ08Y9wJwFfBqFEX76pz/DrCE2u17B6kNySY28pz1OSeEcOiEv7+oc/0/gJ9Qu3IsGfg/AFEUrY39Bk8Du6ltCF8Q63dVRW0vrTHUrnbaSu3b904lmdrgaie1/+8uo3Z7nSRJ6oRCy1dSS5IkdU2xbV07gJsbCX5UjxDCSuAHURQ919G1SJKkzscVTpIkKaGEEKaGEM6IbY37BrVvjlvVwWVJkiR1KwZOkiQp0VwObKJ2q9cUYNoJb16TJElSC7mlTpIkSZIkSa3KFU6SJEmSJElqVSkdXUBb6N+/fzR06NCOLkOSJEmSJKnbWL169Z4oigY0ZWy3DJyGDh3Ku+++29FlSJIkSZIkdRshhN83daxb6iRJkiRJktSqDJwkSZIkSZLUqgycJEmSJEmS1Kq6ZQ8nSZIkSZJ0+iorK9m2bRuHDx/u6FLUgXr06MHgwYNJTU1t9hwGTpIkSZIkCYBt27bRq1cvhg4dSgiho8tRB4iiiOLiYrZt28awYcOaPY9b6iRJkiRJEgCHDx8mOzvbsCmBhRDIzs5u8So3AydJkiRJkhRn2KTW+Ddg4CRJkiRJkqRWZQ8nSZIkSZJUr7yVH7C7sqrV5huQmsL7l49rtfnUebnCSZIkSZIk1as1w6a2mK+ze+655/jqV7/a5PFRFHH33XeTk5PD+PHjWbNmTb3jVq9eTV5eHjk5Odx9991EUQTAnDlzGD16NOPHj2fatGns378fgC1bttCzZ0/y8/PJz8/nS1/6Ussf7hQMnCRJkiRJUpdWXV3d0SWclqqq+oO3xYsXs2HDBjZs2MD3v/99vvzlL9c77stf/jLPPPNMfOySJUsA+NSnPsUHH3zAb3/7W0aOHMkjjzwS/855551HYWEhhYWFfPe73239hzqBgZMkSZIkSeoUSktLuf766zn//PMZN24czz//PJ/5zGfi19944w1uuOEGALKysrj33ns5//zz+d///V8WLVrE6NGjueiii7j77rvj4+rzzW9+k9tuu40rrriCc889l5/97Gd87WtfIy8vj6lTp1JZWQnAfffdx9ixYxk/fjx/+7d/C8Du3bu56aabmDhxIhMnTuStt95q0rPdfvvtfOlLX+Liiy/ma1/7Wr1jFi5cyBe+8AVCCFxyySXs37+fnTt3Hjdm586dHDhwgEsuuYQQAl/4whf47//+bwCuueYaUlJquyddcsklbNu2rUm1tQV7OEmSJEmSpE5hyZIlnH322bz88ssAlJSU8I1vfIPS0lIyMzNZsGAB06dPB2rDqYsvvph/+Zd/4fDhw4wYMYLly5czbNgwZsyYccp7ffTRR7z++uusW7eOSy+9lP/6r//iW9/6FtOmTePll1/miiuu4Oc//zlFRUWEEOLb0/7mb/6G2bNnc/nll7N161amTJnC+vXrm/R827Zt41e/+hXJycn1Xt++fTtDhgyJfx48eDDbt2/nrLPOOm7M4MGDTxpzomeffZZbbrkl/nnz5s1ccMEF9O7dm3/6p3/iiiuuaFLNzeUKJ0mSJEmS1Cnk5eXx6quvMnfuXFasWEGfPn2YOnUq//M//0NVVRUvv/wyN954IwDJycncdNNNABQVFTF8+HCGDRsG0KTA6dprryU1NZW8vDyqq6uZOnVqvIYtW7bQp08fevTowaxZs/jZz35GRkYGAK+99hpf/epXyc/Pp6CggAMHDnDo0KEmPd9nPvOZBsOm1vTP//zPpKSkcOuttwJw1llnsXXrVn7zm9/wne98h8997nMcOHCgTWtwhZMkSZIkSeoURo4cyZo1a1i0aBFf//rXufrqq5k+fTpPPfUU/fr1Y8KECfTq1QuAHj16tCi8SU9PByApKYnU1FRCCPHPVVVVpKSksGrVKpYtW8aLL77IU089xS9/+Utqamp4++236dGjx2nfMzMzs9HrgwYN4uOPP45/3rZtG4MGDTppTN2tcieOee655/jFL37BsmXL4s+Unp4ef96LLrqI8847jw8//JAJEyac9jM0lSucJEmSJElSvQaktu46lVPNt2PHDjIyMpg5cyZz5sxhzZo1fPKTn2TNmjU888wz8e10Jxo1ahSbNm1iy5YtACxYsKDFtR46dIiSkhKuu+46/vVf/5X33nsPqO2T9G//9m/xcYWFhS2+11EFBQX8+Mc/Jooi3n77bfr06XPcdjqoXa3Uu3dv3n77baIo4sc//nF81deSJUv41re+xUsvvRRfkQW1faeONlbftGkTGzZsYPjw4a1Wd31c4SRJkiRJkur1/uXj2vd+77/PnDlz4quOnn76aZKTk7nhhht47rnneP755+v9Xs+ePfn3f/93pk6dSmZmJhMnTmxxLQcPHuTGG2/k8OHDRFHEd77zHQCefPJJ7rrrLsaPH09VVRVXXnllq7317brrrmPRokXk5OSQkZHBj370o/i1/Pz8eLj17//+79x+++2Ul5dz7bXXcu211wLw1a9+lYqKCj71qU8BtY3Dv/vd77J8+XIeeOABUlNTSUpK4rvf/S79+vVrlZobEqIoatMbdIQJEyZE7777bkeXIUmSJElSl7J+/XrGjBnT0WU0y6FDh8jKyiKKIu666y5GjBjB7NmzO7qsLqu+fwshhNVRFDVpH55b6iRJkiRJUpf3zDPPkJ+fT25uLiUlJdx5550dXVJCc0udJEmSJEnq8mbPnn3SiqYf/ehHPPHEE8edmzx5MvPmzWvVe5/ufdqrro7kljpJkiRJkgR07S11al1uqZMkSZIkSVKnYuAkSZIkSZKkVmXgJEmSJEmSpFZl03BJkiRJklSvFSsv5siRPa02X1paf664/NetNp86L1c4SZIkSZKkerVm2NQW89U1dOhQ9uxpu/mbKisr67TGL1myhFGjRpGTk8Ojjz5a75iKigpuueUWcnJyuPjii9myZUv82iOPPEJOTg6jRo3ilVdeiZ+/4447GDhwIOPGjWvWc7SUgZMkSZIkSVIbqqqqqvd8dXU1d911F4sXL2bdunX89Kc/Zd26dSeN++EPf0jfvn3ZuHEjs2fPZu7cuQCsW7eO+fPns3btWpYsWcJXvvIVqqurAbj99ttZsmRJ2z3UKXT7wKmsbAtFv3uAX/3qT3nrV59k/fr7OVS6oaPLkiRJkiRJJygtLeX666/n/PPPZ9y4cSxYsIBFixYxevRoLrroIu6++25uuOEGAIqLi7nmmmvIzc3li1/8IlEUNTjvli1bGD16NLfffjsjR47k1ltv5bXXXmPy5MmMGDGCVatWAfDmm2+Sn59Pfn4+F1xwAQcPHgTg29/+NhMnTmT8+PE8+OCDTXqWN954gyuuuIKCggLGjh1b75hVq1aRk5PD8OHDSUtLY/r06SxcuPCkcQsXLuS2224D4Oabb2bZsmVEUcTChQuZPn066enpDBs2jJycnPizXHnllfTr169JtbaFbh04lZT8hlXvFLB9+/+l/PBWDh/exo6d/8k779zI3r1vdXR5kiRJkiSpjiVLlnD22Wfz3nvv8cEHHzB16lTuvPNOFi9ezOrVq9m9e3d87EMPPcTll1/O2rVrmTZtGlu3bm107o0bN3LvvfdSVFREUVERL7zwAitXruSxxx7j4YcfBuCxxx5j3rx5FBYWsmLFCnr27MnSpUvZsGEDq1atorCwkNWrV7N8+fImPc+aNWt44okn+PDDD+u9vn37doYMGRL/PHjwYLZv397ouJSUFPr06UNxcXGTv98Rum3gFEU1rFs/h+rq0pOu1dRUsG7dHGpqKjugMkmSJEmSVJ+8vDxeffVV5s6dy4oVK9i8eTPDhw9n2LBhAMyYMSM+dvny5cycOROA66+/nr59+zY697Bhw8jLyyMpKYnc3FyuvvpqQgjk5eXFeyJNnjyZe+65hyeffJL9+/eTkpLC0qVLWbp0KRdccAEXXnghRUVFbNjQtJ1TkyZNiteeaLpt4LS/ZDVlZZsbvF5x5I/s3buiHSuSJEmSJEmNGTlyJGvWrCEvL4+vf/3rvPTSS602d3p6evw4KSkp/jkpKSneY+m+++7jBz/4AeXl5UyePJmioiKiKOL++++nsLCQwsJCNm7cyKxZs5p0z8zMzEavDxo0iI8//jj+edu2bQwaNKjRcVVVVZSUlJCdnd3k73eEbhs4VRzeecoxhw/vaIdKJEmSJEnqmtLS+rfrfDt27CAjI4OZM2cyZ84c3nrrLTZt2hRfgbRgwYL42CuvvJIXXngBgMWLF7Nv374W1/fRRx+Rl5fH3LlzmThxIkVFRUyZMoVnn32WQ4cOAbXb23bt2tXiewFMnDiRDRs2sHnzZo4cOcL8+fMpKCg4aVxBQQHPP/88AC+++CJXXXUVIQQKCgqYP38+FRUVbN68mQ0bNjBp0qRWqa2lUjq6gLaS3uOsU47p0ePsdqhEkiRJkqSu6YrLf92u93v//feZM2cOSUlJpKam8vTTT7Nz506mTp1KZmYmEydOjI998MEHmTFjBrm5uVx22WWcc845Lb7/448/zuuvvx7fdnfttdeSnp7O+vXrufTSSwHIysriJz/5CQMHDmzx/VJSUnjqqaeYMmUK1dXV3HHHHeTm5gLwwAMPMGHCBAoKCpg1axaf//znycnJoV+/fsyfPx+A3NxcPvvZzzJ27FhSUlKYN28eycnJQO32wzfeeIM9e/YwePBgHnrooSavzGoNobEu7l3VhAkTonfeWcXbv55CWdmmesekp3+Cyy59g6Sk1HauTpIkSZKkzmn9+vWMGTOmo8s4zqFDh8jKyiKKIu666y5GjBjB7NmzO7qsbq++fwshhNVRFE1oyve77Za6EJIYO+bbJCdnnXQtKakHY8c+ZtgkSZIkSVIn98wzz5Cfn09ubi4lJSXceeedHV2SmqDbbqkD6NMnn4sn/Q9bP36WHTtepKamHICLLlxA797jOrg6SZIkSZJ0KrNnz27yiqbi4mKuvvrqk84vW7aM7OzsVqvpdO/TXnV1Jt06cALo2fMcRo38JklJPdi69Znak6Fja5IkSZIkSa0vOzubwsLCTnef9qqrM+m2W+pOlJmREz8uK/2oAyuRJEmSJEnq3hIncMo8L35cWrqxAyuRJEmSJEnq3hIocDq2wqm0zBVOkiRJkiRJbaXb93A6KiWlF2lpAzlyZBelbqmTJEmSJOmUPrz8Cqr37Gm1+ZL792fkyhWtNp86r4RZ4QTHttWVl2+hpqayg6uRJEmSJKlza82wqS3mq2vo0KHsacP5myorK+u0xi9ZsoRRo0aRk5PDo48+Wu+YiooKbrnlFnJycrj44ovZsmVL/NojjzxCTk4Oo0aN4pVXXgHg448/5k//9E8ZO3Ysubm5PPHEE/Hx3/zmNxk0aBD5+fnk5+ezaNGi03/IJkiswCnWODyKqigv39rB1UiSJEmSpERQVVVV7/nq6mruuusuFi9ezLp16/jpT3/KunXrThr3wx/+kL59+7Jx40Zmz57N3LlzAVi3bh3z589n7dq1LFmyhK985StUV1eTkpLCv/zLv7Bu3Trefvtt5s2bd9y8s2fPprCwkMLCQq677ro2eeaECpwy6jYOL7NxuCRJkiRJnUlpaSnXX389559/PuPGjWPBggUsWrSI0aNHc9FFF3H33Xdzww03AFBcXMw111xDbm4uX/ziF4miqMF5t2zZwujRo7n99tsZOXIkt956K6+99hqTJ09mxIgRrFq1CoA333wzvvLnggsu4ODBgwB8+9vfZuLEiYwfP54HH3ywSc/yxhtvcMUVV1BQUMDYsWPrHbNq1SpycnIYPnw4aWlpTJ8+nYULF540buHChdx2220A3HzzzSxbtowoili4cCHTp08nPT2dYcOGkZOTw6pVqzjrrLO48MILAejVqxdjxoxh+/btTaq7tSRU4JSZcSxwKrOPkyRJkiRJncqSJUs4++yzee+99/jggw+YOnUqd955J4sXL2b16tXs3r07Pvahhx7i8ssvZ+3atUybNo2tWxvfybRx40buvfdeioqKKCoq4oUXXmDlypU89thjPPzwwwA89thjzJs3j8LCQlasWEHPnj1ZunQpGzZsYNWqVRQWFrJ69WqWL1/epOdZs2YNTzzxBB9++GG917dv386QIUPinwcPHlxvMFR3XEpKCn369KG4uLhJ39+yZQu/+c1vuPjii+PnnnrqKcaPH88dd9zBvn37mvQspyuxAqe6K5wMnCRJkiRJ6lTy8vJ49dVXmTt3LitWrGDz5s0MHz6cYcOGATBjxoz42OXLlzNz5kwArr/+evr27dvo3MOGDSMvL4+kpCRyc3O5+uqrCSGQl5cX74k0efJk7rnnHp588kn2799PSkoKS5cuZenSpVxwwQVceOGFFBUVsWHDhiY9z6RJk+K1d4RDhw5x00038fjjj9O7d28AvvzlL/PRRx9RWFjIWWedxb333tsm906owCktbSDJybXNu9xSJ0mSJElS5zJy5EjWrFlDXl4eX//613nppZdabe709PT4cVJSUvxzUlJSvMfSfffdxw9+8APKy8uZPHkyRUVFRFHE/fffH+95tHHjRmbNmtWke2ZmZjZ6fdCgQXz88cfxz9u2bWPQoEGNjquqqqKkpITs7OxGv19ZWclNN93Erbfeyqc//en4mDPPPJPk5GSSkpL4q7/6q/h2wtaWUIFTCIHMzNrG4WVlm4iimg6uSJIkSZKkziu5f/92nW/Hjh1kZGQwc+ZM5syZw1tvvcWmTZviK5AWLFgQH3vllVfywgsvALB48eJW2Rr20UcfkZeXx9y5c5k4cSJFRUVMmTKFZ599lkOHDgG129t27drV4nsBTJw4kQ0bNrB582aOHDnC/PnzKSgoOGlcQUEBzz//PAAvvvgiV111FSEECgoKmD9/PhUVFWzevJkNGzYwadIkoihi1qxZjBkzhnvuuee4uXbu3Bk//vnPf864ceNa5VlOlNIms3ZimRnnceBAIdXVZVRU/IEePc7u6JIkSZIkSeqURq5c0a73e//995kzZw5JSUmkpqby9NNPs3PnTqZOnUpmZiYTJ06Mj33wwQeZMWMGubm5XHbZZZxzzjktvv/jjz/O66+/Ht92d+2115Kens769eu59NJLAcjKyuInP/kJAwcObPH9UlJSeOqpp5gyZQrV1dXccccd5ObmAvDAAw8wYcIECgoKmDVrFp///OfJycmhX79+zJ8/H4Dc3Fw++9nPMnbsWFJSUpg3bx7JycmsXLmS//iP/yAvL4/8/HwAHn74Ya677jq+9rWvUVhYSAiBoUOH8r3vfa/Fz1Gf0FgX965qwoQJ0bvvvlvvtd///nts/OhbAOSf/yOys69sz9IkSZIkSeq01q9fz5gxYzq6jOMcOnSIrKwsoijirrvuYsSIEcyePbujy+r26vu3EEJYHUXRhKZ8P6G21AFkxLbUAZSW2ThckiRJkqTO7JlnniE/P5/c3FxKSkq48847O7okNUFCbqk7qrTUxuGSJEmSJHVms2fPbvKKpuLiYq6++uqTzi9btozs7OxWq+l079NedXUmCRc49ew5hKSkNGpqjlBW6gonSZIkSZK6i+zsbAoLCzvdfdqrrs4k4bbUhZBMRs9hgFvqJEmSJEmS2kLCBU4AGZm12+oqK/dSWdny1yZKkiRJkiTpmIQMnDIz6jQOd1udJEmSJElSq0q4Hk4AmZnHNw4/44wmvdFPkiRJkqSE8u1vf5vS0tJWmy8zM5M5c+a02nzqvBJyhVNGZp0VTvZxkiRJkiSpXq0ZNrXFfHUNHTqUPXv2tNn8TZWVlXVa45csWcKoUaPIycnh0UcfrXdMRUUFt9xyCzk5OVx88cVs2bIlfu2RRx4hJyeHUaNG8corr8TPDx06lLy8PPLz85kwof0X2iTkCqfapuFJQA1lpRs7uhxJkiRJktSNVVVVkZJycgRTXV3NXXfdxauvvsrgwYOZOHEiBQUFjB079rhxP/zhD+nbty8bN25k/vz5zJ07lwULFrBu3Trmz5/P2rVr2bFjB3/2Z3/Ghx9+SHJyMgCvv/46/fv3b5dnPFFCrnBKTk6nZ8/BgCucJEmSJEnqLEpLS7n++us5//zzGTduHAsWLGDRokWMHj2aiy66iLvvvpsbbrgBgOLiYq655hpyc3P54he/SBRFDc67ZcsWRo8eze23387IkSO59dZbee2115g8eTIjRoxg1apVALz55pvk5+eTn5/PBRdcwMGDB4HarYUTJ05k/PjxPPjgg016ljfeeIMrrrii3gDpqFWrVpGTk8Pw4cNJS0tj+vTpLFy48KRxCxcu5LbbbgPg5ptvZtmyZURRxMKFC5k+fTrp6ekMGzaMnJyc+LN0tHYLnEIIU0MIvwshbAwh3FfP9XNDCMtCCL8NIbwRQhhc51p1CKEw9vdSa9RztHH44cPbqa4ua40pJUmSJElSCyxZsoSzzz6b9957jw8++ICpU6dy5513snjxYlavXs3u3bvjYx966CEuv/xy1q5dy7Rp09i6dWujc2/cuJF7772XoqIiioqKeOGFF1i5ciWPPfYYDz/8MACPPfYY8+bNo7CwkBUrVtCzZ0+WLl3Khg0bWLVqFYWFhaxevZpfnNQWAAAgAElEQVTly5c36XnWrFnDE088wYcffljv9e3btzNkyJD458GDB7N9+/ZGx6WkpNCnTx+Ki4sb/X4IgWuuuYaLLrqI73//+02qtzW1S+AUQkgG5gHXAmOBGSGEE+O9x4AfR1E0HvgH4JE618qjKMqP/RW0Rk0ZdRuHl21qjSklSZIkSVIL5OXl8eqrrzJ37lxWrFjB5s2bGT58OMOGDQNgxowZ8bHLly9n5syZAFx//fX07du30bmHDRtGXl4eSUlJ5ObmcvXVVxNCIC8vL94TafLkydxzzz08+eST7N+/n5SUFJYuXcrSpUu54IILuPDCCykqKmLDhg1Nep5JkybFa29vK1euZM2aNSxevJh58+Y1OSRrLe21wmkSsDGKok1RFB0B5gM3njBmLPDL2PHr9VxvVUdXOAGUlbqtTpIkSZKkjjZy5EjWrFlDXl4eX//613nppVbZ5ARAenp6/DgpKSn+OSkpiaqqKgDuu+8+fvCDH1BeXs7kyZMpKioiiiLuv/9+CgsLKSwsZOPGjcyaNatJ98zMzGz0+qBBg/j444/jn7dt28agQYMaHVdVVUVJSQnZ2dmNfv/ofwcOHMi0adPafatdewVOg4CP63zeFjtX13vAp2PH04BeIYTs2OceIYR3QwhvhxD+or4bhBD+Ojbm3bpL7BqSWXeFk43DJUmSJEk6yakCk9aeb8eOHWRkZDBz5kzmzJnDW2+9xaZNm+IrkBYsWBAfe+WVV/LCCy8AsHjxYvbt29fi+j766CPy8vKYO3cuEydOpKioiClTpvDss89y6NAhoHZ7265du1p8L4CJEyeyYcMGNm/ezJEjR5g/fz4FBSdv7CooKOD5558H4MUXX+Sqq64ihEBBQQHz58+noqKCzZs3s2HDBiZNmkRpaWm8/1RpaSlLly5l3LhxrVJzU3Wmt9T9LfBUCOF2YDmwHaiOXTs3iqLtIYThwC9DCO9HUXTcsqQoir4PfB9gwoQJDXcKi8nMPLbCycbhkiRJkiSdbM6cOe16v/fff585c+aQlJREamoqTz/9NDt37mTq1KlkZmYyceLE+NgHH3yQGTNmkJuby2WXXcY555zT4vs//vjjvP766/Ftd9deey3p6emsX7+eSy+9FICsrCx+8pOfMHDgwBbfLyUlhaeeeoopU6ZQXV3NHXfcQW5uLgAPPPAAEyZMoKCggFmzZvH5z3+enJwc+vXrx/z58wHIzc3ls5/9LGPHjiUlJYV58+aRnJzMH//4R6ZNmwbUroj63Oc+x9SpU1tc7+kIjXVxb7WbhHAp8M0oiqbEPt8PEEXRIw2MzwKKoigaXM+154BfRFH0YkP3mzBhQvTuu++esq4VKy/lyJFdZGTkcOklrzTpWSRJkiRJ6q7Wr1/PmDFjOrqM4xw6dIisrCyiKOKuu+5ixIgRzJ49u6PL6vbq+7cQQlgdRdGEpny/vbbUvQOMCCEMCyGkAdOB4zZihhD6hxCO1nM/8GzsfN8QQvrRMcBkYF1rFHV0W115+RZqaipbY0pJkiRJktSKnnnmGfLz88nNzaWkpIQ777yzo0tSE7TLlrooiqpCCF8FXgGSgWejKFobQvgH4N0oil4C/gR4JIQQUbul7q7Y18cA3wsh1FAbkD0aRVHrBE4ZOezb979EURXl5VuP6+skSZIkSZI63uzZs5u8oqm4uJirr776pPPLli0jOzu7nm80z+nep73q6kzarYdTFEWLgEUnnHugzvGLwEnb5KIo+hWQ1xY1ZWQOjx+Xlm00cJIkSZIkJbwoigghdHQZzZKdnU1hYWGnu0971dVaWqP9UnttqeuUMjOOBUxlpTYOlyRJkiQlth49elBcXNwqgYO6piiKKC4upkePHi2apzO9pa7dHfemOgMnSZIkSVKCGzx4MNu2bWP37t0dXYo6UI8ePRg8+KT3uJ2WhA6c0tIGkpycRXX1IUrLNnZ0OZIkSZIkdajU1FSGDRvW0WWoG0joLXUhhPgqp7KyTURRTQdXJEmSJEmS1PV1+xVOUXU1h5Yvp/StX0FNDRmXXEyvq64ipNQ+embGeRw4UEh1dRkVFX+gR4+zO7hiSZIkSZKkrq1bB05V+/bx8Ze+xOH3fhs/t++FF0gfOZIhzzxD6pkDj3szXWnpRgMnSZIkSZKkFurWW+p23nf/cWHTURUffsj2e+4BIKNu4/AyG4dLkiRJkiS1VLcNnCo2b+bQm282eL189WrK3/+AzIxjK5zKfFOdJEmSJElSi3XbwOnw2nWnHvPB+/TsOYSkpDTAFU6SJEmSJEmtodsGTkkZPZswJoMQksnoWfvKx9LSjW1dliRJkiRJUrfXbQOnzEsuIalXrwavh/R0sj75SQAyYo3DKyv3Ulm5r13qkyRJkiRJ6q66beCUlJHBgNn/p8Hr/b/yFZLPOAOAzIw6jcPt4yRJkiRJktQi3TZwAuj3uc9x1qOPkDJkSPxcSE/jE998kOy//qv4uczMY43D3VYnSZIkSZLUMt06cAI44y/+gpxXlpDcPxuAtHPPpe/06YQQ4mMyMuuscLJxuCRJkiRJUot0+8AJICQlkTrwTACq9hSfdL22aXjtT1HmCidJkiRJkqQWSYjACSB5QH8AqvftI6qqOv5acjo9ew4GXOEkSZIkSZLUUgkTOKX0rw2ciCKq9u496frRxuGHD2+nurqsPUuTJEmSJEnqVhIncMruHz+u3rPnpOsZdRuHl21ql5okSZIkSZK6o8QJnPofC5yqik/u43R0hRNAWanb6iRJkiRJkporcQKnAXUCp90nr3DKrLvCycbhkiRJkiRJzZYwgVNydnb8uKqeLXWZmcdWONk4XJIkSZIkqfkSJnBK6T8gflxdfHLglJLSi7S0gQCUuqVOkiRJkiSp2RIncDrFljqAzIzhAJSXb6GmprJd6pIkSZIkSepuEiZwSsrKIqSlAfVvqQPIiG2ri6Iqysu3tlttkiRJkiRJ3UnCBE4hhPib6up7Sx2c0Di8zMbhkiRJkiRJzZEwgRNA8tHAqYEVTpkZxwKnMvs4SZIkSZIkNUtCBU5HVzjVlJRQc+TISdePe1OdgZMkSZIkSVKzJGTgBFBdz7a6tLSBJCdnAXDw0HpqaqrarTZJkiRJkqTuIsECp+z4cX3b6vaXvAtEAJSW/o6Vb01my5bvEkXV7VWiJEmSJElSl5dQgVNynRVOJwZO+/a9zW9+M5Pq6tL4ucrKPXy06dsU/e4b7VajJEmSJElSV5dQgVNKA4FTFEVs2PgwUVT/FrodOxZw8FBRm9cnSZIkSZLUHSRs4FRdJ3AqL9/KwYNrG/3urj++3GZ1SZIkSZIkdScJGzhV7TnWNLzuNrqGVFUfapOaJEmSJEmSupvECpyy628a3rPnuSQnZTT63V5ZuW1WlyRJkiRJUneSUIFTUmYmIaM2WKobOKWkZHLW2Z9p8Htpaf0588zr27w+SZIkSZKk7iChAic4tq2u+oS31OWc9zWysz950vjk5F6cP/4ZkpN7tkt9kiRJkiRJXV3CBk5VJwROyck9OH/8D7kg/8dkZ/9J/PyQIbfRu/f49ixRkiRJkiSpS0u8wCnWx6mmtJSa8vLjroUQ6NdvMrljH4ufO3SoqF3rkyRJkiRJ6uoSL3AaUOdNdcXF9Y5JTe1Lz57nAnDgQCFRFLVLbZIkSZIkSd1BwgVOyf3rBE67dzc4rnfv8wE4cmQPFRU727wuSZIkSZKk7iLhAqeU7DqB0wl9nOrqEwucAEoOFLZpTZIkSZIkSd1J4gVOdbbUVTewpQ6gd+/8+PGBEgMnSZIkSZKkpkq8wOm4LXUNr3Dq1WsMIaQCUHLgvTavS5IkSZIkqbtIvMAp9pY6aHxLXVJSOr2yxgBw8OAH1NRUtXltkiRJkiRJ3UHCBU7HNQ0vbjhwAujdp7aPU03NYUpLP2zTuiRJkiRJkrqLhAucktLTSerVC4DqRrbUwfF9nGwcLkmSJEmS1DQJFzjBsT5OjW2pg+PfVHfAPk6SJEmSJElNktiBU3ExURQ1OK5nz6GkpPQBDJwkSZIkSZKaKiEDp+T+tY3Do8OHqSktbXBcCIHevccDUFq6kaqqg+1SnyRJkiRJUleWkIFTSv8B8eOq3bsbHdsn3scp4sCB99uwKkmSJEmSpO4hQQOnY2+qqy4ubnRsb/s4SZIkSZIknZYEDZyy48enahx+dEsdGDhJkiRJkiQ1RYIGTsdWOFXtbjxwSkvLpmePcwAoOfBeo03GJUmSJEmSlKCBU3LdwKm48cAJoHef2m11R47soqJiZ5vVJUmSJEmS1B0kZOB03AqnU2ypg+P7OJW4rU6SJEmSJKlRiRk49esXP64+xZY6qPumOvs4SZIkSZIknUpCBk4hNZXkvn0BqDrFW+oAsrLGEkIqYOAkSZIkSZJ0KgkZOMGxN9U1ZUtdcnI6WVmjAThw4H1qaqratDZJkiRJkqSuLGEDp6ONw6uKi5v05rmj2+pqasopLd3QprVJkiRJkiR1ZQkbOKX0H1B7UFlJTUnJKcfXbRzutjpJkiRJkqSGJUTgVFNTw+7du9m1axfV1dUApGRnx6+f7pvqDJwkSZIkSZIaltLRBbS1NWvW8Oabb1ISW8XUq1cvLr/8cs7rf3zglJ6T0+g8GRlDSUnpTVXVAUoOFLZpzZIkSZIkSV1Zt17h9Otf/5qXXnopHjYBHDx4kMWLF/NRnbfTVe059ZvqQkiKr3IqLd1AVdWh1i9YkiRJkiSpG+i2gVNFRQW//OUvG7z+3u9/Hz+u2rO7SXMe21YXcfDgBy0pT5IkSZIkqdvqtoHT5s2bqaioaPB6aVpa/Li6CT2cAPrU6eNUYh8nSZIkSZKkenXbwKmysrLR64d79IgfN2VLHZzYONw+TpIkSZIkSfXptoHT2Wef3ej1I2lpkFT7+E15Sx1AWlo2PXoMAeBAiSucJEmSJEmS6tNtA6fs7GxGjRrV4PVzhg0jObsfAFXFTVvhBMe21VUc+SOHK/7QsiIlSZIkSZK6oW4bOAHceOONDBky5KTzn/jEJ7j55ptJye4PNL1pOJywrc5VTpIkSZIkSSfp1oFTRkYGf/mXf8mMGTPi5wYNGsRf//Vf06tXL1L61wZO1cV7iaqrmzRn7z72cZIkSZIkSWpMtw6cAJKSkhg1ahRpsbfSJScnkxTr3XQ0cKKmhur9+5s0X6+sXEJIAXxTnSRJkiRJUn26feB0VM+ePQEoLy+Pn0vpnx0/bmrj8OTkHmRljQbg4MH3iaKmrYySJEmSJElKFAkdOCUfXeEEVO1uWuAEx/o4VVeXUVq6sZUqlCRJkiRJ6h4SMnCKogiAlP4D4teri5seOPWp0zi8xD5OkiRJkiRJx0m4wKm6uprKykqgeVvqAHr3zo8fHygxcJIkSZIkSaor4QInOLatLqWZW+oyMoaRktILgAMHf9tKFUqSJEmSJHUPCRk4lZWVAScETsXFTZ4rhCR69xoPwKFDH1JVVdpKVUqSJEmSJHV9CRk4HV3hlNSnD6SmAlC1Z/dpzdc73sephoMHP2iVGiVJkiRJkrqDhA6cQgikZNf2cao+jR5OAL371OnjZONwSZIkSZKkuIQJnDIyMuLHRwMnOLatrmpP07fUQd0VTlBywD5OkiRJkiRJRyVM4FTfCifg2AqnffuIYm+va4qkkEpycm3j8N27X2HVO3/Bjh0vEkVRK1UsSZIkSZLUNSV84JQ8oE7j8L17mzRXZeV+3l39WaqrD8bORBw8+D7ri+ZS9Lu/N3SSJEmSJEkJLeEDp+PeVNfEPk6bN/8bZWUb6722Y8cC9u37VTOrlCRJkiRJ6voMnLKPBU5NaRweRRE7//CzRsfs3Nn4dUmSJEmSpO4sYQKnlJQUUlNTgRMCpwGnt8KppqaCqqoDjY6pOLKrmVVKkiRJkiR1fQkTOMGxVU71NQ2Hpr2pLikpnbS0gae4zznNrFCSJEmSJKnrS8jAqaysLH4u+TR7OIUQGHT29EbHnOq6JEmSJElSd5aQgVN5eXn8TXIpAwbEr1ft2d2kec4990v07XtpvdfOGz6H3r3zWlipJEmSJElS15VQgVNGRgYA1dXVVFZWApCUmUlIT68934QtdQDJyenkn/8jxoz5fznjjEuBZADS0gYydOiXWr9wSZIkSZKkLiShAqf63lQXQiAltq2uKVvqjkpKSuXss27mogt/wsCBUwA4cmQXZWVbWq9gSZIkSZKkLijhAyegWYFTXdn9rogf7927spnVSZIkSZIkdQ8GThxrHF5z8CA1FRWnPW+/fpfHj4v3rmhBhZIkSZIkSV2fgRPHVjgBVDdjlVOPHmeTkZEDwL59b1NTU9mCKiVJkiRJkro2AyeOD5yav62udpVTdfUhSg4UNrNCSZIkSZKkrs/ACUjpnx0/ripu2pvqTtQvu24fJ7fVSZIkSZKkxGXgxLEeTgBVu5u3wqnvGZMIIQ2wcbgkSZIkSUpsCRs4lZWVxY+P31K3u1lzJydncMYZFwFw4MBvqazc18wqJUmSJEmSurZ2C5xCCFNDCL8LIWwMIdxXz/VzQwjLQgi/DSG8EUIYXOfabSGEDbG/25pbQ5OahjdzSx1Av35Ht9VF7N37q2bPI0mSJEmS1JW1S+AUQkgG5gHXAmOBGSGEsScMewz4cRRF44F/AB6Jfbcf8CBwMTAJeDCE0Lc5daSmppKamgqcEDhl1+nh1MwtdQDZ/er2cXJbnSRJkiRJSkzttcJpErAxiqJNURQdAeYDN54wZizwy9jx63WuTwFejaJobxRF+4BXganNLeToKqe6gVNSRgZJmZlA899SB5CVNZrU1NrwqnjvCqIoavZckiRJkiRJXVV7BU6DgI/rfN4WO1fXe8CnY8fTgF4hhOwmfpcQwl+HEN4NIby7e3fDfZjqC5wAkmNvqmvuW+pqa0giu9/lAFRU7KSs7KNmzyVJkiRJktRVdaam4X8LfDKE8Bvgk8B2oLqpX46i6PtRFE2IomjCgAEDGhzXUOCU0r/2Oy1Z4QR1+zjVrnKSJEmSJElKNO0VOG0HhtT5PDh2Li6Koh1RFH06iqILgL+PndvflO+ejqOBU1VVFZWVlfHzRxuHR2Vl1JSWNnd6+vWbHD+2j5MkSZIkSUpE7RU4vQOMCCEMCyGkAdOBl+oOCCH0DyEcred+4NnY8SvANSGEvrFm4dfEzjVLg2+qq9s4vAXb6tLTB5KVNRqAfft+TU1NRbPnkiRJkiRJ6oraJXCKoqgK+Cq1QdF64D+jKFobQviHEEJBbNifAL8LIXwInAn8c+y7e4F/pDa0egf4h9i5ZmkwcBrQP37c8m11tX2camrK2b9/dYvmkiRJkiRJ6mpS2utGURQtAhadcO6BOscvAi828N1nObbiqUUaCpyS665wamHglN3vSrZu/QFQu62uX7/LWjSfJEmSJElSV9KZmoa3i7qBU1lZWfz4aNNwaHng1KfPBJKS0gH7OEmSJEmSpMST0IFTQ1vqqlsYOCUnp3PGGZMAOHhoLRVHWjafJEmSJElSV5JwgVNGRkb8uMGm4Xua3zT8qOx+V8aP9+19q8XzSZIkSZIkdRUJFzg12MOpf+s1DYdjjcMBiveuaPF8kiRJkiRJXYWBU0xSWhpJffoArRM4ZWaOID3tTKC2j1MURS2eU5IkSZIkqSswcKrj6La6lvZwAgghxFc5HTmym0Olv2vxnJIkSZIkSV1BwgVOqamppKSkAPUETrFtdVV79rTKiqR+2VfEj/e6rU6SJEmSJCWIhAuc4Ngqp4YCp+jIEWoOHmzxffr1nQwEAPYWr2zxfJIkSZIkSV2BgVMdyf1b9011aWn96NUrF4D9Jauori4/xTckSZIkSZK6PgOnOlL6D4gfV+3Z3Sr36tevdltdTc0R9u9/p1XmlCRJkiRJ6swSOnCqrKyksrIyfv7oljponcbhANn96vZxcludJEmSJEnq/hIycMrIyIgf113llNLKW+oA+vS5gOTk2vsV2zhckiRJkiQlgIQMnI6ucIITA6djK5yqWmmFU1JSGn3PuASA0tIPOVzxh1aZV5IkSZIkqbMycKoTOCW3QeAE0C/bbXWSJEmSJClxGDjVXeHUrx+EAEBVcesFTnX7OO3c8SIHD64liqJWm1+SJEmSJKkzMXCqEziFlBSS+/YFoHp36wVOVdWHCSEVgP0l77DqnQLeeedGDh5c12r3kCRJkiRJ6iwMnOoETnCsj1Nrbak7fHgHhYUziaLK484fPLSWNb+ZSXn5tla5jyRJkiRJUmdh4HRS4FT7prqqvXuJampafK+PP36Oysr99V6rqiph68fPtvgekiRJkiRJnYmB0wmBU7xxeFUV1SUlLb7XnuLXG71efIrrkiRJkiRJXY2BU53AqfrgQSp37Ih/3vmNb3C4qKhF94qiqsav1zR+XZIkSZIkqatJyMApNTWV5ORkAMrKygCo3LWLLTd/hvJ3V8fHHXptGZtvupkDixY1+15nnHFx49f7Tmr23JIkSZIkSZ1RQgZOIQQyMjKAYyuc/vjwIxz5/e9PHlxdzY77/46qvXubda9zhvxl/A11J9eRwjlDZjVrXkmSJEmSpM4qIQMnOLatrry8nKq9ezn46qsNjo0qKih56aVm3ScraxTj854mNbXvSddGj3qEXr3GNmteSZIkSZKkzsrAqbycqj/8AaqrGx1f+fG2Zt+rf/8/ZfJlK8nNfZzs7Kvi52tqyhv5liRJkiRJUteU8IFTZWUlUd+TVx+dKGXgwBbdLzm5B584888ZPfof4+f+uOvlFs0pSZIkSZLUGSV84ARQlZVF5hVXNDw4OZk+BX/eKvftkf4J+vS5CID9+1dRUbG7VeaVJEmSJEnqLAycqN1W94m//zuSs7PrHXvm3K+RetZZrXbvMwdeFzuK2LV7SavNK0mSJEmS1BkYOFEbOKUNHcqwF/8/+n7+84Q61z7xzW/S7wtfaNV7Dxg4FQgA7Nq1qFXnliRJkiRJ6mgGTtQGTgCpZ53FJ/7+7zjza3Pi15Kb0N/pdPVI/wRn9JkAwP7971BRsavV7yFJkiRJktRRDJw4FjgdlTZsePz4yOZNbXL/gQOvjR25rU6SJEmSJHUvCRs4ZWRkxI/LysqOu5Y2fFj8uGJTWwVOdbbV/dFtdZIkSZIkqftI2MCpsRVOKQMGkJSVBcCRTZvb5P7p6WdyxhkTAdhf8i6HK/7QJveRJEmSJElqbwZOnBw4hRBIG167re7Ipk1EUdQmNQys87a63bvcVidJkiRJkroHAydODpwA0ocNBaCmrIyqXW3T1HvggGPb6v64a3Gb3EOSJEmSJKm9JWzglJqaSnJyMlB/4HRc4/A26uOUnj6AM86YBECJ2+okSZIkSVI3kbCBUwghvsqp3sCpbuPwzW3TxwngzPi2OtjlKidJkiRJktQNJGzgBDQaOKUPr7vCqe0CpwEDpnD0f8OuXb6tTpIkSZIkdX0GTtQfOKWecw7Etty11ZY6OLqtrvZtdSUlazh8eGeb3UuSJEmS/n/27jy87rLO///zPic52dMmTUv3jS5A2UFwAVzQERl1GEQFR/i6DDIjrjiLouN3fs44zlcdL2ccHFHHkXFEx0FcUVBxARwFSgulpRTolu5N0qZJmvWc8/n9kZM0LU1y2pwly/NxXefqJ5/7Pp/POy3+4eu67/ctSYVg4AT09vaSTCaPGoslEiTmzwfyu6UO4JRZfzh4vb/J0+okSZIkSdLEZuCU0d3d/bzxRGZbXXLPHtKHD+etjpmzhmyr23dP3t4jSZIkSZJUCFM6cKqsrBy87uzsfN74UY3Dt23LWx1liQbq6i4G4FDbWrq7d+ftXZIkSZIkSfk2pQOnoSucitk4HGCWp9VJkiRJkqRJwsAp43iBU2LJkRVOvVvz1zgcYNbMP2Dgn2OfgZMkSZIkSZrADJwyRguc8t04PJFooK7uhQC0ta2lq2tXXt8nSZIkSZKULwZOGccLnErq6ojX1QH531IHcMrQbXVNrnKSJEmSJEkTk4FTxvECJzhyUl3vtm1EqVRe65k58w8IIQ7A/v0/yeu7JEmSJEmS8sXAKWO4wKksc1Jd1NND3549ea0nkZhB3fSBbXVP0NW1M6/vkyRJkiRJygcDp4xhVzgtGXpSXX4bh4On1UmSJEmSpIlvSgdOiUSCWKz/r2D4LXVDGocXIHBqaHglA/8sz23+Rx548EI2PfMJenqb8/5uSZIkSZKkXJjSgVMIYXCV0/Bb6oaucMpv4/Aoiti8+dNAevBeX99Bdu68g9Wrr6Gnpymv75ckSZIkScqFKR04AVRWVgLQ2dl53PHSefMIpaVA/rfUHTz4v+zZ+93jjnV372DL1s/n9f2SJEmSJEm5MOUDp9FWOIV4nMTiRQD0bNuW11r27v3+iOP79v2QKMrvSXmSJEmSJEljZeCUCZx6e3tJpY4f5gw0Dk81N5M6dChvtfSO0qcpleoklTp+MCZJkiRJkjReGDhlc1LdkMbhvVvz18eponLJiOOJxCzi8aq8vV+SJEmSJCkXDJyyCJyGNg7vyWPj8HlzrwXC8OPzriOE4cclSZIkSZLGAwOnbFY4LRlyUt3W/DUOr65ewcoVf3vcsaqq5SxedFPe3i1JkiRJkpQrJcUuoNiyC5yObHXL5wongPnz30rttHPZtfObtLVvoKNjAwCxWAWxWFle3y1JkiRJkpQLrnDKInCKV1dRcsopAPRuyd8KpwG1NWdy+umf4uKLfkh93SUAtLevo719Q97fLUmSJEmSNFYGTlkETnCkcXjvjh1EfX15r2vAvHlvGbzetetbBXuvJEmSJEnSyZrygVNlZeXgdWdn57Dzygb6OCWT9O7Yme+yBjU0vIJEYiYAe/f9kGSyo2DvliRJkiRJOhlTPnDKeoXTkD5O+WwcfqxYrJS5c94IQCp1mH37flSwd0uSJEmSJJ0MA6cT3FIH0FOAPk5DzchMEUkAACAASURBVJ17LRCA/m11URQV9P2SJEmSJEknYsoHTolEglis/69hpMCpbOnSwevePJ9Ud6yKinnMmPFSANo7NtDe/mRB3y9JkiRJknQipnzgFEIYXOU0UuBUcsophEy/p0KcVHeseXOvG7y2ebgkSZIkSRrPpnzgBGQVOIVYjLLFiwHo2bq14NvaZsx4GWVlswHYu+9HJJPtBX2/JEmSJElStgycyC5wAkhkttWl29pItbTkva6hYrES5s59c//7013s2fv9gr5fkiRJkiQpWwZOHAmcenp6SKVSw84rZuNwIHNaXf8/2W6bh0uSJEmSpHHKwImjT6rr7u4edt5RjcO3bstnScdVXj6HhoZXANBxeBOH2tYUvAZJkiRJkqTRGDhxdOA00ra6xJKhJ9UVfoUTwLx5Ng+XJEmSJEnjm4ETUJk5fQ6gs7Nz2HmJRQshBAB6thYncJpRfynl5fMA2L//J/T1tRalDkmSJEmSpOEYOJH9CqdYeTml8/rDnt4tW/Ne1/GEEGfe3GsBSKd72LP3e0WpQ5IkSZIkaTgGTmQfOMGRxuF9u3aRHqHfUz7NmfNGQigB+rfV2TxckiRJkiSNJwZOnFjgVDbQxymK6N2+PZ9lDV9D2UwaGl4JQGfnZlpbHy1KHZIkSZIkScdj4MSJrnAqfuNwgPnz3jJ4vWv3nUWrQ5IkSZIk6VgGTpzgCqfMljqAniIGTnV1L6KiYiEA+/ffR2/vgaLVIkmSJEmSNJSBE2NY4bR1W75KGlUIscHm4VHUyzPP/h1NTb8gmTxctJokSZIkSZLAwAmAsrIyQgjA6IFTvL6e2LRpQHG31AHMnHkF0F/3vn0/ZN2TN/HQb19MY+O/20hckiRJkiQVjYETEEIYXOU0WuAUQqBsSf+2up6tW4nS6bzXdzxRFPH007cCRwdLqVQHzz73D+za/a2i1CVJkiRJkmTglFFZWQmMHjjBkW11UVcXyX378lrXcFpbH+Zg6++HHd+27TbS6WQBK5IkSZIkSepn4JQxsMKps7Nz1LmJJYsHr4vVOLzlwIMjjvf07OXw4WcKVI0kSZIkSdIRBk4ZA4FTT08PqVRqxLllQxuHb9ma17qGFY2+lS+iONv9JEmSJEnS1GbglDH0pLru7u4R5yaWDD2prjgrnOrqXjTieGnpDKqrlheoGkmSJEmSpCMMnDKGBk6j9XFKLJgPJSUA9BRphVN9/SXU1pw97PiiRTcSi5UVsCJJkiRJkqR+Bk4ZJxI4hdJSEgsXAtC7tTiBUwgxzjnnK9RNf+HzxmKxcubNfUsRqpIkSZIkSTJwGnQigRNAYukSAJL79pHqOJy3ukasIdHA+ed/k4te8ENWrvgE9fWXApBOd7N7z3eKUpMkSZIkSZKBU8aJBk5lR/VxKlLj8IyamlXMn/8nnH7apwihFIDGxq+STvcWtS5JkiRJkjQ1GThlnPgKp+I3Dj9Wefkc5sz+YwB6evayd+8PilyRJEmSJEmaigycMk54hVNmSx1Az5bxETgBLFr0Lgb+Wbc33k4UpYpbkCRJkiRJmnIMnDIqKysHr7Na4bTkSODUW6ST6o6nsnIJs2ZdAUBn51b2N/2syBVJkiRJkqSpxsApY+gKp87OzlHnx2triTc0AONnS92AxYv+bPB6+7Z/I4qiIlYjSZIkSZKmGgOnjLKyMkIIQHYrnADKMqucerZuI3nwYN5qO1E1NauYUX8ZAO0dGzhw4MEiVyRJkiRJkqYSA6eMEMLgKqdsAqfup56iZ1tmK10yybOXvZTdt36UVHt7PsvM2qJFfz54vW37l4pYiSRJkiRJmmoMnIbINnDqefZZtl9/A6mm5iM3+/o4dPfdNL7zT0n39uazzKxMn/4Cpk07H4DW1oc5dGhNkSuSJEmSJElThYHTENkGTk23fZH04cPHHetet472n/4057WdqBACixe9e/BnVzlJkiRJkqRCMXAaYiBw6u7uJp1OH3dOlErRfv/9Iz6n7ec/z3ltJ2PGjJdRXX0aAM3N99PRsanIFUmSJEmSpKnAwGmIoSfVdXd3H3dOlEpBX9+Iz4k6s2s6nm8hBBYtvGnwZ1c5SZIkSZKkQjBwGmJo4DTctrpYIkHZypUjPqf87LNyWtdYzJp1JRXlCwHYt+/HdHU1FrkiSZIkSZI02Rk4DZFN4ARQ/7a3DTsWysupe/Obc1nWmMRiJSxcdGPmpzTbG79S1HokSZIkSdLkV7DAKYRwRQhhUwjhuRDCh48zvjCE8KsQwtoQwroQwpWZ+4tDCF0hhMczn7ztC6usrBy8HilwmnbVHzHjppuedz8kEsz/13+ldM6cvNR3subMfgOJxEwA9uy5i56epiJXJEmSJEmSJrOCBE4hhDhwG/Aa4AzguhDCGcdM+xjwnSiKzgOuBb44ZGxzFEXnZj5/lq86h65w6uzsHHZeCIFZH/wAp977U+pvvHHwfsUFF1B9yUvyVd5Ji8fLWLjgHQCk0708uvqPeWzNW9i27d/o7T1Q5OokSZIkSdJkU6gVThcBz0VRtCWKol7g28AfHTMnAmoz19OA3QWqbVC2W+oGJBYv5pQP3ULZ8mUAdK9fTzTM6XbFNn36C4AAQE/PHlpbH2bzls/y8CNX0nH42eIWJ0mSJEmSJpVCBU7zgB1Dft6ZuTfU3wJvDSHsBH4CvHfI2JLMVrvfhBAuPd4LQgjvCiGsDiGsbmo6uS1jJxo4DX7v3PMASLe307t580m9O5+iKGLj0x+hP9M7Wm9vExs2fJAoev6YJEmSJEnSyRhPTcOvA74eRdF84ErgGyGEGLAHWJjZancLcGcIofbYL0dR9OUoii6MoujCmTNnnlQBJx04nXfe4HXn2rUn9e58am19lMMjrGLq6NhIW9v4q1uSJEmSJE1MhQqcdgELhvw8P3NvqHcC3wGIouh3QDnQEEVRTxRFLZn7jwGbgRX5KPJkA6fK848ETl1rxl9w09m1dfQ5naPPkSRJkiRJykahAqdHgeUhhCUhhAT9TcF/eMycRuBygBDC6fQHTk0hhJmZpuOEEJYCy4Et+SiyrKyMEPr7HJ1I4FS6aBHxurr+743DFU6J0vpR55QmZhSgEkmSJEmSNBUUJHCKoigJvAe4D9hI/2l0G0IInwghvD4z7UPAjSGEJ4BvAW+L+hsLXQasCyE8DtwF/FkURXk5Wi0Wi1FeXg6cWOAUQhjcVte7fTvJA+Pr5Lf6+ssoHSF0SiRmUl/34gJWJEmSJEmSJrOSQr0oiqKf0N8MfOi9jw+5fgp4yXG+913gu3kvMKOiooKurq4TCpygf1tdxy9/CfSvcqq5/PJ8lHdS4vEyTlv5dzy5/n1A6nnjS5d8kFgsUfjCJEmSJEnSpDSemoaPC5WVlcCJrXCCoxuHj8dtdbNmXcH553+TGTNeSgilQHxwrPXQI8UrTJIkSZIkTToFW+E0UQw0Du/q6iKdThOLZZfJla9aBaWl0NdH59rH81niSaub/gLqpr+AKIpIp3v4/cN/QHf3Lvbu/QELF/wpNTWnF7tESZIkSZI0CbjC6RhDT6rr7u7O+nux8nLKz+gPbLqffJKotzfnteVKCIF4vJylS2/J3InYvPnTRa1JkiRJkiRNHgZOxxgaOJ1wH6fzzgcg6u2l+6mnclpXPsw+5fVUV58BQMuBBzhw4H+LXJEkSZIkSZoMTipwCiFcGkJ4XoPvyWDglDqA1tbWE/ru0D5O43Vb3VAhxFh26l8N/vzc5v9HFKWLWJEkSZIkSZoMsgqcQgi/DiFckrn+C+Bu4K4Qwl/ns7hCW79+PQ8//PDgz9/85jf58Y9/TF9fX1bfrzjv3MHr8dg4/Hjq6y+hru7FALS3r2ff/nuKXJEkSZIkSZrosl3hdBbw+8z1TcDLgIuBd+ehpqJ46qmnuOuuu47q25ROp1m9ejX/8z//QxRFoz6jdNYsSufPB6Bz7ZqsvlNsIYSjVjlt2fw50unx239KkiRJkiSNf9kGTjEgHUJYCpREUbQhiqJGoD5/pRVOOp3m/vvvH3b8mWeeYceOHVk9a2BbXaqpmb5du3JSX77V1p7FKae8DoCu7kZ27fpWkSuSJEmSJEkTWbaB0/8Cnwc+DXwPIBM+teSproJqaWmhpWXkX2XTpk1ZPWsibqsDOHXphwihFICt2/6VZLK9yBVJkiRJkqSJKtvA6W1AN7AJ+Hjm3hnAF/JQU8GlUqlR5ySTyayeVTmkcfhECpwqKhYwf96fANDXd4DtjV8pckWSJEmSJGmiyipwiqKoKYqiv4qi6KNRFHVk7v04iqJ/ym95hTFjxgwqKipGnLNgwYKsnlW2YgWxykoAOtdMnMAJYPHidxOPVwPQ2Pg1enr2F7kiSZIkSZI0EWV7St37QwjnZq4vCiFsCSE8G0K4KL/lFUZpaSkXX3zxsOP19fWcdtppWT0rxONUnHsOAD3PPEOqoyMnNRZCIjGDRYveBUA63cXWrf9S5IokSZIkSdJElO2Wug8B2zLX/wjcBnwWmDSJxGWXXcYFF1zwvPv19fW89a1vpaSkJOtnVZyb2VaXTtO9bl2uSiyIhQveTiIxC4Dde75DS8sDdHRsIpXqKXJlkiRJkiRposg2cJoeRVFrCKEaOBf4fBRFtwPZLfuZAGKxGK973et4z3vew+LFiwfvX3311dTXn9hhfBVD+jhNtG118XglS5e8H4AoSvH4E2/n4Ueu5KHfvpgtW/6ZKBq935UkSZIkSZrasg2cdoYQLgbeBDwYRVEqhFADTLr0oaGhgXPOOWfw5wMHDpzwMyrOPQdCACZW4/ABJaW1z7uXTLayddu/8PTTHytCRZIkSZIkaSLJNnD6K+BHwCeAv8/cey3waD6KKrYZM2YMXre0tJzw9+M1NZQtXw5A1xNPEGVxCt54EUVpNm/+zLDju/d8h47DzxawIkmSJEmSNNFke0rdj6MomhVF0fwoigZCpu8BV+WvtOIZa+AER7bVpTs66Hluc07qKoSOjk10dTWOOKdp/30FqkaSJEmSJE1E2a5wIoSwNIRwawjhthDCrcC8KIq681hb0VRWVlJeXg6cfOBUef6RPk5da9fkpK5CSKU7s5jTVYBKJEmSJEnSRJVV4BRCuBJ4gv6G4Z3AOcDaEMIf5rG2ogkhDK5yamlpIYqiE37G0MbhE6mPU1XlcmKx8hHn1NacXaBqJEmSJEnSRJTtCqdPAVdFUfSmKIr+MoqiN9O/ne5T+SutuAYCp97eXjo6Ok74+6ULFhDPPKNz7eM5rS2fSktrmTv3jcOOl5XNoaHh8gJWJEmSJEmSJppsA6eFwK+PufdA5v6k1NDQMHh9MtvqQghUnHcuAH2NjSSbm3NWW74tO/UjNDS88rhjpaXTCSHrnZiSJEmSJGkKyjY5eAL4wDH33gesy20540cuGodXnnf+4HXnBNpWF4+XcfZZX+KCC77DwoU3Mm/eW6msWAJAR8dGdu3+dpErlCRJkiRJ41lJlvPeDfw4hPABoBFYACSB1+ersGLL5Ul1AF1rH6f2Va8ac12FEkJg+rQLmD7tAqD/9LpHHn09UZRk8+ZPM7PhVZSVzSxylZIkSZIkaTzKaoVTFEVPASuB64HbgBuA06IoWp/H2oqqvr5+8PpkA6fyVWcQSkuBidU4/Hiqq1eycME7AUgm23n2uU8WuSJJkiRJkjReZd2MJ4qiviiKfh1F0Z1RFP0aIISwJW+VFVkikaC2thaA5pPsvxQrK6N81SoAutevJ93Tk7P6imHJkvdSXj4fgH37fkRLy4NFrkiSJEmSJI1HY+n+HIDFOapjXBrYVnfw4EFSqdRJPaPi/P4+TlFfH90bnspZbcUQj1ewcsXfDv686ZmPk0p1F68gSZIkSZI0Lo31uLEoJ1WMUwOBUzqdprW19aSeMXBSHUz8bXUADQ0vZ+bMKwDo6mpk2/Z/K3JFkiRJkiRpvPF8+xHk5qS6IY3DH5/4gRPAihV/QzxeDcD27bdz+PDmIlckSZIkSZLGkxFPqQshfO1kvzsZ5CJwKmlooHThQvoaG+lcs5Yoiggh5KrEoigvm82pSz/IM8/+HVHUx9Ob/obzz/vmhP+9JEmSJElSboy2wmnXCJ/twD/ktboiy0XgBFCZ2VaXammhb8eOMdc1Hsyffz01NWcC0Nr6MHv33l3kiiRJkiRJ0ngx4iqlKIr+plCFjEfTp08nFouRTqfHFDhVnHceh37wQ6C/j1Ni4cJclVg0IcQ5beXf8+jqq4E0zz73j1RVrQQiKisXU1JSU+wSJUmSJElSkdjDaQTxeJy6ujpgbCucKob0cepcMzn6OAHU1p7F/PnXA9DXd4BHV/8Rj66+igcfupiNT99KMtlR5AolSZIkSVIxGDiNYmBbXVtbG729vSf1jLJly4hV9zfZ7nz4YVKHDuWsvmKbNeuK591Lp3vYvfu/efyJd5JOJ4tQlSRJkiRJKiYDp1E0NDQMXh84cOCknpFsbiGUlwPQu20bz7z4Jez8wAfp27s3JzUW0/bttw87dujQapqbf1HAaiRJkiRJ0nhg4DSKsTYOT7W2sv3660k1Nw+5maL93nvZ/tbrSR48mIsyiyKV6qKl5TcjztnfdG+BqpEkSZIkSePFiE3DB4QQbhhmqAfYCTwSRVFfzqoaR8YaOB345jfpa2w87ljfzp0c/MY3mPm+9510fcWUTvcC0YhzUqmuwhQjSZIkSZLGjawCJ+BdwAuAFvoDpnlAA7AWWAz0hhCuiqJoTT6KHIttXT18aUcTvznQRjqCS+qquWnBLFZUlWf1/bEGTu33/WzE8bZ775uwgVNJSS0VFYvo6to+7JxptecWsCJJkiRJkjQeZLulbg3w4SiK5kZRdFEURfOAvwYeBuYCXwO+kKcaT9qaQ4d51aOb+PquZrZ29bK9u5dv7jnAH6zexEMH27N6RnV1NYlEAoDmodvispTu7BzT+HgWQmDhwj8dYbyUuXPfWMCKJEmSJEnSeJBt4HQ98C/H3PsCcEMURWngU8CqXBY2Vuko4r0bG2lPpZ831p2OeM9TjfSlR94OBv2hysAqp5aWFqJo9O8MVb5q5L+W0cbHu3lzr2PhgncedyyK+mjveLrAFUmSJEmSpGLLNnDaD7zmmHtXAE2Z63IglauicuGRQ4fZ3NUz7Pje3j5+daAtq2cNBE7d3d10nuCKpPobrh95/PqRx8e7EALLl9/KCy/+OUuWvJ/58/8Pc+deNzi+8am/oq+vtYgVSpIkSZKkQsu2h9MHgP8OIawFdgALgPOAN2fGXwh8MfflnbzdPaP3MN+VxRx4fh+nqqqqrOuovOACZv/fj7P37z8JqaMzuVkf/jBVL7w462eNZ1VVS1m65EgvqnSqi737vk9P7z6e3vRxzlz1z4QQilihJEmSJEkqlKxWOEVR9FNgGfB1YCNwB7Asc58oiu6Louhv8lXkyZhbVjrqnHlZzIGxNw6vu+46lv38ZzS8770klp06eL/8tNNO+FkTxcqVf0t52VwA9u+/h337flTkiiRJkiRJUqFku6WOKIr2R1H0H1EUfTKKoq9FUbQ/n4WN1UXTqji1omzY8dmJUl5eX5vVs8YaOAGUzp3LzHe/m9kf/ejgvfb77z+pZ00EJSU1nHHGZ4D+VU2bnvk43d27i1uUJEmSJEkqiKwCpxDCohDCf4YQ1oUQtgz95LvAkxULgS+cvpDq+PN/xfJY4F/PWEhpLLstXrkInAZUXnghsdr+oKv9/l+ccBPyiaSu7oUsXPAOAJLJdp7a+Ff095iXJEmSJEmTWbYrnO4EEsBHgRuP+Yxb50+r4hcvWMl1s+sH7y2tKONnF67kkrqarJ9TXl4+2LdprIFTKC2l+qUvBSC5ew89T0/uU9yWLv0QVVUrADh48Hfs2HlHkSuSJEmSJEn5lm3gdBbwJ1EU/SiKovuHfvJZXC4srijj0ysXDP68oqqMFVXlJ/ycgVVOLS0tpNNjW6VTc/krBq/bfzHu/wrHJB4vY9UZnyOEBACbN3+ajo5NRa5KkiRJkiTlU7aB00PA2fksJJ9KY4GKWP+v2pY8ubBoIHBKpVK0tbWNqZ6qSy4llPY3LJ/MfZwG1NSczqlLPwhAOt3Lhqc+RHf3Xjo6NtHXd6jI1UmSJEmSpFwryXLes8B9IYS7gL1DB6Io+kTOq8qDaSVxunrTtCVTJ/X9hoaGweuWlhamT59+0rXEq6uofPGLOPybB+h5+ml6d+4iMX/eST9vIli48J00t/yK1tZH6OjYyG//9yUAhFDCrJlXsHz5xygrm1nkKiVJkiRJUi5ku8KpHrgPqAGWD/ksy1NdOVdbEgfg0EkGTrlsHA5Q84rLB687fjn5VzmFEGfZso8wcGrdgChKsm//j3lszbX09Y1t5ZgkSZIkSRofsgqcoii6fpjPDfkuMFdqS/p/1fZxEzi9HEJ/+NJ+/y/H/LyJoGn/T4Hjn8rX1bWNXbvuLGxBkiRJkiQpL4YNnEII84dcLxzuU5gyx25ghVNbMkU6On7oMZK6ujpCJiBqbm4ecz0lM2dScc45AHSuXk3y4MExP3O827//3pHHm35SoEokSZIkSVI+jbTCaeOQ623A1syfQz9b81JVHkzLBE5p4HDqxBuHl5SUDPZtysUKJ4DqgdPqUik6fvObnDxzPEumDo84nkp1FqgSSZIkSZKUTyMFTtOGXJcCicyfQz+J/JWWWzWZwAk46cbhA9vqWltbSSaTY6/p8lcOXndMgW11tbVnjTheUzPyuCRJkiRJmhiGDZyiKEoPuU4N9ylMmWM3LYeBE8CBAwfGXFPZ0iUkli4FoOOhh0h3d4/5mePZggVvH3F8zuyrC1SJJEmSJEnKp6yahocQFoUQ/jOEsC6EsGXoJ98F5krtkMBpvJxUB1CT2VYXdXVx+H9/l5Nnjlcz6i9h+fKPMdx/djt2fI0hOackSZIkSZqgSrKcdyewA/goMCEb7eR6hVPuAqfLafnKVwFo/+X9/afXTWILF7ydmQ2vYu/e79Hds5dEaQN7991Nd/duWg48wNZtt7F0yXuLXaYkSZIkSRqDbAOns4DLJtIWumPVjtPAqfzss4nPbCDV1EzHL39FlEoR4vHRvziBVVTMZ8mQUGnWrFez+rFrSKd72Lr1n5lWew4zZlxWxAolSZIkSdJYZLWlDngIODufheRbLrbU1dbWUlLSn9HlKnAKsRg1L+/fVpc6cICuxx/PyXMnkpqaM1i58hOZnyLWb/ggXV27ilqTJEmSJEk6edkGTs8C94UQvhhC+PjQTz6Ly6VcbKmLxWLU19cDuQucAGpeefngdfsUOK3ueObOuYa5c98MQDLZypPrbyad7ilyVZIkSZIk6WRkGzjVA/cBNcDyIZ9leaor52pyEDjBkW11hw8fpqura8x1AVS+8IXEKisBaL//F0RRlJPnTjQrlv9famrOBKC9/Umeefbvi1yRJEmSJEk6GVkFTlEUXT/M54Z8F5gruVjhBNDQ0DB4feDAgTHVNCCWSFB1WX/Por7tjfRu3pyT50408XgZZ515GyUl0wDYtetO9uy5u8hVSZIkSZKkEzVs4BRCmD/keuFwn8KUOXa56OEE+WkcDv2n1Q1o/8X9OXvuRFNRMZ9Vqz4HBACe3vRR1j15M4+teQvrnryZpqafE0Xp4hYpSZIkSZJGNNIKp41DrrcBWzN/Dv1szUtVeVAeC5SG/hCjPXnygUW+Aqfql14GmYbk7fdP3cAJoGHGy1iy+D0ApNO9NDXdS2vrwzQ13cu6J/+MJ9e/l3Q6WeQqJUmSJEnScEYKnKYNuS4FEpk/h34S+Sstt0IIg6uccrXCqbm5ecx1DYjX1lJ10QsA6H7ySfr27cvZsyeiGQ0vH3asqeledu68o4DVSJIkSZKkEzFs4BQN2bcURVFquE9hysyNgT5OY+nhVFlZSUVFBZDbFU4A1UO21XX8cmqeVjdg9+7vjDi+c9d/FagSSZIkSZJ0orJqGh5CiIcQ3h1C+O8Qwv0hhF8OfPJdYC7VlPT/umMJnODIKqeWlpacnihX84pXDF5P5T5OAJ2dW0Yc7+pqZILlnZIkSZIkTRlZBU7A54D3AY8AFwP3APOBh/JUV14MXeE0lqBoIHDq6+ujvb09J7UBlM6ZQ/mqVQAcfuQRUjl89kRTWlo/4nj/SXbZ/ucrSZIkSZIKKdv/x34NcEUURf8EpDJ//hFwWd4qy4OBHk69UUR3euyBE+RjW11mlVNfHx0PPJDTZ08kc2ZfNeJ4Q8MrCZkm8JIkSZIkaXzJNnCqBLZnrjtDCBVRFG0Ezs9PWfkxsMIJoD1HjcNzHTjVXP7KweuD3/o2XY8/TpQ++VP1JqqGhsuZOfPVw463tT1BMjl1V4BJkiRJkjSeZRs4PQ1cmLl+DPh4COHDwO68VJUnNUMCp1ydVJfrwCleN51QXg5A1+rVbLv2Oja/5jV0PDihdi+OWQgxzlz1LyxfdisVFQsBiMdrKS3t/7vv7HyO9evfRzqdLGaZkiRJkiTpOLINnD4IDCyz+RDwIuCNwJ/lo6h8GbrCaSyNw+vrj/QXymXglO7upvEd7yDq7j7qft/2Rnb8+Z/TuWZtzt41EcRiJSxc+E5e/KJf8YqXP8NLL1vDxRfdQ3nZXABaDjzAM8/+fzlt3C5JkiRJksZu1MAphBAHVgDrAaIo2hRF0cuiKLogiqJf57m+nKrN0QqnRCJBbW0tkNvAqe2ee+h9bvPxB5NJmr/4xZy9a6IJIU4IgbKymZxzzleJx6sB2LXrTnbs+FqRq5MkSZIkSUONGjhF/WfPfyGKop4C1JNXtTla4QRHttUdPHiQVGpszxrQ8etfjzh++Le/Jd3bm5N3TWTV1Ss568x/pT8LhWef+xT7m+4rclWSJEmSJGlAtlvq7gkhXJnXSgogV1vqABoaGgBIp9O0traO6VkDor5R+hFFEeQo3JroZsy4lJUrPpH5KWLDhltoNrhz7AAAIABJREFUa1tX1JokSZIkSVK/kiznxYC7QwgPATuAwaY5URS9Ix+F5UOuttTB8xuHD/35ZFVeeMGIq5zKV60iVlEx5vdMFvPmXUtX13a2N36ZdLqbx594JzNnvorOzm3E45XMmnkFs2e/jlisrNilSpIkSZI0pWQbOD0LfCafhRTC0MCpPceBUy5Me8MbaPn3r5E6ePD477zxxpy8ZzI59dS/pLOrkaame+nrO8Du3f89ONbS8it27fom5557B6WltUWsUpIkSZKkqWXEwCmEcF0URd+KouhvClVQPuVrhVNzc/OYnjWgpK6OBV/9Crve+z76du8+aqz6FS+n9opX5+Q9k0kIMZYv+yhNTT/jyEGKR7S1r+O55z7F6ad/qvDFSZIkSZI0RY3Ww+n2glRRILns4TRt2jRisf6/vlyeVFexahWn3ncv82/7V2a8612QeUfvtu1EUTTKt6empub7OF7YNGDvvh+QTLYXriBJkiRJkqa40QKnUJAqCqQqHhv8hdqSwwcU2YjH49TX1wO5DZwAQmkpNZdfzqxbPkjNq14FQO+WLXStXZvT90wWnYc3jzieTvfQ3b17xDmSJEmSJCl3Rguc4iGEl4cQXjHcpyBV5kgshMFtdWNd4QQMBk7t7e1s3bo1LyuQpl9zzeB16//clfPnTwalifrR55TWFaASSZIkSZIEozcNLwP+neFXOkXA0pxWlGe1JXEOJVNj7uG0YcMGtm7dOvjzHXfcwZw5c7jmmmtycmLdgKoXv4iSuXNI7t5D2733csqtHyFeU5Oz508Gs0+5im3bbht2vLJiKWVlswpYkSRJkiRJU9toK5wOR1G0NIqiJcN8JlTYBFBb0v8rt6dOPnDaunUrd911F319fUfd37NnD3fccQfd3d1jqnGoEI8z/eo3ABB1ddF2z09y9uzJoqpqKYsX/fmw451dW9i9x9VhkiRJkiQVymiB06QzsKVuLCucHnjggWG3z7W1tbE2x72Wpl/9xxD6F5m13mVwcjxLl36IM874J2pqVgEQi5VTW3vO4PjGjR9h337DOkmSJEmSCmFKNQ2HIyfVdabS9KVPvOdSKpU6aivd8WzePHIT6xNVOncuVZdcAkD3+vV0b9yY0+dPBiEE5sy+iote8ENe8fJneNlL1/OCC+9m2al/nZmRZsOGD9Lc/Kui1ilJkiRJ0lQwYuAURdGkaxY0sMIJctM4vFCOah5+13eLWMn4F0KckFkRtmjRu1i8+GYAoijJk+tv5uDB3xezPEmSJEmSJr0pu6UOTq6PUzweZ/HixSPOWbo0962tal7+MuKZU/EO/ehHpHPYJ2qyW7rkgyyY/zYA0ukenlj3Lg4dery4RUmSJEmSNIlN6cDpZPs4XXrppcOO1dTUcN55553Uc0cSEgmmXXUVAOm2Ntp//vOcv2OyCiGwfPnHmDvnTQCkUodZ+/gNrH7szfzq16v49W/O5sn176W9fUORK5UkSZIkaXKYcoHTtKFb6vpOLnA69dRTufrqq6moqDjqfkVFBTfccMPz7ufK9GveMHjd+j82Dz8RIQROO+3vmTXrD4H+0OnQodWk092kUofZv/8nPLr6Gg4c+N8iVypJkiRJ0sQ35QKnmqGB00lsqRtw9tlnc8stt/DGN76ReLz/mdOnT2fmzJljrnE4ZUuXUnHBBQB0PvIIvdu25e1dk1EIcU5b+XeEUHLc8Sjq5emnP0oUpQtcmSRJkiRJk8uUC5ym5WBL3YDS0lJWrVrFwoULAdi7dy/dee6tdFTz8O/endd3TUYHDjxIFCWHHe/qbqS1dXUBK5IkSZIkafKZ0oHTyW6pO9ZA4BRFETt37szJM4dT++o/IFZdDUDr979H1NeX1/dNNj29TaPO6e1rLkAlkiRJkiRNXlMucMrVlrqhFi1aNHi9ffv2nDxzOLHKSmpfm+lD1NRMxwMP5PV9k01lxeIs5iwadY4kSZIkSRrelAucjlrhNMYtdQPmz59PLNb/V9nY2JiTZ45k+jVvHLy2efiJqa+/lPLyecOOh1AybI8nSZIkSZKUnSkXONXmsIfTgEQiwZw5cwDYuXMnyeTwPYJyoXzVGZSdfjoAHQ88QN++fXl932QSi5Vw5plfoKSk9rjjUZTksTVvoa19fYErkyRJkiRp8phygVNN/Ejg1J6jwAmO9HFKpVLs3r07Z889nhAC0695Q/8P6TSHvve9vL5vsplWew4XX/xTFi96N9Nqz2PatAtZsvh91NW9BIBkspW1a9/KoUNrilypJEmSJEkT05QLnEpjgcp4/6+dqxVOcCRwgsJsq5v22tcSysoAaL3ru0TpdN7fOZmUl83m1FM/xIUX3sWFF/w3S5e+n3PP+SozZ74agGSynbWP/x8OHvx9kSuVJEmSJGnimXKBExzp45SrHk5wdOCU78bhAPFp06i9oj8c6du5k86HH877Oye7WCzBmav+hVNOeT0AqVQnjz/xDlpafkM63cehQ2s5ePARksn2IlcqSZIkSdL4VrDAKYRwRQhhUwjhuRDCh48zvjCE8KsQwtoQwroQwpVDxj6S+d6mEMKrx1rLwLa6XK5wqqqqoqGhAYAdO3aQLsCKo+nXXDN4vesv/pId776Z1ru/R7q3N+/vnqxisRJWnfFZ5s55EwDpdA+PP3EjDz50Masfu4Y1a6/joYdezLPPfYp0uq/I1UqSJEmSND4VJHAKIcSB24DXAGcA14UQzjhm2seA70RRdB5wLfDFzHfPyPy8CrgC+GLmeSdtYIVTezK3odCiRYsA6O7uZv/+/Tl99vGkk0kIAYBUSwsdv/wle269lW3XXkuqtTXv75+sQohz2mmfZP78GzJ3UiSThwbHU+lOGhu/ytObPlacAiVJkiRJGucKtcLpIuC5KIq2RFHUC3wb+KNj5kTAwNFh04CBztt/BHw7iqKeKIq2As9lnnfSaodsqUtH0VgedZRC9nFKdRxm1/s/AMepv+epjez9h3/I6/snuxBiLDv1w8Ri5cPO2bPnLg4f3lLAqiRJkiRJmhgKFTjNA3YM+Xln5t5Qfwu8NYSwE/gJ8N4T+O4JmVbaHzhFQEcqd6ucCtnHqe0n95Buaxth/KckDx7Maw2TXXv7k6TT3SPOaW65v0DVSJIkSZI0cYynpuHXAV+Pomg+cCXwjRBC1vWFEN4VQlgdQljd1NQ04tya+JHH5rJx+PTp06mt7V+k1djYSJTD1VPH6t2ydeQJySR9O3aMPEcjSqd7Rp+TGn2OJEmSJElTTaECp13AgiE/z8/cG+qdwHcAoij6HVAONGT5XaIo+nIURRdGUXThzJkzRyxmoIcT5DZwCiEMrnJqb2+nNY99lOLTp48+p64ub++fCmpqVhGLlY08p/bsAlUjSZIkSdLEUajA6VFgeQhhSQghQX8T8B8eM6cRuBwghHA6/YFTU2betSGEshDCEmA58MhYiqkdEjjl8qQ6ONI4HPK7ra72tX842DD8eCrOPZfEggXDjmt0paXTmTv3TSPO2b79S/T1Db+1UZIkSZKkqagggVMURUngPcB9wEb6T6PbEEL4RAjh9ZlpHwJuDCE8AXwLeFvUbwP9K5+eAu4Fbo6iaEwp0dDAqT3HgVOhGocn5s+n4eabjz8Yi3HKRz+at3dPJcuXfYRTZr32efcHDkpsbX2Yx9a8me7uPYUuTZIkSZKkcaukUC+Kougn9DcDH3rv40OunwJeMsx3Pwl8Mle15HOF08yZMykvL6e7uzvvjcNnvudmEosW0vIf/0HPUxuPDKTThPh4as81ccViZZx55j+zuOPdNDffTzrdx/TpF1JWNpcnnngHXd2NHD78DKsfu4Zzz/ka1dUri12yJEmSJElFNyVTiWl5DJxisdjgKqeWlhY6Ojpy+vxjTXvd61h6992ctu4J5n7uc4P3m790e17fO9VUV69k8eJ3s3Tp+6mvfwlVVUu44ML/oabmTAB6evby2Jo3c/Dgw6TTvRw48Fv27f8pnZ2jNHeXJEmSJGkSmpKBUz631MHR2+p2FOikuJBIUHvFq0mceioA7T/7GT3PPVeQd09VZYkGzj/vTmbMeCkAyWQ7ax+/ngcevIi1j9/A+vXv4Xe/fyWPP/EOenubi1ytJEmSJEmFM+UDp1yvcILCNQ4/VojFaLjpXYM/N3/5ywV791RVUlLF2Wfdzpw51wAQRSlSqfaj5rS0/Ia1j7+NdLq3GCVKkiRJklRwUzJwGrqlri0PgdOcOXMoKelvj5XPxuHHU3vllZRmTqdr+/E99Bb4/VNRLFbKaSs/RSIxc9g5HR0baWr6eQGrkiRJkiSpeKZk4FST5xVOJSUlzJ8/H4A9e/bQ09OT83cMJ5SUMOPGP+3/IZ2m5StfKdi7p7Jk8gC9vU0jzmlp+XVhipEkSZIkqcimZOBUHgskQgDy08MJjvRxiqKInTt35uUdw5l21VWUzJ4NQOv3f0Df7t0Fff9UFEXpnMyRJEmSJGkymJKBUwhhsI9TPlY4wdGNwwvZxwkglkgw453v7P+hr4+Wf/9aQd8/FSUSM6msXDrKnIYCVSNJkiRJUnFNycAJjvRxykcPJ4AFCxYQMquoCt3HCWD6G68hPmMGAK133UWyaeTtXhqbEAKLF/35iHMad/wHO3b+J1EUFagqSZIkSZKKY8oGTjWDgVN+tjmVlZUxO7OtbefOnSSTyby8Zzix8nJmvP1tAEQ9PbR8/esFff9UNGfO1Sxb9mFisbKj7peUTMtcpXjmmf+Pp5++lXS6cH29JEmSJEkqtCkbOA1d4ZSvFSeLFi0CIJlMsmfPnry8YyTTr72O2LT+sOPgt75N8uDBgtcw1SxaeCOXvOS3nH7a/2P58o9x/nnf5NJLVrNi+d8QQv9/c7v3fIc1a99KT0//qrPOzu20tq6mu2dvMUuXJEmSJClnpmzgNNDDqS+K6ErnJ3Aa2sepGNvq4tVV1N9wPQBRZycHv/GNgtcwFZWW1jF37jUsXPB26upeSCwWY8GCt3HuOf8xuNrp0KE1PPLoa/n9w6/ld79/BY+teTO//e0lPPHEjXR3Fz6clCRJkiQpl6Zw4HTkV8/3SXVQ+MbhA+rf+lZiVVUAHPivb5Jqby9KHYL6+pfwggu/R1XVcgB6e5s5fHjjkBkRzS2/ZM2at9DX11acIiVJkiRJyoEpHDjFB6/zdVJddXU1MzKNuxsbG0mn89MvaiTxadOoe8tbAEi3tXHwzm8VvAYdUVm5iAsvuIvy8nnDzunqbmT37m8XsCpJkiRJknJrygZO04YETvk6qQ6OrHLq7u6mubk5b+8ZSf3b/g+hvByAA1//OunOzqLUoX4lJdUkk10jzmlq+lmBqpEkSZIkKfembOBUU6DAaaBxOBRvW13JjBnUvflNAKQOHmT7DTew+68/zKEf/pB0b29Raprqomjkv/d02n8XSZIkSdLENWUDp0KvcILiNA4fUHvllYPX3es3cOgHP2D3X/01W//4avr27StaXVPVtGnnjTgej1fl7fRESZIkSZLybcoGToXo4QRQV1dHdXU1ULwVTlEUse8fPnXcsd7Nm9n9ob8ocEVauPBPRxxvPfQITz31F6RSbn+UJEmSJE08Bk7kd4VTCGFwW11bWxutra15e9dwutevp+uJJ4Yd71y9mu6nny5gRZpRfwkrV3yCEEqOuh9CKRAA2Lvv+zy6+g10dm4tQoWSJEmSJJ28ktGnTE6F2lIH/dvqNmzYAPSvcpo+fXpe33esbMKk7qefpvy00wpQjQbMn/8nzJz1avbt+xG9PfupqFjIKae8lvb2Dazf8H56e5s5fPgZHnn0Ks44/dNMm3Y+e/beTVfnNhJlM5l9ylVUVS0t9q8hSZIkSdLzTNnAqVBb6uDoxuGNjY2cc845eX3fseI1NaPPyWz7U2GVJRpYuODtR92rq3shF73gRzy5/r0cOrSaVKqDJ9e/m/4FienBedu23caSJR9g6ZL3FrZoSZIkSZJG4ZY68r/CadasWZSVlQGwadMmnnnmGVKp/L5zqKpLLiVWVTXseGxaLVUveUnB6tHoyspmcf55/8XCBe8ccjf9vHlbt36efft/WrjCJEmSJEnKwpQNnKriscFfPt+B0/bt2wcDpo6ODu68804+//nPs2XLlry+d0C8uopZfzl8Y/C6t/wJsYqKgtSi7MVipSxffiv19ZeOOG/Hjv8oUEWSJEmSJGVnygZOsRAGVznlM3Bqbm7mzjvvJJlMHnW/vb2dO++8k/379+ft3UPVXXst8z73T5QtX/68sc6HHyaKooLUoROX7Ds04nh7+5MFqkSSJEmSpOxM2cAJjmyry2cPp9/97nf09fUddyyZTPK73/0ub+8+Vu2VV7Lkhz9g+YMPsOyB31Bx7rkAdK1ZQ9uPflSwOnRi4vHKEcdjsbICVSJJkiRJUnYMnID25PN74+TK1q0jH2k/2niuhRAomTmT0lmzOOVjH4MQANj/mc+S6jhc0FqUnVmzrhxxPJk8zLZtXyKKCtcXTJIkSZKkkRg4kd8VTiET6IxHFWeuYvo11wCQbGqi5Uv/VuSKdDxz5lxNdfXpI8xIs3nLZ3hszXV0dTUWrC5JkiRJkoYzpQOnaZnAqSudpi+dnx5Gp5566pjG823mBz9ArLYWgJY7/pOeAq+40uji8QrOP++/mDPnGmIhkbkbmDHjFSxe9O7BLXWHDj3Gw4/8Ibt2fZuWlt/yxLqb+O1vL+X3D7+GrVu/QN8ovaAkSZIkScqVMBmbRV944YXR6tWrR533vo3b+c7egwBseMmZzEiU5LyWAwcOcPvtt9PT0/O8sUQiwU033cSMGTNy/t4TceAb/8W+T34SgKrLLmXB7beP65VZU1ky2U5Pzz5KS+tJJOoB6Dj8LE899SHa2zeM+N3KyiWcf/63KUs0FKJUSZIkSdIkE0J4LIqiC7OZ6wqnjHydVFdfX8/1119/3FDp9a9/fdHDJoC6664dPL3u8AMP0vHrXxe3IA2rpKSGqqplg2ETQHXVci684C4WL76Zkf4n3dm5leee+8cCVClJkiRJmuqmdOBUOyRwymcfp/nz53PzzTfztre9jfPPP3/w/oEDB/L2zhMRSkr6G4hn7PvUP5I+zoosjV+xWIJTl97C7NlXjThv/757SCZtDi9JkiRJyi8Dp4x8rXAaEIvFWLx4Ma961auIxfr/2jdsGHkLVCFVXXwRNa+5AoC+xkYOfP2OIlekkzPyiYvpqJe+vpYC1SJJkiRJmqoMnDLyHTgNqKioYNmyZQDs27ePpqamgrw3G6f85V8SyssBaP7Sl+jbu7fIFelElZfNGWVGAEoLUYokSZIkaQqb0oFTIXo4Hc+qVasGr9evX1+w946mdO5cGm56FwBRVxf7P/PZIlekEzVnzhvoD5WGE7H6satpar6/UCVJkiRJkqagKR04FaqH07FWrlxJPN7/7g0bNjCeTgqsf8c7KJ0/H4C2e+6h8U9vZNctt3DgjjtItbUVuTqNprJyCcuX3XrcsRD6/5vr7d3PunXvYsNTH6KvrxWAtrZ1bG/8Kjt2fJ3Ozm2FKleSJEmSNEmVFLuAYirGljqA8vJyVqxYwcaNG2lubmbfvn3Mnj27YO8fSaysjJkf+AC7/+IvADj80EMAtP3kpzR/6XYWfPnLVJx1ZjFL1CgWLnwH1dWnsWPnHXS0P0W8pJpTZl3J7Nl/zNZtX2DPnrsA2Lv3+xxoeZDSRAOHD2868oBn/445c67htJV/Tyzm9jtJkiRJ0omb0oFTsbbUQf+2uo0bNwL9q5zGS+AE0PHgg8e9nzp4kJ0338ypv/g5sUSiwFXpRNTXv5j6+hc/7/4Zp/8/Zs16DU8//VF6evbS29dC73GaiO/Zcxcl8WpWrPibQpQrSZIkSZpk3FKXUcgtdQArVqygtLR/9cj69evHzba65MGDtN1zz/Dj+/fT/rOfF7Ai5VrDjJfxwovvpWHGK0act2v3twa33EmSJEmSdCKmdOBUEz8SOLWnChs4JRIJVqxYAcDBgwfZvXt3Qd8/nN4tWyCZHHFOz6anC1SN8qWkpIbp0y8ccU463UNb27oCVSRJkiRJmkymdOBUEgtUxfv/Cg71FTZwAjjzzCO9kDZs2FDw9x9PrKYmizm1BahE+RZCNv2ZRjrxTpIkSZKk45vSgRMc6eNU6B5OAMuWLSOR6YU0Xk6rK1u+nLLly0acU/uaKwpUjfJpxoyXjjpn85bP0Na+vgDVSJIkSZImkykfONUMBE4F3lIHUFpaymmnnQbAoUOH2LlzZ8FrOFYIgVNuvRVKjt9PvnT+PErnzy9wVcqHqqpTmX3KVSPOaW/fwKOP/jHPPPtJksnDAHR0bGLb9tvZuu02WltXj4ugVJIkSZI0vkzpU+qguCucoH9b3bp1/X1y1q9fz4IFC4pSx1BVL3oRi+74Ok3//C90PvJI/82SEkgm6du5i4Pf+hb1b3lLcYtUTpx++j8QL6lm9+7/Jor6gP7+TnPnvIm2tnW0HnoUSLNjx9fYv/8nlJfP49Chx456xvTpF3HWmbeRSNQX4TeQJEmSJI1HYTKuTrjwwguj1atXZzX3reu28IuWNgKw62XnEAuF7VmTTCb57Gc/S3d3N9XV1dxyyy3EYuNn4VmqrY2op4fevXvZft1bIJkklJez5Ht3U7ZkSbHLU4709h6gre0JQqyU6dPOJx6vJIoi9uz5Ls8+9ymSyZFPq6urexHnn/dfBapWkiRJklQMIYTHoiga+QSqjPGTbBRJbWaFUwR0pNIFf39JSQmnn346AB0dHTQ2Nha8hpHEa2spmTmTyrPOouHP/wyAqLub3X/9YaJRTrPTxJFI1NPQ8HJm1F9CPF4J9G+vnDv3Gl70wp8xa9aVI37/4MHfeaKdJEmSJGmQgVMmcAI4VMRtdQPWrx+/DZobbrqJ8rPPBqB73Tqab7+9yBWpEBKJGf8/e/cdH2lV9n/8c0/JpM2kZ9OzPcv23qUqvYvSBFEEFbCAqPj8LNifx0dB8REElSIiIB2lKbC0ZVu2ZftmS7LJbja9zSSZTLl/f2Q3m5BksrskMynf9+vly9z3uebMdWfZMlfOuQ4ZGaF7PQE0NBzfqkIREREREREZ+UZ9wSmhS8EpUn2cxo4dS2xsx6qS7du3E4hAA/PjYdhsZP3Pf2NERwNQc/8DtG4ZugUyGTgWo/92bz5/cxgyERERERERkeFg1BecnNZj34JGX2QKPVarlalTpwLQ0tJCSUlJRPI4Ho5x40j/zrc7LgIBDn33uwRbWyOblAy6xMQF2GzOkDGlpQ+wZ++vO0+zO8o0TYJB32CmJyIiIiIiIkPMqC84JdiPrXBqjuDKomnTpnV+PZS31QEkXX01ccuXA9C+bx9Vv7knwhnJYLNaYxk79taQMabpo7T0AVat/iQVFS/g8exj27Zv8c6701jxzhTWrDmfQ4eeZSQeVCAiIiIiIiLdjfqC01Do4QSQn59PfHw8ADt27MA/hBtyG4ZB5s9/jjUhAYD6v/0N98qVEc5KBlte7peYNPH/Ybcndd6zWmLJy/0SEyd8r3MFVHt7Fdt33MnqNedyuPJFgkEvAG7PLnbs/C579vwyIvmLiIiIiIhI+PTfmGWEc1kj38MJwGKxMG3aNNasWUNbWxv79u1j8uTJEcunP/Yx6WTc/SMO3n4HAIe+810SPn05eL04Jk/Gdd55WI70pZKRwTAM8vK+SHb2tTQ1bcY0/bhcMzoLTZmZl7J3370cOvQ0Hec+9v776UDZX8jM/DTx8QXhS15ERERERETCatSvcBoKTcOP6rqtbtu2bRHM5Pi4zjsP10UXAhCoraXuoT9R99hfqfh/32fPmWfRsn59hDOUwWC1OkhKWkhy8tJufZ2iolI5ZcrPmTP7r/3OcfjwS4OZooiIiIiIiETYqC84uexDY0sdQE5ODi6XC4CdO3fi8w39RsvRU6f1ej/Q0EDZV76Kv64uzBlJpDkcGf3GtLfXhiETERERERERiZRRX3AaSiucjm6rA/B6vezZsyei+fTHNE0annqqz/FgczMNzz4XxoxkKIiOzsBqCb2dsqb2baqqXlcDcRERERERkRFq1BecnEOkh9NR06dP7/z69ddf57XXXqO8vDyCGfUt2NREe2lpyJjWos1hykaGCqs1lozMy0LG+Hx1bNl6K4Xrr6C+fi0+XyN79vyKDz5YytsrJrNq9TmUlT1KMDh0m+eLiIiIiIhI30Z90/BoqwWHxcAbNIdEwampqanz68bGRtasWcOaNWuYO3cuF154IRbL0KkRGlFRYBgQYpWKxREdxoxkqJg44Tu43TtobNzwkRE7Ca4ZNDZ13G9q2sSGjVdjtcYRCHg6o1pa9rC7+Kc0NK5n+rTfYRhD5797ERERERER6Z8+xQGuI9vqIt3DqaGhgWeffbbXsQ0bNrBu3bowZxSaJSaGuE8sDxnj/NSnwpSNDCU2Wzxz5zzBtKn3kJr6SZKSlpCXdxNLl/yb+fOfYf6850hMXNQZ37XY1FVV1avU1LwZrrRFRERERERkgKjgBLiObKtrjnDBaf369QQCfeewevXqIdfzJu1rX8dwOHoftFhwTJwQ3oRkyLBYosjIuIRZMx9k7py/MWniXcTE5AGQkDCbuXOeYNbMP2MY1pDzVBx+IRzpioiIiIiIyABSwYmhs8Lp8OHDIcfr6+tpb28PUzbHJ2bGdPIeeYToGTOO3Ty67S8Y5ODtdxBsaYlMcjKkGYZBSsrpmGbo33fetoowZSQiIiIiIiIDRQUnjp1U1+QPRHQFkaOvlUJHWCwWbLah13Yrdu4cxj3zDya88Tpjn/kHk95/j5i5cwHwFhdTcffdQ25llgwNhmEQEzM2ZEyzeyf79v8ev7+52/22tkPU16+lpaVk8BIUERERERGRk6KCE+A8UnDym9ASDEYsj2nTpoUcnzp1KlZr6O1HkRSVn0/MjBnYUlLIvvderCkDmhWCAAAgAElEQVQpADS9/E8ann46wtnJUJWTfU3IcdP0sX//b1n54emUlj6I272HjZtuYOWHn2DDxqtZtfosCtdfidu9K0wZi4iIiIiISH9UcOLYCieAZn/kCk4FBQWMHz++1zG73c7pp58e3oQ+BvuYdLLvuadze13lz39Ba1FRhLOSoSgn53rSUntvLp+YuBCrNRYAv7+BPXt/xZq151FX9363uMbGQtZvuIbW1vJBz1dERERERET6p4ITx3o4QWT7OFksFq6++mqWLl1KdHR0tzGXy0XKkRVDw0XcooWk33E7AKbPR/k3vom/vj7CWclQY7HYmTHjD0yf/ntSU8/C5ZpFxphLmTf3aebNfZKlS94hL/dGLJajW057Lwr7/Q2UHvhT+BIXERERERGRPhkjsbfO/PnzzcLCwuOO/23JYf57f0fD7n/OncSChLjBSu24+f1+mpubeeWVV9izZw8A119/fZ8roIYq0zQp/9rXcL/5FgBxy5aR+9CDGEN4a6AMTV5vJWvWXoTPV9tnjMORwfJlK8OYlYiIiIiIyOhhGMZ60zTnH0+sVjjRfYVTU4RPqjvKZrORlJTEsmXLOu+tWrUqghmdHMMwyPrlL7Hn5wHgWbmSmj/cT6C5mfaSEgJuT4QzlOHC4RhDVFToVX4+XyOBQFuYMhIREREREZG+qOBE9x5OQ6XgdNTYsWPJyMgAoLi4mOrq6ghndOKsTic5992HcWSbYM3997N70WL2nnsexUuXcuiu7+Gvq4twljIcJCTMCTkeDLby4arTKD3wZwKBFqDjlLudO79PYeEVbNr8RSoqXiAY9IUjXRERERERkVFLBSeOnVIHke3h1BvDMFiyZEnn9erVqyOYzcmLLigg/dvfPnbjyGmAZns7jS++SOl11xNwuyOUnQwXuTk3YBi2kDHt7TXs2fNLVn54GkVbbmXt2gs5eOhJGps2Ulv7Ltt33MnGTZ8nEGgNU9YiIiIiIiKjjwpOfPSUuqFVcAKYNm0aTqcTgM2bN+PxDM9taP7Kyj7H2vfupeGpp8KYjQxH8fGTmTH9/7Bau/dZs1gcTJzwXXKyr8MwogDw+eqorn4d6NmnrqFhDfv2/y4cKYuIiIiIiIxKKjgxdE6p64vNZmPhwoVARzPxE2mIPpQ0vfF66PHXQo+LAKSlfYrly1YypeDn5Od/lcmTf8SypR+Qn38zBQV3s2zpO+TmfgHDCN2Y/tChfxAM+sOUtYiIiIiIyOiighNDu4fTUfPmzcNutwOwdu1a/P7h90E52E+D8OAwXbkl4WezOcnOvoqJE+4kN+d6oqKSO8ccjjFMnvR9kpM/EXIOv78Rn69+sFMVEREREREZlVRwYmieUvdRsbGxzJ49GwCPx8OWLVsinNGJi546tZ/xU8KUiYwG0dHZ/cbs2n03Tc1bO68DAS8Vh19k564fsrv4Z9TVr8I0e27JExERERERkdBUcALirJbOb8RQ3FJ31KJFizq/Xr169bD7IJx8/XUhx61p6WHKREaDzIxL+42prn6ddesuYePG6zlU8SyrVp/N9u3f4uDBJygre4SNGz/Hps1fUINxERERERGRE6SCEx0nwR3dVjdUVzgBpKamUlBQAEBlZSX79++PcEYnJv4TnyD923eCYfQ6Xv/YYzS+/HKYs5KRKiFhLtnZn+t1zGZzER2d03ldV7+SHTu+i9db3iO2ru59dhf/dNDyFBERERERGYlCny8+ijhtVur9gSFdcAJYvHgxu3btAmDVqlWMHz8+whmdmJQbb8R51lk0vPgi/sOV2LOzwWql5r77ADj0vf/CEh+P88wzI5ypjAQFk+/G5ZxOWflfcbt3YrO5GDPmQsaNvYWoqFSqq9+k9MCDNDVtDjlPRcXzTJzwbez2pDBlLiIiIiIiMryp4HTEcFjhBDB27FgyMjI4fPgwxcXFVFdXk5aWFum0TkjU2LGkf/Ob3e4ZFgvVv/0tBAIc/Obt5D70EHGLF/Uxg8jxMQyDrKzPkJX1GUzTxPjI6rr09HNISzubbdvvpLLyxT7nMU0fbvcukpIWD3bKIiIiIiIiI4K21B3hGiYFJ8MwWLJkSef16tWrI5jNwEn58s0kf+ELAJjt7ZTfcgutW7b28yqR4/fRYlPX+/Hxk/t9fUnJAzQ2buq8DgRaOXjwKbZsuY2iLbdQVv44fr97wPIVEREREREZzlRwOuJowak1aNIeDEY4m9CmTZuG0+kEYPPmzXg8nghn9PEZhkH6d75NwhWfBiDY0kLZTTfh3bOHgNuDt7gYf01NhLOUkSo97ex+Y+rqP6Bw/adZV3gFZeWPs3rN+ezc9f+oqn6N6uo32L37blavOZeWluHVW01ERERERGQwqOB0xNGCE0CTf2gXnGw2GwsXLgTA7/dTWFgY4YwGhmEYZP74xzjP7vjwH2hoYP9nPsvupUvZd9HFFC//BAdu/BLePXsinKmMNLGx48jJ6fsUxaioYycoNjVtZPfuu2lrO9AjzuutYMvW24bdCZIiIiIiIiIDTQWnIxK6FZyG9rY6gHnz5mG32wFYu3Ytfr8/whkNDMNqJevX/0vs0o5tg2ZrK7S3d457Vq6k5JpraS8piVCGMlJNnvRDJoy/E7s9pfNeXNxkZs58iOXLVjJr5p9ISlra7zxu904aGkdGEVhERERERORkqeB0hNN27FvROAwKTrGxscyePRsAj8fD008/zbvvvktlZWWEM/v4LFFROM8+p8/xYFMTNQ/8MYwZyWhgGBbGjv0qy5d9wOJFb7B0yQoWLXyVtNSzMAwLqalnMnfO40wY/+1+53I3b+/1vt/vpr29RiugRERERERkxFPB6YiuK5yah0HBCaCgoKDz6+LiYlasWMEDDzzACy+8QCAwPJ6hL54VK0KON73xhj60y6CwWKKIi5tITExer43G451T+p1j77572Lv3N7S1HQKgqamIjZtu4N33ZvH+B4v4cNXplJU9imkO7e27IiIiIiIiJ8sW6QSGiq49nIbDCqdAIMBrr73W69jmzZtxOp188pOfDHNWAyfY0hJy3Gxrg2AQrNaQcSIDLTlpCXZ7Cj5fbZ8xgYCbktL7KSn9I4kJc2ls2oxp+jrH29rK2V38U1paD1Aw+YfhSFtERERERCSstMLpiOHWw2nXrl3U1vb9gXfdunW0d+l9NNxET58ecjxq8iQMFZskAiwWx5EiUc/VTwCJiYuxWuOPXAVpaCzsVmzqqrz8Mdye4sFJVEREREREJIJUcDrCOcwKTuXl5SHHvV4v1dXVYcpm4CVdfRVGVFSf44G6enwVFWHMSOSYMWMuZPash0lImNd5z+WcycwZDzBv7hMsX7aSgoKfEhs7vt+5qipfGcxURUREREREIkJb6o4YbiucbLb+f+mOJ2aoisrLI/u393Lwjm91bJ/7iEBNDaWfu468Rx8hKjc3AhnKaJeSciopKafi93uAIDabs3PMZosnJ/saXM6ZrCu8JOQ8dfWryfU1YrcndN5zu3dzuPJlfL564mInkJl5GXZ70mA9ioiIiIiIyIAbvhWJATbcejhNmTKF9957r8/x5ORk0tLSwpjRwHOeeSYTV7xN40sv0b5vP9bkJOLPOIPKn/6Mtq1b8R082FF0euQRHOPHRTpdGaVstrg+x2Jj87FYogkGexZNj2psXMcHK5cwJv0CsrKuoqr6DcrK/tItZt++e5k+/T5SU88YsLxFREREREQGk7bUHeEaZiucsrKymDZtWp/jp556KhbL8P/ltSUlkXLDDWT+5Mekf/ObxM6aRd4jDxMzdy4A/spKSq+7jrZduyOcqUhPNpuTzMxP9xsXDHqpOPw86zd8tkexCSAQbGHL1ltpbQ29lVZERERERGSoGP4ViQHitA6vghPAZZddxuLFi7Hb7T3GampqIpBReFidTvL+/CdiFy8GIFBby4Hrr6d161ZaCgupfeRR6p/+B/5h3MNKRo5JE+8iMXFRj/s2q5OZM/7I+PF3EB2d3e88waCXg4eeHIwURUREREREBpxhmmakcxhw8+fPNwsLC0/4dRPfK8IdCLI4IY4X504ahMwGR1tbG1VVVfj9fp5++mm8Xi9Wq5VbbrmFlJSUSKc3aIJtbZR/4xt43j2ytdBigWDwWIDNRurNN5H6ta9hGL2fKCYSDqYZoKZmBVXVrxMItJLgmkVm5hVERSV3GX+boi1fCTlPYsIC5s176iNzmzQ2rqet7RCO6EwSE+ZhGPpZgoiIiIiIDDzDMNabpjn/uGJVcDpm7ofbOOT1cUpcNCsWThmEzAbf6tWref311wGYPHky11xzTYQzGlxmezsHv/Utmv/zZp8xY/7reyRff30YsxI5caYZYMU70zHN9hBRBikpp5OV+RlSU8/A7d7Jtu130tKytzMiNnYcU0/5XxIS5gx+0iIiIiIiMqqcSMFJPwbv4mgfp+Gypa43CxYs6GwWvnv3boqLiyOc0eAyoqJwnnteyJjaP/8F0+8PU0YiJ8cwrKSnnd1PlElt7Qq2bL2F9z9YTOH6z3YrNgG0tOxn46bP09JSMmi5ioiIiIiI9EcFpy4SRkDByWq1cu6553Zev/766/hHeLGldf36kOP+qiraDxwIUzYiJ2/cuK9jszp7HYuOziE+/pTOa7+/EdP09RobCHg40EvzcRERERERkXBRwakL55GCU3MgSGAYbzWcMGECU6Z0bAmsra1l7dq1Ec5okHU5YbAvxgg4sU9Gvri4Ccyb9zTJScs671ks0WRnXc2iha+waOG/WLjgZXJyrqe/P75rat7tcc80TZqaijhU8SzVNW8RCHgH+hFEREREREQAsEU6gaEkoUvhwu0PkGAfvt+ec845h+LiYgKBAO+88w4zZszA6ex95cRwF/+JU6n/6+N9jlvi4rBlZoYxI5GTFx9fwJw5f8XrrcbnbyDakYXNFtc57nROo8A5jZqad2hr63vlntd7iO3bv82YjEtITlpCa2s527Z9k6bmos4Yuz2JyZN+SEbGxYP6TCIiIiIiMvpo2UcXri4Fp8ZhvK0OICkpiWXLOlZJtLe389Zbb0U4o8ETt2wpMfPn9Tke9Hgov+VWAm53GLMS+XgcjjTi4yZ1KzZ1lZy8tJ8ZTCoOP8+mTZ/ng5VLWbvuom7FJgCfr55t2++gtvb9AcpaRERERESkgwpOXXQtOA3nPk5HLV++HJfLBcCmTZsoLy+PcEaDw7BYyL3/fpznnguG0Xnf4nRiREcD4Fm5ktLPXYevsjJSaYoMqLzcG7FYovsYtRIVldZ51d5eQyDg6SPWpKTkDwOen4iIiIiIjG4qOHXRveAUjGAmAyMqKoqzzz526tVrr71GMDj8n6s3VpeLnN/ey8Q3/0PO//2evIf/wqSVHzDumX9gy+rYTufduZOSK6+ibdfuCGcr8vHFxY1n1qw/44ga0+1+VFQ6s2c/zPJlHzJ3zt/JyroSwwi9PbihcR2BQFu3e8Ggn+rq/7Bn76/Zv//3uN27BvwZRERERERk5DLMYdwcuy/z5883CwsLT/h1fztUy527ygB4dPo4zk1LGOjUws40TR599FFKS0sBuOSSS5gzZ06EswovX1UVZV/5Ct7tOwCwxMeT9atf4TtcQcuatWAxiP/EqbguOB+LwxHhbEVOTDDoo67ufVrbDhLtyCQl5VQslqhuMZs230Rt7dsh50lImEfGmItJTz8Xv9/D5qIv0dKyr1tMZsblTJnyCywW+4A/h4iIiIiIDH2GYaw3TXP+ccWq4HTMy1UN3LytBIDfTcnjyszkAc4sMg4fPsyDDz6IaZrExMSwbNky7HY748ePJy0trf8JRoCgx0P57bfjea/vXjVREyaQ9/DD2MekhzEzkcFXVvYYu4t/clyxBlYMSxTBYGuv4/n5X2XihDsHMj0RERERERkmTqTgFLYtdYZhnGsYxi7DMPYYhnFXL+P3Goax6cj/dhuG0dBlLNBl7OXBytFlO/btGAk9nI7KyMjoXNXU2trKm2++yWuvvcYf/vAHnnnmGXw+X4QzHHyWuDhy77+fhM9c0WdM+969HLrru2HMSiQ8MjMvx+Ho+6TGuLjJnV+bBPosNgGUlz9OINAyoPmJiIiIiMjIE7qxxwAxDMMK/AH4FFAOrDMM42XTNLcfjTFN8/Yu8V8Duu77ajVNc/Zg5znSmoZ31dLS+wfEbdu2YbPZuOyyy8KcUfgZNhuu88+n8Zln+4xpWbUa7969OCZMCGNmIoPLZnMyd87jbN36DZrd27rcT2Dy5B+SmXEpbW0VVFW9SumBP9HeXt3nXIGAm8bGzSQnL+m819p6gANlj1Jf/yEAycnLyc25gZiYnMF7KBERERERGdLCUnACFgJ7TNPcB2AYxlPAJcD2PuKvBn4Uptw6JYzQglNdXR07d+7sc7yoqIgzzjiDxMTEMGYVGe3Fxf3GeIuLVXCSESc2dhwLFrxEU9NG3J5i7PZEUpJPw2rtOOkuOjqTvLwb8fkb+z21btPmG0hJPpX09HOJikpny9Zbu52C5/EUU3HoWWbPeYwE16xBfS4RERERERmawrWlLhso63JdfuReD4Zh5APjgK4dbqMNwyg0DGO1YRiX9vG6m4/EFFZX9/3T+VC6rnBqHEEFp6MNw/timiYHDhwIUzaRZU3ovxG81eUKQyYi4WcYBgkJc8nOupL0tHM6i01dpaV+st95TNNPTe3bbN/xHTZtvqFbsekof6CZ7du/hWmOzJMxRUREREQktLD1cDoBVwHPmqbZteKTf6Qp1TXAbw3D6LH8xDTNh0zTnG+a5vyTbYTttB4rODUHRk7ByWLp/5fZMIwwZBJ58WecgRHT80N2V55VqzFH0K+/yIlwuWaSnn5+n+Pp6RcSH3/Kcc3V0rKfhoa1Pe4HAl7q6lZSU7MCr/fkfkAgIiIiIiJDW7i21B0Ecrtc5xy515urgFu73jBN8+CR/99nGMY7dPR32jvQSUZbLURbDNqCJo2+kVNwGDduHBaLhWCw95UGFouF8ePHhzmryLC6XKTfeSeVP/1ZnzG1f/oTrVu2kP3r/8WWmhrG7ESGhmlTf43DMYaDB5/qbCAeHZ3LxAnfZsyYC4COYtKePb+iuubfIecqP/h3HI5MYmPzO67Ln2Df/nvx+eoBMAwbGRmXUjD5bqzWmEF8KhERERERCadwFZzWAZMMwxhHR6HpKjpWK3VjGMYUIAlY1eVeEtBimqbXMIxUYBnwq8FK1Gmz0tbuH1E9nFwuFwsWLGDNmjW9jufk5BAXFxfmrCIn+dprsaWkUPPgQ3h37AAgZs4cYubNo/6JJzBbW2lZvZr9l11O9r33EDt/PqZp4q+qBr8PW0YGRpfVcCIjjcXiYPKk7zN+3Ddwe3ZjtUQTH38KhnFstWRs7Diysj7bb8GpquoVqqpeIT5+Cg5HFrW1b3cbN00/FRXP4vPVM2vmQ4PyPCIiIiIiEn5hKTiZpuk3DOM24A3ACjxsmuY2wzB+AhSapvnykdCrgKdM0zS7vPwU4EHDMIJ0bAH8766n2w20BJuV6nY/TSNsS9XZZ5+N1Wpl7dq1+P3+bmPl5eUcOnSIrKysCGUXfq5zz8V17rkEmprAYsEaHw9A4qWXUP71b9C+bx/+6mpKP38DCZddStu27Z3FKVtWJilfvJGka68ZNVsRZXSy2ZwkJszrczw5eTkORwZe7+F+53K7d+J29314QU3NWzQ1FeFyzTypXEVEREREZGgxutd2Rob58+ebhYWFJ/Xa89fvZkNTC8l2K9uXzxjgzCKvpaWFsrKO/u1VVVW89dZbAKSmpnLzzTcTFRUVyfSGhKDHQ8UPfkjTq6+GjEu5+WbS77g9TFmJDE11dSvZXHQTwaC3232LJZppU+/F72+kuvrf1Na9h2n6+5ilw9j8W5gw4Vud18FgO4cPv8jhyn/i9zUSFz+JnOzrSEiYPSjPIiIiIiIioRmGsf5Ij+3+Y1Vw6u6qTXt5p74ZqwHlp80a0StYgsEgTzzxBHv3drTDmj9/PhdeeGGEsxoaTNOk/m9PUPnzn/cdZLEw8a03sWdmhi8xkSHI7d5F6YE/UV/3IRgGyUnLyMv7EvHxkztj6uo+ZOOm60LOYxh20tLOJi3tUyQlLmTr1m/Q0LiuR9zkST8kN/fzA/4cIiIiIiIS2okUnMLVw2nYcNk7evMETGgJBokbwb16LBYLl156Kffffz+tra0UFhYyadIkCgoKIp1axBmGQczMfla4BYM0//vfJH9eH3xldIuPL2Da1F+HjElImIPN5sTvb+4zxjR9nT2fOnZQ937Qwe7in5KcvIy4uIkfI2sRERERERlMlv5DRpcE27EC00hqHN4Xp9PJJZdc0nn90ksv4Xa7I5jR0BE4ju9DwOMJQyYiw5/VGkNuzg19jtvtKURHdz3MtPdiUweTg4ee7nG3sWkzO3f9gM2bb2LX7rtpbh60dn8iIiIiItIPFZw+wtWl4NQ4CgpOAFOmTGHu3LlAR4+nl156iZG41fJEOSZNgn5WuFlidIy7yPEaN+5r5ORcB3TfquxyzWHRwn+xdMkKFi18lfz8r/Q7V2XlKxw69CxebyWmabJnz68oLLycgwf/Tk3t25SXP87adRdRUnL/ID2NiIiIiIiEoi11H+HqUmBo8o2OghPAueeeS0lJCXV1dRQXF7Nu3ToWLlwY6bQiyp6ejuu882j617/6jKn6zT1gQvINn8ewqH4rEophWCmYfDd5uV+ipvZtgoE2EhLmkpAwr7NfXnx8AeNjv0lZ2aMEg219ztXeXsmOnd8FwOHIwus91Gvc3n2/weWaTXLy0oF/IBERERER6ZM+IXfh8QfY6m7pvL7vQBXFnr4/8IwkUVFRfPrTn8ZypGjy73//m6qqqghnFXkZP/ohMfN6HgtvOBwdX/j9VP3qV5R99av46+vDnJ3I8BQTk0NuzvXk599MYuL8HoczWCx2MsZcfNzz9VVsOqr84OM97gUCbRw+/BJ7991DWdmjeL3Vx/1+IiIiIiLSP51Sd0RNu5/LN+5hd0v3ApPdMHhwWj7npyUOZIpD1nvvvcfbb78NQEJCAtnZ2QSDQXJycpgzZw5xcXERzjD8zEAA9/vv417xDqbPR+y8uTjPO4/GZ56h8te/AZ8PAFt6Ohk/+THeHTtoevU1As3NRE+ZQtJ1nyN+2bIIP4XI8OL1VrN+/WdpbTvQYyw7+3Pk5d5Abe171Na9T23tipBz2Wwupk27l6TExVit0dTXr2XL1lvx+eo6YwzDxsSJd5GX+4UBfxYRERERkZHiRE6pU8HpiK9uK+GFqoZex2IsBuuXTiPZPvJ3IAaDQR5++GHKy8t7jEVHR3PttdeSm5vbyytHp9YtWzn4rW/hO9DzQ3FXad+6g9SbbgpTViIjQ3t7LaUHHqLy8D/x+RuJj5tMTs51ZGRc1m1V1IerPklr6/5+57NYHLhcs2ls3IhptvcaM3PGA6SlnT1gzyAiIiIiMpKcSMFJW+qAOp+ff1b3XmwCaA2aPF85OrZLWSwW4uPjex1ra2vjqaeewndkRY9AzIzpjHv+OVznnx8yrvo39+DduzdMWYmMDFFRKUya+D2WL/+QM07fxoIFL5CZeXmPLXgZGZf0MUN3waCXhoY1fRabAEoP/KmX1/moqn6DPXt/TUnJ/Xg8+07sQURERERERiEVnIDytnb8/Sz02tfiDU8yEebxeNi9e3fI8e3bddR4V9b4eDJ+/jOwhV4B1/jCC2HKSGR0ycu9gbi4Sb2OxcdPZ/asx8jP/yrx8VP7nauxcQMNDesIBv0AeDx7Wb3mbLZsuYXS0gfYu+83rF7zKXbu+gGmOXoOlhAREREROVEjf4/YcUiL6v/bkH4cMSNBTU0NwWAwZIyaifcUbGgAvz9kjO9QRZiyERldbDYn8+Y+yb59v6Pi8PMEAh5sNhdZmZ9h3LivYbM5SUlZzsQJd7JhwzXUN6wJOd/6DVdhtcaTmLiAxsYN+P2NPWIOHvw7DkcG48beOliPJSIiIiIyrI2OKko/Mh1RnJoUz3v17l7HLcDlY5LCm1SExMTEDEjMaGNNSsKIjsZs6/tUQ19lJcH2dixRUWHMTGR0sNuTKCi4m0mTvn+k4BSPYVh7xKWnX9BvwQkgEHD324y8rOxR8vNuwmI59nvaNIPU1r1Hff1qDMNKSvJpJCYu6LENUERERERkpNOWuiN+NimHJFvPDycA/zU+k7wYR5gzioy0tDTGjBkTMmbatGlhymb4sERH47rwgpAxrevXs//Sy/CsWdtjbCQ27xeJBIvFht2e0GuxCSAz8zJiYyf0OmYYDiZN/AE5Odf3GdOVz1fHoYpn8Ps9QMfJeusKL2Pz5hs5cOBPlJb+kQ0br2bjpuvx+5tP/qFERERERIYhnVLXRWmrl9+XVvF8ZR0twY7vy1dy0rh7UvZApziklZSU8PjjjxMI9N6f5Oqrr6agoCDMWQ19/vp6Sj93He29NQe328B3bMtdwiWXkHLLV2l89lkaXnyRQHUN9txcEj/7GVI+/3kMrYISGTRebzU7d32fmpq3gI4/6+Pjp1JQcDeJCfM647bv+C8qKp7udz7DsJPgmk1b2yHavAd7jUlPv4AZ0+8bkPxFRERERCLlRE6pU8GpFxuaPJy/vhiAr+am8aOJo6vgBFBeXs6KFSvYe6R4EhcXh8fT8VN8u93OjTfeSEZGRiRTHJICzc3U//1Jml59lWBzM45TTiH5c9cSlZfH4Z/9HPeKLlt0LBbopV9W3KmfIPf++zH6aUIuIh9PW9shWlpLibInExc3uce2t/r6NWzYeM0AvZvB0iXvEhNz7O+TQMBLZeU/qal9CzPoJyFhHllZnyUqKnmA3lNEREREZGCp4PQxC05tgSAT3y/Cb8KyxHiemzNxALMbXtrb2wkEAkRHR/P666+zZk1H7xOXy8VNN92E0+mMcIbDS/Nbb3H4Zz/HXxG6gXjW//w3CZcc37NmIzYAACAASURBVFHvIjI4TNNkc9FNffZymjDh2wQDbdTVfUBj0yaOrpbqS1bWVeTl3khs7Dh8vlo2brwet2dXtxi7PZHZsx7B5Zo5UI8hIiIiIjJgVHD6mAUngLPW7WSbuw2XzcKu5TPU8BUIBoM8+eSTFBd3rP7Kzs7mhhtuwG63Rziz4SXo8VB82ukE3b03qQeIW7qUvIf/EsasRKQ3gUAbxXt+QUXFswSDXgBiYvKZNPEu0tLO7ow7UPYYxcU/Oa45o6LSMQwLXu/hXscdjgyWLlnRrRn5sXxaCQQ82O3JGIbaMIqIiIhIeJ1IwUn/Wu3DTGcsAE3+ICWt7RHOZmiwWCxcccUVpKenA3Dw4EFefPFFgr1sC5O+WeLiMPv5nvlqa8OUjYiEYrVGM6XgJyxftpp5c59m4YJ/smTxm92KTQBj0s/HMI5vG2x7e1WfxSYAr/cw1TVvdbvn9hRTVPQV3nl3Ju9/sIiVH36CkpI/Egz6+5hFRERERCSyVHDqw9GCE8Dm5pYIZjK0OBwOrrnmGuLi4gDYtm0b7777boSzGn4cY8eGHA9UVdG2a3d4khGRftntLhIT5+N0Tu11ZZHDkUZe3k19vn7c2K9zypT/ISPjUuz2/ns07dt3L6UH/kxj4yaamraxvvAzVNf8B+goVnu9h9m773/ZseM7OuVSRERERIYkbanrw4ZGD+dv6Ng6dktuOj+cmDUQqY0YZWVlPProo50n2V1wwQUYhkFraytjxoxh4sSJWCyqZ/al/h//4PAPfxQ6yGol+XPXknrbbViP9Mpq27Wb9tISbKlpxMyehaHvsciQYZomBw48ROmBP+Pz1QHgcGQybtzXyM66sjOusbGIwvWXncDMFo4Wmnozd+5TJCUu6HYvGPTT0LAGr7eSmNh8ElxztTVcRERERD429XAagIJT65HG4QETlifG8+wobhzely1btvDcc8/1OpaUlMSVV16pk+z6YAaDHLrrLppe/mePMVt6Ov6qqs5ra2oqKV/4Ak3//jdtmzd33o/Kzyfjpz8hbuHCsOQsIscnGPTi9hRjYCUubhIWS/etdqZpsnrN2bS07BuQ98vMvIKpp/xP53Vt3Qfs2HEXXu+xwwni4iYzbeo9OJ2nDMh7ioiIiMjopILTABScAM5cu5PtnjYSbFZ2Lp+unw734umnn2bHjh29jsXFxXHbbbcRExMT5qyGB9M0cb/zDo0vvIi/poao3BwSr7ySmNmzaXjuOap/cw+BhoaQcxgOB2OfforoKVPClLWIDIS6upVs2nwjpunrMTZxwnfJyLiMxsb11NS+Q0XFM/3OFx9XQGLSQqIdWezZew/Qc167PYlFC1/F4UgfiEcQERERkVFIBacBKjh9c8cBnjrcsS1i9eJTGBvj+NhzjjT3338/VV1W43zUOeecw5IlS8KY0cjhr6+n+ne/o+Gpp0PGuc4/n+x7fhOmrERkoDQ2bmT//vuorXsfMHHGTyM//2bGjLmwMyYY9PHByiX4fPUD8p5jx97KhPF3dF6bpkl19b85eOhJWltKiXKkkpFxGVmZV/R6Sp6IiIiIjG4nUnA6viN1RqmZzhieOnKQ0ObmFhWcPqKtrS1ksQmgtLRUBaeTZEtKIvPuu2ndtBnvzp19xrnVtF1kWEpImMPs2Y8QDHoxzQBWa2yPGIvFTnb2NZSU/KHXOQzDRkLCfJqbtxAIePp9z4qK50hJPhWncwYWSxS7d/+Y8oOPd463th2gsXED1VWvM2vWn7BY9PeeiIiIiJwcFZxCmNXlpLqi5lYuSU+KYDZDz/FsMbRarWHIZGSzOEJ/4Av6fJimqS2fIsNUf0WdcWNvw+3eRU3Nmx95XTQzpv8fqalnEAz6cbu3s37DVQSD3j7n8noPs37DlRiGnZiY3D77SNXVr6Ss7FHy87/c7b5pmjQ0rKHZvQObNY7U1LOIiko5zicVERERkdFEBacQpsbHYDUgYEJRc0uk0xlyHA4H+fn5lJaW9hkzbty4MGY0MsUuXEhrl2bhPbS3U/alm0j/zneILpjceds0Tcy2Ngy7HcOm3+oiw5XFEsXMGQ9QV7eSyqp/4fc344w/haysz+JwjDkSY8PlmklKyhlUV7/e75ym6eu3aXn5wae6FZxaWw9QtOUW3O5jffsMw874cd8gP/8rKnqLiIiISDfq4dSPM9buZIenjUSblR1qHN5DSUkJjz32GH39d5SVlcV1112nxuEfg6+ign0XXUzQ7Q4daLGQeMUVpN52K81vvkn9Xx+nvaQEw27HefbZpN52Kw4VAEVGtKamIgrXfwbT9PcYs1rjmTTxe7S0ltDYuIHGxg1A6H8DOJ3Tcblm43ROZ//+33U7+a6rU6b8kqyszw7EI4iIiIjIEKam4QNYcPrGjgM8faRx+JrFp5CvPk497Nq1i9dee42GLieq2e12fL6OU5IyMzO57rrriI3t2Z9Ejk/Lxo0cvP0O/IcPd96zxMeT9LnP4X73XbxdTwq02cDf88OmxeVi7JN/xzFhQjhSFpEIqal5m+077sLnq+28FxOTx/Rpv8Plmtl5b/Pmm6mpfWtA3jMmJo8li9/CMCyd9+rqV3Gg9CEaGgsxDBupKWeQn/8V4uMnh5hJRERERIYyFZwGsOD0l/Jq/l/xQQAemjaWi9MTB2TekSYYDFJeXk5rayvp6elYLBYee+wx6uo6inUZGRlcf/31Kjp9DKbPh/v992kvKcWWlorzzDOxxMVhBgI0vvQy1ffei7+6OuQc8aefTu4fHwhTxiISKcFgO7W17+FtryI2Jp+kpCXdikEA1TVvUVR0c59zREfn4PPVEQgc35byCRO+Q2rK6cTFTeLw4RfZvuM7fHQFlcUSw5w5j5GYMO+En0lEREREIk8FpwEsOBU2erhwQzEAt+Wl8/0JWQMy72jQ1NTEY489Rm1tx0/Zx4wZw/XXX09cXFyEMxuZgi0tlH7hC7RtLuo7yDCYvHoV1oSE8CUmIkOSaZrs2HkXFRXP9hhLSJjPnNmPYhh2PJ5idu3+EY2N649rXqs1/kiRKtjreFzcZBYtfLXbFnW/v5lDFc9SW/MOpuknMWkx2VlX4XCkndSziYiIiMjgOJGCk6X/kNFtanxM5zdJjcNPjMvl4oYbbiA1NRWAyspKHn30Uerq6igqKuKDDz5g69atnVvv5OOxxMbiGDs2dJBp4qusDEs+IjK0GYbBKVN+ybSp95CYsICoqDTi46cyadL3mTP7r1itMVgsNpzOU8jLu/G45w0E3PRVbALweHZT37C287q1tZw1ay+iuPhn1NV/QH3Davbv/y2r15xDU1OIArqIiIiIDGla4XQcTl+7k51qHH7S3G43jz32GNVHtnsZhtGtyXhsbCyXX345EydOjFSKI0bNQ3+i+p57QsZYXC6Srr6a5M9diy0tjZYNG6j9y8O0FBZi2GzEn34aKTd+Ccd4NRgXkQ7BoJ/C9Z+muXlrr+OTJ/0IqzWWxqaN1NS8SXt7Tb9zxsTk43ROp6mpiLa2sl5joqOzWbL4bSyWYydtmmaQ2tp3qap6DX/Ag8s5ncysz+CISj25hxMRERGR46YtdQNccPr6jlL+cbgeUOPwk+V2u/nLX/5CfX19r+M2m40vf/nLpKVp+8TH4a+uZs+nzsZsa+s31oiKImbOHFrWroWP/DlgiYsj75GHiZk5s49Xi8ho095ex46dd1FTc6zReFRUKpMm/hcZGZd03qusepWtW782YO87c8ZDpKWdBUAg0MaWLV+ltu69bjFWaywzZ/yR5ORlA/a+IiIiItKTCk4DXHD6c3k13z/SOPxP08ZykRqHn5Qnn3ySXbt29Tk+b948LrroojBmNDI1v/UWB2+/A7O9vdv96DlziJk5g8bnnifodvc7j6OggHEvvqAVfSLSTWvrAZqbd2CzxZOYuACLJarbeDDoZeWHp/a5ysnhyCY+fjLNzVtpbw990AF0NBpPSJiNM34qHk9xj2LTUVZrPMuWvovd3v3vaK+3msqqf9HeXkNs7DjGpJ+P1aoDLEREREROxokUnGz9h8gs57F/mBY1t6jgdJIOHjwYcry0tDRMmYxszrPOYvyrr9Lw9NO07diBJT4e13nn4TzrTAybjbTbbqPhH89Q8+CDBJua+pzHu2sX3h07iJ46NYzZi8hQFxOTR0xMXp/jFouD6dPuY3PRl3qccOdwZDJv7t86X19d/R+Ktnwl5PsFg63U16+ivn5VyLhAwE3F4RfIy/1C570DZY+wZ89/Y5r+znvFxb9gxvTfazWUiIiIyCBTwek4TDvSODwIFDW3RjqdYctiCd2jvr9xOX5ROdmkf+uOXsesTicpN36RoNdLzX33hZyn/eBBFZxE5IQlJS1i0cLXKD/4OA0N6zAMG6kpZ5KdfVW3FUipqWcRE5NHa+uBXucxsBIbO46W1v2YZqDf992373c0Nm7E5ZxGIOBlf8nvesT4/Y1sLvoyixe9TkxMTrcx0zRpatpIa2s5Dkc6iYkLMQz93SQiIiJyMlRwOg6xVguT4qLZ5WmjqLkF0zS1zegkTJ48mVBbHZOSksKYjTjG5vcbc/iHP6K9pISkK6/E6nIRaGig/h/P4H7vXfAHiF24kKRrrsaekRGGjEVkOImJyWHSxO+FjDEMC1MKfsamzTdimj1PLJ0y5edkZX2GQMBLQ+M6Nm36fMj5AoFmqqpeoarqlZBxwWArBw8+wcSJ3+2819y8g23b78Dj2X3sGaLzOOWU/yYpaVHI+URERESkJ/VwOk5f21HKM0cah69dfAp5ahx+wurq6njwwQfxer19xnzqU59i6dKlKuiFQdDrZc8ZZxKoq+s31hIXh/O8c3G/+x6B6u49Vyzx8eQ+9BCxc+cMVqoiMsI1NRWxv+QP1Na+i2kGSExcwNj8L5OSclq3uA0brqW+YXWf89jtSfh8vR9O8VE2m4vs7GtxOqfhcKSzadNNBAKNPeIslmgWzH+B+PjJ3e6bZpD6+lW43buw2VykpX2yR/8oERERkZFGTcMHoeDUtXH4n6eN5UL1cTophw4d4qWXXqKysrLzntPppLm5ufN6wYIFnHfeedpiFwaeDz+k7JZbe5xqZ01OJm7JEprffBMzRIHwKFtGBhP/828Mu32wUhWRUaDj3yRmn9vYmpq3sn79VQSDPbe3Z2V+llNO+SVebxXNzdso2vLVXldNnYyMjMuYNvXXndcezz62bL0Fj6e4857F4mDC+DvJy/vigLyniIiIyFCkgtMgFJzWNri5eOMeAL6el85/Tcga0PlHE9M0qaiooLm5mYSEBDIyMtiwYQP//Oc/OfrfY0FBAZ/+9KeJiorqZzb5uNpLS6n72xO0FBZi2GzEn34aSVdfjS05GX9tLfVPPEHdXx/v92S7nD/8H86zzgpT1iIyWjU372DfvnuoqV0BmDgcmeTm3kBe7he7Faq2brudysqXB+Q9LZYo8vO/SnzcZKKjc9hc9GXa2w/3Gjtt6r1kZFzca96NjRuwWKJISTkNhyN9QHITERERCScVnAah4OQJBJj03haCwOlJTp6aPWFA5xcoLi7mmWeeob29HYDs7GwuvPBCdu3aRU1NDbGxscyaNYvs7OwIZzr6NPzrX1Tc+e2QMam33Ubabbf2uO+rqCDY0oI9JweLQ1tRRWRgBAItBAIt2O3Jva6IcnuKKSy8vMdJeQCxsROYO+cJWtsO0Ny8jX37fovf33M73cmIi5vMooWvdm4N9/nq2brtdurq3u+MMQwbubk3MHHCd9WUXERERIYVFZwGoeAEcOqanexuaSPZbmXbsunqMzQIKioqeOKJJ3CHWE2zYMECzj//fH3/w6hl40ZKr74mdJDNRsLFF5N09dXEzJhOS2EhVf/7a1o3bwbA4nKRdNVVpN12K4ZWrolIGDQ1FbFr149oai46cscgNfUsphT8DIcjrTNud/HPKSt7eMDeNyvzShISZhMXN4XiPT+nsbH3f5OMG/dNxo/7Wq9519a9D2aQpOSlJLjm6u88ERERGRJUcBqkgtNt20t5trKjGem6JVPJjdaH5sHQ0NDAY489Rn19341fL7roIubNmxfGrEY3Mxhk3/kX0F5SclzxUePH015aCoGex5g7zzmH7N/eqw9PIhI2Hs8+2ttriInNI9rR81TN1tYy1qy9gEDA02PMMOzMmvUwFsOK27ObsrJHaG0tHZC8bDYny5d9iNUaC4Df72Hrtm9QW7uiW1xS4mJmzLgfuz1hQN5XRERE5GSdSMFJ67hPwCxnbOfXRc09l+jLwEhMTGT8+PEhY9asWROmbATAsFjI+MmPMfrYEuc89xxsmZmd1+379vVabAJofuMNWjduHJQ8RUR6Exc3nqSkhb0WmwBiYnKZPfsRoqO7b9m221OYOeMBUpKXkpS0iNyc6xg37uv9vNvxF9P9/ma2bP0aBw78hdra99m2/Vs9ik0A9Q2r2bb99h73A4EWysoeY/2Ga1hXeDm7dt2Nx7P3uN9fREREZDDZIp3AcDLTGdP5dVFzKxek6aS6wVJTUxNyvKqqimAwqJPswihu4ULGPv0UtQ89hPuddzEDAWIXLiTlxhuJW7wI0+/H/d571D36GC1r14acq/mNN4idOzdMmYuI9C8xYR5LFr9NXf0HtLWW43Ckk5JyGhZL90L7mPTzKC19EI9ndy+zGMyc8RDRMVm43Ts5ePDvNDauD/m+tbXvUFv7Tr/51da+i9u9i/j4AgDa22vZsPHabiflNTVt5uChp5gx/T7S0s7ucy7TNLXKVERERAadCk4nYHp8DAZgohVOgy06OjrkuM1m0z+WIyB6yhSy77mn1zHDZsN55plEjRvHvvPODzmP+/33cV1wAdEzZmAYBsG2NhqefpqGl14iUFNL1LhxJF11Jc5zz9Wvs4iEjcViIzXl9H5iHMyZ8zg7tn+b2rr3Ou87osYwafIPSEs7EwBn/BRiY8dRWHj5gOW3afONJCbMIzZ2PHX1H3QrNh1lmj62bruD5cvex25P6rzf3l5DSckDHK58CZ+vntjYCeRkX0tOzucwDOuA5SgiIiJylHo4naBPrNlBcYtXjcMHWVFREc8//3yf4xaLhYsvvphZs2bp12CICba3s+fU0wg0NPQb65g0EddFF9P8n//QtmVLj/Gka65mzA9+oF9jERmSWlr243bvxmZzkpi4AIvF3iNm0+Yb+1zBNHHC90hPPxePZzeHK/9FZeVLA5bb+HF3MHbsLRiGgbe9hvXrP0Nr64EecWPGXMy0qff0+udsa2sZbW2HiI7OJCYmb8ByExERkeFLTcMHseDUtXF44ZKp5Khx+KAIBAL89a9/pbQ0dGPWGTNmcOGFF+Loo7eQREb1ffdRc/8DvQ8aBpzAnzu5f/4z8cuXDVBmIiLh5fd72Lnr+1RW/gsIAmC1xjE2/xby87/cWejx+5t5/4MlBIOtfc5ls7nw+5uO+71ttkTi4ibg8zXS0rKnz7jZsx4mJeW0zuuWlv3s3PUD6utXdd5LTFzElIKfERcXuseiiIiIjGwqOA1iwemhsip+uOcQAA9PH8v56uM0aNrb21mxYgUbNmzA6/ViGAYFBQW4XC7WdukRlJyczBVXXEFGRgb79+/n0KFDREVFMWXKFBISdKJPJJg+H4fu+h5Nr7zS7b41OZmcP/wfgYYGGp9/nuYV74DfH3Iu1/nn9bmNT0RkuGhrO0Rj40YsFgdJSYuw2Zw9Yvbv/z/27b+319fn5X2JiRPuwuero75hHVu33jpguSUkzGNKwU+JiRmL39/E2nUX095e1SMuKiqVBQte6tF8PRj0U1v3Lh53MXZ7Iunp53TbziciIiIjhwpOg1hwWt3g5tKNHT8l/Gb+GO4an9nPK+Tj8vv9uN1uYmJiOlcy7d+/n+eeew632w10bLGLjY3tvAYwDIOlS5fyyU9+UluyIsA0TdqKimh6/Q2CHg/R06aRcOEFWOLiOmN8NTXsWf6JkPPYMjLIe+RhHOPGdczb3k7dE3+n4Zln8JWVYRszhoTLLyPlhhuwxMaGnEtEZCgzTZMDZX+mpOSP+P0d25JtNie5uV9k3NjbMIxjB2Vs3Hg9dfUr+5jJIDXlDNq8FbS07CMY9B5nBhZstviQq6jy8r7EpInf67xubt5G0ZZbaGsrPzaLEcWEid8hL/cLx/m+IiIiMlyo4DSIBSePP8DE97dgAmckO3ly1oRBeR/pn8fj4cUXX6S4uGfT1K7OPvtsli5dGqas5ETtOfMsfIcO9RsXPX06rvPPo/m992hdvabn+KyZ5D/6KJaYmF5eLSIyfAQCbTQ3b8U0g7hc07FaexbTPZ49FK6/srMw1dWECd9hbP6XgY7VRys/XE57e/WA5Ga1xjN+/DeJi51AVFQaGzZeh99f32vsjOl/ID393G73Ghs3cKDsEZoaN2GxxpCWdja5uTfgiEodkPxERERkcKngNIgFJzjWODzFbmPrsmlaPRNBpmnywgsvUFRU1GdMfHw8t99+O1arTuEZiqrvv5+a+34/IHOl3XEHqTffNCBziYgMda2tBygp/SNVVW8QDLbgcs4iL+9G0tI+1S2upOR+9u77TZ/zjM2/lUCwBY9nD/V1H2ISGJD8nM7pLFxwrBF6RcVzbN/xXTrO+z3G4RjDvLlP9dqYvLX1IPUNqzAwSEpaQnR01oDkJiIiIifnRApOtsFOZiSa6YyluMVLrc/PIa+PbDUOjxjDMPptGO52u6mrqyMtLS1MWcmJSPniF2n5cBUtvRSJE6+8kpjZs2l65RU8q1ZBIPSHoMYXX1TBSURGjZiYPE6Z8gtOmfKLkHF5eV+ioXEDtbUreoxNnvQDcnNv6Lzetu1bHK58cUDya27eyrrCK4iLHUdUVBqlB/7MR4tNAF5vJbt2/5jZs/7SeS8Y9LJz1w+pqHieo83WwUJW5hUUFPwYi0X/9hIRERnqVHA6CTOdMTx35KS6ouYWFZwizGKx9Bvj76cxtUSOJTqa3If/QsM/nqHx5ZcJ1NQQNXYsiVddifNTn8IwDBIvuxR/TQ37Lr2MQE1Nn3O1l5XhWbuW2PnzMSwWTNOk6V//ou7xv+HdvRur04nrggtI+dKN2FK1fUNERgeLJYpZMx+kqvrfHD78Ij5fPXGxE8jOuRaXc3q32NzcGzhc+U/odZWT8f/Zu88AqaqzgeP/e6fPTtnZ3ndZYJfeEQQVwdjQEBsqKvYaS9TkNWpi1MTEEmPX2FERK3aiKHYB6bDUZSnbe5tt0+fe98PCwDAzC+pSPb9v99xnzj2XZXdmnnvOcxgy+ElkjR5X11aqql/H4+l5SXR7+2ra21fvdYzNzd9S3zAfu204BkPqjmTT3D2iFGpq30GSNAwYcF/YGbe7kvKK52ls/BJF8WC3jyQn+woSEsQup4IgCIJwsIgldT/Dj85OztxROPyW3FT+LAqHH1Tbtm1j9uzZPcZYLBZOOOEEhg8fHkpQqaqK2+1GkiRMou7PYaHyut/T+U3kE/o9adPTsZ9+Ov6GBto/+ijivC4zk9w33kCXmrI/hikIgnBYq6+fx6ZNdxBUXKE2WTYxcMC/SEubFmqrqXmHTcV3ROsCAI3GjKoqKIrnJ11fkvSoqq+HCA0TJ34f2i2vs7OElatmRK1nNaDwPjIzZ0S0B4MeWlp+wO9vw2IpwGodKkokCIIgCMI+EDWc9nPCqTMQpP+OwuFTEqy8IQqHH1SKovDaa69RVla219j09HROOeUU2tvb+eGHH2ho6N72OSsri0mTJtG/f//9PFrhl+j87jsqr7m2V/qyn3EGGQ/c3yt9CYIgHGn8/nbqG+bh8dRgNKSTmno6Op09LCYY9LBs+e9wubZGvF6StIwaOQe7fRReXwNlZc9SXd3zw6GfwmTMwWobismUTUPDZ7jd5VHjJEnHMRMXot+tKHlt7QeUbLkvLEFls41gyODHMJmye22MgiAIgnAkEgmn/ZxwAjhm6Sa2urwk6bSsE4XDDzqv18u8efNYv349O/9PWywWJk2aRH19PStXrmRf/q+fc845DBkyZK9xwsGhqioNDzxIy6uvRpyznnYa8WeeSfu8eXR88QWKyxWlh10kvZ6C5cuQ91IDTBAEQYjN621gU/HtNDd/F2ozGXMoLLyHxMRJoTafr5mFi46JOXPJYhlISvIpuD2VtLWtwuXa3mtjTE+fTk72ZZhMObS2LqFo7ZVR40zGHMaN+1/YroDBoJvKylnU1M7F663DaMwiI+NcsrMuFnWkBEEQhF8lkXA6AAmn328s5/0ddZxWHT2IDFHH6ZDQ0dFBbW0ter2e7Ozs0M509fX1fP7552zf3vMHWLGj3aFPVVW6Fi/G+e5c/JWVaFNTsZ95BtYTTkDasVxScbspv/gSPOvW9dhXxr8fwnbqqUja7nJ2XcuW0fLyLNyrVyPp9VimTCHxyivQZ4sn3oIgCD1xucpxubah08Vjs41AkiLrK9bWvr9jlzolrN1gSNuxS13331qPp4ZFiydFxO1Oo4kjGOz6yeOUJB2q6o95fvcleMGgh9VrLqGtLfIzZULCsQwf9gKyrAtrDwQ6qa+fh8u1HZ0ugbS0aWJnPUEQBOGIIhJOByDh9GxFA/ds6y6U+cqQPpySbN/LK4SDTVVV5s+fz9KlS3uMmzlzJn37imWSh7uGhx+m+cWX9hqnSUzEdtpU5DgLzf/9b8R52WYjd/ZrGAsL98cwBUEQflXa2tZQUfky7e1FyLKRlOSTyM6+JGzJG8Cm4jupqXk7ah9ZmTMpKLibQKCNtrbVMWcs/RwajQVH/FEYTZm43dU0N38dM3bggPvJyDg3dNzcspD1628kEGjfLUqmb/6t5OVdF7WPYNCL212ORhOHyZTZW7chCIIgCPvNT0k4iV3qfqZh1l3TrYs6XCLhdBiQJAmHw7HXOI/npxU3FQ5N9rPPpvnlWaDEfkIOEGxupvW12HVFlPZ26u6+h7y33ow4jNfAUQAAIABJREFUpyoKvvJyUFX0OTmhmVKCIAhCdHb7CIban9hrXGHBPUjIVNe8w84d8yRJS0bG+fTv/xckSUKniycpaTKJiZNpbo6+oYQsG8jPvxWvtx5X1zaaW76LGrdTMNhJUw9Jpt2VlT2DTufAaMpGQsPatdeiKO49ohS2bX8YkzmX1JSpu1oVP6Wlj1NVPSeUoLJah9Cv3+0kOI7ep+sLgiAIwqFOzHD6mTp2FA4HOCHBxpzh+fv1ekLvKC0t5dUo9X92V1hYyOmnn47Vaj1AoxL2l9a33qLu3r/DHn/njMOHk3LzH2j/4gs6Pv2MYFvbXvvK//RTDPl9QsfODz6k6emn8VdVAaBNSyPx6qtwzJgharoJgiD0Eo+3DmfrMgAcjvEYDJG7i3o8NaxcNQOPpyqsXZJ0DB3yJMnJJ4balq84i/b2opjX02qtBAIdvTT6XazWIRw1tnvXVFVV2bDhZuob5kXESZKWEcNnkZAwIay9rW0N5RXP09q6BEnSkJh4HLk512CxFPT6WAVBEAShJ2JJ3QFIOAVVlVGLN1DvC6CV4OwUB5dlJTPCZt77i4WDRlEUnn322dDudLHo9XqOPfZYjj76aGRZZvv27WzevJlgMEhOTg6DBw9Gp9P12IdwaHCvW0/rG2/g3bwZ2WbDNvVU7Gecgazvrrum+nxU33YbHfM/77EfXW4u1uOPxzzuKLzbt9P48H+ixiXffDNJ117T6/chCIIgxOb3t1Fd/SaNTV+gKF7s9lFkZV2MJS5899mmpq8pWntV1D6MxmzGHfU/JEmL11tD0dprcLm29doYtdp4TMZMNNo4nM5lMeOs1qEcNfbD0HFD4+esX38TqhoIi5NlEyNGzMIRPzaiD7e7kqbmb1HVAPH20dhsw3rtPgRBEIRfN5Fw2s8JJ6+icNm6Ur5uiXwC9vd+GVydHfn0TTh0NDU1MXv2bNr2mNVis9lQVZWOjl0/1/j4ePR6fUSCKj4+nosuuoikpPCaE8Lhyfnee9T+5a+90pdkMND/u2/RxMf3Sn+CIAhC76qtfY+SLfeF1VqyWocydMgTmEw5obaa2rls2vTnmP3k5V6P3pCMx11JXf1H+HxNvTbG5OSTiTPno9MnsW3bIyhK9ALpJlMeR49fECrSrigBSrbcS3X1m8Cuz/gOx9EMHfIkOl1kaQFVDdLWtppAoAOLpVAUORcEQRB6JBJO+znh9FhZHQ+U1sU8/+WYAoZYxUynQ5nX62XdunWUlpYiSRL9+/dn0KBBqKrK4sWLWbhwIYFAoMc+EhMTuf7665HlyJ14hMNLsLOLrZMno3REX0Yh2+2oXi/qPtb3Sn/gfuLPOCN0HGhupuW12XR88QWKx4Np6FASLrkY8+jRvTJ+QRAE4acJBt00N39PINBGXFwBNtvwiOXQqqqwcdP/UVf3YcTrs7JmUtD/7tBrqqvfonjzX2Jez2BIQ69PwuOpwe9v6dV7yc//I0mJkzGZsikre5ryiuejxsXHH8WokW+E3WdD4+dsKbkPj7dmR4tEcvLJDBzwT3S66A9OVFVBUTzIskksIRcEQfgVEgmn/ZhwUlWV0T9upMYbe0vdSzISebBQbKN+OHM6nXz22Wds3ry5x7gLLriAggJRP+FI0PHNN1Tf9AdUf/jvtjYjndzXXkOXkoJ7/XpaZr9Ox2ef9djXzqV7lkmT0OfkUHHlVQRqa8ODJIm0v9+LY/r03r4VQRAEoZeoqkpT0wJqaufi9dZhNGaSkXEeiQmTwpItwaCHFSvPprOzOKIPjcbMmDHvh5b3VVXNYXPJ3w7YPexu1Mi3cDi6l+A1N3/PmqIrgMjNNWzWYYwe/S6yvGszDL+/le2lT1BX9wGBQAd6fTIZGeeRl3sdGo3xQN2CIAiCcJCJhNN+TDi5gwp9vl/bY8yxDgvvjui3X64vHDjbtm1j9uzYu5cBTJ48mUmTJh2gEQn7m3frVlpmv4579WokvR7rCVOIP/98tLvtbugtLWX7qVN76GUPshx7pzytln5ff4UuRSzDFQRBONz5fC2UlNxLQ+P8UL0lu20kBYX3YLMOCcUFgy4W/zgFn68xaj+5OdeSk3MZHm8dDQ2fU17+TK+NUZI0GAwZGAwpdHVtJRCIvWnG0CHPkJJyMtBdI2vFyulRa1rFx49j5IhXkGV9WLvTuYLq6jdxucsx6JNISz+T5KQTQ8v/BEEQhMPTT0k4iT28fyKDLGHVyHQEY2+1nqwXxaSPBHq9fq8xW7ZsYdiwYTgc4TUR2tvb8Xg8oRpQwuHB0K8f6ffe03NMnz5Yjj+ezm+/jXpek5AAikLQ6exuiJVsAggEaP/kExKvuCLUpHR10fLGG7R//AkBZyuG/L44ZszAevJJYumCIAjCIUyvT2DIkMfx+ZpxuyvQ6RIwm3Mj4jQaM8OHv0hR0ZURSaeUlNPIz78ZWdah1ydhiSukru59vN7opRxMplxysq/A7amgvW0tzrbYxcihu16Tx1OJx1O51/spLr6TuroPMBjT6eosiVlA3elcSl3dx2RknBNq2176BKWlj4fFNTYtICVlKkMGP4YkacLO+f1Oaus+oLNjIxptHCkppxFvHyPe9wRBEA5zYobTz3BHSRWzqmMXhnxzWD6TE2377frCgaEoCo8//nhEcfE9ybLMsGHDOPbYY3G5XHzxxRdUVnZ/kNPr9YwePZopU6aIXe2OIIHWVqquvQ53UfjW2oYBA8h+/jm0iYm4i9bifG8ube+932Nf2pQUHDPOJ27CBHS5uVRefgWejRsj4hIuuYTUO27v1fsQBEEQDp5g0EV9/f9o71iLRmMmJWUqNuuwiCSL07mCNUWXEwyGFw7X6RIZNfJ1LJaCHf15WbhoAoGAM+Y1rdZhBIMdeDx1KIq71+7FYEgnJ/syDMZ0/L7WHpcMFhTcQ3bWzNBxa+tS1q69hkAwvI5iSspUBg96BFkO//ykKAGam7/B6VyOJOtISpyM3T5aJKcEQRAOELGkbj8nnBp9fqat2kKp2xdxbkqClTnD8sWb3hFi48aNvPPOO1HP6XQ6/LvV+9n5M4/2O9W/f39mzJghCowfQVRFoeuHH+hcuAgUhbijx2M5/ngk7a6Jo4GmJrYcN6nnWU67kfR6VF/k35Wdcme/hnls+PbXqqriLS7GX1uLLi0Nw8CB4u+PIAjCEcbtrqKq6jVaW5eAJJOYeBxZmTMxGJLD4ioqXmbL1n9G7SMlZSpDhzwJdD9U+3HJlB5nOmk0ZoJBV+/dxA4GQxqDBz2CwZCCLBtZuvTUiGTTTn3ybiQ//+bQscdTw5qiK+jqKgmLS0ycxNAhT6HRRG7a09W1neaW71DVAI748dhsQ3v3hgRBEH5lRMJpPyecAJp8AZ4sr+e9+laa/YHQxrOnJNl4ZWj+fr22cGBt3ryZr7/+mvr6egCsVisTJkxgzJgxrF69moULF9Le3r6XXmDmzJn07dt3fw9XOMRUXn8DnV99Ff2kJHXXeAoG96kv2xlnkPnA/aFjz+bN1N75FzwbNoTaDIMGknHffRgHDfpF4xYEQRAOP6qqUlH5ImVlTxMIdCdxJElLevrZFPS/G43GEIqtrnmb4uI7o/ZjMKQxftwXSJKM11tHUdFVuNylB+QedqfRWBkz+h2MxnRkOY4VK8+go2ND1Nj0tLMZNOih0LGieNlUfGfELoMJjokMGfIkOp09oo9g0EVLy0ICQRdW6+BQoXdBEARhF5FwOgAJp90FFYXJy0socXVvmf7dUQMojBO7dRxJVFWlra2NYDBIfHw8Gs2u2gOBQIAVK1Ywf/78HvsYNWoU06ZN299DFQ4x/poayi66iEBN5C516ff9A+tJJ+FaupTOHxbifPvtnjvTaIg7ZiLmkaPQ5+VSe/c9KFGWfMo2G33efx99VmYv3okgCIJwuAgGXTidy1EUPzb7CAz6pIgYVVUpK3uK0rKnQkXOAczmPgwb+ixxcbs2wKmpncumTX+Oeb1BA/+DXp+Ax1tLefnzuN1lvXo/ALJsQFG8PURoOOqoeVji+iFJMsWb/0Z19ZyokQkJxzJyxCthbVVVc9i2/d+hRN3OuMGDHkYf5d8PumdP+XxNmMw5GA1pP/WWBEEQDksi4XSAE04A79a1cOOmCgCmpzl4cmBkkUjhyNXW1sajjz7aY0xaWhqXXnopRuOuZGQwGKSsrIyOjg4SEhLIzs4Wy6GOQIGWFlpff52OBQtQXG6Mw4aScPHFmEeODMWoqsqWY44l2NzcK9d0XHQRaX/9S1hb56JFtL7xJt6tW9DEx2M/7XTiz52ObBQJckEQhF8rr7eBxsYvCAQ6sFgHkphwbERRb1VVKd78V2pq3op4fb++t5Gbe03ouL5+Hus3/CHm9ZIST8AePxqvt57mpm9weyp672bontGl0yXsKMge+3vOmDHvY7cNB6C27kM2bvxj1DirZTBjxryPLO9aMt/evpbNm++mvWPnztUSSUknMKDwHxgM0XeedburcHsqMRrSMJv7/Kx7EwRBOBSIhNNBSDj5FZWjl26kyuNHK8GP4weRbRS7k/1aKIrCI488QmdnZ49xer2eESNGcNRRR9HW1sZHH30UthwvJSWFs846i7Q08ZTs16jhkUdpfv75mOd1mZn4q6v3qS/Z4SDnuWcxDhiApNfT+PTTND35VESccfgwcl56GY0l7mePWxAEQTjyqapKq3MJtbXv4fM2YjLnkJFxHjbrkD3igqxd93uamr6M6MMSV8jo0W+j1VoBaHUuZ9Wq82Ne02TKISlxCl5fAx0d63G7ezc5pdcnodcn43KV9VhEfejQZ0hJPhmAzq4trFhxVtT6VmZzPmPHfIhWu+s91e2upLj4r7S0Lgy12e2jGFB4HxZLYdTrBQIduN0VaLU2TKbsn3t7giAI+4VIOB2EhBPArOom7iipAuDyzCT+VZB1wMcgHDwLFy7kyy8jP1z9VGazmeuuuw6r1doLoxIOJ0pXFxWXXxGx+x1A4jXXkHLLzQSdTlyrV1Pz59tR9qF2mKTXo+/TB+/mzTFjEq+8gpQ//Smi3V9Tg3v9emSTCfOYMcgm00+7IUEQBOFXSVH8VFW9RnXNm7hc5ej1SaSnn0VuzjXodLt2clZVlfUbbqKh4dOIPmTZyOhRb2KzDQPA7a5m8Y+TiDVrSZZNpCSfhM/XhMtdhsezbw9o9oUsGzGb8zEYkunqKsXTw6ysgoK7yc66GACfr4Vly6fh9dZGxGm18Rw19sOwhFIw6GLL1georX0PReku1WGzjaB//zuJt4+Oej2vr4mO9rVIsp54+xg0GjFrWRCE/UsknA5SwskdVBj740aa/AGMssTyoweRrNft/YXCEUFRFD755BNWr14d1m4ymZg2bRq1tbWsWLECl2vvO74cd9xxTJkyZX8NVTiEKR4Pznfn0vbJJwRbWzHk5+O4YAaWSZPC4mr+8hfa3nu/V64p2+0U/LgYaccuisHOLur+9jfa588P7bAn220k33gTCRdd2CvXFARBEAToTk6VlT1NVfUc/P4WAByOCfTr+3+hZNNOJSX/oLLqlaj9DBzwABkZ0wEIBLpYuOhogsGumNeNjz+KQKATr7cWv7+1d26G7uSU1ToEvT4Rr6d2t2V3kTIzL2RA4d+B7plhq1dfTKtzSZQ+DYwe9VbYv0cw6KGk5F5q694P1eDSau3k97mJrKxLopZoCAQ6aWtfA6qCzTYiLPknCIKwr0TC6SAlnACeLK/nn9u7n2L8ITeVO/LTD8o4hIOnpqaG9evX4/V6SUtLY+jQoaG6TYFAgPXr1/PRRx/R0+9eRkYGV1999YEasnAY8pSUUHbOdFSfL/KkVkvyzX8g2NiEu6ioe8bUXv7Wm0aMwDxmNMYRI2h95VVcMf6Gpv3j7zimT49oV9xuPJs2AWAcNEjUhRIEQRB+EkXx4fXWo9HEodcnRI1RVYWy8v9SWTkrlCQyGrPJz7+Z9LQzwmLLyv7Ltu0PR+0nPX06gwY+EOpz8eIpeLyVMcdmNGajKB58viZ6qgv1U0losNtHoTckEwx6aG7+OmbsnoXO1677PY2Nn0eNLSi4h+ysmaFjVVUoLXuaiooXQkk4WTaRnTWT/Pw/htWn2ikQ6KClZRHBoBubbRhxcWKnZUEQuomE00FMOLUHgoxevIGOoIJNK7Py6MFYtZq9v1D4Vbn//vvxemPvtGI0Gjn//PPJzc0NPaHy+Xxs3LiRlpYWLBYLgwcPJi5O1N35Nev87jtq/nw7Qacz1Cbb7WTc/y+su82Qq7nzTtre/6BXrqlNS6PflwuQtN0fTlVFofn5F2ie9TJKW3toDElXXUnCFVeIIviCIAhCrwsGvbhcW5EkLXFx/SKKnEP3cr3yiucpL/9vaOc5WTaQmTGDfv1uR5Z3rUKoqXmXTcW3R71WXFwBR439GFnWoSgBVqycTkcPs5a0WhuqGuxxdtXPZbUORq9PBiSam7+JGafTJXLMxIXIcnc92W3bH6WsLLKOI0BW1iUUFvwtdKyqKhWVL1Fa+kTYPSQmHs/gQQ+j0zki+vB666mv/x9+fwvmuH6kJJ8ilvYJwhFMJJwOYsIJ4F/baniiogGAv+Snc2Nu6kEbi3Boevvtt9m0YzZIT1JTUxk/fjxxcXF88MEHuN27ClpqNBpOPfVUxozZp9914QileDx0fPUVgdpatGnpWH9zQsTsItfq1ZTPuCBmH5qkJIJtbeD379M1Ey69FMtxx2IYMICW116j+dnnosYl3XADyTdcH33cPh8EAshm8z5dUxAEQRB+jmDQhbNtFaoawG4bgU4XHzWuouIltm1/NKx4uN0+hqFDnsBg2PVZvrHxS9auuyZaF8iyifHj5mMyZREMutm46XYaGubFHJssmwA1VK+pNxn0qRiNGWh18TQ3fw8EY42CYyYuDN1jVfUbbN58V9RIu20ko0e/gyTJobay8ufYtu0/Yf3rdIkMG/oM8fGRn1Hd7kpqa9/D46nGYEwnPe1szGaxu7cgHE5EwukgJ5wafX7G/rgRj6KSrNeybPwgTBp57y8UfjWqq6t56aWXUHbUx9mdJEk9Lrfb00UXXUS/fv16c3jCEaj+wYdomTUrot1QUEDu7NeQDAY8GzbQ+PjjuJYt75VrSkYj/b/7Fo3dHmpzr99A01NP0fn996AoGPr3J+GKy7H/7ndiNpQgCIJwUAUCHTQ1fUMw2IXVOgSbbWjUuIrKWWzd+kCodhJ0FwEfOuQJEhImhto6u7awbNlvUdVoD3QkRo18g/j4sQSDXZSWPkVF5QsxxyZJWjQaC4GAM2bMz6XVWjEaM9FpHbS1r+4xATZ82AskJXXPoq6vn8f6DX+IGqfRWDh6/AIMhpRQW2XVa5SU/APY/fOvTP9+d5CTc3lEH23tRVRXzaGraws6XTypqdNITT09bHbanhTFjyRpwpJigiD0LpFwOsgJJ4A7SqqYVd0EwAMFWVyamXRQxyMcejZv3szHH39MV9eu6coOh4Ozzz4bn8/HkiVLKCkp2Ws/+fn5XHzxxWFt1dXVLF26lLq6OgwGA4MHD2bUqFHo9fpevw/h8KCqKh1fLKB1zhy827ahiY/HfvppOGbORGOxhOI6v/+eyqujP7n9ORwXXYhjxgz0eXm4i4qouOxy1CjLSZNuvIHk6yNnQyldXXT+8ANBZxuGggJMI0eIxJQgCIJw0Hm9jdQ3zMPna8ZsziM1ZSoaTeSs3camr9iw4VaCwc5QmywbKCz8Oxnp54TafL4WflxyAoFA9B1oC/rfRXb2pShKgMamr1i//vc9jE7GZMrG52sOu25v0Wqt6LQOvL7GsBlhe8rMnEl+n5vQ6ew4nStZtXpGzNiRI14LS9ZVVr5CyZZ/RMQ5HBMYPuxFNBpDWHtd3cdUVL5IR8cGJElPcvJv6JN3IxZLQdTrqaoS2snQaMwUCSpB+AlEwukQSDhVuL0cvXQTQRVyjHoWjxuIVhZfkoRwgUCArVu30tnZicPhoE+fPsjyrje85uZmXn755bCk1J4kSeLss88mNzcXq9XKypUr+eSTTyLiUlNTueSSSzCLJUxCD1RFoeyc6Xg2box6PuHqqzCPHIm3uJiOr77Cs37DPvUrmU2gguqO8cFUlun35QJ0GRmhJud771F//wMonbs+LBsGDSTr0UfR54rp94IgCMLhIRDopKHhU9zuSgyGNFJTp0atheR0rmDtumsjds3LyrqEgv53hR64qKrKipXTaW9fHdEHQF7udfTt+ycAurpKWbL0RHoqdm425xMMdOHzN6GqsZbe/RIykqSJMdOrm802nIL+f0Wnc+D1NrFq9fkxY/vk3Uh+/s2h49LSJ9le+lhEnEZjZtTINyJmqtXUzqWs7Gnc7goATKY8+vS5MaLw/E7BoAdn2wqUoAebbVjYrC1B+DUSCadDIOEEcMPGcubWd79hPD0wh7PTou+4IQg9mTVrFuXl5fsUa7fbaWtri3l+1KhRTJs2rbeGJhyhAo2NVP3hZtyrVu1q1GpJvPQSkm+9FWlHUtS7fTvbp57Wa9e1nnwSCRddhC43F/fatVTfcGPUOF1GBvmffIy8R9F8X3k5bR99hL+uHl1mBvFnnIEuM7PXxicIgiAI+1sg0EV9wzw6O4vRam2kpp6OJa5/RJzP18S6dTfgbNt9GbxEZuaFFBb8LayQ+oaNf6KuLvrmIUlJv2H4sO5ajMGgn8WLj8Xnb4w5Ppt1OJIk4/c7cblLf95N/kKybKJv3z+i08ajqAGKi+8gVkLNbh/FmNHvho4rKmexZct9UWP33N0PupcBbt/+GIHAzs/XGtLTfkdh4b0RM9oUJUBD42fU18/D73disRSQmXkhVsuAn32vgnAoEgmnQyThVNzl5vhlmwEYEGfk67GFyGIpiPATLVq0iAULFvRKX1qtlttuuy1iaZ3f76epqQlZlklOTg6bZSX8OqmqimftWtxFRUhGI5bjj0eXEvlEr/K639P5TfSdcszjx2ObeiqeDRtxLV+Ob/v2fR+ALEOUGmc7pd39Nxwzdk3Nb37pZRoefhh2f0/TaEi76y4c55+379cVBEEQhMOEqqq0t6/G2bYKWTaQlDgZkykrIq67gPltNDR8GtaemHg8QwY/hlZrDbVVVs2mpOSeqNezWYcxZsx7oeVnRWuvoanpy5jjS3Acg96QhN/fSmvr0v1SHH1fJCediNGUjUZjprz8eVTVFzVOo7FwzMTFaLXdD7Sqq9+kePNfo8YmJU5h+PBdNbeCQS9r115NS+vCPSJlBgy4j8yM8M8iqqrS1PQlNbVz8XprMRqzyMw4j4SE42KWDlDVIH6/E63WgiwbosYIwoEgEk6HSMIJ4NJ125nf1L0W+7WhfTgpyb6XVwhCOI/Hw3PPPUdra2vEOZ1OxxlnnEFbWxsVFRVs2bKFYLDnqdBTp05lxIgR6PV6FEVh4cKF/Pjjj6Ed8OLj45k8eTLDhw/fL/cjHFmC7e1U3fQHXEuWhLXHTZxI5uOPhepDBZqb2XLscT0mkX4KTWIi9tNPQ5/fF6Wzk4Z//zt6oCSR+8YczCNHhjX76+pwvv8+vtIyNI547NN+h2nI4F4ZmyAIgiAcijo7N9PSshAVhQTHBKzWyPc9VVUpL/8vpWVPoSi7ai46HBMYMvhR9PpddWk7OjawYuW5URNJlrhCxox5H42me+fc0tKn2F76aMyxJSYcj90+Ar/fSVPT17g9Fb/kVn82kykPszkXrdZOY+OCHmtUjRjxKo74cciyjm3b/kNZ+TMxImWOHv8FZnMfoLt+1MZNt0WddZaVNZOC/neHJZ2CQS9lZU9RXfMWfn8LkqQjNeU0+vb9I0ZjRkQfAH5/G51dJWg1ZiyWgaJGldCrRMLpEEo4rWrrYuqqLQCMsJp4pDCbNKOeBJ32II9MOJy0tbXxySefsHXr1lBbeno6p59+Opm7LRn64osvWLx48V7702q19O/fH7/fH9bn7qZNm8aoUaN++eCFI56qqrhXr6Hrx8UgSVgmTMA4fHjEE7rqP/6J9v/9L2ofst1O6p13EqipxrNlKx2ffho17uewTZ1K5iP/CR23f/opNX++HdUfXkvCcdFFpP7lzqhPFoMdHXi3bkU2x2Eo6C8KlwuCIAhHNL+/jebm7wgqbmy24TGXhTmdK9hcci+dnTtrP8qkJJ9MYeG96PWJu/XXzoqVZ+FyRS7DM5lyGDvm/VBdq5bWH1m9+qKYY0tOPonMjBmh5FR9Q2Tt0gNJknQ7diyM/b06IWES2Vkz0esTcTqXsWXr/TFjhw19luTkE4HuZXpFRVdEmTkFen0qY8e8h9GYHmoLBt1s2fovamvmouyYyWUy5dC/3x0kJ58U0Ucw6KGh4X+0ti4BSSYxcRLJSSf2uBOgIIiE0yGUcAL47coSlre7Qsca4JRkO3/vl0mmUewaJuw7p9NJS0sLFouFlCjLmxoaGnjmmVhPV36auLg4brnlFrTa8OSooig0NTWhKApJSUkR5wUhlqDTScUVV+LZEF5oXLZayX7uWcy7JTjLzjsfd1FR7M4kKXz5XE80GuKOPhpDYQGaeAeNjz4ac6ZV2j/+jmP69NCx4vXS8NC/cb73Hqqn+wmuPi+PlD/fhnXy5H27viAIgiAcwVRVpatrC35/K2Zzn5hFtb2+JrZuuZ/6hv+hqn4kSUtKylT697sj7DWqqlJcfCc1te9E9GEy5TB69LsYdsy0CgQ6WbT42Ji7+6WmTKOg4C78/lba2laxqfj2Hu9Fq7ESCHbSU/JofzMZc8jKugit1kZXVwkVlS/HjM3IOJ+BA/4JdP+7Fa29kubmb6NESgwf9jxJSVNCLW53BavXXBIqnr6T1TKYESNeDpvNBuDx1FJZOYum5q9RlADx9tHk5FwedaZc93iCtLWvIeBvx2IpjDkbSzj8iITTIZRwavUHmLS0mAZ/IOJcllHH/NGFJOnFF3ah93z77bd8++23Ee0Y3clAAAAgAElEQVQ2m43TTz+dyspKNm7cSHNz8177mjlzJn379g0dr1mzhm+//Ran0wmA2WxmwoQJTJw4Ucz4EPaJ4vPRMX8+HV9+heL1YB45kvhzzkGbFP6hpvP776m85tqoSSVdbg55b79NoL4e37Zt1N5zL0p79A+aP5U+L4/8Tz5G0ulQVZWq318fvUaVLJP932ewTJoUcUpVVQINDUiyjCYpSfxuCIIgCMJuAoEOfL4m9PqksPpRu1NVhZrad6mqmk1X11Z0unjSUqeRm3tN2MwpgJaWRRStvSZi+ZvFMohRI2ej08WH2laumoHTuSzqNR2OCYwaORtVDeL1NrBk6ckEg7F3inY4JqDRmAkGOml1LuXgJKlkkpKmoNM5CAbdNDTMixkZZy5g3LhPkSQJVVVZtnzabjPTwiUmTmLE8F2Jrs6uLaxadQF+f0tYnCRpGTL4CVJSTg5rb2z8gpKSf+Dx1oTGmZx8EgMH/DPs5xHqv3MztbXv4fU1YjLlkJE+PWo9MuHQIBJOh1DC6eHSOh4uq4t5/pbcVP6cnx7zvCD8HJs2bWLp0qXU1tZiMBgYMmQIEyZMwLKjno6qqsyfP5+lS5f22I9GoyEvL4++ffvi8Xj4/vvvo8ZNnDiRE088sdfvQ/h1a/tkHvX/+hfB3eqXmUaOJPPhf4ftPtfwn0dofuGFaF0AoHE4CDqd+z4jSpbRpqWisdnxFhfHDDMUFtLnww/CEkrODz6k+bnn8JWVdccMHEjyDddjPeGEfbu2IAiCIAg/mcdTQ3X1m7S1r0EjG0lOPpnU1N+i0YQX1/Z661mz5jI6uzaHtVssgxgxYlZo5hTA9tInKC19POr19iyg3tNOgAB98m5Cp3fg8zVTVTV7t13vDixZNqDVWpEkLV5v7O+oAIUFf8diKUSrtbFx0210dKyLGtddbH0RWm3394zm5u9ZU3QFEDmb3GYdxujR7yLL3RMuVFVl+/ZHotS/0jBwwH1kZJwb1qooAaqrX6e65i1crjL0+iTS088iN+eqmMlLl6t8x66PFuLjxyLLYoXRLyUSTodQwun4ZcUUd8XekaG/2cAP4wYewBEJQrfS0lJeffXVXulLkiRuueUWbDZbqM3lcrFkyRI2btyI1+slIyODcePGkZ+f3yvXFH4dFK+XrsWLUdrbMfTvj3HQoIiYQEsLZedMx19TE3FOn59P3ttvIWm1eLduo/auu3pMIv1UqXf9Fcsxx6DLzqZl1isxi5dnPPQg9mnTwsfd2EjL63Po/OYb1EAA06iRJFx8McaCgl4bnyAIgiAI4RTFT2PTl7S2LAIgIeFYkpJOCCVBdlJVhS1b/kll1Wvsnjyx20czdMjTGAzJoTaPp5YVK6fj9dZGXC8j/VwGDPhX6AFVecWLbO2hhlNe3g044scRCLRTVTWbVueSmLEgcTCX/+1kt4/Bah2MVmuhtvb9qP8OOw0Z/CSpqVMBqG/4jPXrb4gRKTF2zAfYbEOB7p/HuvU30Nj4eUSkxTKI0aPeCEs6+XwtbCq+naamr0JtOl0i/fvfSXraGRF9eDw1VFW/gdO5DEnSkpQ0hYz0c9HpbBGxO/n9bahqAJ0u4Vc1o10knA6hhNNRP26kwhN9602ADIOOVRPEzkjCgaeqKs8//zy1tdHfEJKTk5Flmfr6+n3qb+zYsUyePBmz2Ux7ezsvv/xyaOnd7k455RTGjx//i8YuCHvy19VR/+CDdCz4EgIBJJ0O29SppNz2f2gTd029b/vkE2r+77aY/ej79EGXmYm/uhpfefk+76onmc2obnfMWVSaxET6f/M1kr77qZq3tJTymRcTbGoK70enI/OJxyPqQ6mBAB1ffkX7Z5+hdLRjGDAQx3nnos/N3afxCYIgCILw83g8NTQ1fY2ieLHbR2KzjYyaXPB66ykte5r6+o8JBDqIi+tPVuZMMjNnhO0SFwx6WL3mYtraVkb0kZBwLMOHvRAq2t3ZWcLSZacRbbYQdM9CysiYjt/vpKb2PbZvfzjmfUiSDrt9JMFgF15vPT5fU8zY/U2WTWi1VgKB9qi7HO5kt48hN+dKtForbe1FbNv2UMzYvLwb6Jt/C9A9E2rFyrPp6FgfNXbo0GdISd61DNDpXMGaoisIBjvD4ozGLEaNnBOxvK+5+Xu2lz5Be/tqAMzmfuTlXRc1kaUoAZqavqS+4VOCQRdW62AyM84PK/a+J1VVUdVgRBL0UCESTodQwunK9aXMa4w9ZfLUJDuzhvY5gCMShF3a2tqYM2cODQ0NYe39+vXj3HPPRa/X09nZycKFC1mypKenK7vEx8ejKArtMWrqSJLETTfdhMPhCGtXFIXq6mq6urpITEwkOTk56usFoSfB9nYCzc1ok5PR7FhCujvV76f8sstwr4j8kKdJTCTv7bfRZ3Uv13N+8AG1d9zZa2OzTJ6MaeRIdOlpNL34Ir7NJVHjZJuN/t9+g2w2A6B4PFRedx2uH/f4HdRqyXz439hOOSWij2BHBx1ffEGgsRFddjbW3/wG2WCIiBMEQRAEofepqhKWZNpTMOimsnIWNbXv4vXWYzRmkpFxHtlZF0cs+aqpncumTXcCwbD27plT/wxdJxj0smTpiXg81VGvmZ9/K33yrgfA729l4aKJKIo3aqxGY6Fv/q0Egy5c7jJqa+fu660fNLJkICn5BLRaKz5fM01NX8aMNZv7MnrUu2i1cYDCosWT8PkaosY6HEczauTroeOGhvmsW38D0WaW9et3O7k5V4WOg0EXRUVXRcxSk2UjQ4c+TVLi8WHtPl8LZeXPUFv7AYGAE5Mxh8zMGWRnXxZ150BVDdLSshC3uwqDIZXExOMOyJJBkXA6hBJOy5ydTFsdfdt5gNeH9uE3SfYDOCJBCBcMBtmyZQvl5eVoNBoKCgrIzs4Oe3LT1tbGY489Rm/9vTjuuOOYMmXXLhmlpaXMmzcvrJB5bm4uZ5xxRkRiShB+KcXlovHxJ3C+9x5KZydoNFinTCHl//6EPidnV5zXy7ZTTyVQE30WoP3sszEOGoi3uJjOHxYSqOu5FsJPYT/rLGwnn4Q2JQXn3Lm0znkjapyk09H3ywXoUlNDbW0ff0zt3fd0z7jaQZOQQOYj/yEuyuxCNRCgc+FC/BUVaJOSsEyejGwy9dq9CIIgCILwy7hc5dTUvI3LtR2dPoG0tDOJt4+JmGnV1bWNorVX4XaXh7VnZl5IYcHdSJIm1FZe8QJbtz4Q9XqDBv0nbLbOihXn0LZjNs+eZDmOcUd9AqgEAp0Urb0any/2CgmLZQBarZ1AoIPOzs3smUg70CRJh6r6e4zJyb4CkykXjSaOki33EQi0Ro2TJT3HHLMYna77+8vmkr9TVRW9hIlGY2bihO9DsX5/KytWTsflKo2ITUr6DcOGPhP282trW8X6Dbfg8VSF2vT6JAYOeICkpMidlFtaFlNV9RqdnZvR6mykpp5OZsYFO5JukdzuCtra1iDLehyOCWFLC0XC6RBKOAHMrmnijpIqAlH+qc9JdfDkwJxf1ZpP4fA0d+5c1q+PPi01MTGRYcOGUVtbS2VlJV1dsXf0ANBqtfTv35/s7GzMZjMff/wxSpSlS3a7nWuvvRbTHl9+29vbWbNmDU6nE6vVyvDhw0lISPj5Nyf8Kik+H8HGRmS7PepsKADv1q1UXnsd/qqqsPb46dNJu+duJE33G79r+XLKZ16838ccjf3ss0i67vfoUlNwrV5NxSWXRl3aJxmN5H/8UVhSzV1URPUtt4bVv5JtNtLvvQfbqaceiOELgiAIgtCLFCVAc/PXtLevRaOJIzn5ZOLiotdQramdS1nZf3G7ywCwWAaS3+cPJCeHbwbUvUvdDPz+8ERLtF3qqmveprg4+gxxgyGN8eO+CCU5Srb8k8rKl6PGAqSnn4vdNoxAoIPa2vfocsWeyHGo0Gji0GntSLJhR+IvdnmG5OSTSUk+Ba3WSl3dh9T3sMvgkCFPkZrS/dnM7a5i6bKpUXdRlCQdY8bMxWYdEmorL3+OrVGWI1osAxk1cg463a4JMIFAJ5uK76Ch4dNQmyybyMu9lry865EkSSScDrWEE0C1x8fculaqvD7sWpk3a1tp9gcAeGxANuenJ+6lB0E4uLxeL2+99RalpeFZ99TUVC688MJQwXCPx8ODDz7Ya7OhTjrpJCZMmBA6XrNmTdQE1YknnsjEiRN75ZqCsDvV76fjq6/xrF+HZDZjO+kkDP36hceoKqVnnY1306aofZiPmUj63/6Gv7YOd9EaGh95tPcHKklIOh2qL3bdwPjzziXtnnuQJAl/XR3bfzsNpaMjMlCWyZ39GubRo8Oa2xcsoPW12XiKi5Hj4rCdcgqJV1yOViyBFQRBEITDkqqqeL11SJIGvT455kQIj6eWyspZNDV/jaL4ibePITvnsrDExs7+SsueoqzsKVQ1EGo3m/swbOizxMXt+gzl9TayfMWZUYuMW62DGT3qbTSa7gfPra1LWLX6wpj30b/fXWRkTCcQ7KCu9gO29VDPSqOxkJQ0pXvJoKsUl2tbzNhDhVYbj90+Aq3GQmdXCV1d0UszACQ4jqWw8B40Wgsedw0rVp4ZMzYrcyaFhfcA3T+7oqLLaW6JvjP5ziWDIuF0CCac9rSwtYPpa7ahAiZZZv6YAgrjjAd7WILQI1VVKS8vZ8uWLQSDQfLy8igoKECWw9eo9zQbCrpnLrW17dt2sGazmTFjxpCYmIiqqnz44YcxYy+44AIK9tjhS1EUtmzZQkNDA0ajkYEDB2KJMZtFEH4JX2UlFZdfgb+yMqzdMHAgOS++EFa8vPTc8/CsXRu1H8lgIO3ee1E6OvBVV9P66qsxi5H/LDod2sRE1GCQYGNjzDDL8ceT/ex/Q8eNTz1N01NPRcRp09PJm/M6uoyMsPauJUtofuFFXCtXImm1WCZNIvGaq8UufIIgCILwK+D1NtDY+AWBQAcW60ASE44NWxK2k8dTw9atD9LQOB9VDaCRzaSln0Hf/P+L2CFu2/ZHKCt7OqKP5OSTGTL4iVCR7WDQy9Jlp+B2V0Qd28AB95ORcS4Afn87CxdNQFHcUWONxmwGDXyIYLCTrq7tbN0We4dBAJOpD7KsJRh0xayndSiRJA2pqdPQai0E/B3U1cf+rqXTOZg4YRFarVEknA71hBPAQ6W1PFLWvb51QJyRT0cXYNbELi4nCIeLjo4OZs2aRUtLS8S50047jbFjx9LR0UFFRQXvvvtur103Pz+fiy/etaypvr6et99+O2wcsiwzefJkjj322F67riDspHg8tH82H9eSJaDRYDnuWKwnnICkCy/06Nm8mfKZF6PsWVxfoyHjoQexn3ZaqKnqxhu7d9+LIf7c6YCEv66WrkWLIdh7tRBMY0ajz8pGjjPHrCMFYJt6KpmPPBI6dr7/AbV/+UtEokwyGsl5+SXMo0aFtauqiqeoiI7vvoNAEPNRRxE3cQKSHPs9MdDSguJyoUtNjfj3FQRBEATh8BIIdOL3O9Hrk9BoYk/EaG1dQnXNW7hcZRj0yaSln0VK8skRRdrd7grWrruezs6NoTZZ0tOnzx/Izb0mbCZXTc07bCq+I+JasmxkxIhXcMSPDbWtKbqC5uZvo47NbO7H+HGfhcayevUltLQujHkvBf3/hk4Xjz/QTmnpE/j9kd+ddpIkbdiMsYNl9Ki3cTjGioTT4ZBwCqoq56zZyo/O7rWXF6Yn8J8BOXt5lSAcHtxuN8uWLWPjxo14vV4yMjIYN24cuXts4/7qq69GLNP7uSRJYvDgwaSlpZGYmMi8efNi1pM644wzGDFiRER7IBCgpaUFrVaLw+EQ9dWE/cZXVU3LrFl0fPM1qt+PefQYEi+7FNPw4eFxZWWUzbiAYGtkgcr4c88l/e/3ho5r7riTtg8+iHlNw4ABaBzxBJua8G4v7b3klCSRcOUV6NLSkM1mau++B2Is7TP070efjz8O/W4pLhdVt9xC13fh07eNgweT/ex/I5bruYuKaPjPI7iWLQO6C6I7LryApGuuQdJG3z5Y8XgItrejdThEckoQBEEQfiVUVcXpXEZH50a0GgtJSSeg10ev+9rc/APlFc/jdC5HkrQkJU0mL/f3WK0Dw+K8vibWrLmUzs7wMgoGQzojR7xKXFzfUFtHZzErV55HMNgZcb3dl7IBVFa+QsmWf8S8lzFj3sdmHUow6GLr1oeorpkTMzYurgCHYzzBoAunc3lEEflfQiScOHwSTgC1Xh8nLN9Mi7/7Q/8zg3I5K1XsyiX8emzbto3Zs2dHPWcwGLj88svx+Xw0NTXx1Vdf0dkZ+Qf750hKSuL666/f9aVXUVi0aBE//vgjLpcLgOTkZE444QQGDBgQtQ9FUWhpaUFVVRISEtBoIqcJC0Jv8FVW0vTU07TPn4/q9aLv0wfHRRfimDEjbBaQd/t2yqafixIl0apNS6PPB++j3bHzY9Ozz9L42OMxrylbLKh+P6o3+pbJv0TSDTcQN34cuqwsGh55lPaPP44aZxo5ktw35oR+T91r1lB+yaVRx2T77W/JeOjBsCSxv6aGhkcfo2P+fFS/H9luI/6cc0i+4QaxE58gCIIgCD+LovhoaJhPc/N3KKofh2M8aam/i7rjW2fXFkq3P05j0wJUNYDJlEt29mVkZV4U9plFVYOs33ALDQ3/i+ijX787yM25MnTs8dSydNnpBALOiFhZNjJ2zPtYLIUAdHRsYtny02PeS1ra2fTvd9uO5NQKNm76v5ixWq2NYyYuRqs1H3oJJ0mSTgEeBzTAi6qqPrDH+UeBnfv3mYEUVVXjd5y7BPjrjnP3qaoafW/BHQ6nhBPA183tXLB2OwBxGpkFYwrJNxsO8qgE4cBZvXo1n376KX7/ri1JbTYb06dPJzs7O9S2aNEiFixYELMfnU4X1sfeHHPMMeTl5ZGRkcE333zD8uXLo8ade+65DBo0KGLM3333HU6nMzTeY445hrFjx0adFaWqKk1NTfj9fpKSktDr9fs8TkHYSQ0GUQMBJL0+5uw799q11N71N7ybN4fazGPHkv7P+8J2qAu0tlI67XcEotRxkvR68t56E8OAAQQam2h87LEeZ07tT0k33IBxQCGSwUj9w//Gtzl2kcy8t98KzRDz19dTdu55BOojt2Y2jx1Lzssvhc12Un0+Wua8gfOdd/BXVaFNScF+1pkkXnYZstkc9XqK242/pgbZakWXkvIL71QQBEEQhCOVovhQFC8ajSXmZzhVVWlq/pq6ug/x+Zowm/LIzJyBzTYsIrajYyMbNt5KV9eWUFt3vakHcDjGh8VuL32C0tLIh4xmc19Gj3orNOtLVVVWr76IVueSqOPLz7+VPt071R1aCSepuzpYCXAiUAUsB2aoqroxRvyNwEhVVS+XJCkBWAGMAVRgJTBaVdXItQU7HG4JJ4D7ttXwVEUDAEMsJuaO6EuzP4BdqyVJH32JgCAcSTweD8XFxXR2dpKYmEhBQUHEjCGPx8OLL75IU1NTxOutVitXX301ALW1tXz11VfUR/mi+XPEx8dz0003hYqjL1u2jE8//TRq7JQpUzjuuOPC2kpKSliwYAGNO77Y6/V6xowZw5QpU9DGWAIkCL+Eqqp4N20i0NSELisbQ36fqHHe7aXU/PnPeNatC7XpsrJIv+8fxI3f9WHFX13N1pNOjrkEz3baaSRceimB5iY6v/sO55tv9e4N7SNdZibmcePQJiXhWrEC96pVMWMzHnwA++9+B4AaCFB5/fURy/oAjMOHkTtrVljSSfF4aHz0MZxz54Zmk5nHjiXlttswDR0S0Qd073boq6xC0uvRZWaI5bqCIAiCIPwiqqrS1r4Kj7sKvSEFR/xRUQuzAzQ1fU1l1Wt0dhaj1dpITf0t2VmXRBRm9/vb2LDxjzQ3fxNqkyQdOTlX0Df/j0iSfEgmnI4G7lFV9eQdx3cAqKoatcS7JEmLgbtVVV0gSdIM4HhVVa/Zce454FtVVd+Mdb3DMeHkV1TOXL2FFe3dS3m0EgR2/GiOd1i5u18GAy1i+r8gdHV18fnnn7NhwwaCwSCSJFFYWMjJJ5+Mw7FrOeq6det47733eu26GRkZJCQkYDKZWLVqFcEYX7y1Wi233nor5h1fTktKSnjzzTeJ9rd28ODBTJ8+PWo/bW1tuFwuHA4HRqPYwVLYf3Ymp3zl5WiTkjCNHh21WHfru+9S97e7IwqBGwYMIPeVWWji4wFQfD62nnACwcbIxDB079qXeNml+KqqcK9cRdeiRb1/U/tAk5yMfepUtCnJ+MorcL7zTszY5FtvJenqqwBQFYXKq66OOm7JaCTvjTkYd5sRqSoKzS+9RMurrxHckSw3FBSQfMvNWCdPjuhDVVVcy5bTtXAhqArm8UcTN+HoHguoC4IgCIIg9KbOzhLa2lYhywYSE49Fr08KnTsUE07nAKeoqnrljuOZwDhVVW+IEpsLLAGyVFUNSpL0J8Coqup9O87fBbhVVX14j9ddDVwNkJOTM7q8vPcKYx0oZS4vxyzbFEo07c6mkZk3uoCCOPHFUxCge7ZTe3s7FosllNzZXSAQ4MUXX6Suri7inCRJTJ8+Hb1eT01NDUVFRTQ3N/fa2BITE0lJSSEuLo6NGzeGakJFc9VVV5GZmRk6rq2tZf78+ez8G6bRaBg2bBgnn3xy1MST0+mkqKgIp9OJ3W5n+PDhYYk3QehN7nXraX39dTzFxcgWC7ZTTiH+7LMilpy5li+n4pprUff4v6/NSCf31VfR71gqq3g8bJl0PEpbW/QLyjKpd9yOZDSidHbR+PhjqJ7erym1N3JcHLZpv0VjtxNoaqZt7tyYsXGTjiPnuedCx3X3/ZPW11+PDJQkMp94HNuJJ4aagu3tVN1wY6gg+k7GYcO6C6gnhBc6Dba10frmW3QsWIDi8WAaMgTHxTMxDR4cc3yB1laCTmd3gXdRx0oQBEEQhJ/ocE84/ZnuZNONO473KeG0u8NxhhPAvAYnV24oi3l+Wko8zw/OO2DjEYTDXVdXF5988gnFxcWhtvj4eE499VQKCwtDbcXFxbz1Vs9LgGRZRlGUXh9j3759Oe6440hNTaW9vZ0XX3wRX5TdvTIzM7nsssvCluAtX76cTz/9NGz2lCRJnHLKKYwbNy6ij87OTtasWUNjYyMmk4lhw4aRkZHR6/ckCNC9DK/1zTdxLV8BOi3W448nfvp0NHZ7WFzLnDnU/+O+qH0kXHIxqXfs2qa44T+P0PzCC1FjZbudvvM/g0CAQFMTVTfehL+qqvdu6CeImzgBbUoqkk7X48wpXXY2fT+fH5q9VHXjjXQs+DJqrHn8eHJfmRU69tc3UD5zJv6KivBAjYaMB+7H/tvfhjV7t26l/sGH6PrhBwBksxn7mWeScustyHGRRU7VQICuH3/EX12DLj3t/9k77/go6vz/P2e2JZtk03tIAknoTQhYUFRUEEQUBEEQiSAWsJ3end+788565+mdnqcHFmxYQBSQYqGjiFTpEHpCek82yW62zszvj90M2exu9L5f736WfT0eeSQzee/MZz7zmc/OvOb1fr2JGDEiaBVAAMlixVVRgSYmGl1yctC4EEIIIYQQQgjhp40fI+H0vVPqBEE4AMxXFGWHd/kXkVIHcOexc6yp9Xeab4dWgOKRg9CJId+HEEL4d2A2m6mtrSUsLIyMjAzVi6kdkiSxcOHCoCqnIUOGcP3112O32ykvL+eDD4KXIQVPWp3b7f632/ldpucTJ05kkNcQuaSkhLfffjto7KxZs+je/bxvz8mTJ1m+fLnf9vPz8xk3bpxfn7S0tLBv3z7Ky8vR6XT07t2b/v37d+k55XK5EAQh5EsVwr+Npo8+on7BQtXgW4w2ETdrFgl33+2TSqY4nVQ8/LAfKSOaTHR7ZSHGoUPVdebly6l69I+BdygIZH3wvkexVFdHzbPP4Th+PHDsfxiahAS0sTEIOj32woDWlipS//w0Yf0HoDFFUf3U01i2bAkYJ+j15G7dgjY+HgBHcTHnpk5Dbmnxiw3PH0rW22/7GKi37d1LxSOP4K6sUtdpU1JI++tfibjIl8yWLFZq//Y3mlevRrHbAY+fVfLv/scntbAj3PX12I8fRzAYMA4ejBAqohBCCCGEEEIIPxn8GAknLR7T8KuACjym4dMVRTnWKa43sA7orngb5jUN3wcM8Ybtx2Ma3hhsfz9VwmnGoSI2N/rfDHbE2csGEKENlV4PIYQfGvX19bz//vtq1bl25ObmcvPNN/tUlVuyZAmnTgWulJWZmcns2bNxOp2YzWYWLVr0b1XO6wqiKBIREYHBYMBisWD3PtwFQrdu3bjxxhuJiIjAZrOxYMGCoCTYuHHjGD58uLpcXFzM0qVL/ZRWqamp3HbbbYR3SsM5c+YM27Zto9SrtMjKymLkyJHk5OQE3F9dXR0nT57E7XaTkZFBjx49/AivEH55UNxu7CdPgiRh6NkTMYh3maIotO3dS+u69chWK2H9+hF9wwQ/5ZSiKNQ89TRNS5b4bkCjIfXJJ4m5aZK6yvLVV5TddXfQtiX/8VEiL78CqdlM05KlNHflDyeKHp+r/1IV4GAIHzaMyEsuRowy0fzpp9gPHgwam/b834m+7jrAQ04VT7oJxWbzixPCwui+YjkG77WtuFyUzCoIaM4uGI1kf7iUsJ491XWyzUb100/TvHoNeOcjTUICSb96kJibbvLbhrupCfOyj7Du3AmCQMQllxAzZTLaIGnD7oYGLF9/jeJwEj54MGG9egaMCyGEEEIIIYQQ/vf40RFOAIIgjANeBDTAW4qi/FkQhCeBbxVFWeONeRxP+tz/dPrsbOD33sU/K4oS/LU+P13C6dmiKv5REryqllaADwfmcGlc1H+xVSGE8MuB2+2msLCQsrIyNBoNvXv3Jisry6+aVFtbG0uWLKG8U7pOcnIyt956K1FR56/RTZs2sX379oD7i4yMZPTo0dedCrYAACAASURBVNTX11NdXR2UxPq/QhCEgKblHdsxbdo0jEYjWq2WBQsW4HAE9skZOHAgkyadf1A/fPgwK1euDBg7efJk+vc/X7FLkiQ+/fRTDhw44BOXnJzM9OnTie5EGICnr0+cOIHNZiMpKYmcnJyg5FRDQwMHDx6kpaWFmJgYBg8eHPKzCgHbwYM0r1mDu74BfXY2MZNvQp+Z6ROjKAq1zz5H4zvv+H3edP31pD37V1Vp5W5o4OzoMWp1us5IeuQR4mbeitTURPPnn1P7zF+DN04Q0Hfvjmy3ITWZA5I8/w2IRiOGvDzEaBPOkhJcJaVBY40jLiH+9tkIWi1t3+6l/l8LgsZGXXMNGS+/BHj6uPzue7B89VXA2I5VAwHsp05RevtspE7KU01CAlmL31FJr/Zt1/3znzS++RZKB4I/YsQI0v7+Nz+Cyt3YSMOiN2j59FOklhYMeXnEzphB9I03BKweKJnNtHzxBa7qGnTpaZjGjkUTFboXCyGEEEII4ZeJHyXh9N/ET5VwqrA7GbH7OHa563MyMy2eP+akYQopnUII4f8bZFnm7NmznDlzBkVR6NGjB3l5eWg0vtelJEmsWrWKIx3KzgNER0czY8YMkpKS1HULFy6ktrY26D5NJhPh4eE4HA6am5u7JJH+UxAEgauvvpqoqCi0Wi2rVq0K6DkFYDQaeeihh9QUu67It+TkZO666y4fMmnXrl1s2rTJR5kVFxfH1KlTSe7kEfPNN9+wceNGv7aOGzeOYcOGBdxnW1sbZrMZo9FIjLfCWjDIsozFYkGn0/kpvEL4eUBRFNp27qTpo49xlZejTUoiZtJEIkeN8qsQZ929h/J770VubfVZH3PzzaQ8/pgar7jdFF03HmeQQiYxUyaT+tRTgCc17fTIkX5G6ypEkfi77gK3C1dNLS1r1vwfj/i/g/DBg9HEx4MsYdn6ZdA4bUoKPT5dq/pJFV1/Pc4zZwPGGnr3pvsnK1VyqH7RIuqefyHw/ocOJev999RYd10d56bPwFVW5hcbO306KX/yTcNsXr2aqsceV9MFwWMin/rXZ3wM38FTkbB51WqaPvwQZ3ExmthYoidMIK5gFprISL/9yU4nrRs2Yj9yBDHCSNTo0YT17h2siwCQWlsRRDGg71YIIYQQQggh/DcQIpx+ooQTwKaGFu48eo62TubEWWF6SuznH+pS9Dqe7ZVBpEbktbI69rW0Ea4RGJcQw93dEkkLC/khhBDCjwmVlZUcP34cl8tFeno6ffr08fM6OnjwIKtWrQr4eZ1Ox/z581Vi5Msvv+TLL78Mur/s7GySkpKwWq0UFxd3WSnvP4m4uDiioqLQ6XQUFRV1abw+efJk+vXrhyAIHD16lOVBKoFFRkZy7733qlX7Tp8+3aWv1uzZs8nsoGhpa2tj3bp1HD16VG1PVlYWY8eOJSUlxeezsiyzc+dOdu/eTYvX/6ZHjx5cddVVPtUF29Hc3MzevXspLi5GFEVyc3PJz88noouHQ4vFgiRJREVFhVILf0KQWlpoXr0Gx6lTiKYoTOPGBawO5ygqpmzuXFwVFT7rI0aMIOPll3wq/DW89Ta1zz0XcH8J8+4h8f771eXSO+/Euu3rwI0TRTLffgtBr0dubaXy0T8idUFmi1FRIAgeAu3/932hICAYDD4ETyBEXX01uvQ00Gppev8DlCCqTIDkP/2JiGH5COHh1L74Iq2ffhY0NmvJEoxDLgCgbf9+Sm6dCYHmLa2W7iuWE+YtQKEoClV//CPNy/3TLQ29e5P13rs+qij7yVOU3X037qoqn9joiRNJfepJP4P2lnXrqX/9NRyFHq+x8KFDSZh3D5EjRvjtrz0V0bJ1K4rLRfjQocTdOgN9VlbAY1YUBcfp08gWC/ru3YOmLHaMddfVoe/WzU8tGEIIIYQQws8fIcLpJ0w4ATQ43XxU3chJq51YnYZJybEMiDLyaa2Z350up87ZtRlxok7LqiG55BgD+1+EEEIIP04oisLmzZv9VEAGg4EpU6aQm5urrmtra2PRokU0NTX5bScmJoa5c+eqJEdhYSEfdVEpKyUlhd69e2O1WikpKelSZfWfhkajITIyEqvV2qXxem5uLunp6ciyzJEjR/y8tzqiV69eTJs2DUEQcLlcvPnmm1RXV/vFGQwG7rjjDhITE9V1a9euZd++fX6xWq2WWbNm0a1bN3VdeXk57733nl86YmRkJAUFBSQkJPisP3PmDFu2bKGyshLwnLcRI0aQn58fMK2nvr6ewsJC7HY7KSkp9OnTB10Ho+eOMJvNHDlyhNbWVuLi4hgwYECXpJfT6aS1tRWj0RhScP0HIDsctK5bR9v+AwgGPVGjrsJ44XC/86woCuYPP6T+lVdxe69DTXw88XPmEHd7gU+8s7SUc9NnINXX++0v+Y+PEjdjhrrc9NFHVP/psYBtE/R6enz+GfqMDBRJouqxx2kOQvYChA8ZgunaMSguN5Zt22jbvTv4gWu1iAZD0PTDHyM08fGE9eqJEG7EfuwY7gBzRTuMl1xC4r3zESMisBcep6pDRcXOiL9jDkm//jUAst3O2WvHBt12wvz5JN53vpBz45Il1Dz5lH+gKJL+j39gGjNaXeUsL6dk5m1+RJYQFkbGgn/5EVTWHTuoeeYZHKfPeOJ0OkwTrif5d79HE+k7Z9iOHqP6scewHztvwRpxycWkPPkU+gx/Al62WmlZtw5nSSnahARM48ai7TQPtqPdm81x/DhiZCSRo0Z1SXyBpx8RRcTvYTov2+0gSSFlWAghhBDCD4AQ4fQTJ5y6QpPLzRNnKvmwOqhnOgCXx0axbHBgw94QQgjhx436+nqOHDlCW1sbiYmJDBgwICAJ0NLSwoYNGygsLESWZURRpE+fPowePdrHD0mWZd577z2Ki4v9thEWFsacOXNUkqWpqYl//vOfQduWmJjINddcg91up6qqip07d3Z5LOHh4bjd7h/MOP1/C51OR0xMDIIgdEmo9ezZk7FjxyIIAnV1dV0qp7p168acOXMAT+rkSy+9RHNzc8DYtLQ05s6dqxIGJ06cYNmyZQHTIi+77DKuuuoqdVlRFNavX8+uXbt84qKiorjllltIS0vzWb9jxw42btzos22tVsvEiRPp10mBY7PZ2LhxI4cPH8btdiMIAr169WL06NHExcX5ta2kpIQ9e/ZQVVWFwWCgX79+5Ofnq2qzjpAkiVOnTlFaWoooivTs2ZPMzMyAZFoIvlBcLhxnzqDIMmF5eUGruLlqaml8+21aN2xAttkIGziA+IICIi6+2Hd7ikLNM8/Q9O57PuuFsDDSX3ieqFGj1HXO0lKKJ04KSBIJRqPHNNxbAdNVU8PZa8cG9Z5KeeIJYqfejOx00vLpZ1T9/vcB4wDQ6Yi64gpkhx1XRSXOs4HT6X6y0GgwDh2KEGZAamj0IW06QwgPJ/Xpp9GYolBkmfL7H4AgCi5tYiK5WzarVQZLCm6nrdNcoTYhJobcrVsQvd8nbXv3UlJwO0iSX6wxP5/Mxe8geNPEHcXFnJtyM7LF4t+GtFR6rFyJpkNqsuXr7VQ8/LBPZURBpyPpd/9D3PTpPp93lpVRfv8DPpUiBb2ehHvvJX7uHX5zRsv6DTQsWoT96FGPmfyIESTMm6cq0zrCumcP9QtfUfskrF8/4u+804ek6wjZ6cT27bfIbW0YevcJSKS1Q1EUnGfPevzhsjLRpaYGjQWQ29pw19ejiYsLmGIZQgghhPBTQYhw+hkTTu349YlS3q/qmnTad3Ff0kOpdSGE8LOHzWbDYrEQGRkZVJ3idDrZsmULBw4cUBU4ubm5jB492sdHCmDr1q18FcDYV6fTUVBQoKaSKYrCa6+9FlAtBL4ki8vl4qWXXqK1k+dNR+Tm5iJJEq2trdQHUG38GKHX69FqtSiKgu07DJ/z8/NJSEjAYDCwcePGoGmOgiDw4IMPqqThrl27WLduXcBYo9HI/fffrxI+p06dYknnqmxeiKLIXXfdpfpfuVwu3nrrLao6KSHAo8qaO3euD3G5Z88ePv/8c7/YxMRECgoKfBRUZrOZDz74gLq6Op/Y3NxcpkyZgsFg8Fnf1tbGvn37OH36NLIsk52dzbBhwwIaySuKQlFREYWFhTidTlJTUxk8eDDGDqlp7ZAkicOHD3P48GEsFgvx8fHk5+f7qAU7w2az0draSkRERJeqMIDq6mpqamoIDw+ne/fuQRVn/0koikJdXR0ul0sdX8FgP3GC5rVrkZrMGHJ6EH3jjWjj4/3ibAcPUvHIIz7m4brMTNL++gzGIUN8Yi3ffEPFAw/6ERFxBQUkPfJblSxQJIniiZNwBCmOkHD/fSTOmwd4FGFnrrgSKYCCE0CMj6P70qUgy0gWK6Vz5iAHIXsBom+ahDY2FqnNhvnjj6ELAlzQ60FRfMzHf+wQjEYPiSQIARVvHRFx2aWEDxiIGBFB07JluEqDG8Qn/vY3RAwZgqIo1L/2GtYvAxu+AyQ+9BAJd84FPKRl0fUTgqY5dlv0OpGXXQZ4znXR9ROCtiP16aeImTxZXW589z1q/vIX/0Ctlm6vvkrkpecVXK1bt1J+730BCbXkRx8l7tYZPuvMq1ZR+7e/nzeqFwSirr6K1Kee8iHTAGyHD1P9xJPniUNBIPKKK0h5/HF0yb7fqe6mJmqff56WtZ96+kSrxTR6NEm//Q26TmncAPbjx2lashT7qZNoTNGYxo0j+rpxAYlnRZax7tyJ7cBBBL2eqFFXYuhiflMUBamxEWQZTULCD/oCQPFeN4JOF3qxEEIIP3OECKdfAOH02OkKXiuv6zLmzf5ZXJfoK0c+ZrGxrbEVAbg8Loo+kaHUiRBC+CXB5XLR0tJCeHh4wAd08Nw0Hj58mB07dlBTU4MoivTq1YvLL7/cz+Oovr6ed999V/U3akd0dDSzZs3yUcl05VHVt29fbr75ZnX57bffpiSI0TLAyJEj6d27N6Iosnv3br/Kdx2RkpKCXq+nqampS8Lrx4Tw8HBMJhN6vZ6qqqou0wt79OihqpyOHDkSVGUFHq+q/Px8dDodxcXF7O4iHSo/P5/x48cD0NjYyMsvvxzUqH7w4MHceOONgGf8vP766wGJLIBBgwYxceJEdbmhoYHFixf7jSGDwcCMGTN8/Lfcbjcff/wxJ0+e9Iu95ZZbyM7O9oldunQpZwMoZS699FKuvvpqn3UWi4X169dz7NgxZFlW1V5jxozxq3bY3NzMypUrfcao0WhkzJgxDBo0yG9/jY2N7Nmzh7KyMrRaLb169WLIkCEBlWEAra2tnDt3DkVRyMrKCki8AZw8eZKNGzeqBK1er2fo0KFcddVVfh5xiqJQUlLC4cOHsdlsJCQkMGTIkICVHCVJYv/+/ezbvp0WqxWTMYIhl45g6NChfoURANzNzRR+9BG1FRWEG40MvP56ovr08YtzVVVRPv9ebIWF2MPC0EgSereb2OnTSf7971Q1DXhULGUPPURVSjLVyZ55J6W6mtSaGjJf+idRHVSAje++S81fngnYRxGXXEy3N99UH4Brn3+ehkVvBIxFEOjx6VoMOTkobjfVTz2NedmywLFAxGWXYRw6FNlqpXXTJpwBVKQdty0YDB7C4Wd4740goEmIR2OKRm5pwV0X/B5Vl5lJ7NSpIAjYjh2j9bPgnlqaxETS//43xPBwZIeT0tmzgxKG2m7dyFm7BkEUUSSZM+PGIlVVIwMtJhOKIGBqbUUjywhhYeR99SUa77XVsm49FQ8+iC0sjIr0dNw6LbGNTSTV1mIcNIisJR+cV3udPk3x1GkBDf712dlkL1+upiPKVivnbpkekGjVpqXS/aOPfNIMzcuXU/Wnx/x8w4z5+XR7/TUfzzdXdTXl8+ZjLyz0iY2+4QZSn35KVb21o2XjRuoXLMRx4oSnrbk5JNx1N9HXj/drm7uujoZ33qF14yYUm42wAQOImzWLiAuH+8W6amupX7CQlrVrkdva0KWnEzt9OnGzbvPzIlMUBev27ZiXr8BVWYkuJYWYyTcRMXJkQJLKXV9P86pVOE6fQRMTg2n8eMIH9PeLa4f95ElsBw8hhocRcdll3+lH5q6rQxCE70W+SWYzstOJNiHBr5BEZzjLy5HMzeizMkMVLUP4WSJEOP0CCKc3yut49HTFd8aNjI3k1rQELouJ4IETZWxo8L2hH5cQzct9M4kIcAMZQgghhOByuRBFMeBDZjvsdjsHDx6kqKgIgJycHAYNGhTwYfrAgQNs2bJFJX60Wi1Dhgzhmmuu8VGHFBcX8+677wYkOBITE7nzzjvVeKvVyqJFiwL6OCUkJHDHHXeobVm/fn2XaYBGo5H+/ft7UpBqaijt4u2/IAhkZGSoFey6Inp+qhAEQT33XZFegiCQk5ODTqfDbrcHTN/sGDtt2jTi4+MJDw9n6dKllJeXB4w1mUw88MADahs2btzIN998EzA2LCyMBx54QFX5bd++nU2bNgVtR0FBgUpQ2e123njjjYDKuqioKO68806ivA8NLpeLV199lYZ2FUQn3HLLLfTyGkkDnD17lqVLl/r1X2xsLAUFBT5kkiRJrF+/nm+//VY1tBcEgcGDB3Pdddf5kEgnT55k6dKlAdvQt29fpkyZoj5AybLMmjVrOHjwoE+cKIrceOONDBw40KcNH330kR+pB5CXl8e0adN85oO6ujo+/vhjn1RVg8HAtddeywUX+KY4KYrC7t272fHlV7TYPYrArNRUrho71odYBA/p9t6iRdR2IiKTo6OZOXcukR1SkhRFofq119m3bh0VSYlIGg3xDQ0MSksj76mnfB74ZJuNkrlzKaqopDQrE6dOT3RLMzlnz5Lz4K98VC+uykqKb5qM2e2iOLs7NmM4EVYr2cXniImMpPvKFapCrG3/AYpnzOBsTg5nc3NojYpC73SSfe4cvY+fIOOeu0m45x6PV9GePZTOKqAyJYVz3bNpjYrC4HCQWlVNVkkJkQkJJMy7B8Vmx3HyBOaPl9MWFkZzTAwunQ5RljC22YhqbUXndhPWry9otMitrTiLi1EAWRSRRRFRlhFlmZDexAOXVosiCOhcLgw9emDIzUU0mWjdtIlKg56K9AwcYQbc3jijxUr/Y8fodvddGAcPBkGk/pXzKXpt4WE4dXrCbTYMXiIs7o47iLlhAmg0mFetovH1RQAogCSKHsLL256O3l7O0lLOjh0HkkRrVCTm6Bj0LieJtXWIikLc7Nkk//Y3nm3JMsWTJ6sm8p0RV1BA8v88oi6bP1kV1GMs+Q9/IG7mreqys7yckhm3YrZaKE/PQNJqiGto9BC+jz9G7LRp5/uztpaSabfgqqz0HJ9Gg0aSEICoMWNI/8cL56t2Kgo1Tz1F05KltERF0RZhxGhtw9TaSsyUKaQ8+YQP6dO6ZSsVDz2EbLerY1kAYqZNJeVPf/IhfdyNjZT+5jccr6yiMT4OjSSRUV1D74kTSbzvXv+0zM8/p27hQrUSpiEvj4T58zFdO8avf6q2bWPbyk84p9chiyIJVisXDhjIgADpnraDB6l+5hksR4/h1moxALETJpD0yCN+nmgAksVC6/oNuGqq0aenE3XNNT6kYmfYT57CUXQWbVw8xvyhPiR9R8gOBy3r1tF65Chao5GYMaMDFrVQ++PgIcp2fIMgasgedSXGnj2DxoJHtac4XWiTEr+TqJNtNuS2NjSxsd9J1P2n4HK5MJvN6PX6oC9w/tOQJImKigqcTifJycnqPcVPFSHC6RdAODW63AzZcQy7/P3On14QcAY515OSY1nYN3DlkhBCCCGEHxqSJFFZWYnb7SY5OTmo0urEiRN8/vnnPsqX3NxcbrjhBr8v6paWFtavX8/x48eRZRmNRkP//v0ZPXq0T1pUU1MTCxYsCEqe3HTTTQwYMADwkBD//Oc/g6bKXXTRRVx77bWAJ2XxhRdewB6kspbJZGLy5Mk4nU7MZjOffvppkN45H68oCg6HA6fT2WXszx2pqamYTCYANeUuGJKSkoiOjkaWZUpKSrokyeLj4xkwYAAGg4GysjIKO6kEOmLAgAEMHz4cWZY5efIkO3bs6LINt912GwaDAVmWefHFF4OOoby8PGZ0MPf+4osvgqrOOqvIFi5c6Jey2BF33HEHGRkZQNdpmaIoMm/ePNXUfv/+/axZsybodsePH09+vuce0263s2DBgqDKwenTp9Ozw4PLunXr/LzIwFMsYObMmT4KtXfeeYdz584F3G5OTg4zZ85Ul61WK4sXL/bzZ9PpdEyfPp3uXs8p8Mw/Hy9bxolOihNREJg4aZJ6/bfjq9Wr2bp/P3R4qBIUhdGXXMLFY84/nCqKwpJnnuF0gOs12unkzt/+lghvWpaiKHz2yCN8G2DuM9jtzLjySjK921Zkma2zZrGtRw+fNgCIksToujouevVVdbunJ09ma1w8VekdvN0UhdjGRi7du5e+ixaBICK1tnDy93/gSEYGGllG53KhkSQQQEHA1NJM3+49MA4eBIKAedVq2srLqUpJwRwbAwiE2e3ENTYQ19iEPjkZTWwscnMzrpqagGlsIQSGGBGhqt/cbW20GY04DAYkjQZZFAEFU0sLRrcb07XXIhg8PmCWrVtxabU0xsVhjYhAEQSiWlqIa2xEq9GQ9pc/o4mJBY1Ixf0P4GxroyotlZrkZA9xUl9PZkkpWr2evG1fofXOs6V338O2lhbO9MzzaafJ3MxlO75h8Nq16Lyp2VV/eoyqNWs42r8/pVmZuHU6jFYruafP0OvkSTIXLiDqyisBj2rx+B8fZe+wYdR3KM4RX1/PsD176fPYn4i+7jrAQ/aeHncdp7IyOZ2XhzUyEq3LRWZpKf2OHqP7ww+r5LAiyxwsuJ0NKcnYOl1TaRUV3HDxxSTPnq2ua1q6lOonnvR81ruu/cpKeepJYqdMUWNLN2zg/a1bcQZIVb5Cr+eKDr50tmPHOHrnXRzu1ZPyjAwUUSTMZiP3zBmGhIXTffE7Poov8+efs+u11yhKS6PNaCTCaiW3sooLf/UgJm+ftcNeUsKWZ5/luCGMVpOHoO5RX8+o6dNJ7ODBB2A7cYKtTzzJ8dQUWk0mBFkmraKSixMT6ff0Uz5tcDc38/ljj3M0PEw9xjCbjaEIjHricTSdXhxad+zg+GuvUWy1IosiKQj0nzSJuKk3+xFPTUePsuXddzmrKLh0OmKtbQzt0YPh9873I8raGhrY9PrrHG9pwabXE2W3MzAllSvvvgttJ6sIRVFo3raNw6tW02RpJcJgYOCoUSROmOCnqHO73Wz69FP2HzqkPgenmkyMvekmMgNU7WyprWPnR8soq65GFAR65uQwbMoUDEHS6x1FRVi//hrFLWEcPozwTt8f7Th69CgbNmxQ72cFQaB///5cd911AV/ONjQ0cHDXLixNTSRkZDB42LCAKf6yLHPo0CG+3b0bs9lMVFQUF+TnM3ToUD+VM3heEO3Zs4fa6mrCjEYGDBhAnz59Ar5UdrlcHDt2jNLSUlWZ3aNHD/U8hwinXwDhBLCqpon5hSV0/kpPN+iYk5HAp3XN7G/57lLoArD7oj5khp+fTCVFYWN9C1/UN+OQZYaaIrg5JZZonf/gDSGEEEL4T0GWZUpLS7Hb7SQmJhIfwG+mI+x2O1arlcjIyKBeNqdPn2b58uV+1eSuuOIKrrjiCp91JSUlLF261I9Iys3NZerUqT6qrGPHjrF8+XI/VZZWq2XGjBk+D73Lly/n6NGjAdsXFxfH/Pnz1RuAN998k7KysqDHPHbsWLKyshAEgW3btnGsCzPiwYMH06NHD1wuFwcOHAiqLAKPYig9PR1JkmhoaPjJpCP+mCAIQtA0xHYMGDCAyMhIVf3TVXxOTg4ul4vW1taAFSo77zssLAy9Xo/FYkHqggAwmUzExsYiSRI1NTVdmvwbDAZyc3PR6/U0NDR0qQJMSEhQlVmtra1dVsuMi4tj7NixuFwuamtr+fLLL7s8vokTJ5Keno7RaGTt2rUcPx5Y6REeHs6vfvUr9F7/m6+++oqtW7cGjBVFkfvuu09NMzx9+nSXhQNmz56tKrO6UpwBXHzxxYzxkkiNjY289NJLQWM7EpFOp5Pn//53HEGI51iTifsefBDRqxxY+c5iDp8r9qTtdXoATAgLY95vf6vGvvXsc5TaAtwnej87c8oUcryqiLpt23hnzRqsAcyu4xsbmXP//Ri93kHVb77FsoMHsEZGonc60blcCF51isFu53JBJGPuHSArtOzZzYa939JmNCIonhhRktFKbjSSRG5dPT1vugnF4cBx8iQnjh1T0zE1koTQ4XqJam0lbcAARGM4bouVs+fO4dZokDQa3DodCqD1fk6UJbSKgqIoqhpMEjXq34KiYLS1Ed5mw9jWht7p/PmrxLRaBI0Gt8uFW6tFFkUUQUARBARFQVAURFkmTBQ9flKShOQlHtrPg9ChPwH0oogmLg5Bq8XV0ECzTouk0aox7dtHUUh2OonKH4oganCcPUNFcwuWyEhPjCggCyKKKCBKMt0bG0mafguiToe9tIxdJ05gDwvzxKnnU0TWaEitb+DKhx9CNEaguN2U3XUXDWFhnM7LpTYpCUUQSKyto+epUyS4XPRYvRoxPAwFWPzYY9R2GvNqX0gSsydOJDo7G0VROPnoH9kSFYlbp/MwWQK4dDqcej2plVXcPGUK0V4FlfXAAZYuWEC596VAR2SfK2Ha739HmJesd7e08N7//I6SpES/2KjWVgpuvpn4Cy8EPGqij+69lxMdKuiqp9flYlJ8PH0fekhdt+LXv+ZIEAP7i2WZMU8+qS43bdrMig+X+rU5usnMjd2z6f7gg+q6+kOHePuDD7AGINUvcLqY8OenVeKiraGRRc89S1MAD9IMu53bn3wSjXf+VhSFg399lvXmJuwd4jUuFyOsVq587jnV60xRFN5fsICzHZXL3rlNVBRm3XYbWTnnC2ydVrRVEgAAIABJREFU272bJavX4NT7pqKa7A5unz+P2A59KjudnH30UQ4UF1OZmoYiesbQoMgI+v7tb2qqLnhenn744Ye+B+ZtR2a3bhTcfrs6JwNs+uQTth865BOuVRQmTphAv6FDz7dBllm+dCmFp0/79Vt2aiq3zpnjQzodPHiQ1atW0fkOIzstjRm33+5zP1tXV8d7b75JS6d73+y0NKYXFKDX60OE0y+FcAI40trGG+X17G+xEi6KjE2MpiA9gVgvMXTMYuOx0xVsN/tXFemIgrR4fpWdQrJBR6tb4tbDRexu9q1QE6/TsmRQDwZF+U8esqJwoKUNs1uiZ0QY3UJm5SGEEMKPGHa7ncOHD9PQ0IDR+5YnUFU28BhaHzx4kIqKCvR6PX379iUnJ8fnBqEdJSUlfPPNNxQXFyOKIrm5uVx66aWkdqpeZLPZWLJkiR+RZDKZuPXWW32M3EtKSli8eHFAZU9WVhazZs1S22I2m3n99dcDGpJ3Ti+sqanh1VdfDUpwTJ48mf79PV4Z9fX1/Otf/woYBx7SZOLEiciyTENDA6+88krQWL1ez8UXX4zD4aC+vp4zZ84Ejf3fQhTFLpVQIfyykJycrKZRnD17tkvyLTExkdTUVARBoKioqEuiNSIigoSEBNxuN/X19X4kdme0k+But7vLNrTHCoKAJEnfWeUzMTGRyMhIRFEM6FnWEUlJSYiiiMPh+E7S0mQy0aNHDyIjI6moqAicKut9cLrmmmsYNGgQiqKwf/dutm7fHnS7PbtlMuLqq5AkiTOnT7OjqzRnrZZrvKoXc1MTX23bFpBMAw8R0CM3F4fDQXNz8w9KkouyjEmvJ1KjwVFVhSzLaF0udN4fQVYQFRlBVsgQBboNHoygKFR+s4MaUVD/L0qySlgIKEQ5XcRkZuJ2u7FUV2MTzlt8CYDW7ULv9OxD63arSjTxB35+UwC3VovDYMCp1+M06HHoDTgNeiTRkyandbs9RKDb83egdaIsowjnSR9ZFM+TQILou95LZLWnfao/HfpJTQlVFA9xqNXi1mlxaXX+v7VaXDrPb73LSWSrhajWViItFvT/ZhEABbAZw2mOjsYcHUNzdDTN0dG49Dr0TicGhwO904ne4fnb4HSof3f8f3vbZVFE1GoJS09HYzLRVFlJTViYh5Ay6HHpPOl6CqCIIr0QyJt4I2i0FG/dwv5WC4rg+Z8idBhPskyGzUb+xEkoLhd1e/awv6YGUZYQvTGKcJ6sC3M4GHjllShaLW119Rw4exZZFFBEERkvuYji3YfMhAkTCE9LA52ODS+8QHFyMpIoInlTThXvdRhtNjP7wQfRe30+lz3xBMXee432mHZiURFFbhs5kqyrrgJB4PPnnmNvIBWw9zq/NjubiwoKAKjcvJm3tmzxkHoB4m9MTmHwvHsAOHPoEO9/8knQ+SJVELjrsccAcDud/OOxx7AGeVGZ4XRyR4diBYWPP85qux1HJ3WSxu1mdGMTFy5c4G2SwoLnn6c+QHXP9nZ1VAIf/OorVm3dGrDNoixzz913k+j17Dy8dy8rP/vMP9a7fPUll3DpaE81zsbGRl7+5z/Vc9E59uKBAxkzaRLgUQG//OyzmDu/5PDGDuzRg0m33RYinH5JhNP3wfLqRu49HvwNZEfkGg0oCpy1Bb5pSjXo2HVRHwwdHrQ2N7Twh9PlnLN5BqYAjE4w8fde3UjU//cr9oQQQggh/BQgyzKnTp3i5MmTSJJEVlYW/fv3D6jMKioqYt26dWrKkEajYdCgQYwZM8Yvvr6+ni+++EJ98BRFkT59+jB27Fgf3xuAw4cPs3r1ar+H35EjR3LllVf6SOS//vprNm/e7Ne2mJgYZs+eraa9QfDUKYAJEyYwxFvpTJZlFixYENQPSa/X89BDD6HX65FlmcWLF3ep9pozZw4ZGRkIgsDmzZv5+uuvg8aOHz+elJQUHA4H27Zt69KkPjY2lgsuuABRFKmsrOwy/S48PJxevXrhcDioq6v7wSsu6nQ6jEYjLS0tXaqhkpKS0Ov1OJ1Ov1SzzhAEAZ1OhyiKOJ3OEFkXQgg/cmgEAdmr8FJ/ZMWjElMUREUhMT4erSDislpobmlR/y8qCrIgIGm0KoEjeRVNP1cY7HYPAWVp9SGiolpbkUWR5pholVQ6Ty79cC/PBVn2I9d8CDVZ8aSzesm6djKnqx/Ah0BQfuDzJ8iyqlDUOV3oXU50XvJT73IiSjKSRoOk9agIO/64tVqf5fZj8mtzp3Xtv7VuNxq3G62X1NQLIjpZBpsNQZYQJQmN5Ok7t1aDW6vDpdMia7SI0SacoohNlnF52xKIcEJR0AI6RUFwu1G8hGn7j0aSPG3wkqs5ySlER5swSDKn9uzGoTdAB+WdJIo49XpkQaBn//7YoqJoaGujpLoat0aD0oUXapheT5heT5vZ7MlcUklpb9qnt/1ROh3xGRm4JYnqigo1TbBjTDs0QEJyMoIg0Go2Y7Hb/fvBuywqCr379UMURVqamiitCO4TLcoyDz/yCJGRkSHCKUQ4nUetw8WQncdw/0Cn+pm8dGalJyAKAjvNFqYcPBNw230jwvgiv6cPOeWUZZZUNbK0qoEqh4tuYXpmpMUzNSUOTRDTuXqnm1K7g3idlqzw4OWeQwghhBB+zlAUhbq6OhwOB/Hx8UG9r9rR0tKCxWIhOjo6YO5/x7iDBw/S2NhIREQEgwYN8lFYdcTp06fZtWsX1dXVGAwG+vXrx0UXXeS3fVmW2b59Ozt37lT9i2JiYhg1apSPQTXAuXPneP/99/38lgRBYOLEiT7x5eXlvPPOOwG9mTpXv3M4HLzzzjsBq+UNGjSIG2+8USXUKisrWbRoUVACZ8aMGeTlefxMnE4nCxcuDGhSDzBlyhT6edOQrFYrL7zwQlA1S3JyMrfccgtutxuz2cwHH3zQJYk0b9489dx0RajFxMRw7733qnL6VatW+RmGd8TNN99M3759ATh06BCffPJJ0NgJEybQr18/nE4n+/bt6zL1LTY2lvz8fNxuNxUVFZwKUKmrHaIoMmrUKHVcf/bZZ0H7TaPRMGLECJxOJ01NTQENzv9b0Ov1aLVaXC5Xl0okjUZDeno64Lnmgo2fdqSnpyOKIna7vUuvLvCoob6Paqq9HeHh4ej1esxm88+SXAwLC8NgMHxnIYeLLrqIhIQERFFk69atgRVR3rf6SUlJapXX79PPIfz8oPEqzNxabXASI4RfFHReNZsiCLQZjT846fdjQlRzM61epfD00WPoNeKSEOEUIpx88diZCl4rC3zDMi0llj6R4ewwW9jeZMEqfffNh04QSA/T0eSSaHYH/+L9V59MJqd40lRcssJtR4rY2uj/hX59Ygyv9svyIZ0anG4ePV3O2jqzSmgNM0Xw557pDAyQ1ldmd/JuRT2HWtswakTGJ8YwISkGfZCLv9nlZn9LGxpBYGi0sctKfYqiUGJ34pIVssMN6MTQl0wIIYQQwnfB7XZTV1eHRqNRH+wCobq6mu3bt3Pq1ClkWSY7O5sRI0b4+F61o6Kigs2bN6tVESMjIxk+fDiXXnqp3/YdDge7du3i8OHDWCwW4uPjyc/PZ/DgwX6xBw8eZO3atT4Pk4IgcPXVVzNixAif2MbGRj7++GMfMkuv13PNNdcwbNgwn9i9e/fyWYCy73q9noKCAtLSzps7f/755+zZsydgHw0cOJBJXsk7eGTvq1ev5vDhwz5xMTExzJgxg8QOprxms5lFixZhtfqmyoPHj2z69Olqf8iyzMqVKwN6jPXt25fJkyersQ6Hg1deeSUgeSIIAjNnzqRHjx6Ax4D0xRdfDNgGgCFDhjBhwgR1+ZtvvmHjxo0BY6+99louuugidfn111+nsrIyYKwgCDz44IOqEf+KFSu69DmbOnUq2V5Plq+++iqoiTvA1VdfzaWXXgp4xuWiRYu+V6zFYuHFF18MamrfkTyVJImXXnopKHmSlpbG3LlzEQQBt9vNiy++iCVQ+oYX999/v5o+3JWflV6v57777lPbu2LFii7VerGxsWRnZyMIAqWlpV3Gmkwmhg8fjkajoba2lgMHDnQZ2+591dzczIYNG4LG6nQ6HnroIbVS5YoVKzhy5EjA2IyMDObMmaOSzoWFhUE9xrp168btXp8VRVFYtGhR0PEGkJKSQnp6OrIsU1RU1CXxFRERQf/+/dW+6CrFOCwsjFxvymB9ff13pkT+O0hKSiIxMZHw8HCOHjmCPVB6qJd8u+iii4iKisLpdLJr586g/mLgSffMyspCFEWKioq6HBfxcXHk5OaqRR+6Ilr1Wi0jr7gCg8GAzWZjy5YtQVOndIJAXFISjY2N35meCh7yOykpieTkZM4dOEBzkO8tjdvN7LlzUTQarFYrK5ctw9EFgRsdFUV0bCyyLFNdVobbq2BRfci8y+3Km6joaERBwNrUhMvrgSYoilfOcv5YRSA5PR0EAUuTmWZbm0cF1b5jb+oiXuVbXs+e6DQabM3NnCsv9/HfalfUtKuooqKjcSkKdrcbhyx/L0JFoyggSX6pkuePD7QC6A1hCCjY29o8llcKnnQ+QBY1qnpK9qqU/NLBgqA99VTndPopsxDArdGqqiy3Vuu3LHWlivpfQOtyEWGxEmm1EGG1YnA4cGl1uPQ6nDq9z+92zy/5O6rHi6oX3Xk1lijLKHj8zjoq4QIp5lTF2b9xjCazmRZv4YuZF19C7rVjQoRTiHDyhaQoPFdczaKyOtq8k2GERuTOjER+0z0F0Tvgjra2cfW3wd9A/rtI0Gm5Kt5EtzA952x2ltcEf6O3oE8mN3nJKZskM37/KY5Z/Cs+RWpEPhvak14R5/NmNzW0MOdoMY5OVfuGmox8OCiHKO35C9ctK/y5qJK3K+rVKn9RGpF7M5O5PyvJr8rC53Vmnimq4nSb58s3Ua/l7m5J3NMtUe23dljcEu9U1LOm1kyzW6J/VDiz0xMYEetf+lJRFHaYLXxU3USt00VWuIFbU+PoH4BMAw9BtrLWzLk2B4l6LZOSY0kL4pUlKwrfNFk4YrERpRW5NiG6y/TGU1Y7u5staAWBK+KiSDUElxG3uiV2mS1ICgyNNna5XYcss9tsxSJJ9I8M9zGm7wxFUThhtVPndNPdaPhOH7AGp5tKh5NkvY4kQ9epm1ZJoszuJFqr6fLYwDM+yh1O9IJAqkHXZblXRVGod7lRFM+4+K7SsFa3hF1WiNNpvjPWLsm0ShJxOm1Q9V87JEXBKslEaMTvjAXPedELwne2ATz9IQr4jfVAkL3fJ98nVlE8NxXfJ7Y9/vu0N4RfDmw2G06nk6ioqKBE1r+LlpYWDh06RFNTEyaTiYEDBwb19lIUhbKyMmpqaggLC6Nnz55dGtXv2LGDsrIyNBoNvXv35tJLL/UhhcBDLnzxxRfs27fPR+k0aNAgxo8f72Pq2Y7q6moKCwtxuVykp6fTu3fvgNVpmpqa2Lx5M4WFhciyjNFoZOjQoVx++eV+8bIsc/ToUQ4cOIDZbCY6OpoLLriAAQMG+PV1U1MTK1eu9El1jIiIYNy4carSqx3l5eV88MEHflX7srKymD59ul//7du3j23btqkP7DExMVx++eVccMEFfttdvHhxwIfJjkQPeMjCN998MyDx1a9fPyZPnqzONVarlUWLFgUk1Dp7ogFs376dTZs2+cXm5eUxdepUn34+evQoK1eu9FMYJSUlUVBQ4KNgLCsr4/333/fziIqMjGTWrFk+46grhdqwYcO4zuuHBJ7xtnLlSj8CTq/XM23aNJUsBDhy5AgrVqwIuF3wrYr4Xf5wHVWAkiTx+uuvU1NTEzC2o5ccwLJly4IaxHcu/OBwOFi2bJlKTrcjOTmZGTNm+KQBgyfFeMOGDSphJwgCffr04frrr1dJLIDjx4+zbNmygG3QarXMmzdPnTeKiop49913A8YCTJo0SVVxOhwOFixY4FOZtSM6VlG1WCy88MILQRVqKSkp3HnnncB5sjdYtUy9Xs/DDz+sXn/Hjh3j448/Dhibl5fH9OnT1WukK2JYEATuueceVZVZVVXFokWLArZZFEXmzJmjqgDr6upYuHCh/xjykkqjRo1i5MiR6upVy5dzsCNJ7o0TgOlehaqiKBQWFgY9tvbP3H777WR5K5g1VFTw1sKFWDvNvYIsc8OIEQzuUKly27ZtQYkvjSBw3wMPEON9YP9m/Xo27twZ1H9nwtVXM8Q7b5UWFvJWIDLUGzsyN49R3op9bW1t/OOvz+Lq/GLcGzsoI4OJd9yhrn798cfxo069sXkGAzN+9zvAc53+7YknsAf6rvXG3zV3LsmpqYiiyMfPPMOxYJ52isK8mTNJ8hYZ2Pbqq2yprg7aFzfn59N3/Hjcbjcb33mH3V0UO8nT65nhrRzYtHcvr6xYgUuvV9NO27crAINaWxn3l2cAhdbycha+8855Qq0jCadAmNPBiGuvxWa3U1dTy6niItVvq50Ia0+/07lc9ExMJDcri0hZ5sy771GZkODjq9XOBuqdTgakphDeqxcoULhlC80GT1qeJGpQRMHDLXrTL9MFyB42HEEQqPzmG0o0ompQr7ZX9vjDJdsdZAwfDpJE49GjVLic3rTa8x5yipfkC3M4SO7ZC0VRsNfWUuM1C1e815AiwJEBA2iJiSGytZX7HvwVYZndQoRTiHAKjBa3xL5mK4IAQ00RPkQMeG6gr9h7kpPWwKW9AcYnRtPkkii1Oyiz/3smfF0hSa/lhqQYTFoNp60O1tQFJ6duSIzmtf6eN9+NLjfDdhYGVWbNTIvnb73OVxZ45GQZiysD+4X8tnsKD2WnqMurapq4uzCwr8ddGYk8kZeuLje63Ew8cCZg3z2Rm8Zd3c6nqCiKwiOnynk3QDv+mJPG/EzfdJYN9c3cU1jic4wab+zdnWKL2xzcfrSYEx3aoRMEHspO5sGsZJ+H9la3xH3HS1hXf/7mRgPcnpHAE7npPuSFoii8cK6GBaW1KmmpEwSmp8bxZF66T+okwLKqRp46W0m9y/MGVwDGJkTzfO9uqql9O/Y1W/ntqTIfgnFUXBTP9+7mRxDVOFw8erqCz+s8ec4CMCrOxNN56XQ3+j6s2CSZvxRVsqSqUe27i6IjeCw3nQtMvsSerCi8XlbHq2V1VDs943pAZDi/65HKqHjfm1KAL+rMPH+uhqMWzw1cT2MYv8pOZmJyrF/swZY2ni2u4svGVhQgM0zPXd0SmZ2e4EeilNgc/LWoik/rmnEpCvE6LbelxfNgdrJfH7e4JZ4/V82yqkbMbolorYapKXE8nJ3sV1FSUhTeKK/j7Yp6ztmcRGhEJibF8nD35IAk3KqaJhaW1XK41YZeELg2MZqHspPpHeFfRWRHk4WXSmr42uxRL46MjeKBrGQuivGvfHLCauMf52pYX9+MQ1YYYjIyLzOJ6xJj/GKrHS5eLKnhk5ommt0SvSLCmJ2ewMy0eD+iyipJLCyt5cOqRp903TszEgnT+PabpCgsrqhncWUDZ9vsxOu0TE6J497MJL+xCbCm1syisjoOtrYRoRG5LjGa+7OSA6b47jZb+FdpLd+YLYjAqHgT92cmBSSST1rtvFxSw4aGZlyywvDoSOZnJjEyzp+grnW4eLm0htW1ZlrcEv0iw7kjI5Ebk2L8xpBVknitrI6PqhupcbjJDtdza1o8s9IS0Ha6CZUUhfcqG3i3op5im5MkvZabU+K4u1siEVr/N2yf1pp5o7zOQ2ZrNExIimF+ZhLJAUjfXWYLC0tr2eUls6+Jj+bezCTyIvxLAJ+02vlXaQ2bG1pwKwqXxEQyPzOZYdH+qYA1DhcLS2tZW2fGIkkMjDQyt1siYxKi/WJb3RKvltWyvLqJOpebnHADt6XHMyPVfwy5ZYV3K+t5v7KBUruTFIOOqSlx3JGRSHinMaQoCqtrzbxVUc9xiw2TVsONybHM65ZEvN5/DO1osvBKWS3fNlvRiwJjEqKZn5kUcAwdt9h45UQxZ4qKAZms7Gzu7pMTsGBHpd3JwrJaPqtrxibJXGAycle3RK6I85+zWtwSr5XV8kllHa12O5nRJgoykpicHOs3hpyyzOKKBpZUNVDhcJJm0HNLahwF6Ql+85CiKHxSa+bDY6cwN9SjM4RxWZ9e3JOdErCy7ebKOlbs3I29phq0WlJz87jroqFkGv3HxdHWNhaUVLO/ogYBGJqRyvysZPpG+s9DB0vLWLFpC5SVoFFk7LEJDLv4Ym4aPtQv9lxdHe99vh7XuSJERcYZbiR70AXMuWaUX3noerOZN9Z8RlvxGY/3jSgSlduTO68fT3SU7xynKAqLDxXy7bd7EVqakQ1hZPbpxz2XXogpwMuZtaeK2LRzF0p9HbJWS3xOHndcPoJuUf7jfmdFDcu/3o6zsgIEAWO3LGZcfikDE/0J0S379vPl1q2IFs+cLOn19B6Szy2jr/YjDOsdLl7Zf4SzhYXgcmKIT+C6C4cxIdO30IGiKHy+fj17O3uzCQJjx47lwuHDfVYfPnyYT1avRumUfnbJyJGM7lTGvbW1lTc++pjmsg5+o3oDV159FZd32q7T6WTxyk8oP3GCdg2HImoYOPxCJgY4vtNWG6/vO0J50VmQZeIzunH78AsYGhN4nl1YUs3O02dwO5ykpKRS0DOLawLMLV9u386Xmzefd/gGRL2eaVOm0NObftuOrV9+xVdf+ivJhuQP4/rrxvlcf/X19byxZCn2xg73hxoNI0ddxagRl/h8fteuXaxbt85vu1qdjoJZs1QCEDznY+XKlZ7+8q5r3+t1113np8z8at9+tmzejNDmIWZlUSS9Tz9uv+F6teIjeIiIDz/+mNMnTvi1Y/z48eTn+z6PHiksZOUnq1Bc51VRgk7HjTfcwKAOxCJ4COe1n37q08cAPfJ6MmPaVJ9rVZZl3t+wiTPf7kV0e+7hlOhYxl07hgv79PaJW7hwYVClVXJyMnfffbfPOTlV38iqJUtxVnuoGTE8gqHXT2DMYN/2SpLEh8tXcPq4r8efIIpMvukmHwLe7Xbz3qJFlAQgWntmZjKtoMBnLH+1ciVbO6lZAbJNJmY+8IBPX5z2VkXrnH+SEh3N7fPm+RD7luZm3vrHizR2ql2WqtMx6+GHfQj1fTt2sDaIwnBQTg4TZ85Ul20tLSx69lkaA7wkuSovj8u8VTgBJJeLDx5/nKIAsUMNBq73kl4AbVYrC559FqsonieovL81sszc2bNJyc5W4w88+ihfSBLOTi8zssvKuPnXv8HYYWxsfv4Fvm71J3tFSWLiwIEMmDIF8MyFL//5zzQGUajqFYVf/+EP6nVS98orbNm6leN9+vgQaqbmZi4/VsgFqz5B9L5gKFmxgg++/dbT3k4EXGyTmTn3zifSO7/Yjh3jo+eeoziAEjytooKpt84k+nIPKesoK2fZH/5AUQ//2DCbjSnJyeR4qxdKFivL5s/nVPfz/dixr69qaOCyf/0rZBoeIpz+b/i6sZXph4twBRgb87ol8adcj/xfURSu2HOCk23Bq7JowG/C+6GgEwSMGo+8uaWLNEANcGe3RGK0WuyKzIvnavxKQrYjTBR4f2APIjUaUBRuO1JMrSvwhAKwY3hvuhs9lWR+faKM96sCE1kCsP3C3uR4b6iXVTXywIngRu6rL8jlQu/D+pk2O6P2nFSN4Trj3QHdGe29GbJJMpfvOUGpPbC8+W+9MpiZlqAuTz90li0BUhwB7s1M4tGc86keL5yr5rni6oCxU1JieblPlrq8ptbMncfOBYwdajKyZkieSmYdt9gYt+8UNtn/+LqH69mY34tI74Nvi1vi2m9PURTA1D5Jr2V9fk+VPJEVhVsOFfFVk//xhYsCq4fk+aRmPna6gtfK/eXbIvD2gO4+D7MfVTdyfxAj/j/npTMn4/zb5m+brdx08Iyf+g5gbkYCT+WdvyEsszu5bt8pap3+Y25UXBTvDeyh9ptVkph44AyHW/3fWPaLDGP1BXlqvymKwrzCEj75f+2deZhdRZn/P3W33vctnc7S2TcSIIR9U9xxARFURBSB0XFGccaFGXVGcV9HHZ1RRnZcQEFkEZFFRUAggSQEspB966ST9L5v997z++Oce/vce6rq3jg9cvPz/TxPP+nb/U11nXpPvW/Ve+pUHQkmcacXRXnwpAUZSafv7jnENzS2LguHuOfE+RkT3weO9PKhTXvI7oFh4MbjWnmTL5G0vn+Yi1/YoU0Of27edP7BlzxtHxvnzWu3c3AsmNS+rLmWby+aOXmkbiLJJS/sYG1/8GS2s2vK+dmKuelXax3H4R+37OOew8FXERaUFnHfygXU+ibJpvu+Nhrm/pULmO+bJN93pIcPb9obaIuikOJnK+Zylm+l47r+IS55Yae2Lb6zaCbvmV6X/nxgdJy3rdvOAU1b/OOsRv7d10+HEgkuXr+T9QPBtji/voobjmtN30NJx+HvN+/lfs19saKihHtOmJ++hwC+ubud7+wJDo6nF0W5b+WCjFWJvzrUzUe37Au0RVk4xF3Hz2OlL5G0pneQd2/YlU5kpwgDP1w2mwsaJ5O4+0bGuGDdDtrHg23xmbnNXDO7Kf15IO72kVRS2M/FTTX8YMms9D2UcByu3riHhzqDr76cWlXGncfPy0g6fWnnQf57X3Aj7tnFMR5YuSBj1eUd7V18/OX9gbhTFQnz6xPnZyRP/twzwGUv7kqvvk0RVYqbj2vNmPjuGh7jgnXb6dDEqC8vaOFqnx/qnYhz4fodGQ8iUrx/eh1fXzgj3RYTSYf3v7RLGxvOrangJyvmZLyq/m/b27ixLTh50/WnWw908q/bgk+n66IR7ls5P6M//bGrn/e/tDsQ+2LK7U9n+xKzW4dGuXDddnriifTT69RT528vmsl7ff2pczzOheu3s2N4jFAySSSZYDzsvj7x4ZkNfH7+5IOksWSSyzbs4qneQaLxCUomxhmOFREPR3hDfSUMhMpPAAAgAElEQVQ3LZuTTuI6jsMnt+7nZ+3dges7rryEe06cT6WvP12/7wjX7Qy+ljUtFuG+lQsykpEPd/Zx1cbdgf0yS0Mh7jx+Lqf4kvsvDQzzjvU76I8nqBkeIJRM0ltWgRMK833fNgfgJm/fum67dszw8dYmrp0zmXQaTiR51ws72dPWxqLD+ygdH6W/uIwtzbN5VetMfrR0djqJm3QcPrplHw/ta2fh4X1UjQwxEitia9NM5k1r4q4T5mVsY5CKOTVD/dQP9jEejtBW00hzWTH3n7ggYzX3vYd7+IfNeykdHWZaXxfJUIgD1Q3Eiou564T5nOB7mLS2b4h3bgj62YiC/1nWmvGgY//oOBes08ecz85t5qM+3zIYT/COF3awo7Ob+UfaKBkfo6+0nB0NLVw0o4nvLZ6Z4Vs+uGkPa3btYXH7XipHhxiKlbB12ixmz57NncfPy3go8rVd7fznnkNM7+2kbqiP8UiU3XXNzKis4L6V8zNWlf+ivZvvPrWa4/dvp6m/m2QoxJ66ZrbPXcKtZ5/EEp9vebpnkE/98c+s2P0yTQNu/Ossr2LD7EV8/ryzeK3vwdru4THetm47nePjNPb3EknG6SqvYjRapPUtb1+3naH9e1lweD8lE+P0lFWwubmVCxbO5WsLWjJ8yxUv7eaJI13M7TxI+egIg0Ul7GqYzpkNtdye5Vv+fXsbd2/dxdL2PdQMDTAajbGjcQaxma3ce1Kmb7ntQCf/sq3NPU1wdIh4KMxgUQl1sWjAt/xm206evOuXlExkjiWHo0Wc965LeeP81vTPtg6N8vb12+meCM5m/mPRTC7L9i3rtjFy8ADzj+ynKD5Bd2klLzfP5vIFswO+5b3rt9O9ZRMr9m+ndHyMoaISNsxawOxlx3Hj8nkZvuVTW9t49OVtnL1lPVWjQ4yFozw3bynRBYu5Z+WCgG/51uadLN+/gxndRxiLRNk0Yy4j02dx76pFQd/y0i5a2/ezrG0nSUK80LqQrsbp3HnCvAzfsnFgmE89+Bgnbt1AzHHbIwlsmj6XS99yPhf75heHxya4cPUmlqx5kjmd7ShgNBzhiSUrecuZpwZ8y6XrtlHz+KMsbt9LSEEcxbrWRdSd99qAb/mnZ1+k9OH7KfbNMBNJ2HnqudzwxnMyfcuug+y66WZO276RZCxCOB6nq7iCR9/yDq5/y3mZvuVQN4/ecCNzug8xWFWJSiYp6x/gxbnL+PgV78nwLY9t2caTd/48+Eaa4zDv9W/i8jMnX/neNzTCAx//F0577inaWmYQj0ao6e4hNDbOnq98kyvOm3xlf2Aizv98/suUdR/isHfiXyiRoGV/G31nvYZ/vuqyDN/ynR/ezNIH7mZf62yGysooGRlh1t69bDn9VVzz6U9k+JbbbvoJ1ffexc758+ivqiIyMcGsffsIx+F1N1xPY/nkuOzBex5g+Ke3snPBfEa91Z3VPT3M3LOPpd/+NksWzJOEkySc/vc82zvIN3a380yv+2RjVnGMv5/ZwAeyVmT86lA3/2iYeNdHIzx5yiLGHDeQ/+PmPeybwhVRhYICYiGlTSj4aYq5m56HFbw0MMKgJUk2qzjGmTXlRJTi+b4htlhWnM0sjnFZs7vp+ksDI9aVYbWRMNd6r1AeGBvnP/eaTy6KKsU3Fs6gLBJiPOHwyW37rdf47UUzmVYUBcfh2m37OThmTtRd2zqNVVVlhJQ7oX+6V7+nB8Dfz2zg7U01hIA727u56YD5/f93NtVwTWuTm+DrGeRfNBObFKdXlfG9JbMAd0J/0Qvmo6RnFEe5/0Q3STaWSPK657fSZ7BfiVI8ceri9OrBd27YqU0KpXjopAUs8AZC125t454j5j0Zfrh0Fm+or0KhuHF/B1/dHdwMOcWn50zjI94A+fGufi57SXOctcelzbV8d7HbFvtGxjjt2S2BREGKkyvLeOAk9+nKaCLJiU9vcid5GhpiEdaevjQ9gHzj89t4QZMIATe58PwZS9OJr2u27OWXh8xt8esT53O6NxD6r72H+fIuc1v4J5yPdPbxPktb+Fcu7hoe44zV+lc3AF5dW8Edx88D3AHTiU9vMu5pN7s4xjOnLSGkFI7j8Nrnt2pfGQY38b3+jGXp1VYf3rRHmyxM8diqhekVVKZkYYrrl87mQm8V3m+O9HK1ITEM8InWJj7lDQpfHhrhVWvMGzO/rbGaHy9rBdxEzwlPbzKuOl1SVswfTl6E8trinDUvp19ZzqYyHGL9mcvSA8grX9rNbzVJIXB98Z9PXcJcb6XjV3ce5PuapFAKf7I+12mu/pWnGwaGeYPl1fN3T6tN+5buiTgnPr3J6DtXVZbym5Pc45ATjsPpz24xPjCoj0ZYe8bS9AqjSzfs1O6JCG5/WnP6Ulq8wbQpoZ7i7hPmpROiPz3YxSe3mk8C/NrCGXygxZ1YrO4d5IL15j1nrmyp56sLvdesxiZY9cxm7cMsgHNqyvnlCe4rFhNJh1XPbOKwJvkO0FIUZfVpS9MTsgvXbefZPn0ciSnF2jOWpifq127dr11dnOK3Jy1gZaU78L5hfwf/vsN8Ws9/Lp7Fu5rdBM7j3f28e8Muo9b/EGfvyBinW/zsG+oruW25+zrbSCLJSoufnVdSxJOnLk77ljeu3cYGQ8wpDYV44cxl6cnpRzbv5W5N8j3FH09elE5afH/vYb5q8bM3LGvlrY1uAuehjl4+sHGPUetfTb5taJRz1gRXx6S4sLGa6z3fMuj5FtP46bjyEh5dtTDtW85ds5Vtw3o/WxUJs/6MZZR6E7KrNu7mwQ6zb3n61CXpVdTf2NXOd/fqXwEE+OmKuekEzj1egszE5+dN58Oeb3lxYJjXW3zLe5pr+Y4Xq3s835JOTqf6lTdG98fqhONwxrNb2Ov5llh8HOXAWNT1D9m+xfYwMgw8d/rS9ETdtkcswK9OmJfeVuJnB7v4hMW3fH3hDK7wfMua3kHeZvEtV7XU85W/0Lec/MxmegcHWdq+hxk9R3BQtNU2sqV5NnXl5aw5fWn6wUwu37LujGXUe6taC8G35BrD+X1LrjGc37cAvOH5rWwYGEE5SeoG+wk5SbrLKomHI6+Ib6keHqDSS2Z3lFeDUhm+ZfvQKGenfIvjUDE8xGgsxoR337+9sZof+XzLiU9vYiDh7jVV19fDRCRCf7nbj7N9y6ue28qhjg5O2rOV1q52lONwsLqeta1LGKut1/qW1gP7OWf9akrHRtk+cw5PnHgK8WhU61tmHG7ndc89ReXwIO21DTx86jn0VVRqfUtNfy+vW/0UzZ1H6Kyu4dFTz+ZIbb3Wt7QcaeetT/6euW17GSwt44+rzuDPx6/i3S31Ad/ScLCNt//xYRYc3Ec8FOL5xSu4/5zXsri5iQdOWiAJJ0k4TR29E3HGkg4NsYhxr5X/3neEr+9qz3D0M4tj3Lp8Dst8T1dSTx9MfGvhDM6sqaAvnuBH+45YEyeVkTAnVJQwnEiyZ2Q8/dqWIAh/Gf7enSsqhFP/wcm9gjGsIORtA5nrpMwwpAd5phV9fm005L5bPpp0rHUOQfopz1gimbPOZZ52POkYB7B+rQLijhNYlZJNaThEREHCIefhDMUhRcybTPfH7doipSjx6tEXTxgHmuAmkisi7vUNxJPW6wsB1VF38DicSOa8vppIGKVgLOnkvL6qSJiwgrjjrly0UR4OpdtC95TZT0lIpVci9UwkrPdFTCnKIyEUiv54wtoWYUgnAAcTiZxtUe9pR5NJ68MFcFfLhZViIunQm6MtKiMhikIhkg505Yh7ZeFQ+l7uGI9b26I4pNIThZ6JOBMWcUSRXlnQH7e3hcJNPIN7D+Vqi7pohLB3D9kOJAGojoSJhRQJx6Erx31RHg6lB/+6FaR+Snxt0TURt/qtqFLUen2kL0dbhJhsi8FEMmcfaYhGCCnXv+Vqi9pImGhIEc+jLSrDIUrCIRxyt0VpKJT2F53jcavvjClFjdcWvfGE9QFVGNKT9IFEkuEcbdEYi6CAkaST01/URcNElGLCcXL6i6pImOKQIonbR2yUhUOU+/qTrcZFIUV1uj8lrPEsrCb9xUA8GVjpmU1TLIJCMZxIWFf3g9ufIsqNZaakQorqSJiikCLpoF016afc51uO5OFbqr29y7om4lY/G1WKOq8t+uJx7cr3FAr3vgA3nubyLfU+35LLz9akfQs55xcV4RBl4TAOTs62KAkpqry26JyYeMV8S2MsQgjFaDKZsy1qo2FiKkTc27PURmUkTGkohINjfFiQojQcotJ7kNQxPmH1LUVKpeNvTzye07ekHi4MJBI52yLVn0aSyZx+ti4aJqpCTDjJnH62KhKmJBQi6d0XNsrCISq8tjgyPpHTt9R491D3RNzqWyIK6r3XFPvjiSn1LX8+dTHzy0ok4SQJp78uHeMTPNjRR99EgsXlxbymtjKwV0g86XD1pt0Z+wWluLiphu8vmZVOau0fHeec1VuMweZ/fK9Z5Ho6tqqylG8smslIIsmWwRE+ZUl6hYAPzWygNBxiIJ7ghrZOa/BYVVlKZSTMSNLh2d5Bq7YopCgKKeIOOQdXgiAIgiAIgiAIglBI3H3CPM6urcw74RTczVEQ/gIaYtH0MlgTkZDixmVzuOtwN79o7+agt7Hve6fXcUFjdcYKKneF1Fz+buPujCxrCHcptn9Pj4VlxfzjrEbtfhqV4RDfXDQzvUfGqqoyftfZz++79SeAvLu5NuMd6954wvhaT2p/itSKjI9t2ccvDgX3bkjxyKpF6ZP1cr3q8aUF07mwsYa44/DTA138h2Xp9uvrKrlmdhNJx+GZ3kG+ZnmdZnZxjC/NbwEF+0bG+TfLct4ipfjawhaioRD98QSf334AW47+Y7MbaS6KkXQcvrm7nV7LqoxLmmpYXlFC0oGbD3QaXyEBOKWqjLNrykk67rvmm62vF0Y539uTYX3/MGsMy6DBfaLwLm8vi70jY/zesHwc3PvuoqYaikMheuJx47L7FK+uraAuGmHCcXjgSK/1acWJFaW0lrjLex/p6rc+jZlXUsSyihIcB1b3DVqfmtRFI5xcVYoDvDw4ml5Kr6MopDiruhwHODg2od3nJYUCzqwuJxpS9E4ktHsF+VlZ4SZlx5xk+hVdEwtLi2guiuHg8EzvkPVJaEtRlDnePgQvDgxbn8ZUR8JpH7BjeNTabkVKsbLKfT3t4OiEtd0UsKqyjLByV4XY9rMDd0l2WTjEaDJpfOUlxbySIupiEZKOw7r+Yes91FwUpcXbM2jT4Ij1qXB1JMw8b+n2ruEx61Pv4pBKr1I9ODZBu2ZvkxQKOL6ilJDXFrtHzO0G7mt1xaEQI8mk9X4Ddy+3qojbFi8NjlgT+81FUabFJtvC9vSvNhpmdrHbFjtHRq0ryUpCikVlqbYYt95DIWB5havtnojnPFxjcVkxsZBiOJFkR457aE5JjIpImHjSsfpCcPfWqo9F3JNvhkasT9Nro2FmeK/I7Bwes/qhsnAofQ8dGJ2wrrQKK1hWVgLKXSGj2x/Hz5KyYqJKMZBI5LyH5pUUURYJMZF0rK+dg+sv6ry22DQ4Yn2aXh8NM917tXf78Ki1P5WHQ8z1/ND+0XFrf4oqWOLdQ0fGJziU46n30rJiIkrRF09Y/RDA/NIiSkMhxpIOWw2vkKWYWRylJhJx917J0Z8aohH3VXlg2/CodWVBZTiU3htm7+i4dSVSkVLp8dCh8Qlrf1LA0vJiwih64wnreAHcsVnKt+TqT7OKY1RHwiRw2Dw4am2LpliEJs+3bBkatcan6kiYWV5/2j0y5r6mY6A4pFjovVZ/cGzCuookhNsWCkX3RFy7r5+fRWXFFCnFUCLJTs3el35aS2JUhsPEndy+ZVosSkMs4o4vcviWmsikb9k1YvctpSGfbxkbt65QCwNLyktQuKuQbPEJXD8bVYrBPHzL3JIiysIhxh3HeogSuH62Luq2xebBEWusrvP5lh15+JY5Pt9iW4kUVaQPdDkyPpFzddHSsmLCyl3Zm8u3zCspojQcYiyZZFuO/jSjKEpNNEIyj/70f+lbFnq+5XCeviXk+Zb9U+xbqiJhEo4bn3L5lkbPt7ycw7dU+XzLnjx8S2rLjvYp9i25ThLPRlY4CQXNYDzBr4/0sGN4jPpohAubarQ3ueM43Hmomx/v72DL0CgxpTi/oYpPtE4LnIjUNxHnQ5v28njWZtIXNlbzvcWzMjZYG4on+MDG3TzRM5ihbS2JcceKeRmnonWMT3DBuh3aDa0/PaeZj7VObjjZMxHnreu2ax3WGdXl3Hn85CbHg/EEb1q7TbvHSVUkzMOrFtLqOeJ40uGt67YbkwC3ZW1+bduz4FOt0/jEnMkT+z67rc24f9Jraiv52fGTxyebNoYFN3j98ZRF6et7tLOPyw176pSFQ/zplMXpwcpGb+8UU6i58/i56dOZOsfjnP7sZqMz9p8cGE86nLl6izHw+vdkAbh4/Q6e6h3UaldUlPDwSQvTe53Z3umviYR5/vSl6dPAbmzr4N+265OAEQV/OmVy4/nfd/Vz2Yvmd/r9tt4+NMq5a142DoT+dc40/sl7530wnuCkZzYblxZf1FTDD5e6G8TnOtXS/8472O+3mkiY589Ymt6r5+u72vmeIdEaAp70bcJ/l7dBtQn/Xj3r+4d501rzHhn+zWE7xic4+ZnNxqXs75xWw/e9zfLjSYfTVm+mzZBcOKGilIfcd97d//vCjoBfSVEXjfD86UvTr4Z9ZlsbNxv6Xhh4+rQl6cHYzW0dfMZwDwHcsWIur/be/3+qZ4CLLXuXfXH+dD7o9ZH9o+Oc/uxm48TCfxroaCLJqmc2Gwc3p1aVcd/KyVOczl+7jXWaDd/BHYg959sHzLa3V1Qp1py+JL0PWK69ve49cX76JMWHO/t4v2Vvr28unMH7vIcqO4ZHOXv1y8YBpP8wgMF4gpVPbzImRM+tqeAXJ7j7gDmOw6uf22pMwM0odvctSj3kuHrjbn5j6E9FIcXa0yf3Fsm158xDJy1Mn9z568M9fNiy58z3l8zinV6y/qWBYV5n2XPmmlmNfMbbW6RnIs7KpzcZJ1lvrK/kVm9vkYTj+uQ9honhgtIi/nTK5N4il23YZXyQVB4Ose6Myb1Fcu05499bJNeeM/69RXLtOePfW+TwmOtbTAlR/94iE0mHU57dbJxQZ/vZt6/fbkzuV0fCrPPtLWKLTyHcPdFS4xxbfILMfYv+1D3AuzaYfYt/b5F93n5Wprh+WXMt/+HtLTKSSHLSM5uMiQj/vkWO4/CmtduN+wY2xFw/m9q36KNb9nKXwbdEFKw+bXJPtB/sPcxXLL7Fv2/R7zr6uGKj2bf49y3a5sVqk2/x71uUseeMBv++RVPtW54/fXJPtFy+xb9vUS7f4t+3KJdv+eisRj77F/oW/35W2cwvLeIJn2+x7WdVFg6xXnwLUJi+Ze/IGGdYfIt/TzTxLUfvW45mD6fwddddl4/umOLHP/7xdR/84Adf6WoIU0AsFOL4ilJeXVvJqdXlVGmO6QZQSrG8opQrWur56OxGPtE6jbc21miPpy4Oh7h4Wi2vq69kTkkR59VW8IUFLVzREjwyPBYKcXFTDadXl1MdibC8wj1y+msLZ1CfdcRxWTjMRU01lIRCdIxPEFWKU6vK+PKCGVzqO8UCoCQc4oLGGkaSSXYOjzLuONRHI3xoZgPfXDQzI+kVC4V4a2M1baMT7ByezJKfUV3ODctaMxJqIS/Rtnt4LCOZ1RSL8O1FM3lLY+bR86+rq+Lg2Dgv+55EFIcUH53VxCfmTMvYIP7smnIOj08ETnx6bV0lP1o2O+Oo7OMrSgjhbnjud93Ly0u4fcUcan1tN6+0mLpYhKe6BzK0ddEItxw3h2UVk/uANRZFmVtaxKOd/RnaMPClBS1c1DR5+k5pOMSqyjJ+19kXeHLyvul1/Muc5vT1hZTinNoKHunsCzjYs6rL+cGSWRknp5xTW8Hj3QOBCXVrSYzbl8+l2ndyyhnV5TzXPxRIRFSGQ9y+Yi5zfCenHF9RStvoeGAz6YhyB2Jn+k45m1taRNJbCZTNNbMa+YDvFJm6WITmoiiPdvUHgs3r6yr58oIZ6YFmLBRiRUUpD3b0BpILy8qLuX5ZazoRopTi9OoyHjzSF3g3vCkW4fblc6nz2fqM6nJ+3zUQWBFRElLcfNwc5vvu5VWVZazrHwoMChXuJuDn+o59X1pWTPvYBC9pTiP70IwGrpoxedhBc1GUopDiSU2y53V1lXxxQUu6LcrCYeaXFvNQZ18gWbesvJgfLZ1si5BSrKos44EjvYxlDfSaYhFuWzEnoy1Oqy7nd519gSd1paEQNy9vTSfTwF3l91TPIIeyTmZLtYX/tK7l5aVsHR7VJqj/fmYDV/gOfphVUsRYMqldCfjG+ko+N2+yLaoiYaYVRXm0M3gPLS8v4QdLJpP1kZBiRUUJDxwJ3kPNsSg3L5+T3o8hdX0PdvQGnnyXhUPcvHxOOqHuasv5Q3d/oO+FgO8snpVOIAGcUFnKiwMj7NY8BPjn2U28u3nSL88rKaJ7IqEdQL6loYrPzJuenoDURiNUR8PaSciJFaV8b8mstD+MhUIsKivmNx3Be6ilKMqNx7Wm/YVSipOr3HsoO8lZEQ5x6/K56eQ7wKlV5TzS2R9YVRMGfrBkdsZJgCdVlrGmb1CbEL12zjQuappcNbyorJi20Qk2afrTO5pq+GTrZGxosvSnU6rK+NbimWnfWRIOMbe0mIc6g6s+ZxfHuGFZa/qwhZBSrKws1fanqkiYW5bPyThZ89TqMh7S9KeoUvxo6ez0pvrgThye6h3UTrI+N296epUswLLyEnYMj2mT6pc11/KRWY3ptmgpjpFwHO0Gw2dVl/O1hTPSY4xyb9XHw519gf60oLSI65e1ppPvYaU4vqKU33T0Bp5810Uj3Lp8Dg2+UxFPrSrntx19gT1tikKKHy9rTa8sAji5qozHuwe0qwC+sqCF83wnly0vL2XT4Ih2pcxVLfVc7fOzs4tjDMaT2pNDz6utyPCzVdEIjUURHusKJgyXlBXzX0tnp/1sNKQ4rtxti2zf0hiLcMvyOen9xVL96TdHegPxqSSkuOm4OczN8LPlPNbVH4hPIeBbi2dmxN8TKktZ3z+sTVp8ZFYj7/Wd1jWvtIiO8bh2VeubG6r47NxJ31IXi1AZCWkPA1hRUcL3l8zO8C2LLb7lpuPmZPiWVV5/yvYt5eEQt2h8y8OdfYFVNWHg+0tmc5LPt6ysLGN17yBtmv70qdZpvMN3KuKismL2jY6zWXNgxkVNNXxqTqZviSqlfbh3cmUZ3/5f+JaTKku5/0hvIClTFQlzy3FzaPa3RXU5v+3sDayAjSrFDzW+5UlNrIaj8y3vaa7lo3n6ljOry/l6nr5lfmkR/6PxLQ8cCfqW2mhY61se7OgN+JaYUvz4uMLzLdXRCA0xs2/57yzfsqy8hAc6ekn8lXyLAr59DPuWL3zhC+3XXXfdjwMFapAVToLwCpNwHEYTSUrDoYwEj46O8Qn2j47TEIvmXM64b2SMLUOjlIdDnFLlvgJl4sDoOM/1DRENKc6sLs9ImGSzd2SMP3YPEHcczqguzzjSW1ffRzr7GUwkWFFRymlVZcZr7ByP80BHL53jE8wrLeZN9VUZx5Bnl3v3oR72jY7TXBTlHU016acD2fTHE9x9qJuXh0apjIS5sLE6Y4DgZziR5N4jPTzfN0QsFOKN9ZWcU1Oh3TB/LJnkwY4+/tjdj+PAWTXlXNBYo61zwnF4tLOf33T0MpJMckJFKZc216VXH/hxvEHFrw710D0RZ0FZMe9prs04xtbP+v5h7mjvom10nJbiGJdOq82YaPp5eWiE2w90sW1olJpohIuaqnlDfZX2+tpGx7n9QCfr+ocpDoc4v6GKixprMpKhKbrG4/y8vYsnvFWD59ZUcNn0uoykQorhRJK7DnXzu84+RpNJVlWW8f6W+ozBbop40uGBjl5+fbiHvniCRWXFXNFSr73nHMfh990D3NHexaGxCWYUx3hvcx1n1ZRr77nn+4a47WAnO4bGqI9FeEdTDW9trE4PVPxsHxrllgOdvDAwTFk4xFsaqrlkWm36SZ6fg6Pj3Hqgkz/3DhJWilfXVnD59Hqtrfsm4vysvZtHOvsYdxxOrSrjipZ6ra1HE0nuOtzNvYd7GUgkWFZewgda6lmhuZcT3qucvzjUTcd4nNklMS6fXse5NRXatniie4DbD3aye2SMxliUd06r5W2GtnhxYJib2zrZODhCeTjEBU01vMvQFntGxri5rZNnegeJKMXr6yt53/R67YOAjvEJbj/QxWNd/e5pbTXlXGloi6F4gp+3d3P/kd60b7lqhr4t4kmHXx/p4e5DPXSMTzC3tIj3T6/PSNKlcByHx7r6+Wl7F/tGxplWFOVdXlvo+sjaviFuPtDJ5sERqiJhLvTaQucDtg2NcmNbB8/1DVEUCvGG+kre31Kfcax3ivaxcW5p6+SP3QM4uL7lypZ6Zmnaoj+e4PYDnfy2s889KbGylKta6rU+bjyZ5K5DPdx9uJvuiQQLSou4oqU+fTpddlv8trOPnx3s4sDYBNOLorynuY43N+j9xdM9g9xyoJOtQ6PURsO8vamGdzfXZjyISLF5cIQb2zpc3xIK8eaGKi6fXqeNO/tHx7m5rYMnegZQKM6treDKlnqtv++ZiHP7gS4e8nzLyVVlXNlSn3E0fIqRRJI72ru490gvvd7ekx9oqU+ffOkn6Tjcd6SXO9u7OTg2nt4K4E31Vcb+dOuBTnYMj1EXC3NJUy0XT6vJeGiR4sWBYW5s6+CF/hHXtzRW877pdenJsZ89I2Pc1NbBUz2ubzmvtoIrZzSkX0vx0zke59YDnTzS1eeuYqgq4+oZDYEV3wBDiQR3tHdz7+Ee+uNJjqso4cqWelZp4kiqP/2ivS6Ij1IAABzcSURBVJvD4xO0lhRx+fQ6XldXGWiLlE/+ycFOdg1P+pZ3NNUEHu4BrOsb4qYDnbw0MEJFJMQFjdVc1lyXXgHsZ+fwKDe2dfJs7yBRpXhtfSUfaKlPr7rxc3hsglsOdPL7rn4mvHHL1TMa0qdZ+hmMJ/jpwS7u7+hlMJ5kRUUJV81oSK/+8zORdLjrcHfat8wrLeJ90+szJtL+tni40/Ute0fGaPZ8y4VNNVo/u7p3MOBb3tNcp/WzWz3fsqZviKKQ4g11VVzRovezB0fHuflAJ493D5B0HM6uqeDKGXo/2zcR5/aDXTzY4fqWlZWun12u8S1jySS/PNTNrw710DURZ2FZMVcY/GzScXiwo4+fe+OW6UUx3jO9lrc26P3sUz0D3HKgk21Do9RGI65vmVarHYtsGhzhprYO1vYPUxxSvLnB7U8m33JTWwdPdA+gFJxbU8mVM/Rjke6JOLcd6OQh7wHmqspSrp7RYPUtvz7cS288zuKyEq6cofctCcfh/iO93NHeRXtq3DK9jvMNvuVP3QPcdqCT7cOj1EUjXDKtlksMvmXDwDA3ZfmWy6fXpVdj+dkzMsaNbR38uWdy3HKVwbd0jE9w64FOHu3sZ9yZ9C0Lp8C33OP5liPjE8wuKeJ9Ft/yWFc/PznYxe6RMRpiUd45rYaLm2qNvuXGA51s9HzL2xqrea/Bt+zI9i117n0xFb7lJwe7eOAofMtdh7rpHI+nxy023/KTg13sGx1jWlGUd/+VfIucUicJJ0EQBEEQBEEQBEEQhCnlaBJO+uUDgiAIgiAIgiAIgiAIgvAXIgknQRAEQRAEQRAEQRAEYUqRhJMgCIIgCIIgCIIgCIIwpUjCSRAEQRAEQRAEQRAEQZhSJOEkCIIgCIIgCIIgCIIgTCmScBIEQRAEQRAEQRAEQRCmFEk4CYIgCIIgCIIgCIIgCFOKJJwEQRAEQRAEQRAEQRCEKUUSToIgCIIgCIIgCIIgCMKUIgknQRAEQRAEQRAEQRAEYUqRhJMgCIIgCIIgCIIgCIIwpUjCSRAEQRAEQRAEQRAEQZhSJOEkCIIgCIIgCIIgCIIgTCmScBIEQRAEQRAEQRAEQRCmFEk4CYIgCIIgCIIgCIIgCFOKJJwEQRAEQRAEQRAEQRCEKUUSToIgCIIgCIIgCIIgCMKUIgknQRAEQRAEQRAEQRAEYUqRhJMgCIIgCIIgCIIgCIIwpUjCSRAEQRAEQRAEQRAEQZhSJOEkCIIgCIIgCIIgCIIgTCmScBIEQRAEQRAEQRAEQRCmFEk4CYIgCIIgCIIgCIIgCFOKJJwEQRAEQRAEQRAEQRCEKUUSToIgCIIgCIIgCIIgCMKUIgknQRAEQRAEQRAEQRAEYUpRjuO80nWYcpRSHcBeza/qgc48i/m/0hZKPY41baHUoxC0hVKPQtAWSj0KQVso9TjWtIVSj0LQFko9CkFbKPUoBG2h1ONY0xZKPQpBWyj1KARtodSjELSFUo9jTVso9SgEbaHUoxC0f+16zHYcpyGv/+04zt/MF/D8K60tlHoca9pCqUchaAulHoWgLZR6FIK2UOpxrGkLpR6FoC2UehSCtlDqUQjaQqnHsaYtlHoUgrZQ6lEI2kKpRyFoC6Uex5q2UOpRCNpCqUchaAupHtlf8kqdIAiCIAiCIAiCIAiCMKVIwkkQBEEQBEEQBEEQBEGYUv7WEk4/LgBtodTjWNMWSj0KQVso9SgEbaHUoxC0hVKPY01bKPUoBG2h1KMQtIVSj0LQFko9jjVtodSjELSFUo9C0BZKPQpBWyj1ONa0hVKPQtAWSj0KQVtI9cjg/8tNwwVBEARBEARBEARBEIRXjr+1FU6CIAiCIAiCIAiCIAjC/zGScBIEQRAEQRAEQRAEQRCmlv/NEXfH0hfwRmArsAP4V4vuZuAIsDGPMmcCfwQ2A5uAj1m0xcAaYIOn/UIe5YeB9cBvcuj2AC8BL5DHsYVANXA38DKwBTjdoFvklZn66gf+yVLuP3vXthG4Ayi2aD/m6TZll6mzAVALPAps9/6tyaG/xCs7CazKof2W1xYvAr8Gqi3aL3m6F4BHgOm57hvgE4AD1FvKvQ444Gvr823lAh/16rwJ+GaO6/uFr9w9wAsW7QnAs6l7CTjFoj0eeMa79x4AKm39QmdDizZgP4s2YD+LNmA/k1ZnP0u5JvsZy862oaXsgP0s2oD9LNqA/TD4KWAOsBrXf/4CiFm0H/F06Xve5gOBn+H65o2491nUor3J+9mLuD6s3KT1/d3vA4M56nArsNvXzidYtAr4CrAN139eY9E+6SvzIHBvjnq8Bljn6Z8C5lu053najcBtQMQUO3T2s2i19jNoA7azaAO2yxXr/LazlBuwXQ59wH4WrdZ+Bm3AdhatzXZ7yIrnGOKfQWuKfTqtKfbptKbYF9BaYp+u3OvQ+05tuZhjn65sU+zTaU2xT6c1xb7A2MpiO53WZDudVms7i95kP+N4UGM/Xbkm+2nL1dnPUK7JdjqtyXY6rcl22vGuzn4WrW7cYtLqxi0mrW7cYh2fkzluMZVrsp2x7Gz7WcrWjVtMWt24xaQ12S8w/8AQ9wxa07hFp7XFPZ1eG/t0WlPsM5R7K5rYZ9Bq455Baxq36LS2uKfTa2MfmjkhZt+p05p8p05rins6rclv2uaw2X5TV+51aPqerWz0vlNXtsl36rQm36nTavtevl95C4/lL9zB3k5gLu5EaQOw1KA9B1hJfgmnZmCl930Fbkc2lauYdDJRXCd4Wo7yPw78nPwSTvW56uvT3wZc7X0fwzdQydGGh4DZht+34Dq+Eu/zL4ErDNrjvBu5FIgAj5HppAI2wA1s/+p9/6/AN3Lol+AGrMfJdD467euZdHrfSJVt0Fb6vr8GuN523+BO9h8G9jLpfHTlXgd8Mp/7EXi112ZF3ufGfO9f4D+Az1nKfgR4k/f9+cDjFu1zwLne91cCX7L1C50NLdqA/SzagP0s2oD9TFqd/Szlmuxn0gdsaKtHtv0s5QbsZ9EG7IfBT+H253d7P78e+LBFeyLQSpZfsujP936ncAcltrL99vsO7n1k9K3AKuAnTCacTOXeClyc1dYm7QeA24GQz3Y5/TvwK+B9OcreBizxfv4PXr102jOA/cBC7+dfBK7y/a2M2KGzn0WrtZ9BG7CdRRuwnUmrs52l3IDtcugD9rPVQ2c/Q7kB2+m0uCvLbbbTtbs2/hm0ptin05pin05rin0Brc53Wsq9Dr3v1GltsU9bj2zfaSnbFPt0WlPsC4ytLLbTaU2202m1trPoTfbTjgcN9tOVa7KfTqu1n6kOBtvpyjXZTqfV2i7r76XHuyb7GbRa+xm0RvtptFrb6bQm2xnK1drOojf2P109dPYzlKu1n0GrG7do5x/oxy0mbSDuWbTauGfR68YtxjkTwXGLqdxbCY5bTFrduCXnvA0v7lnK1cY9g/5KNLEPw5wQ/ZzBpNXNGUxa3ZzBpNXNGYxzWIJzBlO516H3mya9bs5gnUv7+56lXN2cwaTN6TttX38rr9SdAuxwHGeX4zjjwJ3ABTqh4zhPAN35FOo4TrvjOOu87wdws8YtBq3jOM6g9zHqfTmmspVSM4A3AzfmU5d8UUpV4SYPbvLqNe44Tm8e//U1wE7HcfZaNBGgRCkVwb1RDxp0S4DVjuMMO44TB/4EXJT6pcEGF+AOHvD+vdCmdxxni+M4W7P/sEH7iFcPcDO9Myzaft/HMjwbWu6b7wLX4rP1Ud5jOu2Hga87jjPmaY7kU7ZSSgHvxA2QJq2Du9IFoArPhgbtQuAJ7/tHgXd4WlO/CNjQpNXZz6IN2M+iDdgvRz/OsN/R9Pkc+oANc5Xtt59FG7CfRRuwn8VPnYf7ZA4mbafVOo6z3nGcPZq2MOl/6/3OwV3JM8Oi7fe1RclksUGtUiqM+yTr2lx1yK5rDu2HgS86jpP0dEdylauUqvTa8N4cZevsp9MmgHHHcbZ5P0/3v+zY4bVVwH46rVc3rf0M2oDtLNqA7Uxane1MWhsGfcB+ucrOtp9Bq/WdGm0dBttZMMa/bHS+06LVxj6DVhv7LARi3xRgjH02smOfAa39DAR8p2VsFbCdSauznUWrtZ1FH7BfjvFghv2OZuxo0Qbsl6tcv+0s2oDtLFrtuCUL/3g3V99La/Poe35trr7n1+bqe9njc1vfy2csb9Ln6n+Bsi19z6/N1ff8WpP9sucf7RjinkZ70BT3DFpt3LPotbFPpzXFPp1WU1ebVhv3bOVmxz2D1ma7bP0Q+thnmhPq+p5Wa+h7Jq2u75m0ur5nm8Nm9z3rfFeDSa/re9ays/qeSauzn0mbj+808reScGrBzaqmaMMySfxLUEq14mbIV1s0YaXUC7ivJj3qOI5RC3wP96ZN5vHnHeARpdRapdQHc2jnAB3ALUqp9UqpG5VSZXn8jXdjGaw5jnMA+DawD9fR9zmO84hBvhE4WylVp5Qqxc2qzszx95scx2n3vj8ENOVR57+EK4GHbAKl1FeUUvuBy3AzxybdBcABx3E25Pm3P6KUelEpdbNSqsaiW4jbfquVUn9SSp2cZ/lnA4cdx9lu0fwT8C3v+r4NfNqi3cRk4vYSNDbM6hdWG+bTh/LQBuyXrbXZz6/NZT9NHaz2y9JbbWi4Pq39srRW+2VptfbL9lO4q0N7fQE67T+P0qdZ9UqpKHA58DubVil1C+79sxj4gUX7EeB+3z2Xqw5f8ez3XaVUkUU7D3iXUup5pdRDSqkFebTFhcDv/YMXg/5q4LdKqTavLb5usMkaIKKUWuUVdzGT/S87dtRhsJ9Ga8OozbadSauznUGrtZ2lDgHbWfRa+9muj6D9dFqt7TTaTsy2A308N/nOo4n9ubR+36nVGnxnQGvxnaY66HynTmvzm7bry/adOq3Jd+q0Ot9pGlvpbHc047B8tH7bGfUa+2m1BvvZ6pFtP5NWZ79c1+e3nUmrs51Jm3PcQuZ4N9fY0zo2zlOrG3dmaA19L6DNNW7R1CHXuNOvzzX21F2fadzp1+Yad/q1Afvp5h/AWjRx72jmKrm02XHPps+OfRZtIPblqEdG7LNoA3Evj7ZIxz2LVhv3DDb5JfrYZ5oT6vre0cwf89Gm+p5Rq+l7Wq2h79nqoOt7Jr2u7+W6Pn/fM2l1fc+kzcd3mnGOYjnUsfqFe1Pf6Pt8OfBfFn0rebxS59OX4zq3i/LUV+PuqXKc4fdvAX7off8qcr9S1+JMLrHbAJxj0a4C4sCp3uf/JMeyONylyJ24nd+kqQH+ADTgPoG/F3ivRX+V12ZPAD8CvmezAW7g8P++Jx+boVnabNF+Fvd9XpXPvYDbMb+g0+Jm81cDVd7nPWS+XpR9fU24y4ZDuO9Z32zRbsSdrCnc1Xu786mz186fyNHO38dd6QJuZvwxi3Yx7nLMtcDngS5bv7DZMFubw34mrc5+xr6psV9am4f9sq/NaD+D3mhDy/Xp7Jddrs1+2dpc9kv5qbNwV4imfj4z+/5C49Oy2ywP/Q1k+QGLNgz8EPiAQXsO7n4CqWXTg7ZycV87VEAR7pO07OX/fu1gyg7evfJkHvV9KGWXHPW4h0nf/Cl8cUujPR13r4U1wJdx38EPxA7cZd0B++m0WX8rbb88tGnb5aFN285Q3+k625nKNdnOog/YL486p+1nKTdgO4s2YDvf3wrEcwy+U6f1aR4n87UsmzbDd9q02b7TUF+t7zRotb7ToLX5Tdv1ZfhOQ9la32nQBnwnhrGVznYmrc52eWizbZdzjJeyn0H7LZ39LNcXsJ9Fq7NfrutL285SbsB2Fm2uuJcx3tXZz6Q19b0cWt24xTjmJjhuSWvJPW7JvrZc45Zsva3/ma5PN27JLtc2bsnW6vqedv6BPu5Z5ypk+qtc2owxSx56f+zTad+HPvaZri8Q+yxaXdzLVV9/3DOVqx2zWPTa2IdmTog57hnnjwTjnk2b7TtzzUv9cS9b+2PMcU93bbY5n06v7Xs5ri877unKNcU9ndbqO3N95S08lr+8G/zhrJvm0xZ9K3kmnLyO9DDw8aOs0+cwvDsNfA03G78HN6s7DPw0z3KvM5Xr/X4asMf3+WzgwRxlXgA8kkNzCXCT7/P78AbaedT5q8A/2GyAu0Ffs/d9M7A1H5uRZ8IJ913kZ4DSfO8FYFZWHdNaYDnuaoQ93lccN9M/LY9ys689+/PvgFf7Pu8EGnJcXwQ4jPu6ku1v9THpfBXQn2dbLATW2PqFyYY6rcl+Jq3OfrZys+2XrbXZL49ys9tU1xZaG1quL2A/Q7la++VR5wz7+X7+OdxBRCeTg6AMf5ql/aTv8x7se6qk9bjB6168/QVyle397Bz0e+18zivvkM9+SXyDzxzlvspS7idxN22c42vjvhzXVo87KLYdopBq551Z9+fmPOv8etynh7rY8TOd/Qzan/rKTNvPps22Xa5y/bYzaHt0tsuz3LTtTHqd/XJcX4b9DNoHdbbLs86vB35puC+uw73nrPHPr/V9fhzNPjLZWgyxz1Su7/p08fY64N+xxL4c5bZayv0kOWKf4fq0sU9TtjH25ajzQtzJk3ZspbOdSauznU2rs12usv32M2h/b7Df8jzKbbWU+6DBfsss15dhO0u5Advl2Q6BuEfWeFdnP5PW1vd0Wp39bOXq+p5fS+5xp63cVoLjxuy2MPY/w/WZxp3Z5drGnbY6p/qebv7xI/RxzzpXITPuGbVoxiy5yvZ+lop9Ou1u9LEvn3JfZSn3h+jjnu36suOeqY21Y5Y866yNfXhzQvKLexnzR+xxL60ld9zTzUtNce+ruJts5xP3cs53DW2RM/ZlXV+uuJcqN5+4p6uzds5g+/pbeaXuOWCBUmqOUiqGuzzz/v9toUophft++BbHcb6TQ9uglKr2vi8BXofrAAI4jvNpx3FmOI7T6tX1D47jvNdQbplSqiL1PW4H3miqh+M4h4D9SqlF3o9egzswtnEpuZcM7wNOU0qVeu3yGty9YrQopRq9f2fhZtt/nqP8+4H3e9+/H7gvhz5vlFJvxH3l4W2O4wzn0C7wfbwAsw1fchyn0XGcVs+ObbgbNx8ylNvs+/h2LDbEDXKv9v7fQiafANl4LfCy4zhtOXQHgXO978/DPR1Ci8+GIeDfcDfTs/WLgA2Psg9ptTr7WbQB++m0JvvhOnBduVr7Wa7PZENTW2TYz1JuwH6WtgjYz+CntuCuqrnY+68p2+Xt0zyNVq+Uuhp4A3Cp4+0vYNBuVUrN913/27z/r9OudRxnms9+w47jzLfUodlX7oXARsv1pW3ntfW2HG1xMW4SZDRHW2wBqrz7gdTPLHVO2a8I+BfcjWR1seMynf2OJs6YtDrb6bTA5TrbGcqt0dnOUoeA7Wx11tkvR1tk2M9wfRfobGepc8B23mdTPNf5zrxjv0lr8J0mrc536rTPGXzngKHcgO+0XJvWb+Zoi2zfadLqfKepLQK+0zK2CtjuaMZhJq1p3GLRB+xn0K4zjF1eMpQbsJ/l+nT222xpiwzbWcoN2M7SDtpxi4/s8a5t7JnP2FirNdnPoLWNO9PaPMad2eXmGndmX59t7KlrC9O4M1trG3dm11lnP938YzOauGfQmuYqWq0u7uXQB2KfQfsdXeyzlKuLfabrC8S9HG2RPW4xtXEg7uVoC1Ps080JtX3PoNWi05r6nkGr7Xsa7W2mvmco19j3DNdnin2mtgj0PYNW2/cMdc7lO+0cTXbqWP7CfQdxG25W8LMW3R2475tOeDfMVRbtWbjv96eOTMw42jBLuwL3WOQXvRvrc3nW+1VYXqnDPXlvA5NHZhuvzfd/TsA9/vBF7yausWjLcLPcVXmU+wXczrgR94SFIov2SVxntQF4TS4b4O5D8nuvMzwG1ObQv937fgw3SfCwRbsDd4+vlA2vt2h/5V3fi7jHQraYtFnXtIfJpya6cn+Ce9Tki7hOttmijeE+qd+Ie7zoebnuX9zTLP4+j3Y+C3e55Abc5aEnWbQfw+1T23Df205lybX9QmdDizZgP4s2YD+LNmA/k1ZnP0u5JvuZ9AEb2uqRbT9LuQH7WbQB+2HwU7h+Zo3X1nfhLuE2aa/xbBfHDWapJdYmfRzXL6fq9jmdFnfp8Z+9dt6Iu3Kn0lRulv0Gc9ThD75yf4r7+qFJW437VP0l3Cdkx9vqgPvE7Y35xAPc+/4lz36Pe+1u0n4Ld3C3laxjebNjh85+Fq3WfgZtwHY6rcl2+cQ69K9D+usQsF0OfcB+tnro7GcoN2A7i1ZrOwzxHL3vNGl1vtOk1flOk1bnO3OOP5j0naZyA77TotXGPls9CPpOU9k632nSmmJfYGyls51Faxq36LTacYtFbxq7WMeDZI5ddOWaYp9Oa7Kftg7ZtrOUaxq36LRa23n6wHjXYj+d1mQ/ndY07tRpTbazjs+zbKcrV2s7i95kP209DPbTlWuyn05r6nuB+QeGuGfQmsYtOq0x7hn02tin05pin6FcbewzaLVxz1QH9OMWXbnGuGfQm2JfYE6Iue/ptKa+p9Oa+p5Oa+p7xjmspu/pyrX1PZ3e1Pe09UDf93TlmvqeTmv0nfl8pTqqIAiCIAiCIAiCIAiCIEwJfyuv1AmCIAiCIAiCIAiCIAh/JSThJAiCIAiCIAiCIAiCIEwpknASBEEQBEEQBEEQBEEQphRJOAmCIAiCIAiCIAiCIAhTiiScBEEQBEEQBEEQBEEQhClFEk6CIAiCIAgFjFLKUUrNf6XrIQiCIAiCcDRIwkkQBEEQBOEoUErtUUqNKKUGfV//9UrXSxAEQRAEoZCIvNIVEARBEARBOAZ5q+M4j73SlRAEQRAEQShUZIWTIAiCIAjCFKCUukIp9Wel1H8ppfqUUi8rpV7j+/10pdT9SqlupdQOpdTf+X4XVkp9Rim1Uyk1oJRaq5Sa6Sv+tUqp7UqpXqXUfyullPf/5iul/uT9vU6l1C/+ipcsCIIgCIJgRFY4CYIgCIIgTB2nAncD9cBFwD1KqTmO43QDdwIbgenAYuBRpdROx3H+AHwcuBQ4H9gGrACGfeW+BTgZqATWAg8AvwO+BDwCvBqIAav+ry9QEARBEAQhH5TjOK90HQRBEARBEI4ZlFJ7cBNKcd+PPwVMAF8FWhxvgKWUWgP8AHgc2ANUO44z4P3ua0Cz4zhXKKW2Atc6jnOf5u85wNmO4zzlff4lsM5xnK8rpW4HRoEvOo7T9n9wuYIgCIIgCH8R8kqdIAiCIAjC0XOh4zjVvq8bvJ8fcDKf5u3FXdE0HehOJZt8v2vxvp8J7LT8vUO+74eBcu/7awEFrFFKbVJKXfkXXo8gCIIgCMKUIgknQRAEQRCEqaMltb+SxyzgoPdVq5SqyPrdAe/7/cC8o/1jjuMcchzn7xzHmQ58CPihUmr+X1Z1QRAEQRCEqUMSToIgCIIgCFNHI3CNUiqqlLoEWAL81nGc/cDTwNeUUsVKqRXAVcBPvf93I/AlpdQC5bJCKVWX648ppS5RSs3wPvYADpCc6osSBEEQBEE4WmTTcEEQBEEQhKPnAaVUwvf5UeA+YDWwAOgEDgMXO47T5WkuBa7HXe3UA3zecZzHvN99ByjC3QC8HngZeHse9TgZ+J5Sqsr7ex9zHGfX/+bCBEEQBEEQpgLZNFwQBEEQBGEKUEpdAVztOM5Zr3RdBEEQBEEQXmnklTpBEARBEARBEARBEARhSpGEkyAIgiAIgiAIgiAIgjClyCt1giAIgiAIgiAIgiAIwpQiK5wEQRAEQRAEQRAEQRCEKUUSToIgCIIgCIIgCIIgCMKUIgknQRAEQRAEQRAEQRAEYUqRhJMgCIIgCIIgCIIgCIIwpUjCSRAEQRAEQRAEQRAEQZhS/h8eZnJxoULcXgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#Plot training loss over Epochs:\n", + "color = sns.color_palette()\n", + "#Draw Weight Variance Ratio\n", + "dataplot3 = {\"svrg_mse\": [], \"sgd_mse_lr_0.001\": [], \"sgd_mse_lr_0.0025\": [], \"sgd_mse_lr_0.005\":[]}\n", + "with open('sgd_0.001.json') as sgd_data, open('svrg_0.025.json') as svrg_data, open('sgd_0.0025.json') as sgd_data_2, open('sgd_0.005.json') as sgd_data_3:\n", + " sgd = json.load(sgd_data)\n", + " svrg = json.load(svrg_data)\n", + " sgd_lr = json.load(sgd_data_2)\n", + " sgd_lr_2 = json.load(sgd_data_3)\n", + " for epoch in range(100):\n", + " dataplot3[\"svrg_mse\"].append(svrg[str(epoch)][\"mse\"])\n", + " dataplot3[\"sgd_mse_lr_0.001\"].append(sgd[str(epoch)][\"mse\"])\n", + " dataplot3[\"sgd_mse_lr_0.0025\"].append(sgd_lr[str(epoch)][\"mse\"])\n", + " dataplot3[\"sgd_mse_lr_0.005\"].append(sgd_lr_2[str(epoch)][\"mse\"])\n", + "\n", + "x3 = list(range(100))\n", + "plt.figure(figsize=(20, 12))\n", + "plt.title(\"Training Loss Over Epochs\")\n", + "sns.pointplot(x3, dataplot3['svrg_mse'], color=color[9])\n", + "sns.pointplot(x3, dataplot3['sgd_mse_lr_0.001'], color=color[8])\n", + "sns.pointplot(x3, dataplot3['sgd_mse_lr_0.0025'], color=color[3])\n", + "sns.pointplot(x3, dataplot3['sgd_mse_lr_0.005'], color=color[7])\n", + "color_patch1 = mpatches.Patch(color=color[9], label=\"svrg_mse_lr_0.025\")\n", + "color_patch2 = mpatches.Patch(color=color[8], label=\"sgd_mse_lr_0.001\")\n", + "color_patch3 = mpatches.Patch(color=color[3], label=\"sgd_mse_lr_0.0025\")\n", + "color_patch4 = mpatches.Patch(color=color[7], label=\"sgd_mse_lr_0.005\")\n", + "plt.legend(handles=[color_patch1, color_patch2, color_patch3, color_patch4])\n", + "plt.ylabel('Training Loss', fontsize=12)\n", + "plt.xlabel('Epochs', fontsize=12)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/example/svrg_module/linear_regression/common.py b/example/svrg_module/linear_regression/common.py new file mode 100644 index 000000000000..14a144f40ce2 --- /dev/null +++ b/example/svrg_module/linear_regression/common.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import mxnet as mx +import logging +from mxnet.contrib.svrg_optimization.svrg_module import SVRGModule + + +def create_lin_reg_network(train_features, train_labels, feature_dim, batch_size, update_freq, ctx, logger): + # fit a linear regression model with mxnet SVRGModule + print("Fitting linear regression with mxnet") + train_iter = mx.io.NDArrayIter(train_features, train_labels, batch_size=batch_size, shuffle=True, + data_name='data', label_name='label') + data = mx.sym.Variable("data") + label = mx.sym.Variable("label") + weight = mx.sym.Variable("fc_weight", shape=(1, feature_dim)) + net = mx.sym.dot(data, weight.transpose()) + bias = mx.sym.Variable("fc_bias", shape=(1,), wd_mult=0.0, lr_mult=10.0) + net = mx.sym.broadcast_plus(net, bias) + net = mx.sym.LinearRegressionOutput(data=net, label=label) + mod = SVRGModule(symbol=net, context=ctx, data_names=['data'], label_names=['label'], logger=logger, + update_freq=update_freq) + return train_iter, mod + + +def create_metrics(metrics): + metric = mx.metric.create(metrics) + return metric + + +def create_logger(): + logger = logging.getLogger('sgd_svrg') + logger.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(message)s') + fh = logging.FileHandler('experiments.log') + fh.setFormatter(formatter) + logger.addHandler(fh) + return logger + + +################################################################################ +# Functions below are for benchmark purpose to calcuate expectation, variance of +# gradients per epoch for each parameter. These calculations will be helpful when +# benchmarking SVRG optimization with other optimization techniques, such as SGD. +# Currently it only calculates the expectation, variance for single context but +# can be extended to multi-context in later iterations. +################################################################################ + +def accumulate_grad(grad_dict, mod): + param_names = mod._exec_group.param_names + + for index, name in enumerate(param_names): + if name not in grad_dict: + grad_dict[name] = mod._exec_group.grad_arrays[index][0].copy() + else: + grad_dict[name] = mx.ndarray.concat(grad_dict[name], mod._exec_group.grad_arrays[index][0], dim=0) + + +def calc_expectation(grad_dict, num_batches): + """Calculates the expectation of the gradients per epoch for each parameter w.r.t number of batches + + Parameters + ---------- + grad_dict: dict + dictionary that maps parameter name to gradients in the mod executor group + num_batches: int + number of batches + + Returns + ---------- + grad_dict: dict + dictionary with new keys mapping to gradients expectations + + """ + for key in grad_dict.keys(): + grad_dict[str.format(key+"_expectation")] = mx.ndarray.sum(grad_dict[key], axis=0) / num_batches + + return grad_dict + + +def calc_variance(grad_dict, num_batches, param_names): + """Calculates the variance of the gradients per epoch for each parameter w.r.t number of batches + + Parameters + ---------- + grad_dict: dict + dictionary that maps parameter name to gradients in the mod executor group + num_batches: int + number of batches + param_names: str + parameter name in the module + + Returns + ---------- + grad_dict: dict + dictionary with new keys mapping to gradients variance + + """ + for i in range(len(param_names)): + diff_sqr = mx.ndarray.square(mx.nd.subtract(grad_dict[param_names[i]], + grad_dict[str.format(param_names[i]+"_expectation")])) + grad_dict[str.format(param_names[i] + "_variance")] = mx.ndarray.sum(diff_sqr, axis=0) / num_batches diff --git a/example/vae-gan/vaegan_mxnet.py b/example/vae-gan/vaegan_mxnet.py new file mode 100644 index 000000000000..38e7e2ecc92f --- /dev/null +++ b/example/vae-gan/vaegan_mxnet.py @@ -0,0 +1,739 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +''' +Created on Jun 15, 2017 + +@author: shujon +''' + +from __future__ import print_function +import logging +from datetime import datetime +import os +import argparse +import errno +import mxnet as mx +import numpy as np +import cv2 +from scipy.io import savemat +#from layer import GaussianSampleLayer + +###################################################################### +#An adversarial variational autoencoder implementation in mxnet +# following the implementation at https://github.com/JeremyCCHsu/tf-vaegan +# of paper `Larsen, Anders Boesen Lindbo, et al. "Autoencoding beyond pixels using a +# learned similarity metric." arXiv preprint arXiv:1512.09300 (2015).` +###################################################################### + +@mx.init.register +class MyConstant(mx.init.Initializer): + '''constant operator in mxnet, no used in the code + ''' + def __init__(self, value): + super(MyConstant, self).__init__(value=value) + self.value = value + + def _init_weight(self, _, arr): + arr[:] = mx.nd.array(self.value) + +def encoder(nef, z_dim, batch_size, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12): + '''The encoder is a CNN which takes 32x32 image as input + generates the 100 dimensional shape embedding as a sample from normal distribution + using predicted meand and variance + ''' + BatchNorm = mx.sym.BatchNorm + + data = mx.sym.Variable('data') + + e1 = mx.sym.Convolution(data, name='enc1', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef, no_bias=no_bias) + ebn1 = BatchNorm(e1, name='encbn1', fix_gamma=fix_gamma, eps=eps) + eact1 = mx.sym.LeakyReLU(ebn1, name='encact1', act_type='leaky', slope=0.2) + + e2 = mx.sym.Convolution(eact1, name='enc2', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef*2, no_bias=no_bias) + ebn2 = BatchNorm(e2, name='encbn2', fix_gamma=fix_gamma, eps=eps) + eact2 = mx.sym.LeakyReLU(ebn2, name='encact2', act_type='leaky', slope=0.2) + + e3 = mx.sym.Convolution(eact2, name='enc3', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef*4, no_bias=no_bias) + ebn3 = BatchNorm(e3, name='encbn3', fix_gamma=fix_gamma, eps=eps) + eact3 = mx.sym.LeakyReLU(ebn3, name='encact3', act_type='leaky', slope=0.2) + + e4 = mx.sym.Convolution(eact3, name='enc4', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=nef*8, no_bias=no_bias) + ebn4 = BatchNorm(e4, name='encbn4', fix_gamma=fix_gamma, eps=eps) + eact4 = mx.sym.LeakyReLU(ebn4, name='encact4', act_type='leaky', slope=0.2) + + eact4 = mx.sym.Flatten(eact4) + + z_mu = mx.sym.FullyConnected(eact4, num_hidden=z_dim, name="enc_mu") + z_lv = mx.sym.FullyConnected(eact4, num_hidden=z_dim, name="enc_lv") + + z = z_mu + mx.symbol.broadcast_mul(mx.symbol.exp(0.5*z_lv),mx.symbol.random_normal(loc=0, scale=1,shape=(batch_size,z_dim))) + + return z_mu, z_lv, z + +def generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim=100, activation='sigmoid'): + '''The genrator is a CNN which takes 100 dimensional embedding as input + and reconstructs the input image given to the encoder + ''' + BatchNorm = mx.sym.BatchNorm + rand = mx.sym.Variable('rand') + + rand = mx.sym.Reshape(rand, shape=(-1, z_dim, 1, 1)) + + g1 = mx.sym.Deconvolution(rand, name='gen1', kernel=(5,5), stride=(2,2),target_shape=(2,2), num_filter=ngf*8, no_bias=no_bias) + gbn1 = BatchNorm(g1, name='genbn1', fix_gamma=fix_gamma, eps=eps) + gact1 = mx.sym.Activation(gbn1, name="genact1", act_type="relu") + + g2 = mx.sym.Deconvolution(gact1, name='gen2', kernel=(5,5), stride=(2,2),target_shape=(4,4), num_filter=ngf*4, no_bias=no_bias) + gbn2 = BatchNorm(g2, name='genbn2', fix_gamma=fix_gamma, eps=eps) + gact2 = mx.sym.Activation(gbn2, name='genact2', act_type='relu') + + g3 = mx.sym.Deconvolution(gact2, name='gen3', kernel=(5,5), stride=(2,2), target_shape=(8,8), num_filter=ngf*2, no_bias=no_bias) + gbn3 = BatchNorm(g3, name='genbn3', fix_gamma=fix_gamma, eps=eps) + gact3 = mx.sym.Activation(gbn3, name='genact3', act_type='relu') + + g4 = mx.sym.Deconvolution(gact3, name='gen4', kernel=(5,5), stride=(2,2), target_shape=(16,16), num_filter=ngf, no_bias=no_bias) + gbn4 = BatchNorm(g4, name='genbn4', fix_gamma=fix_gamma, eps=eps) + gact4 = mx.sym.Activation(gbn4, name='genact4', act_type='relu') + + g5 = mx.sym.Deconvolution(gact4, name='gen5', kernel=(5,5), stride=(2,2), target_shape=(32,32), num_filter=nc, no_bias=no_bias) + gout = mx.sym.Activation(g5, name='genact5', act_type=activation) + + return gout + +def discriminator1(ndf, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12): + '''First part of the discriminator which takes a 32x32 image as input + and output a convolutional feature map, this is required to calculate + the layer loss''' + BatchNorm = mx.sym.BatchNorm + + data = mx.sym.Variable('data') + + d1 = mx.sym.Convolution(data, name='d1', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=ndf, no_bias=no_bias) + dact1 = mx.sym.LeakyReLU(d1, name='dact1', act_type='leaky', slope=0.2) + + d2 = mx.sym.Convolution(dact1, name='d2', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=ndf*2, no_bias=no_bias) + dbn2 = BatchNorm(d2, name='dbn2', fix_gamma=fix_gamma, eps=eps) + dact2 = mx.sym.LeakyReLU(dbn2, name='dact2', act_type='leaky', slope=0.2) + + d3 = mx.sym.Convolution(dact2, name='d3', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=ndf*4, no_bias=no_bias) + dbn3 = BatchNorm(d3, name='dbn3', fix_gamma=fix_gamma, eps=eps) + dact3 = mx.sym.LeakyReLU(dbn3, name='dact3', act_type='leaky', slope=0.2) + + return dact3 + +def discriminator2(ndf, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12): + '''Second part of the discriminator which takes a 256x8x8 feature map as input + and generates the loss based on whether the input image was a real one or fake one''' + + BatchNorm = mx.sym.BatchNorm + + data = mx.sym.Variable('data') + + label = mx.sym.Variable('label') + + d4 = mx.sym.Convolution(data, name='d4', kernel=(5,5), stride=(2,2), pad=(2,2), num_filter=ndf*8, no_bias=no_bias) + dbn4 = BatchNorm(d4, name='dbn4', fix_gamma=fix_gamma, eps=eps) + dact4 = mx.sym.LeakyReLU(dbn4, name='dact4', act_type='leaky', slope=0.2) + + h = mx.sym.Flatten(dact4) + + d5 = mx.sym.FullyConnected(h, num_hidden=1, name="d5") + + dloss = mx.sym.LogisticRegressionOutput(data=d5, label=label, name='dloss') + + return dloss + +def GaussianLogDensity(x, mu, log_var, name='GaussianLogDensity', EPSILON = 1e-6): + '''GaussianLogDensity loss calculation for layer wise loss + ''' + c = mx.sym.ones_like(log_var)*2.0 * 3.1416 + c = mx.symbol.log(c) + var = mx.sym.exp(log_var) + x_mu2 = mx.symbol.square(x - mu) # [Issue] not sure the dim works or not? + x_mu2_over_var = mx.symbol.broadcast_div(x_mu2, var + EPSILON) + log_prob = -0.5 * (c + log_var + x_mu2_over_var) + log_prob = mx.symbol.sum(log_prob, axis=1, name=name) # keep_dims=True, + return log_prob + +def DiscriminatorLayerLoss(): + '''Calculate the discriminator layer loss + ''' + + data = mx.sym.Variable('data') + + label = mx.sym.Variable('label') + + data = mx.sym.Flatten(data) + label = mx.sym.Flatten(label) + + label = mx.sym.BlockGrad(label) + + zeros = mx.sym.zeros_like(data) + + output = -GaussianLogDensity(label, data, zeros) + + dloss = mx.symbol.MakeLoss(mx.symbol.mean(output),name='lloss') + + return dloss + +def KLDivergenceLoss(): + '''KLDivergenceLoss loss + ''' + + data = mx.sym.Variable('data') + mu1, lv1 = mx.sym.split(data, num_outputs=2, axis=0) + mu2 = mx.sym.zeros_like(mu1) + lv2 = mx.sym.zeros_like(lv1) + + v1 = mx.sym.exp(lv1) + v2 = mx.sym.exp(lv2) + mu_diff_sq = mx.sym.square(mu1 - mu2) + dimwise_kld = .5 * ( + (lv2 - lv1) + mx.symbol.broadcast_div(v1, v2) + mx.symbol.broadcast_div(mu_diff_sq, v2) - 1.) + KL = mx.symbol.sum(dimwise_kld, axis=1) + + KLloss = mx.symbol.MakeLoss(mx.symbol.mean(KL),name='KLloss') + return KLloss + +def get_data(path, activation): + '''Get the dataset + ''' + data = [] + image_names = [] + for filename in os.listdir(path): + img = cv2.imread(os.path.join(path,filename), cv2.IMREAD_GRAYSCALE) + image_names.append(filename) + if img is not None: + data.append(img) + + data = np.asarray(data) + + if activation == 'sigmoid': + data = data.astype(np.float32)/(255.0) + elif activation == 'tanh': + data = data.astype(np.float32)/(255.0/2) - 1.0 + + data = data.reshape((data.shape[0], 1, data.shape[1], data.shape[2])) + + np.random.seed(1234) + p = np.random.permutation(data.shape[0]) + X = data[p] + + return X, image_names + +class RandIter(mx.io.DataIter): + '''Create a random iterator for generator + ''' + def __init__(self, batch_size, ndim): + self.batch_size = batch_size + self.ndim = ndim + self.provide_data = [('rand', (batch_size, ndim, 1, 1))] + self.provide_label = [] + + def iter_next(self): + return True + + def getdata(self): + return [mx.random.normal(0, 1.0, shape=(self.batch_size, self.ndim, 1, 1))] + +def fill_buf(buf, i, img, shape): + '''fill the ith grid of the buffer matrix with the values from the img + buf : buffer matrix + i : serial of the image in the 2D grid + img : image data + shape : ( height width depth ) of image''' + + # grid height is a multiple of individual image height + m = buf.shape[0]/shape[0] + + sx = (i%m)*shape[1] + sy = (i//m)*shape[0] + sx = int(sx) + sy = int(sy) + buf[sy:sy+shape[0], sx:sx+shape[1], :] = img + +def visual(title, X, activation): + '''create a grid of images and save it as a final image + title : grid image name + X : array of images + ''' + assert len(X.shape) == 4 + + X = X.transpose((0, 2, 3, 1)) + if activation == 'sigmoid': + X = np.clip((X)*(255.0), 0, 255).astype(np.uint8) + elif activation == 'tanh': + X = np.clip((X+1.0)*(255.0/2.0), 0, 255).astype(np.uint8) + n = np.ceil(np.sqrt(X.shape[0])) + buff = np.zeros((int(n*X.shape[1]), int(n*X.shape[2]), int(X.shape[3])), dtype=np.uint8) + for i, img in enumerate(X): + fill_buf(buff, i, img, X.shape[1:3]) + cv2.imwrite('%s.jpg' % (title), buff) + +def train(dataset, nef, ndf, ngf, nc, batch_size, Z, lr, beta1, epsilon, ctx, check_point, g_dl_weight, output_path, checkpoint_path, data_path, activation,num_epoch, save_after_every, visualize_after_every, show_after_every): + '''adversarial training of the VAE + ''' + + #encoder + z_mu, z_lv, z = encoder(nef, Z, batch_size) + symE = mx.sym.Group([z_mu, z_lv, z]) + + #generator + symG = generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim = Z, activation=activation ) + + #discriminator + h = discriminator1(ndf) + dloss = discriminator2(ndf) + symD1 = h + symD2 = dloss + + + # ==============data============== + X_train, _ = get_data(data_path, activation) + train_iter = mx.io.NDArrayIter(X_train, batch_size=batch_size, shuffle=True) + rand_iter = RandIter(batch_size, Z) + label = mx.nd.zeros((batch_size,), ctx=ctx) + + # =============module E============= + modE = mx.mod.Module(symbol=symE, data_names=('data',), label_names=None, context=ctx) + modE.bind(data_shapes=train_iter.provide_data) + modE.init_params(initializer=mx.init.Normal(0.02)) + modE.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'wd': 1e-6, + 'beta1': beta1, + 'epsilon': epsilon, + 'rescale_grad': (1.0/batch_size) + }) + mods = [modE] + + # =============module G============= + modG = mx.mod.Module(symbol=symG, data_names=('rand',), label_names=None, context=ctx) + modG.bind(data_shapes=rand_iter.provide_data, inputs_need_grad=True) + modG.init_params(initializer=mx.init.Normal(0.02)) + modG.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'wd': 1e-6, + 'beta1': beta1, + 'epsilon': epsilon, + }) + mods.append(modG) + + # =============module D============= + modD1 = mx.mod.Module(symD1, label_names=[], context=ctx) + modD2 = mx.mod.Module(symD2, label_names=('label',), context=ctx) + modD = mx.mod.SequentialModule() + modD.add(modD1).add(modD2, take_labels=True, auto_wiring=True) + modD.bind(data_shapes=train_iter.provide_data, + label_shapes=[('label', (batch_size,))], + inputs_need_grad=True) + modD.init_params(initializer=mx.init.Normal(0.02)) + modD.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'wd': 1e-3, + 'beta1': beta1, + 'epsilon': epsilon, + 'rescale_grad': (1.0/batch_size) + }) + mods.append(modD) + + + # =============module DL============= + symDL = DiscriminatorLayerLoss() + modDL = mx.mod.Module(symbol=symDL, data_names=('data',), label_names=('label',), context=ctx) + modDL.bind(data_shapes=[('data', (batch_size,nef * 4,4,4))], ################################################################################################################################ fix 512 here + label_shapes=[('label', (batch_size,nef * 4,4,4))], + inputs_need_grad=True) + modDL.init_params(initializer=mx.init.Normal(0.02)) + modDL.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'wd': 0., + 'beta1': beta1, + 'epsilon': epsilon, + 'rescale_grad': (1.0/batch_size) + }) + + # =============module KL============= + symKL = KLDivergenceLoss() + modKL = mx.mod.Module(symbol=symKL, data_names=('data',), label_names=None, context=ctx) + modKL.bind(data_shapes=[('data', (batch_size*2,Z))], + inputs_need_grad=True) + modKL.init_params(initializer=mx.init.Normal(0.02)) + modKL.init_optimizer( + optimizer='adam', + optimizer_params={ + 'learning_rate': lr, + 'wd': 0., + 'beta1': beta1, + 'epsilon': epsilon, + 'rescale_grad': (1.0/batch_size) + }) + mods.append(modKL) + + def norm_stat(d): + return mx.nd.norm(d)/np.sqrt(d.size) + mon = mx.mon.Monitor(10, norm_stat, pattern=".*output|d1_backward_data", sort=True) + mon = None + if mon is not None: + for mod in mods: + pass + + def facc(label, pred): + '''calculating prediction accuracy + ''' + pred = pred.ravel() + label = label.ravel() + return ((pred > 0.5) == label).mean() + + def fentropy(label, pred): + '''calculating binary cross-entropy loss + ''' + pred = pred.ravel() + label = label.ravel() + return -(label*np.log(pred+1e-12) + (1.-label)*np.log(1.-pred+1e-12)).mean() + + def kldivergence(label, pred): + '''calculating KL divergence loss + ''' + mean, log_var = np.split(pred, 2, axis=0) + var = np.exp(log_var) + KLLoss = -0.5 * np.sum(1 + log_var - np.power(mean, 2) - var) + KLLoss = KLLoss / nElements + return KLLoss + + mG = mx.metric.CustomMetric(fentropy) + mD = mx.metric.CustomMetric(fentropy) + mE = mx.metric.CustomMetric(kldivergence) + mACC = mx.metric.CustomMetric(facc) + + print('Training...') + stamp = datetime.now().strftime('%Y_%m_%d-%H_%M') + + # =============train=============== + for epoch in range(num_epoch): + train_iter.reset() + for t, batch in enumerate(train_iter): + + rbatch = rand_iter.next() + + if mon is not None: + mon.tic() + + modG.forward(rbatch, is_train=True) + outG = modG.get_outputs() + + # update discriminator on fake + label[:] = 0 + modD.forward(mx.io.DataBatch(outG, [label]), is_train=True) + modD.backward() + gradD11 = [[grad.copyto(grad.context) for grad in grads] for grads in modD1._exec_group.grad_arrays] + gradD12 = [[grad.copyto(grad.context) for grad in grads] for grads in modD2._exec_group.grad_arrays] + + modD.update_metric(mD, [label]) + modD.update_metric(mACC, [label]) + + + #update discriminator on decoded + modE.forward(batch, is_train=True) + mu, lv, z = modE.get_outputs() + z = z.reshape((batch_size, Z, 1, 1)) + sample = mx.io.DataBatch([z], label=None, provide_data = [('rand', (batch_size, Z, 1, 1))]) + modG.forward(sample, is_train=True) + xz = modG.get_outputs() + label[:] = 0 + modD.forward(mx.io.DataBatch(xz, [label]), is_train=True) + modD.backward() + + #modD.update() + gradD21 = [[grad.copyto(grad.context) for grad in grads] for grads in modD1._exec_group.grad_arrays] + gradD22 = [[grad.copyto(grad.context) for grad in grads] for grads in modD2._exec_group.grad_arrays] + modD.update_metric(mD, [label]) + modD.update_metric(mACC, [label]) + + # update discriminator on real + label[:] = 1 + batch.label = [label] + modD.forward(batch, is_train=True) + lx = [out.copyto(out.context) for out in modD1.get_outputs()] + modD.backward() + for gradsr, gradsf, gradsd in zip(modD1._exec_group.grad_arrays, gradD11, gradD21): + for gradr, gradf, gradd in zip(gradsr, gradsf, gradsd): + gradr += 0.5 * (gradf + gradd) + for gradsr, gradsf, gradsd in zip(modD2._exec_group.grad_arrays, gradD12, gradD22): + for gradr, gradf, gradd in zip(gradsr, gradsf, gradsd): + gradr += 0.5 * (gradf + gradd) + + modD.update() + modD.update_metric(mD, [label]) + modD.update_metric(mACC, [label]) + + modG.forward(rbatch, is_train=True) + outG = modG.get_outputs() + label[:] = 1 + modD.forward(mx.io.DataBatch(outG, [label]), is_train=True) + modD.backward() + diffD = modD1.get_input_grads() + modG.backward(diffD) + gradG1 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays] + mG.update([label], modD.get_outputs()) + + modG.forward(sample, is_train=True) + xz = modG.get_outputs() + label[:] = 1 + modD.forward(mx.io.DataBatch(xz, [label]), is_train=True) + modD.backward() + diffD = modD1.get_input_grads() + modG.backward(diffD) + gradG2 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays] + mG.update([label], modD.get_outputs()) + + modG.forward(sample, is_train=True) + xz = modG.get_outputs() + modD1.forward(mx.io.DataBatch(xz, []), is_train=True) + outD1 = modD1.get_outputs() + modDL.forward(mx.io.DataBatch(outD1, lx), is_train=True) + modDL.backward() + dlGrad = modDL.get_input_grads() + modD1.backward(dlGrad) + diffD = modD1.get_input_grads() + modG.backward(diffD) + + for grads, gradsG1, gradsG2 in zip(modG._exec_group.grad_arrays, gradG1, gradG2): + for grad, gradg1, gradg2 in zip(grads, gradsG1, gradsG2): + grad = g_dl_weight * grad + 0.5 * (gradg1 + gradg2) + + modG.update() + mG.update([label], modD.get_outputs()) + + modG.forward(rbatch, is_train=True) + outG = modG.get_outputs() + label[:] = 1 + modD.forward(mx.io.DataBatch(outG, [label]), is_train=True) + modD.backward() + diffD = modD1.get_input_grads() + modG.backward(diffD) + gradG1 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays] + mG.update([label], modD.get_outputs()) + + modG.forward(sample, is_train=True) + xz = modG.get_outputs() + label[:] = 1 + modD.forward(mx.io.DataBatch(xz, [label]), is_train=True) + modD.backward() + diffD = modD1.get_input_grads() + modG.backward(diffD) + gradG2 = [[grad.copyto(grad.context) for grad in grads] for grads in modG._exec_group.grad_arrays] + mG.update([label], modD.get_outputs()) + + modG.forward(sample, is_train=True) + xz = modG.get_outputs() + modD1.forward(mx.io.DataBatch(xz, []), is_train=True) + outD1 = modD1.get_outputs() + modDL.forward(mx.io.DataBatch(outD1, lx), is_train=True) + modDL.backward() + dlGrad = modDL.get_input_grads() + modD1.backward(dlGrad) + diffD = modD1.get_input_grads() + modG.backward(diffD) + + for grads, gradsG1, gradsG2 in zip(modG._exec_group.grad_arrays, gradG1, gradG2): + for grad, gradg1, gradg2 in zip(grads, gradsG1, gradsG2): + grad = g_dl_weight * grad + 0.5 * (gradg1 + gradg2) + + modG.update() + mG.update([label], modD.get_outputs()) + + modG.forward(sample, is_train=True) + xz = modG.get_outputs() + + #update generator + modD1.forward(mx.io.DataBatch(xz, []), is_train=True) + outD1 = modD1.get_outputs() + modDL.forward(mx.io.DataBatch(outD1, lx), is_train=True) + DLloss = modDL.get_outputs() + modDL.backward() + dlGrad = modDL.get_input_grads() + modD1.backward(dlGrad) + diffD = modD1.get_input_grads() + modG.backward(diffD) + #update encoder + nElements = batch_size + modKL.forward(mx.io.DataBatch([mx.ndarray.concat(mu,lv, dim=0)]), is_train=True) + KLloss = modKL.get_outputs() + modKL.backward() + gradKLLoss = modKL.get_input_grads() + diffG = modG.get_input_grads() + diffG = diffG[0].reshape((batch_size, Z)) + modE.backward(mx.ndarray.split(gradKLLoss[0], num_outputs=2, axis=0) + [diffG]) + modE.update() + pred = mx.ndarray.concat(mu,lv, dim=0) + mE.update([pred], [pred]) + if mon is not None: + mon.toc_print() + + t += 1 + if t % show_after_every == 0: + print('epoch:', epoch, 'iter:', t, 'metric:', mACC.get(), mG.get(), mD.get(), mE.get(), KLloss[0].asnumpy(), DLloss[0].asnumpy()) + mACC.reset() + mG.reset() + mD.reset() + mE.reset() + + if epoch % visualize_after_every == 0: + visual(output_path +'gout'+str(epoch), outG[0].asnumpy(), activation) + visual(output_path + 'data'+str(epoch), batch.data[0].asnumpy(), activation) + + if check_point and epoch % save_after_every == 0: + print('Saving...') + modG.save_params(checkpoint_path + '/%s_G-%04d.params'%(dataset, epoch)) + modD.save_params(checkpoint_path + '/%s_D-%04d.params'%(dataset, epoch)) + modE.save_params(checkpoint_path + '/%s_E-%04d.params'%(dataset, epoch)) + +def test(nef, ngf, nc, batch_size, Z, ctx, pretrained_encoder_path, pretrained_generator_path, output_path, data_path, activation, save_embedding, embedding_path = ''): + '''Test the VAE with a pretrained encoder and generator. + Keep the batch size 1''' + #encoder + z_mu, z_lv, z = encoder(nef, Z, batch_size) + symE = mx.sym.Group([z_mu, z_lv, z]) + + #generator + symG = generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim = Z, activation=activation ) + + # ==============data============== + X_test, image_names = get_data(data_path, activation) + test_iter = mx.io.NDArrayIter(X_test, batch_size=batch_size, shuffle=False) + + # =============module E============= + modE = mx.mod.Module(symbol=symE, data_names=('data',), label_names=None, context=ctx) + modE.bind(data_shapes=test_iter.provide_data) + modE.load_params(pretrained_encoder_path) + + # =============module G============= + modG = mx.mod.Module(symbol=symG, data_names=('rand',), label_names=None, context=ctx) + modG.bind(data_shapes=[('rand', (1, Z, 1, 1))]) + modG.load_params(pretrained_generator_path) + + print('Testing...') + + # =============test=============== + test_iter.reset() + for t, batch in enumerate(test_iter): + + #update discriminator on decoded + modE.forward(batch, is_train=False) + mu, lv, z = modE.get_outputs() + mu = mu.reshape((batch_size, Z, 1, 1)) + sample = mx.io.DataBatch([mu], label=None, provide_data = [('rand', (batch_size, Z, 1, 1))]) + modG.forward(sample, is_train=False) + outG = modG.get_outputs() + + visual(output_path + '/' + 'gout'+str(t), outG[0].asnumpy(), activation) + visual(output_path + '/' + 'data'+str(t), batch.data[0].asnumpy(), activation) + image_name = image_names[t].split('.')[0] + + if save_embedding: + savemat(embedding_path+'/'+image_name+'.mat', {'embedding':mu.asnumpy()}) + +def create_and_validate_dir(data_dir): + '''Creates/Validates dir + ''' + if data_dir != "": + if not os.path.exists(data_dir): + try: + logging.info('create directory %s', data_dir) + os.makedirs(data_dir) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise OSError('failed to create ' + data_dir) + + +def parse_args(): + '''Parse args + ''' + parser = argparse.ArgumentParser(description='Train and Test an Adversarial Variatiional Encoder') + + parser.add_argument('--train', help='train the network', action='store_true') + parser.add_argument('--test', help='test the network', action='store_true') + parser.add_argument('--save_embedding', help='saves the shape embedding of each input image', action='store_true') + parser.add_argument('--dataset', help='dataset name', default='caltech', type=str) + parser.add_argument('--activation', help='activation i.e. sigmoid or tanh', default='sigmoid', type=str) + parser.add_argument('--training_data_path', help='training data path', default='datasets/caltech101/data/images32x32', type=str) + parser.add_argument('--testing_data_path', help='testing data path', default='datasets/caltech101/test_data', type=str) + parser.add_argument('--pretrained_encoder_path', help='pretrained encoder model path', default='checkpoints32x32_sigmoid/caltech_E-0045.params', type=str) + parser.add_argument('--pretrained_generator_path', help='pretrained generator model path', default='checkpoints32x32_sigmoid/caltech_G-0045.params', type=str) + parser.add_argument('--output_path', help='output path for the generated images', default='outputs32x32_sigmoid', type=str) + parser.add_argument('--embedding_path', help='output path for the generated embeddings', default='outputs32x32_sigmoid', type=str) + parser.add_argument('--checkpoint_path', help='checkpoint saving path ', default='checkpoints32x32_sigmoid', type=str) + parser.add_argument('--nef', help='encoder filter count in the first layer', default=64, type=int) + parser.add_argument('--ndf', help='discriminator filter count in the first layer', default=64, type=int) + parser.add_argument('--ngf', help='generator filter count in the second last layer', default=64, type=int) + parser.add_argument('--nc', help='generator filter count in the last layer i.e. 1 for grayscale image, 3 for RGB image', default=1, type=int) + parser.add_argument('--batch_size', help='batch size, keep it 1 during testing', default=64, type=int) + parser.add_argument('--Z', help='embedding size', default=100, type=int) + parser.add_argument('--lr', help='learning rate', default=0.0002, type=float) + parser.add_argument('--beta1', help='beta1 for adam optimizer', default=0.5, type=float) + parser.add_argument('--epsilon', help='epsilon for adam optimizer', default=1e-5, type=float) + parser.add_argument('--g_dl_weight', help='discriminator layer loss weight', default=1e-1, type=float) + parser.add_argument('--gpu', help='gpu index', default=0, type=int) + parser.add_argument('--use_cpu', help='use cpu', action='store_true') + parser.add_argument('--num_epoch', help='number of maximum epochs ', default=45, type=int) + parser.add_argument('--save_after_every', help='save checkpoint after every this number of epochs ', default=5, type=int) + parser.add_argument('--visualize_after_every', help='save output images after every this number of epochs', default=5, type=int) + parser.add_argument('--show_after_every', help='show metrics after this number of iterations', default=10, type=int) + + args = parser.parse_args() + return args + +def main(): + args = parse_args() + + if args.test and not os.path.exists(args.testing_data_path): + if not os.path.exists(args.testing_data_path): + raise OSError("Provided Testing Path: {} does not exist".format(args.testing_data_path)) + if not os.path.exists(args.checkpoint_path): + raise OSError("Provided Checkpoint Path: {} does not exist".format(args.checkpoint_path)) + + create_and_validate_dir(args.checkpoint_path) + create_and_validate_dir(args.output_path) + + # gpu context + if args.use_cpu: + ctx = mx.cpu() + else: + ctx = mx.gpu(args.gpu) + + # checkpoint saving flags + check_point = True + + if args.train: + train(args.dataset, args.nef, args.ndf, args.ngf, args.nc, args.batch_size, args.Z, args.lr, args.beta1, args.epsilon, ctx, check_point, args.g_dl_weight, args.output_path, args.checkpoint_path, args.training_data_path, args.activation, args.num_epoch, args.save_after_every, args.visualize_after_every, args.show_after_every) + + if args.test: + test(args.nef, args.ngf, args.nc, 1, args.Z, ctx, args.pretrained_encoder_path, args.pretrained_generator_path, args.output_path, args.testing_data_path, args.activation, args.save_embedding, args.embedding_path) + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + main() diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py index efdd02a3be6a..8c98fd73561c 100644 --- a/python/mxnet/__init__.py +++ b/python/mxnet/__init__.py @@ -56,6 +56,7 @@ from . import random from . import optimizer from . import model +from . import metric from . import notebook from . import initializer # use mx.init as short for mx.initializer diff --git a/python/mxnet/callback.py b/python/mxnet/callback.py index 7ada7fe029f8..cf030509fed4 100644 --- a/python/mxnet/callback.py +++ b/python/mxnet/callback.py @@ -84,7 +84,7 @@ def _callback(param): logging.info('Iter[%d] Batch[%d] Train-%s=%f', param.epoch, param.nbatch, name, value) if auto_reset: - param.eval_metric.reset() + param.eval_metric.reset_local() return _callback @@ -135,7 +135,7 @@ def __call__(self, param): if param.eval_metric is not None: name_value = param.eval_metric.get_name_value() if self.auto_reset: - param.eval_metric.reset() + param.eval_metric.reset_local() msg = 'Epoch[%d] Batch [%d-%d]\tSpeed: %.2f samples/sec' msg += '\t%s=%f'*len(name_value) logging.info(msg, param.epoch, count-self.frequent, count, speed, *sum(name_value, ())) diff --git a/python/mxnet/contrib/svrg_optimization/svrg_module.py b/python/mxnet/contrib/svrg_optimization/svrg_module.py new file mode 100644 index 000000000000..eecb87cf25bb --- /dev/null +++ b/python/mxnet/contrib/svrg_optimization/svrg_module.py @@ -0,0 +1,579 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# coding: utf-8 +"""A `SVRGModule` implements the `Module` API by wrapping an auxiliary module to perform +SVRG optimization logic. +""" + +import time +import logging +import mxnet as mx +from mxnet.module import Module +from .svrg_optimizer import _SVRGOptimizer + + +class SVRGModule(Module): + """SVRGModule is a module that encapsulates two Modules to accommodate the SVRG optimization technique. + It is functionally the same as Module API, except it is implemented using SVRG optimization logic. + + Parameters + ---------- + symbol : Symbol + data_names : list of str + Defaults to `('data')` for a typical model used in image classification. + label_names : list of str + Defaults to `('softmax_label')` for a typical model used in image classification. + logger : Logger + Defaults to `logging`. + context : Context or list of Context + Defaults to ``mx.cpu()``. + work_load_list : list of number + Default ``None``, indicating uniform workload. + fixed_param_names: list of str + Default ``None``, indicating no network parameters are fixed. + state_names : list of str + states are similar to data and label, but not provided by data iterator. \ + Instead they are initialized to 0 and can be set by `set_states()`. + group2ctxs : dict of str to context or list of context, or list of dict of str to context + Default is `None`. Mapping the `ctx_group` attribute to the context assignment. + compression_params : dict + Specifies type of gradient compression and additional arguments depending \ + on the type of compression being used. For example, 2bit compression requires a threshold. \ + Arguments would then be {'type':'2bit', 'threshold':0.5} \ + See mxnet.KVStore.set_gradient_compression method for more details on gradient compression. \ + update_freq: int + Specifies the number of times to update the full gradients to be used in the SVRG optimization. For instance, \ + update_freq = 2 will calculates the gradients over all data every two epochs + + Examples + -------- + >>> # An example of declaring and using SVRGModule. + >>> mod = SVRGModule(symbol=lro, data_names=['data'], label_names=['lin_reg_label'], update_freq=2) + >>> mod.fit(di, eval_metric='mse', optimizer='sgd', optimizer_params=(('learning_rate', 0.025),), + >>> num_epoch=num_epoch, kvstore='local') + """ + + def __init__(self, symbol, data_names=('data',), label_names=('softmax_label',), + logger=logging, context=mx.cpu(), work_load_list=None, + fixed_param_names=None, state_names=None, group2ctxs=None, + compression_params=None, update_freq=None): + super(SVRGModule, self).__init__(symbol, data_names=data_names, label_names=label_names, logger=logger, + context=context, work_load_list=work_load_list, + fixed_param_names=fixed_param_names, state_names=state_names, + group2ctxs=group2ctxs, compression_params=compression_params) + + # Type check update_frequency + if isinstance(update_freq, int): + if update_freq <= 0: + raise ValueError("update_freq in SVRGModule must be a positive integer to represent the frequency for " + "calculating full gradients") + self.update_freq = update_freq + else: + raise TypeError("update_freq in SVRGModule must be an integer to represent the frequency for " + "calculating full gradients") + + self._mod_aux = mx.mod.Module(symbol, data_names, label_names, logger, context, work_load_list, + fixed_param_names, state_names, group2ctxs, compression_params) + + self._param_dict = None + self._ctx_len = len(self._context) + + def _reset_bind(self): + """Internal function to reset binded state for both modules.""" + super(SVRGModule, self)._reset_bind() + self._mod_aux._reset_bind() + + def reshape(self, data_shapes, label_shapes=None): + """Reshapes both modules for new input shapes. + + Parameters + ---------- + data_shapes : list of (str, tuple) + Typically is ``data_iter.provide_data``. + label_shapes : list of (str, tuple) + Typically is ``data_iter.provide_label``. + """ + super(SVRGModule, self).reshape(data_shapes, label_shapes=label_shapes) + self._mod_aux.reshape(data_shapes, label_shapes=label_shapes) + + def init_optimizer(self, kvstore='local', optimizer='sgd', + optimizer_params=(('learning_rate', 0.01),), force_init=False): + """Installs and initializes SVRGOptimizer. The SVRGOptimizer is a wrapper class for a regular optimizer that is + passed in and a special AssignmentOptimizer to accumulate the full gradients. If KVStore is 'local' or None, + the full gradients will be accumulated locally without pushing to the KVStore. Otherwise, additional keys will + be pushed to accumulate the full gradients in the KVStore. + + Parameters + ---------- + kvstore : str or KVStore + Default `'local'`. + optimizer : str or Optimizer + Default `'sgd'` + optimizer_params : dict + Default `(('learning_rate', 0.01),)`. The default value is not a dictionary, + just to avoid pylint warning of dangerous default values. + force_init : bool + Default ``False``, indicating whether we should force re-initializing the + optimizer in the case an optimizer is already installed. + """ + + # Init dict for storing average of full gradients for each device + self._param_dict = [{key: mx.nd.zeros(shape=value.shape, ctx=self._context[i]) + for key, value in self.get_params()[0].items()} for i in range(self._ctx_len)] + + svrg_optimizer = self._create_optimizer(_SVRGOptimizer.__name__, default_opt=optimizer, + kvstore=kvstore, optimizer_params=optimizer_params) + + super(SVRGModule, self).init_optimizer(kvstore=kvstore, optimizer=svrg_optimizer, + optimizer_params=optimizer_params, force_init=force_init) + + # Init additional keys for accumulating full grads in KVStore + if self._kvstore: + for idx, param_on_devs in enumerate(self._exec_group.param_arrays): + name = self._exec_group.param_names[idx] + self._kvstore.init(name + "_full", mx.nd.zeros(shape=self._arg_params[name].shape)) + if self._update_on_kvstore: + self._kvstore.pull(name + "_full", param_on_devs, priority=-idx) + + def _create_optimizer(self, optimizer, default_opt, kvstore, optimizer_params): + """Helper function to create a svrg optimizer. SVRG optimizer encapsulates two optimizers and + will redirect update() to the correct optimizer based on the key. + + Parameters + ---------- + kvstore : str or KVStore + Default `'local'`. + optimizer: str + Name for SVRGOptimizer + default_opt : str or Optimizer that was passed in. + optimizer_params : dict + optimizer params that was passed in. + """ + + # code partially copied from mxnet module.init_optimizer() to accomodate svrg_optimizer + batch_size = self._exec_group.batch_size + + (kv_store, update_on_kvstore) = mx.model._create_kvstore(kvstore, self._ctx_len, self._arg_params) + if kv_store and 'dist' in kv_store.type and '_sync' in kv_store.type: + batch_size *= kv_store.num_workers + rescale_grad = 1.0 / batch_size + + idx2name = {} + if update_on_kvstore: + idx2name.update(enumerate(self._exec_group.param_names)) + else: + for k in range(self._ctx_len): + idx2name.update({i * self._ctx_len + k: n + for i, n in enumerate(self._exec_group.param_names)}) + + # update idx2name to include new keys + for key in self._param_dict[0].keys(): + max_key = max(list(idx2name.keys())) + 1 + idx2name[max_key] = key + "_full" + + optimizer_params = dict(optimizer_params) + if 'rescale_grad' not in optimizer_params: + optimizer_params['rescale_grad'] = rescale_grad + optimizer_params["default_optimizer"] = default_opt + optimizer_params["param_idx2name"] = idx2name + optimizer = mx.optimizer.create(optimizer, **optimizer_params) + + return optimizer + + def bind(self, data_shapes, label_shapes=None, for_training=True, + inputs_need_grad=False, force_rebind=False, shared_module=None, grad_req='write'): + """Binds the symbols to construct executors for both two modules. This is necessary before one + can perform computation with the SVRGModule. + + Parameters + ---------- + data_shapes : list of (str, tuple) + Typically is ``data_iter.provide_data``. + label_shapes : list of (str, tuple) + Typically is ``data_iter.provide_label``. + for_training : bool + Default is ``True``. Whether the executors should be bound for training. + inputs_need_grad : bool + Default is ``False``. Whether the gradients to the input data need to be computed. + Typically this is not needed. But this might be needed when implementing composition + of modules. + force_rebind : bool + Default is ``False``. This function does nothing if the executors are already + bound. But with this ``True``, the executors will be forced to rebind. + shared_module : Module + Default is ``None``. This is used in bucketing. When not ``None``, the shared module + essentially corresponds to a different bucket -- a module with different symbol + but with the same sets of parameters (e.g. unrolled RNNs with different lengths). + """ + # force rebinding is typically used when one want to switch from + # training to prediction phase. + super(SVRGModule, self).bind(data_shapes, label_shapes, for_training, inputs_need_grad, force_rebind, + shared_module, grad_req) + + if for_training: + self._mod_aux.bind(data_shapes, label_shapes, for_training, inputs_need_grad, force_rebind, shared_module, + grad_req) + + def forward(self, data_batch, is_train=None): + """Forward computation for both two modules. It supports data batches with different shapes, such as + different batch sizes or different image sizes. + If reshaping of data batch relates to modification of symbol or module, such as + changing image layout ordering or switching from training to predicting, module + rebinding is required. + + See Also + ---------- + :meth:`BaseModule.forward`. + + Parameters + ---------- + data_batch : DataBatch + Could be anything with similar API implemented. + is_train : bool + Default is ``None``, which means ``is_train`` takes the value of ``self.for_training``. + """ + super(SVRGModule, self).forward(data_batch, is_train) + + if is_train: + self._mod_aux.forward(data_batch, is_train) + + def backward(self, out_grads=None): + """Backward computation. + + See Also + ---------- + :meth:`BaseModule.backward`. + + Parameters + ---------- + out_grads : NDArray or list of NDArray, optional + Gradient on the outputs to be propagated back. + This parameter is only needed when bind is called + on outputs that are not a loss function. + """ + super(SVRGModule, self).backward(out_grads) + + if self._mod_aux.binded: + self._mod_aux.backward(out_grads) + + def update(self): + """Updates parameters according to the installed optimizer and the gradients computed + in the previous forward-backward batch. The gradients in the _exec_group will be overwritten + using the gradients calculated by the SVRG update rule. + + When KVStore is used to update parameters for multi-device or multi-machine training, + a copy of the parameters is stored in KVStore. Note that for `row_sparse` parameters, + this function does update the copy of parameters in KVStore, but doesn't broadcast the + updated parameters to all devices / machines. Please call `prepare` to broadcast + `row_sparse` parameters with the next batch of data. + + See Also + ---------- + :meth:`BaseModule.update`. + """ + self._update_svrg_gradients() + super(SVRGModule, self).update() + + def update_full_grads(self, train_data): + """Computes the gradients over all data w.r.t weights of past + m epochs. For distributed env, it will accumulate full grads in the kvstore. + + Parameters + ---------- + train_data: DataIter + Train data iterator + """ + param_names = self._exec_group.param_names + arg, aux = self.get_params() + self._mod_aux.set_params(arg_params=arg, aux_params=aux) + train_data.reset() + nbatch = 0 + padding = 0 + for batch in train_data: + self._mod_aux.forward(batch, is_train=True) + self._mod_aux.backward() + nbatch += 1 + for ctx in range(self._ctx_len): + for index, name in enumerate(param_names): + grads = self._mod_aux._exec_group.grad_arrays[index][ctx] + self._param_dict[ctx][name] = mx.nd.broadcast_add(self._param_dict[ctx][name], grads, axis=0) + padding = batch.pad + + true_num_batch = nbatch - padding / train_data.batch_size + for name in param_names: + grad_list = [] + for i in range(self._ctx_len): + self._param_dict[i][name] /= true_num_batch + grad_list.append(self._param_dict[i][name]) + if self._kvstore: + # If in distributed mode, push a list of gradients from each worker/device to the KVStore + self._accumulate_kvstore(name, grad_list) + + def _accumulate_kvstore(self, key, value): + """Accumulate gradients over all data in the KVStore. In distributed setting, each worker sees a portion of + data. The full gradients will be aggregated from each worker in the KVStore. + + Parameters + ---------- + + key: int or str + Key in the KVStore. + value: NDArray, RowSparseNDArray + Average of the full gradients. + """ + # Accumulate full gradients for current epochs + self._kvstore.push(key + "_full", value) + self._kvstore._barrier() + self._kvstore.pull(key + "_full", value) + + self._allocate_gradients(key, value) + + def _allocate_gradients(self, key, value): + """Allocate average of full gradients accumulated in the KVStore to each device. + + Parameters + ---------- + + key: int or str + Key in the kvstore. + value: List of NDArray, List of RowSparseNDArray + A list of average of the full gradients in the KVStore. + """ + for i in range(self._ctx_len): + self._param_dict[i][key] = value[i] / self._ctx_len + + def _svrg_grads_update_rule(self, g_curr_batch_curr_weight, g_curr_batch_special_weight, + g_special_weight_all_batch): + """Calculates the gradient based on the SVRG update rule. + Parameters + ---------- + g_curr_batch_curr_weight : NDArray + gradients of current weight of self.mod w.r.t current batch of data + g_curr_batch_special_weight: NDArray + gradients of the weight of past m epochs of self._mod_special w.r.t current batch of data + g_special_weight_all_batch: NDArray + average of full gradients over full pass of data + + Returns + ---------- + Gradients calculated using SVRG update rule: + grads = g_curr_batch_curr_weight - g_curr_batch_special_weight + g_special_weight_all_batch + """ + for index, grad in enumerate(g_curr_batch_curr_weight): + grad -= g_curr_batch_special_weight[index] + grad += g_special_weight_all_batch[index] + return g_curr_batch_curr_weight + + def _update_svrg_gradients(self): + """Calculates gradients based on the SVRG update rule. + """ + param_names = self._exec_group.param_names + for ctx in range(self._ctx_len): + for index, name in enumerate(param_names): + g_curr_batch_reg = self._exec_group.grad_arrays[index][ctx] + g_curr_batch_special = self._mod_aux._exec_group.grad_arrays[index][ctx] + g_special_weight_all_batch = self._param_dict[ctx][name] + g_svrg = self._svrg_grads_update_rule(g_curr_batch_reg, g_curr_batch_special, + g_special_weight_all_batch) + self._exec_group.grad_arrays[index][ctx] = g_svrg + + def fit(self, train_data, eval_data=None, eval_metric='acc', + epoch_end_callback=None, batch_end_callback=None, kvstore='local', + optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), + eval_end_callback=None, + eval_batch_end_callback=None, initializer=mx.init.Uniform(0.01), + arg_params=None, aux_params=None, allow_missing=False, + force_rebind=False, force_init=False, begin_epoch=0, num_epoch=None, + validation_metric=None, monitor=None, sparse_row_id_fn=None): + """Trains the module parameters. + + Parameters + ---------- + train_data : DataIter + Train DataIter. + eval_data : DataIter + If not ``None``, will be used as validation set and the performance + after each epoch will be evaluated. + eval_metric : str or EvalMetric + Defaults to 'accuracy'. The performance measure used to display during training. + Other possible predefined metrics are: + 'ce' (CrossEntropy), 'f1', 'mae', 'mse', 'rmse', 'top_k_accuracy'. + epoch_end_callback : function or list of functions + Each callback will be called with the current `epoch`, `symbol`, `arg_params` + and `aux_params`. + batch_end_callback : function or list of function + Each callback will be called with a `BatchEndParam`. + kvstore : str or KVStore + Defaults to 'local'. + optimizer : str or Optimizer + Defaults to 'sgd'. + optimizer_params : dict + Defaults to ``(('learning_rate', 0.01),)``. The parameters for + the optimizer constructor. + The default value is not a dict, just to avoid pylint warning on dangerous + default values. + eval_end_callback : function or list of function + These will be called at the end of each full evaluation, with the metrics over + the entire evaluation set. + eval_batch_end_callback : function or list of function + These will be called at the end of each mini-batch during evaluation. + initializer : Initializer + The initializer is called to initialize the module parameters when they are + not already initialized. + arg_params : dict + Defaults to ``None``, if not ``None``, should be existing parameters from a trained + model or loaded from a checkpoint (previously saved model). In this case, + the value here will be used to initialize the module parameters, unless they + are already initialized by the user via a call to `init_params` or `fit`. + `arg_params` has a higher priority than `initializer`. + aux_params : dict + Defaults to ``None``. Similar to `arg_params`, except for auxiliary states. + allow_missing : bool + Defaults to ``False``. Indicates whether to allow missing parameters when `arg_params` + and `aux_params` are not ``None``. If this is ``True``, then the missing parameters + will be initialized via the `initializer`. + force_rebind : bool + Defaults to ``False``. Whether to force rebinding the executors if already bound. + force_init : bool + Defaults to ``False``. Indicates whether to force initialization even if the + parameters are already initialized. + begin_epoch : int + Defaults to 0. Indicates the starting epoch. Usually, if resumed from a + checkpoint saved at a previous training phase at epoch N, then this value should be + N+1. + num_epoch : int + Number of epochs for training. + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + validation_metric: str or EvalMetric + The performance measure used to display during validation. + """ + assert num_epoch is not None, 'please specify number of epochs' + + self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label, + for_training=True, force_rebind=force_rebind) + if monitor is not None: + self.install_monitor(monitor) + self.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, + allow_missing=allow_missing, force_init=force_init) + self.init_optimizer(kvstore=kvstore, optimizer=optimizer, optimizer_params=optimizer_params) + + if validation_metric is None: + validation_metric = eval_metric + if not isinstance(eval_metric, mx.metric.EvalMetric): + eval_metric = mx.metric.create(eval_metric) + + ################################################################################ + # training loop + ################################################################################ + for epoch in range(begin_epoch, num_epoch): + eval_metric.reset() + tic = time.time() + if epoch % self.update_freq == 0: + self.update_full_grads(train_data) + + train_data.reset() + data_iter = iter(train_data) + end_of_batch = False + nbatch = 0 + next_data_batch = next(data_iter) + + while not end_of_batch: + data_batch = next_data_batch + if monitor is not None: + monitor.tic() + + self.forward_backward(data_batch) + self.update() + + if isinstance(data_batch, list): + self.update_metric(eval_metric, [db.label for db in data_batch], pre_sliced=True) + else: + self.update_metric(eval_metric, data_batch.label) + + try: + # pre fetch next batch + next_data_batch = next(data_iter) + self.prepare(next_data_batch, sparse_row_id_fn=sparse_row_id_fn) + except StopIteration: + end_of_batch = True + + if monitor is not None: + monitor.toc_print() + + if end_of_batch: + eval_name_vals = eval_metric.get_name_value() + + if batch_end_callback is not None: + batch_end_params = mx.model.BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=eval_metric, locals=locals()) + for callback in mx.base._as_list(batch_end_callback): + callback(batch_end_params) + + nbatch += 1 + for name, val in eval_name_vals: + self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) + toc = time.time() + self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc - tic)) + + # sync aux params across devices + arg_params, aux_params = self.get_params() + self.set_params(arg_params, aux_params) + + if epoch_end_callback is not None: + for callback in mx.base._as_list(epoch_end_callback): + callback(epoch, self.symbol, arg_params, aux_params) + + # ---------------------------------------- + # evaluation on validation set + if eval_data: + res = self.score(eval_data, validation_metric, + score_end_callback=eval_end_callback, + batch_end_callback=eval_batch_end_callback, epoch=epoch) + for name, val in res: + self.logger.info('Epoch[%d] Validation-%s=%f', epoch, name, val) + + def prepare(self, data_batch, sparse_row_id_fn=None): + """Prepares two modules for processing a data batch. + + Usually involves switching bucket and reshaping. + For modules that contain `row_sparse` parameters in KVStore, + it prepares the `row_sparse` parameters based on the sparse_row_id_fn. + + When KVStore is used to update parameters for multi-device or multi-machine training, + a copy of the parameters are stored in KVStore. Note that for `row_sparse` parameters, + the `update()` updates the copy of parameters in KVStore, but doesn't broadcast + the updated parameters to all devices / machines. The `prepare` function is used to + broadcast `row_sparse` parameters with the next batch of data. + + Parameters + ---------- + data_batch : DataBatch + The current batch of data for forward computation. + + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + """ + super(SVRGModule, self).prepare(data_batch, sparse_row_id_fn=sparse_row_id_fn) + self._mod_aux.prepare(data_batch, sparse_row_id_fn=sparse_row_id_fn) diff --git a/python/mxnet/gluon/__init__.py b/python/mxnet/gluon/__init__.py index f43da1dae738..3de1f5f0ad0b 100644 --- a/python/mxnet/gluon/__init__.py +++ b/python/mxnet/gluon/__init__.py @@ -19,8 +19,6 @@ # pylint: disable=wildcard-import """Neural network module.""" -from . import metric - from .parameter import * from .block import * diff --git a/python/mxnet/gluon/block.py b/python/mxnet/gluon/block.py index 921d54d71d4a..6e735316cae2 100644 --- a/python/mxnet/gluon/block.py +++ b/python/mxnet/gluon/block.py @@ -30,8 +30,7 @@ import numpy as np from ..base import mx_real_t, MXNetError, NDArrayHandle, py_str -from .. import symbol, ndarray, initializer, autograd, _deferred_compute as dc -from ..symbol.numpy import _symbol as np_symbol +from .. import symbol, ndarray, initializer, np_symbol, autograd, _deferred_compute as dc from ..symbol import Symbol from ..ndarray import NDArray from .. import name as _name diff --git a/python/mxnet/gluon/contrib/data/text.py b/python/mxnet/gluon/contrib/data/text.py index 916b41880d45..0536ac585484 100644 --- a/python/mxnet/gluon/contrib/data/text.py +++ b/python/mxnet/gluon/contrib/data/text.py @@ -29,7 +29,7 @@ from ...data import dataset from ...utils import download, check_sha1, _get_repo_file_url from ....contrib import text -from .... import ndarray as nd, base +from .... import nd, base class _LanguageModelDataset(dataset._DownloadedDataset): # pylint: disable=abstract-method def __init__(self, root, namespace, vocabulary): diff --git a/python/mxnet/gluon/contrib/data/vision/dataloader.py b/python/mxnet/gluon/contrib/data/vision/dataloader.py index 3213398b2214..0c71d90453d8 100644 --- a/python/mxnet/gluon/contrib/data/vision/dataloader.py +++ b/python/mxnet/gluon/contrib/data/vision/dataloader.py @@ -21,9 +21,9 @@ import logging import numpy as np -from ..... import ndarray as nd +from ..... import nd from .....util import is_np_array -from ..... import numpy as _mx_np # pylint: disable=reimported +from ..... import np as _mx_np # pylint: disable=reimported from ....nn import HybridSequential, Sequential, HybridBlock, Block from ....data.vision import transforms from ....data import DataLoader diff --git a/python/mxnet/gluon/contrib/data/vision/transforms/bbox/bbox.py b/python/mxnet/gluon/contrib/data/vision/transforms/bbox/bbox.py index 65a18aaf80cd..1629c212957f 100644 --- a/python/mxnet/gluon/contrib/data/vision/transforms/bbox/bbox.py +++ b/python/mxnet/gluon/contrib/data/vision/transforms/bbox/bbox.py @@ -23,7 +23,7 @@ from .......base import numeric_types from ......block import Block from .......util import is_np_array -from ....... import ndarray as nd, numpy_extension as npx, numpy as np +from ....... import nd, npx, np from .utils import _check_bbox_shape, bbox_crop, bbox_translate from .utils import bbox_resize, bbox_random_crop_with_constraints diff --git a/python/mxnet/gluon/contrib/estimator/estimator.py b/python/mxnet/gluon/contrib/estimator/estimator.py index c47e02b7213f..ed8a53d7c3a6 100644 --- a/python/mxnet/gluon/contrib/estimator/estimator.py +++ b/python/mxnet/gluon/contrib/estimator/estimator.py @@ -33,7 +33,7 @@ from ...trainer import Trainer from ...utils import split_and_load from ....context import Context, cpu, gpu, num_gpus -from ...metric import Loss as metric_loss +from ....metric import Loss as metric_loss from .batch_processor import BatchProcessor __all__ = ['Estimator'] diff --git a/python/mxnet/gluon/contrib/estimator/event_handler.py b/python/mxnet/gluon/contrib/estimator/event_handler.py index 5709a803a610..338c7f00e05e 100644 --- a/python/mxnet/gluon/contrib/estimator/event_handler.py +++ b/python/mxnet/gluon/contrib/estimator/event_handler.py @@ -25,8 +25,8 @@ import numpy as np -from ...metric import CompositeEvalMetric, EvalMetric -from ...metric import Loss as metric_loss +from ....metric import CompositeEvalMetric, EvalMetric +from ....metric import Loss as metric_loss from .utils import _check_metrics __all__ = ['TrainBegin', 'TrainEnd', 'EpochBegin', 'EpochEnd', 'BatchBegin', 'BatchEnd', diff --git a/python/mxnet/gluon/contrib/estimator/utils.py b/python/mxnet/gluon/contrib/estimator/utils.py index dc0c4bf8f081..d9126a2f6763 100644 --- a/python/mxnet/gluon/contrib/estimator/utils.py +++ b/python/mxnet/gluon/contrib/estimator/utils.py @@ -20,7 +20,7 @@ """Gluon Estimator Utility Functions""" from ...loss import SoftmaxCrossEntropyLoss -from ...metric import Accuracy, EvalMetric, CompositeEvalMetric +from ....metric import Accuracy, EvalMetric, CompositeEvalMetric def _check_metrics(metrics): if isinstance(metrics, CompositeEvalMetric): @@ -31,7 +31,7 @@ def _check_metrics(metrics): metrics = metrics or [] if not all([isinstance(metric, EvalMetric) for metric in metrics]): raise ValueError("metrics must be a Metric or a list of Metric, " - "refer to mxnet.gluon.metric.EvalMetric: {}".format(metrics)) + "refer to mxnet.metric.EvalMetric: {}".format(metrics)) return metrics def _check_handler_metric_ref(handler, known_metrics): diff --git a/python/mxnet/gluon/contrib/nn/basic_layers.py b/python/mxnet/gluon/contrib/nn/basic_layers.py index 4335c5cd3431..945867b909c8 100644 --- a/python/mxnet/gluon/contrib/nn/basic_layers.py +++ b/python/mxnet/gluon/contrib/nn/basic_layers.py @@ -24,8 +24,8 @@ 'PixelShuffle3D'] import warnings -from .... import ndarray as nd, context -from ...block import HybridBlock +from .... import nd, context +from ...block import HybridBlock, Block from ...nn import Sequential, HybridSequential, BatchNorm class Concurrent(Sequential): diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py index c51981678367..d991bc769ac9 100644 --- a/python/mxnet/gluon/data/dataloader.py +++ b/python/mxnet/gluon/data/dataloader.py @@ -39,7 +39,7 @@ from . import sampler as _sampler from . import batchify as _batchify -from ... import ndarray as nd, context +from ... import nd, context from ...util import is_np_shape, is_np_array, set_np from ... import numpy as _mx_np # pylint: disable=reimported diff --git a/python/mxnet/gluon/data/vision/datasets.py b/python/mxnet/gluon/data/vision/datasets.py index 028d846c6bee..c88648cbb73e 100644 --- a/python/mxnet/gluon/data/vision/datasets.py +++ b/python/mxnet/gluon/data/vision/datasets.py @@ -30,7 +30,7 @@ from .. import dataset from ...utils import download, check_sha1, _get_repo_file_url -from .... import ndarray as nd, image, recordio, base +from .... import nd, image, recordio, base from .... import numpy as _mx_np # pylint: disable=reimported from ....util import is_np_array, default_array from ....base import numeric_types diff --git a/python/mxnet/gluon/nn/basic_layers.py b/python/mxnet/gluon/nn/basic_layers.py index 8e364532a2f7..a424a32837c3 100644 --- a/python/mxnet/gluon/nn/basic_layers.py +++ b/python/mxnet/gluon/nn/basic_layers.py @@ -27,7 +27,7 @@ from .activations import Activation from ..block import Block, HybridBlock from ..utils import _indent -from ... import ndarray as nd, symbol as sym +from ... import nd, sym from ...util import is_np_array diff --git a/python/mxnet/gluon/metric.py b/python/mxnet/metric.py similarity index 66% rename from python/mxnet/gluon/metric.py rename to python/mxnet/metric.py index 5b081ceac4d8..eb8f99a66d48 100644 --- a/python/mxnet/gluon/metric.py +++ b/python/mxnet/metric.py @@ -22,12 +22,11 @@ import math from collections import OrderedDict -from .. import numpy -from ..util import use_np +import numpy -from ..base import numeric_types, string_types -from .. import ndarray -from .. import registry +from .base import numeric_types, string_types +from . import ndarray +from . import registry def check_label_shapes(labels, preds, wrap=False, shape=False): @@ -90,6 +89,7 @@ def __init__(self, name, output_names=None, self.name = str(name) self.output_names = output_names self.label_names = label_names + self._has_global_stats = kwargs.pop("has_global_stats", False) self._kwargs = kwargs self.reset() @@ -148,6 +148,13 @@ def reset(self): """Resets the internal evaluation result to initial state.""" self.num_inst = 0 self.sum_metric = 0.0 + self.global_num_inst = 0 + self.global_sum_metric = 0.0 + + def reset_local(self): + """Resets the local portion of the internal evaluation results to initial state.""" + self.num_inst = 0 + self.sum_metric = 0.0 def get(self): """Gets the current evaluation result. @@ -162,13 +169,25 @@ def get(self): if self.num_inst == 0: return (self.name, float('nan')) else: - res = self.sum_metric / self.num_inst - if isinstance(res, numpy.ndarray) and len(res.shape) == 0: - # currently calling ' c = mxnet.numpy.array([1,2,3]).sum() ' would get - # ' array(6.) ', a ndarray with shape () - # In this case, returning a 'float' in .get() is more explicit. - res = res.item() - return (self.name, res) + return (self.name, self.sum_metric / self.num_inst) + + def get_global(self): + """Gets the current global evaluation result. + + Returns + ------- + names : list of str + Name of the metrics. + values : list of float + Value of the evaluations. + """ + if self._has_global_stats: + if self.global_num_inst == 0: + return (self.name, float('nan')) + else: + return (self.name, self.global_sum_metric / self.global_num_inst) + else: + return self.get() def get_name_value(self): """Returns zipped name and value pairs. @@ -185,6 +204,24 @@ def get_name_value(self): value = [value] return list(zip(name, value)) + def get_global_name_value(self): + """Returns zipped name and value pairs for global results. + + Returns + ------- + list of tuples + A (name, value) tuple list. + """ + if self._has_global_stats: + name, value = self.get_global() + if not isinstance(name, list): + name = [name] + if not isinstance(value, list): + value = [value] + return list(zip(name, value)) + else: + return self.get_name_value() + # pylint: disable=invalid-name register = registry.get_register_func(EvalMetric, 'metric') alias = registry.get_alias_func(EvalMetric, 'metric') @@ -219,9 +256,9 @@ def create(metric, *args, **kwargs): >>> def custom_metric(label, pred): ... return np.mean(np.abs(label - pred)) ... - >>> metric1 = mx.gluon.metric.create('acc') - >>> metric2 = mx.gluon.metric.create(custom_metric) - >>> metric3 = mx.gluon.metric.create([metric1, metric2, 'rmse']) + >>> metric1 = mx.metric.create('acc') + >>> metric2 = mx.metric.create(custom_metric) + >>> metric3 = mx.metric.create([metric1, metric2, 'rmse']) """ if callable(metric): return CustomMetric(metric, *args, **kwargs) @@ -256,9 +293,9 @@ class CompositeEvalMetric(EvalMetric): -------- >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] >>> labels = [mx.nd.array([0, 1, 1])] - >>> eval_metrics_1 = mx.gluon.metric.Accuracy() - >>> eval_metrics_2 = mx.gluon.metric.F1() - >>> eval_metrics = mx.gluon.metric.CompositeEvalMetric() + >>> eval_metrics_1 = mx.metric.Accuracy() + >>> eval_metrics_2 = mx.metric.F1() + >>> eval_metrics = mx.metric.CompositeEvalMetric() >>> for child_metric in [eval_metrics_1, eval_metrics_2]: >>> eval_metrics.add(child_metric) >>> eval_metrics.update(labels = labels, preds = predicts) @@ -269,7 +306,8 @@ class CompositeEvalMetric(EvalMetric): def __init__(self, metrics=None, name='composite', output_names=None, label_names=None): super(CompositeEvalMetric, self).__init__( - name, output_names=output_names, label_names=label_names) + name, output_names=output_names, label_names=label_names, + has_global_stats=True) if metrics is None: metrics = [] self.metrics = [create(i) for i in metrics] @@ -331,6 +369,14 @@ def reset(self): except AttributeError: pass + def reset_local(self): + """Resets the local portion of the internal evaluation results to initial state.""" + try: + for metric in self.metrics: + metric.reset_local() + except AttributeError: + pass + def get(self): """Returns the current evaluation result. @@ -353,6 +399,28 @@ def get(self): values.extend(value) return (names, values) + def get_global(self): + """Returns the current evaluation result. + + Returns + ------- + names : list of str + Name of the metrics. + values : list of float + Value of the evaluations. + """ + names = [] + values = [] + for metric in self.metrics: + name, value = metric.get_global() + if isinstance(name, string_types): + name = [name] + if isinstance(value, numeric_types): + value = [value] + names.extend(name) + values.extend(value) + return (names, values) + def get_config(self): config = super(CompositeEvalMetric, self).get_config() config.update({'metrics': [i.get_config() for i in self.metrics]}) @@ -366,7 +434,6 @@ def get_config(self): @register @alias('acc') -@use_np class Accuracy(EvalMetric): """Computes accuracy classification score. @@ -393,7 +460,7 @@ class Accuracy(EvalMetric): -------- >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] >>> labels = [mx.nd.array([0, 1, 1])] - >>> acc = mx.gluon.metric.Accuracy() + >>> acc = mx.metric.Accuracy() >>> acc.update(preds = predicts, labels = labels) >>> print acc.get() ('accuracy', 0.6666666666666666) @@ -402,7 +469,8 @@ def __init__(self, axis=1, name='accuracy', output_names=None, label_names=None): super(Accuracy, self).__init__( name, axis=axis, - output_names=output_names, label_names=label_names) + output_names=output_names, label_names=label_names, + has_global_stats=True) self.axis = axis def update(self, labels, preds): @@ -420,26 +488,25 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred_label in zip(labels, preds): - pred_label = pred_label.as_np_ndarray().as_in_ctx(label.ctx) - label = label.as_np_ndarray() if pred_label.shape != label.shape: - pred_label = pred_label.argmax(axis=self.axis) - pred_label = pred_label.astype('int32') - label = label.astype('int32') + pred_label = ndarray.argmax(pred_label, axis=self.axis) + pred_label = pred_label.asnumpy().astype('int32') + label = label.asnumpy().astype('int32') # flatten before checking shapes to avoid shape miss match - label = label.reshape(-1) - pred_label = pred_label.reshape(-1) + label = label.flat + pred_label = pred_label.flat check_label_shapes(label, pred_label) - num_correct = (pred_label == label).sum().astype('float64') + num_correct = (pred_label == label).sum() self.sum_metric += num_correct + self.global_sum_metric += num_correct self.num_inst += len(pred_label) + self.global_num_inst += len(pred_label) @register @alias('top_k_accuracy', 'top_k_acc') -@use_np class TopKAccuracy(EvalMetric): """Computes top k predictions accuracy. @@ -468,7 +535,7 @@ class TopKAccuracy(EvalMetric): >>> top_k = 3 >>> labels = [mx.nd.array([2, 6, 9, 2, 3, 4, 7, 8, 9, 6])] >>> predicts = [mx.nd.array(np.random.rand(10, 10))] - >>> acc = mx.gluon.metric.TopKAccuracy(top_k=top_k) + >>> acc = mx.metric.TopKAccuracy(top_k=top_k) >>> acc.update(labels, predicts) >>> print acc.get() ('top_k_accuracy', 0.3) @@ -478,7 +545,8 @@ def __init__(self, top_k=1, name='top_k_accuracy', output_names=None, label_names=None): super(TopKAccuracy, self).__init__( name, top_k=top_k, - output_names=output_names, label_names=label_names) + output_names=output_names, label_names=label_names, + has_global_stats=True) self.top_k = top_k assert(self.top_k > 1), 'Please use Accuracy if top_k is no more than 1' self.name += '_%d' % self.top_k @@ -502,89 +570,43 @@ def update(self, labels, preds): # we do not care about the order of top k elements. It is # much faster, which is important since that computation is # single-threaded due to Python GIL. - pred_label = pred_label.as_np_ndarray().as_in_ctx(label.ctx).astype('float32') - pred_label = numpy.argpartition(pred_label, -self.top_k) - label = label.as_np_ndarray().astype('int32') + pred_label = numpy.argpartition(pred_label.asnumpy().astype('float32'), -self.top_k) + label = label.asnumpy().astype('int32') check_label_shapes(label, pred_label) num_samples = pred_label.shape[0] num_dims = len(pred_label.shape) if num_dims == 1: - num_correct = (pred_label.reshape(-1) == label.reshape(-1)).sum() - self.sum_metric += num_correct.astype('float64') + self.sum_metric += (pred_label.flat == label.flat).sum() elif num_dims == 2: num_classes = pred_label.shape[1] top_k = min(num_classes, self.top_k) for j in range(top_k): - num_correct = (pred_label[:, num_classes - 1 - j].reshape(-1) == label.reshape(-1)).sum() - self.sum_metric += num_correct.astype('float64') + num_correct = (pred_label[:, num_classes - 1 - j].flat == label.flat).sum() + self.sum_metric += num_correct + self.global_sum_metric += num_correct self.num_inst += num_samples + self.global_num_inst += num_samples -def predict_with_threshold(pred, threshold=0.5): - """Do thresholding of predictions in binary and multilabel cases. - - Parameters - ---------- - preds : ndarray - predictions in shape of (batch_size, ...) or (batch_size, ..., num_categories) - - preds : float or ndarray - threshold（s) in shape of float or (num_categories) - """ - if isinstance(threshold, float): - return pred > threshold - elif isinstance(threshold, (numpy.ndarray, ndarray.ndarray.NDArray)): - num_classes = pred.shape[-1] - assert threshold.shape[-1] == num_classes, \ - "shape mismatch: %s vs. %s"%(pred.shape[-1], threshold.shape[-1]) - return pred > threshold - else: - raise ValueError("{} is a wrong type for threshold!".format(type(threshold))) - - -def one_hot(idx, num): - return (numpy.arange(num).astype(idx) == idx[:, None]).astype('int32') - - -@use_np -class _ClassificationMetrics(object): +class _BinaryClassificationMetrics(object): """Private container class for classification metric statistics. True/false positive and true/false negative counts are sufficient statistics for various classification metrics. This class provides the machinery to track those statistics across mini-batches of (label, prediction) pairs. - - Parameters - ---------- - class_type : str, default "binary" - "binary": f1 for binary classification. - "multiclass": f1 for multiclassification problem. - "multilabel": f1 for multilabel classification. - beta : float, default 1 - weight of precision in harmonic mean. - threshold : float, default 0.5 - threshold for deciding whether the predictions are positive or negative. - """ - def __init__(self, class_type="binary", threshold=0.5, beta=1): - self.class_type = class_type - self.threshold = threshold - self.beta = beta - self.reset_stats() - - def _set(self, num, ctx): - if self.num_classes is None: - self.num_classes = num - self.true_positives = numpy.zeros(num, dtype='float64').as_in_ctx(ctx) - self.false_negatives = numpy.zeros(num, dtype='float64').as_in_ctx(ctx) - self.false_positives = numpy.zeros(num, dtype='float64').as_in_ctx(ctx) - self.true_negatives = numpy.zeros(num, dtype='float64').as_in_ctx(ctx) - else: - assert self.num_classes == num, \ - "Input number of classes has changed from {} to {}".format(self.num_classes, num) - - def update_stats(self, label, pred): + def __init__(self): + self.true_positives = 0 + self.false_negatives = 0 + self.false_positives = 0 + self.true_negatives = 0 + self.global_true_positives = 0 + self.global_false_negatives = 0 + self.global_false_positives = 0 + self.global_true_negatives = 0 + + def update_binary_stats(self, label, pred): """Update various binary classification counts for a single (label, pred) pair. Parameters @@ -595,107 +617,92 @@ def update_stats(self, label, pred): pred : `NDArray` Predicted values. """ - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) - label = label.as_np_ndarray().astype('int32') - if self.class_type == "binary": - self._set(1, label.ctx) - if label.max() > 1: - raise ValueError("Wrong label for binary classification.") - if pred.shape == label.shape: - pass - elif pred.shape[-1] > 2: - raise ValueError("The shape of prediction {} is wrong for binary classification.".format(pred.shape)) - elif pred.shape[-1] == 2: - pred = pred.reshape(-1, 2)[:, 1] - pred_label = predict_with_threshold(pred, self.threshold).reshape(-1) - label = label.reshape(-1) - - elif self.class_type == "multiclass": - num = pred.shape[-1] - self._set(num, label.ctx) - assert label.max() < num, "pred contains fewer classes than label!" - pred_label = one_hot(pred.argmax(axis=-1).reshape(-1), num) - label = one_hot(label.reshape(-1), num) - - elif self.class_type == "multilabel": - num = pred.shape[-1] - self._set(num, label.ctx) - assert pred.shape == label.shape, \ - "The shape of label should be same as that of prediction for multilabel classification." - pred_label = predict_with_threshold(pred, self.threshold).reshape(-1, num) - label = label.reshape(-1, num) - else: - raise ValueError( - "Wrong class_type {}! Only supports ['binary', 'multiclass', 'multilabel']".format(self.class_type)) - - check_label_shapes(label, pred_label) - + pred = pred.asnumpy() + label = label.asnumpy().astype('int32') + pred_label = numpy.argmax(pred, axis=1) + + check_label_shapes(label, pred) + if len(numpy.unique(label)) > 2: + raise ValueError("%s currently only supports binary classification." + % self.__class__.__name__) pred_true = (pred_label == 1) - pred_false = (pred_label == 0) + pred_false = 1 - pred_true label_true = (label == 1) - label_false = (label == 0) + label_false = 1 - label_true - true_pos = (pred_true * label_true).sum(0) - false_pos = (pred_true * label_false).sum(0) - false_neg = (pred_false * label_true).sum(0) - true_neg = (pred_false * label_false).sum(0) + true_pos = (pred_true * label_true).sum() + false_pos = (pred_true * label_false).sum() + false_neg = (pred_false * label_true).sum() + true_neg = (pred_false * label_false).sum() self.true_positives += true_pos + self.global_true_positives += true_pos self.false_positives += false_pos + self.global_false_positives += false_pos self.false_negatives += false_neg + self.global_false_negatives += false_neg self.true_negatives += true_neg + self.global_true_negatives += true_neg @property def precision(self): - if self.num_classes is not None: - return self.true_positives / numpy.maximum(self.true_positives + self.false_positives, 1e-12) + if self.true_positives + self.false_positives > 0: + return float(self.true_positives) / (self.true_positives + self.false_positives) else: return 0. @property - def micro_precision(self): - if self.num_classes is not None: - return self.true_positives.sum() / \ - numpy.maximum(self.true_positives.sum() + self.false_positives.sum(), 1e-12) + def global_precision(self): + if self.global_true_positives + self.global_false_positives > 0: + return float(self.global_true_positives) / (self.global_true_positives + self.global_false_positives) else: return 0. @property def recall(self): - if self.num_classes is not None: - return self.true_positives / numpy.maximum(self.true_positives + self.false_negatives, 1e-12) + if self.true_positives + self.false_negatives > 0: + return float(self.true_positives) / (self.true_positives + self.false_negatives) else: return 0. @property - def micro_recall(self): - if self.num_classes is not None: - return self.true_positives.sum() / \ - numpy.maximum(self.true_positives.sum() + self.false_negatives.sum(), 1e-12) + def global_recall(self): + if self.global_true_positives + self.global_false_negatives > 0: + return float(self.global_true_positives) / (self.global_true_positives + self.global_false_negatives) else: return 0. @property def fscore(self): - return (1 + self.beta ** 2) * self.precision * self.recall / \ - numpy.maximum(self.beta ** 2 * self.precision + self.recall, 1e-12) + if self.precision + self.recall > 0: + return 2 * self.precision * self.recall / (self.precision + self.recall) + else: + return 0. @property - def micro_fscore(self): - if self.micro_precision + self.micro_recall > 0: - return (1 + self.beta ** 2) * self.micro_precision * self.micro_recall / \ - (self.beta ** 2 * self.micro_precision + self.micro_recall) + def global_fscore(self): + if self.global_precision + self.global_recall > 0: + return 2 * self.global_precision * self.global_recall / (self.global_precision + self.global_recall) else: return 0. - def binary_matthewscc(self): + def matthewscc(self, use_global=False): """Calculate the Matthew's Correlation Coefficent""" - if not self.total_examples: - return 0. + if use_global: + if not self.global_total_examples: + return 0. + + true_pos = float(self.global_true_positives) + false_pos = float(self.global_false_positives) + false_neg = float(self.global_false_negatives) + true_neg = float(self.global_true_negatives) + else: + if not self.total_examples: + return 0. - true_pos = float(self.true_positives) - false_pos = float(self.false_positives) - false_neg = float(self.false_negatives) - true_neg = float(self.true_negatives) + true_pos = float(self.true_positives) + false_pos = float(self.false_positives) + false_neg = float(self.false_negatives) + true_neg = float(self.true_negatives) terms = [(true_pos + false_pos), (true_pos + false_neg), @@ -708,21 +715,32 @@ def binary_matthewscc(self): @property def total_examples(self): - if self.num_classes is None: - return 0 - return int(self.false_negatives[0] + self.false_positives[0] + \ - self.true_negatives[0] + self.true_positives[0]) + return self.false_negatives + self.false_positives + \ + self.true_negatives + self.true_positives + + @property + def global_total_examples(self): + return self.global_false_negatives + self.global_false_positives + \ + self.global_true_negatives + self.global_true_positives + + def local_reset_stats(self): + self.false_positives = 0 + self.false_negatives = 0 + self.true_positives = 0 + self.true_negatives = 0 def reset_stats(self): - self.num_classes = None - self.true_positives = None - self.false_negatives = None - self.false_positives = None - self.true_negatives = None + self.false_positives = 0 + self.false_negatives = 0 + self.true_positives = 0 + self.true_negatives = 0 + self.global_false_positives = 0 + self.global_false_negatives = 0 + self.global_true_positives = 0 + self.global_true_negatives = 0 @register -@use_np class F1(EvalMetric): """Computes the F1 score of a binary classification problem. @@ -750,34 +768,28 @@ class F1(EvalMetric): label_names : list of str, or None Name of labels that should be used when updating with update_dict. By default include all labels. - class_type : str, default "binary" - "binary": f1 for binary classification. - "multiclass": f1 for multiclassification problem. - "multilabel": f1 for multilabel classification. - threshold : float, default 0.5 - threshold for postive confidence value. - average : str, default 'micro' + average : str, default 'macro' Strategy to be used for aggregating across mini-batches. - "macro": Calculate metrics for each label and return unweighted mean of f1. - "micro": Calculate metrics globally by counting the total TP, FN and FP. - None: Return f1 scores for each class (numpy.ndarray) . + "macro": average the F1 scores for each batch. + "micro": compute a single F1 score across all batches. Examples -------- >>> predicts = [mx.nd.array([[0.3, 0.7], [0., 1.], [0.4, 0.6]])] >>> labels = [mx.nd.array([0., 1., 1.])] - >>> f1 = mx.gluon.metric.F1() + >>> f1 = mx.metric.F1() >>> f1.update(preds = predicts, labels = labels) >>> print f1.get() ('f1', 0.8) """ def __init__(self, name='f1', - output_names=None, label_names=None, class_type="binary", threshold=0.5, average="micro"): + output_names=None, label_names=None, average="macro"): self.average = average - self.metrics = _ClassificationMetrics(class_type=class_type, threshold=threshold) + self.metrics = _BinaryClassificationMetrics() EvalMetric.__init__(self, name=name, - output_names=output_names, label_names=label_names) + output_names=output_names, label_names=label_names, + has_global_stats=True) def update(self, labels, preds): """Updates the internal evaluation result. @@ -793,149 +805,36 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): - self.metrics.update_stats(label, pred) + self.metrics.update_binary_stats(label, pred) - if self.average == "micro": - self.sum_metric = self.metrics.micro_fscore * self.metrics.total_examples - elif self.average == "macro": - self.sum_metric = self.metrics.fscore.mean() * self.metrics.total_examples + if self.average == "macro": + self.sum_metric += self.metrics.fscore + self.global_sum_metric += self.metrics.global_fscore + self.num_inst += 1 + self.global_num_inst += 1 + self.metrics.reset_stats() else: self.sum_metric = self.metrics.fscore * self.metrics.total_examples - self.num_inst = self.metrics.total_examples + self.global_sum_metric = self.metrics.global_fscore * self.metrics.global_total_examples + self.num_inst = self.metrics.total_examples + self.global_num_inst = self.metrics.global_total_examples def reset(self): """Resets the internal evaluation result to initial state.""" self.sum_metric = 0. self.num_inst = 0 + self.global_num_inst = 0 + self.global_sum_metric = 0.0 self.metrics.reset_stats() - -@register -@use_np -class Fbeta(F1): - """Computes the Fbeta score of a binary classification problem. - - The Fbeta score is equivalent to harmonic mean of the precision and recall, - where the best value is 1.0 and the worst value is 0.0. The formula for Fbeta score is:: - - Fbeta = (1 + beta ** 2) * (precision * recall) / (beta ** 2 * precision + recall) - - The formula for precision and recall is:: - - precision = true_positives / (true_positives + false_positives) - recall = true_positives / (true_positives + false_negatives) - - .. note:: - - This Fbeta score only supports binary classification. - - Parameters - ---------- - name : str - Name of this metric instance for display. - output_names : list of str, or None - Name of predictions that should be used when updating with update_dict. - By default include all predictions. - label_names : list of str, or None - Name of labels that should be used when updating with update_dict. - By default include all labels. - class_type : str, default "binary" - "binary": f1 for binary classification. - "multiclass": f1 for multiclassification problem. - "multilabel": f1 for multilabel classification. - beta : float, default 1 - weight of precision in harmonic mean. - threshold : float, default 0.5 - threshold for postive confidence value. - average : str, default 'micro' - Strategy to be used for aggregating across mini-batches. - "macro": Calculate metrics for each label and return unweighted mean of f1. - "micro": Calculate metrics globally by counting the total TP, FN and FP. - None: Return f1 scores for each class. - - Examples - -------- - >>> predicts = [mx.nd.array([[0.3, 0.7], [0., 1.], [0.4, 0.6]])] - >>> labels = [mx.nd.array([0., 1., 1.])] - >>> fbeta = mx.gluon.metric.Fbeta(beta=2) - >>> fbeta.update(preds = predicts, labels = labels) - >>> print fbeta.get() - ('fbeta', 0.9090909090909091) - """ - - def __init__(self, name='fbeta', - output_names=None, label_names=None, class_type="binary", beta=1, threshold=0.5, average="micro"): - super(Fbeta, self).__init__( - name=name, output_names=output_names, label_names=label_names, - class_type=class_type, threshold=threshold, average=average) - self.metrics = _ClassificationMetrics(class_type=class_type, threshold=threshold, beta=beta) - - -@register -@use_np -class BinaryAccuracy(EvalMetric): - """Computes the accuracy of a binary or multilabel classification problem. - - Parameters - ---------- - name : str - Name of this metric instance for display. - output_names : list of str, or None - Name of predictions that should be used when updating with update_dict. - By default include all predictions. - label_names : list of str, or None - Name of labels that should be used when updating with update_dict. - By default include all labels. - threshold : float or ndarray, default 0.5 - threshold for deciding whether the predictions are positive or negative. - - Examples - -------- - >>> predicts = [mx.nd.array([0.7, 1, 0.55])] - >>> labels = [mx.nd.array([0., 1., 0.])] - >>> bacc = mx.gluon.metric.BinaryAccuracy(threshold=0.6) - >>> bacc.update(preds = predicts, labels = labels) - >>> print bacc.get() - ('binary_accuracy', 0.6666666666666666) - """ - - def __init__(self, name='binary_accuracy', - output_names=None, label_names=None, threshold=0.5): - self.threshold = threshold - EvalMetric.__init__(self, name=name, - output_names=output_names, label_names=label_names) - - def update(self, labels, preds): - """Updates the internal evaluation result. - - Parameters - ---------- - labels : list of `NDArray` - Each label denotes positive/negative for each class. - - preds : list of `NDArray` - Each prediction value is a confidence value of being positive for each class. - """ - labels, preds = check_label_shapes(labels, preds, True) - - for label, pred_label in zip(labels, preds): - pred_label = predict_with_threshold(pred_label, self.threshold) - - pred_label = pred_label.as_np_ndarray().astype('int32').as_in_ctx(label.ctx) - label = label.as_np_ndarray().astype('int32') - # flatten before checking shapes to avoid shape miss match - label = label.reshape(-1) - pred_label = pred_label.reshape(-1) - - check_label_shapes(label, pred_label) - - num_correct = (pred_label == label).sum().astype('float64') - self.sum_metric += num_correct - self.num_inst += len(pred_label) + def reset_local(self): + """Resets the internal evaluation result to initial state.""" + self.sum_metric = 0. + self.num_inst = 0 + self.metrics.local_reset_stats() @register -@use_np class MCC(EvalMetric): """Computes the Matthews Correlation Coefficient of a binary classification problem. @@ -966,6 +865,10 @@ class MCC(EvalMetric): label_names : list of str, or None Name of labels that should be used when updating with update_dict. By default include all labels. + average : str, default 'macro' + Strategy to be used for aggregating across mini-batches. + "macro": average the MCC for each batch. + "micro": compute a single MCC across all batches. Examples -------- @@ -984,9 +887,9 @@ class MCC(EvalMetric): [0.]*(false_positives + true_negatives) + [1.]*(false_negatives + true_positives) )] - >>> f1 = mx.gluon.metric.F1() + >>> f1 = mx.metric.F1() >>> f1.update(preds = predicts, labels = labels) - >>> mcc = mx.gluon.metric.MCC() + >>> mcc = mx.metric.MCC() >>> mcc.update(preds = predicts, labels = labels) >>> print f1.get() ('f1', 0.95233560306652054) @@ -995,10 +898,12 @@ class MCC(EvalMetric): """ def __init__(self, name='mcc', - output_names=None, label_names=None): - self._metrics = _ClassificationMetrics() + output_names=None, label_names=None, average="macro"): + self._average = average + self._metrics = _BinaryClassificationMetrics() EvalMetric.__init__(self, name=name, - output_names=output_names, label_names=label_names) + output_names=output_names, label_names=label_names, + has_global_stats=True) def update(self, labels, preds): """Updates the internal evaluation result. @@ -1014,35 +919,72 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): - self._metrics.update_stats(label, pred) + self._metrics.update_binary_stats(label, pred) - self.sum_metric = self._metrics.binary_matthewscc() * self._metrics.total_examples - self.num_inst = self._metrics.total_examples + if self._average == "macro": + self.sum_metric += self._metrics.matthewscc() + self.global_sum_metric += self._metrics.matthewscc(use_global=True) + self.num_inst += 1 + self.global_num_inst += 1 + self._metrics.reset_stats() + else: + self.sum_metric = self._metrics.matthewscc() * self._metrics.total_examples + self.global_sum_metric = self._metrics.matthewscc(use_global=True) * \ + self._metrics.global_total_examples + self.num_inst = self._metrics.total_examples + self.global_num_inst = self._metrics.global_total_examples def reset(self): """Resets the internal evaluation result to initial state.""" self.sum_metric = 0. self.num_inst = 0. + self.global_sum_metric = 0. + self.global_num_inst = 0. self._metrics.reset_stats() - -#################### -# REGRESSION METRICS -#################### + def reset_local(self): + """Resets the internal evaluation result to initial state.""" + self.sum_metric = 0. + self.num_inst = 0. + self._metrics.local_reset_stats() @register -@use_np -class MAE(EvalMetric): - """Computes Mean Absolute Error (MAE) loss. +class Perplexity(EvalMetric): + """Computes perplexity. - The mean absolute error is given by + Perplexity is a measurement of how well a probability distribution + or model predicts a sample. A low perplexity indicates the model + is good at predicting the sample. + + The perplexity of a model q is defined as .. math:: - \\frac{\\sum_i^n |y_i - \\hat{y}_i|}{n} + b^{\\big(-\\frac{1}{N} \\sum_{i=1}^N \\log_b q(x_i) \\big)} + = \\exp \\big(-\\frac{1}{N} \\sum_{i=1}^N \\log q(x_i)\\big) + + where we let `b = e`. + + :math:`q(x_i)` is the predicted value of its ground truth + label on sample :math:`x_i`. + + For example, we have three samples :math:`x_1, x_2, x_3` and their labels + are :math:`[0, 1, 1]`. + Suppose our model predicts :math:`q(x_1) = p(y_1 = 0 | x_1) = 0.3` + and :math:`q(x_2) = 1.0`, + :math:`q(x_3) = 0.6`. The perplexity of model q is + :math:`exp\\big(-(\\log 0.3 + \\log 1.0 + \\log 0.6) / 3\\big) = 1.77109762852`. Parameters ---------- + ignore_label : int or None + Index of invalid label to ignore when + counting. By default, sets to -1. + If set to `None`, it will include all entries. + axis : int (default -1) + The axis from prediction that was used to + compute softmax. By default use the last + axis. name : str Name of this metric instance for display. output_names : list of str, or None @@ -1054,18 +996,21 @@ class MAE(EvalMetric): Examples -------- - >>> predicts = [mx.nd.array([3, -0.5, 2, 7])] - >>> labels = [mx.nd.array([2.5, 0.0, 2, 8])] - >>> mean_absolute_error = mx.gluon.metric.MAE() - >>> mean_absolute_error.update(labels = labels, preds = predicts) - >>> print mean_absolute_error.get() - ('mae', 0.5) + >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] + >>> labels = [mx.nd.array([0, 1, 1])] + >>> perp = mx.metric.Perplexity(ignore_label=None) + >>> perp.update(labels, predicts) + >>> print perp.get() + ('Perplexity', 1.7710976285155853) """ - - def __init__(self, name='mae', + def __init__(self, ignore_label, axis=-1, name='perplexity', output_names=None, label_names=None): - super(MAE, self).__init__( - name, output_names=output_names, label_names=label_names) + super(Perplexity, self).__init__( + name, ignore_label=ignore_label, + output_names=output_names, label_names=label_names, + has_global_stats=True) + self.ignore_label = ignore_label + self.axis = axis def update(self, labels, preds): """Updates the internal evaluation result. @@ -1078,28 +1023,64 @@ def update(self, labels, preds): preds : list of `NDArray` Predicted values. """ - labels, preds = check_label_shapes(labels, preds, True) - + assert len(labels) == len(preds) + loss = 0. + num = 0 for label, pred in zip(labels, preds): - label = label.as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + assert label.size == pred.size/pred.shape[-1], \ + "shape mismatch: %s vs. %s"%(label.shape, pred.shape) + label = label.as_in_context(pred.context).reshape((label.size,)) + pred = ndarray.pick(pred, label.astype(dtype='int32'), axis=self.axis) + if self.ignore_label is not None: + ignore = (label == self.ignore_label).astype(pred.dtype) + num -= ndarray.sum(ignore).asscalar() + pred = pred*(1-ignore) + ignore + loss -= ndarray.sum(ndarray.log(ndarray.maximum(1e-10, pred))).asscalar() + num += pred.size + self.sum_metric += loss + self.global_sum_metric += loss + self.num_inst += num + self.global_num_inst += num - num_inst = label.shape[0] - mae = numpy.abs(label - pred).reshape(num_inst, -1).mean(axis=-1).sum() + def get(self): + """Returns the current evaluation result. - self.sum_metric += mae - self.num_inst += num_inst + Returns + ------- + Tuple of (str, float) + Representing name of the metric and evaluation result. + """ + if self.num_inst == 0: + return (self.name, float('nan')) + else: + return (self.name, math.exp(self.sum_metric/self.num_inst)) + + def get_global(self): + """Returns the current global evaluation result. + + Returns + ------- + Tuple of (str, float) + Representing name of the metric and evaluation result. + """ + if self.global_num_inst == 0: + return (self.name, float('nan')) + else: + return (self.name, math.exp(self.global_sum_metric/self.global_num_inst)) + +#################### +# REGRESSION METRICS +#################### @register -@use_np -class MSE(EvalMetric): - """Computes Mean Squared Error (MSE) loss. +class MAE(EvalMetric): + """Computes Mean Absolute Error (MAE) loss. - The mean squared error is given by + The mean absolute error is given by .. math:: - \\frac{\\sum_i^n (y_i - \\hat{y}_i)^2}{n} + \\frac{\\sum_i^n |y_i - \\hat{y}_i|}{n} Parameters ---------- @@ -1114,17 +1095,19 @@ class MSE(EvalMetric): Examples -------- - >>> predicts = [mx.nd.array([3, -0.5, 2, 7])] - >>> labels = [mx.nd.array([2.5, 0.0, 2, 8])] - >>> mean_squared_error = mx.gluon.metric.MSE() - >>> mean_squared_error.update(labels = labels, preds = predicts) - >>> print mean_squared_error.get() - ('mse', 0.375) + >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] + >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] + >>> mean_absolute_error = mx.metric.MAE() + >>> mean_absolute_error.update(labels = labels, preds = predicts) + >>> print mean_absolute_error.get() + ('mae', 0.5) """ - def __init__(self, name='mse', + + def __init__(self, name='mae', output_names=None, label_names=None): - super(MSE, self).__init__( - name, output_names=output_names, label_names=label_names) + super(MAE, self).__init__( + name, output_names=output_names, label_names=label_names, + has_global_stats=True) def update(self, labels, preds): """Updates the internal evaluation result. @@ -1140,25 +1123,29 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): - label = label.as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + label = label.asnumpy() + pred = pred.asnumpy() - num_inst = label.shape[0] - mse = ((label - pred)**2.0).reshape(num_inst, -1).mean(axis=-1).sum() + if len(label.shape) == 1: + label = label.reshape(label.shape[0], 1) + if len(pred.shape) == 1: + pred = pred.reshape(pred.shape[0], 1) - self.sum_metric += mse - self.num_inst += num_inst + mae = numpy.abs(label - pred).mean() + self.sum_metric += mae + self.global_sum_metric += mae + self.num_inst += 1 # numpy.prod(label.shape) + self.global_num_inst += 1 # numpy.prod(label.shape) @register -@use_np -class RMSE(MSE): - """Computes Root Mean Squred Error (RMSE) loss. +class MSE(EvalMetric): + """Computes Mean Squared Error (MSE) loss. - The root mean squared error is given by + The mean squared error is given by .. math:: - \\sqrt{\\frac{\\sum_i^n (y_i - \\hat{y}_i)^2}{n}} + \\frac{\\sum_i^n (y_i - \\hat{y}_i)^2}{n} Parameters ---------- @@ -1173,62 +1160,18 @@ class RMSE(MSE): Examples -------- - >>> predicts = [mx.nd.array([3, -0.5, 2, 7])] - >>> labels = [mx.nd.array([2.5, 0.0, 2, 8])] - >>> root_mean_squared_error = mx.gluon.metric.RMSE() - >>> root_mean_squared_error.update(labels = labels, preds = predicts) - >>> print root_mean_squared_error.get() - ('rmse', 0.612372457981) + >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] + >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] + >>> mean_squared_error = mx.metric.MSE() + >>> mean_squared_error.update(labels = labels, preds = predicts) + >>> print mean_squared_error.get() + ('mse', 0.375) """ - def __init__(self, name='rmse', + def __init__(self, name='mse', output_names=None, label_names=None): - super(RMSE, self).__init__( - name, output_names=output_names, label_names=label_names) - - def get(self): - if self.num_inst == 0: - return (self.name, float('nan')) - else: - return (self.name, math.sqrt(self.sum_metric / self.num_inst)) - - -@register -@use_np -class MeanPairwiseDistance(EvalMetric): - """Computes Mean Pairwise Distance. - - The mean pairwise distance is given by - - .. math:: - \\sqrt{\\frac{(\\sum_i^n (y_i - \\hat{y}_i)^p)^\\frac{1}{p}}{n}} - - Parameters - ---------- - name : str - Name of this metric instance for display. - output_names : list of str, or None - Name of predictions that should be used when updating with update_dict. - By default include all predictions. - label_names : list of str, or None - Name of labels that should be used when updating with update_dict. - By default include all labels. - p : float, default 2 - calculating distance using the p-norm - - Examples - -------- - >>> predicts = [mx.nd.array([[1., 2.], [3., 4.]])] - >>> labels = [mx.nd.array([[1., 0.], [4., 2.]])] - >>> mpd = mx.gluon.metric.MeanPairwiseDistance() - >>> mpd.update(labels = labels, preds = predicts) - >>> print mpd.get() - ('mpd', 2.1180338859558105) - """ - def __init__(self, name='mpd', - output_names=None, label_names=None, p=2): - super(MeanPairwiseDistance, self).__init__( - name, output_names=output_names, label_names=label_names) - self.p = p + super(MSE, self).__init__( + name, output_names=output_names, label_names=label_names, + has_global_stats=True) def update(self, labels, preds): """Updates the internal evaluation result. @@ -1244,30 +1187,29 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): - label = label.as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + label = label.asnumpy() + pred = pred.asnumpy() - label = label.reshape(label.shape[0], -1) - pred = pred.reshape(pred.shape[0], -1) - - dis = (((label - pred) ** self.p).sum(axis=-1)) ** (1./self.p) - dis = dis.sum() - num_inst = label.shape[0] + if len(label.shape) == 1: + label = label.reshape(label.shape[0], 1) + if len(pred.shape) == 1: + pred = pred.reshape(pred.shape[0], 1) - self.sum_metric += dis - self.num_inst += num_inst + mse = ((label - pred)**2.0).mean() + self.sum_metric += mse + self.global_sum_metric += mse + self.num_inst += 1 # numpy.prod(label.shape) + self.global_num_inst += 1 # numpy.prod(label.shape) @register -@use_np -class MeanCosineSimilarity(EvalMetric): - """Computes Mean Cosine Similarity. +class RMSE(EvalMetric): + """Computes Root Mean Squred Error (RMSE) loss. - The mean cosine similarity is given by + The root mean squared error is given by .. math:: - cos_sim(label, pred) = \frac{{label}.{pred}}{max(||label||.||pred||, eps)} - (calculating on the last dimension of label and pred.) + \\sqrt{\\frac{\\sum_i^n (y_i - \\hat{y}_i)^2}{n}} Parameters ---------- @@ -1279,23 +1221,21 @@ class MeanCosineSimilarity(EvalMetric): label_names : list of str, or None Name of labels that should be used when updating with update_dict. By default include all labels. - eps : float, default 1e-8 - small vale to avoid division by zero. Examples -------- - >>> predicts = [mx.nd.array([[1., 0.], [1., 1.]])] - >>> labels = [mx.nd.array([[3., 4.], [2., 2.]])] - >>> mcs = mx.gluon.metric.MeanCosineSimilarity() - >>> mcs.update(labels = labels, preds = predicts) - >>> print mcs.get() - ('cos_sim', 0.8) + >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] + >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] + >>> root_mean_squared_error = mx.metric.RMSE() + >>> root_mean_squared_error.update(labels = labels, preds = predicts) + >>> print root_mean_squared_error.get() + ('rmse', 0.612372457981) """ - def __init__(self, name='cos_sim', - output_names=None, label_names=None, eps=1e-8): - super(MeanCosineSimilarity, self).__init__( - name, output_names=output_names, label_names=label_names) - self.eps = eps + def __init__(self, name='rmse', + output_names=None, label_names=None): + super(RMSE, self).__init__( + name, output_names=output_names, label_names=label_names, + has_global_stats=True) def update(self, labels, preds): """Updates the internal evaluation result. @@ -1311,27 +1251,23 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): - label = label.as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + label = label.asnumpy() + pred = pred.asnumpy() if len(label.shape) == 1: - label = label.reshape(1, label.shape[0]) + label = label.reshape(label.shape[0], 1) if len(pred.shape) == 1: - pred = pred.reshape(1, pred.shape[0]) + pred = pred.reshape(pred.shape[0], 1) - sim = (label * pred).sum(axis=-1) - n_p = numpy.linalg.norm(pred, axis=-1) - n_l = numpy.linalg.norm(label, axis=-1) - sim = sim / numpy.maximum(n_l * n_p, self.eps) - sim = sim.sum() - num_inst = len(label.reshape(-1, label.shape[-1])) # numpy.prod(label.shape[:-1]) is not supported - self.sum_metric += sim - self.num_inst += num_inst + rmse = numpy.sqrt(((label - pred)**2.0).mean()) + self.sum_metric += rmse + self.global_sum_metric += rmse + self.num_inst += 1 + self.global_num_inst += 1 @register @alias('ce') -@use_np class CrossEntropy(EvalMetric): """Computes Cross Entropy loss. @@ -1346,15 +1282,9 @@ class :math:`k`. Parameters ---------- - eps : float, default 1e-12 - Use small constant for the case that predicted value is 0. - ignore_label : int or None, default None - Index of invalid label to ignore when - counting. By default, sets to -1. - If set to `None`, it will include all entries. - axis : int (default -1) - The axis from prediction that was used to - compute softmax. By default use the last axis. + eps : float + Cross Entropy loss is undefined for predicted value is 0 or 1, + so predicted values are added with the small constant. name : str Name of this metric instance for display. output_names : list of str, or None @@ -1368,17 +1298,17 @@ class :math:`k`. -------- >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] >>> labels = [mx.nd.array([0, 1, 1])] - >>> ce = mx.gluon.metric.CrossEntropy() + >>> ce = mx.metric.CrossEntropy() >>> ce.update(labels, predicts) >>> print ce.get() ('cross-entropy', 0.57159948348999023) """ - def __init__(self, eps=1e-12, ignore_label=None, axis=-1, name='cross-entropy', + def __init__(self, eps=1e-12, name='cross-entropy', output_names=None, label_names=None): super(CrossEntropy, self).__init__( - name, output_names=output_names, label_names=label_names) - self.ignore_label = ignore_label - self.axis = axis + name, eps=eps, + output_names=output_names, label_names=label_names, + has_global_stats=True) self.eps = eps def update(self, labels, preds): @@ -1394,97 +1324,22 @@ def update(self, labels, preds): """ labels, preds = check_label_shapes(labels, preds, True) - loss = 0. - num = 0 for label, pred in zip(labels, preds): - assert label.size == pred.size/pred.shape[-1], \ - "shape mismatch: %s vs. %s"%(label.shape, pred.shape) - label = label.reshape((label.size,)) - pred = ndarray.pick(pred.as_in_context(label.ctx), label.astype(dtype='int32'), axis=self.axis) - label = label.as_np_ndarray() - pred = pred.as_np_ndarray() - if self.ignore_label is not None: - ignore = (label == self.ignore_label).astype(pred.dtype) - num -= ignore.sum() - pred = pred * (1 - ignore) + ignore - loss -= numpy.log(numpy.maximum(self.eps, pred)).sum() - num += pred.size - self.sum_metric += loss - self.num_inst += num + label = label.asnumpy() + pred = pred.asnumpy() + label = label.ravel() + assert label.shape[0] == pred.shape[0] -@register -@use_np -class Perplexity(CrossEntropy): - """Computes perplexity. - - Perplexity is a measurement of how well a probability distribution - or model predicts a sample. A low perplexity indicates the model - is good at predicting the sample. - - The perplexity of a model q is defined as - - .. math:: - b^{\\big(-\\frac{1}{N} \\sum_{i=1}^N \\log_b q(x_i) \\big)} - = \\exp \\big(-\\frac{1}{N} \\sum_{i=1}^N \\log q(x_i)\\big) - - where we let `b = e`. - - :math:`q(x_i)` is the predicted value of its ground truth - label on sample :math:`x_i`. - - For example, we have three samples :math:`x_1, x_2, x_3` and their labels - are :math:`[0, 1, 1]`. - Suppose our model predicts :math:`q(x_1) = p(y_1 = 0 | x_1) = 0.3` - and :math:`q(x_2) = 1.0`, - :math:`q(x_3) = 0.6`. The perplexity of model q is - :math:`exp\\big(-(\\log 0.3 + \\log 1.0 + \\log 0.6) / 3\\big) = 1.77109762852`. - - Parameters - ---------- - eps : float, default 1e-12 - Use small constant for the case that predicted value is 0. - ignore_label : int or None, default None - Index of invalid label to ignore when - counting. By default, sets to -1. - If set to `None`, it will include all entries. - axis : int (default -1) - The axis from prediction that was used to - compute softmax. By default use the last axis. - name : str - Name of this metric instance for display. - output_names : list of str, or None - Name of predictions that should be used when updating with update_dict. - By default include all predictions. - label_names : list of str, or None - Name of labels that should be used when updating with update_dict. - By default include all labels. - - Examples - -------- - >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] - >>> labels = [mx.nd.array([0, 1, 1])] - >>> perp = mx.gluon.metric.Perplexity(ignore_label=None) - >>> perp.update(labels, predicts) - >>> print perp.get() - ('Perplexity', 1.7710976285155853) - """ - def __init__(self, eps=1e-12, ignore_label=None, axis=-1, name='perplexity', - output_names=None, label_names=None): - super(Perplexity, self).__init__( - name=name, eps=eps, ignore_label=ignore_label, axis=axis, - output_names=output_names, label_names=label_names) - - def get(self): - if self.num_inst == 0: - return (self.name, float('nan')) - else: - return (self.name, math.exp(self.sum_metric/self.num_inst)) - + prob = pred[numpy.arange(label.shape[0]), numpy.int64(label)] + cross_entropy = (-numpy.log(prob + self.eps)).sum() + self.sum_metric += cross_entropy + self.global_sum_metric += cross_entropy + self.num_inst += label.shape[0] + self.global_num_inst += label.shape[0] @register @alias('nll_loss') -@use_np class NegativeLogLikelihood(EvalMetric): """Computes the negative log-likelihood loss. @@ -1515,7 +1370,7 @@ class NegativeLogLikelihood(EvalMetric): -------- >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] >>> labels = [mx.nd.array([0, 1, 1])] - >>> nll_loss = mx.gluon.metric.NegativeLogLikelihood() + >>> nll_loss = mx.metric.NegativeLogLikelihood() >>> nll_loss.update(labels, predicts) >>> print nll_loss.get() ('nll-loss', 0.57159948348999023) @@ -1524,7 +1379,8 @@ def __init__(self, eps=1e-12, name='nll-loss', output_names=None, label_names=None): super(NegativeLogLikelihood, self).__init__( name, eps=eps, - output_names=output_names, label_names=label_names) + output_names=output_names, label_names=label_names, + has_global_stats=True) self.eps = eps def update(self, labels, preds): @@ -1541,21 +1397,21 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): - label = label.as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + label = label.asnumpy() + pred = pred.asnumpy() - label = label.reshape(-1) + label = label.ravel() num_examples = pred.shape[0] assert label.shape[0] == num_examples, (label.shape[0], num_examples) prob = pred[numpy.arange(num_examples, dtype=numpy.int64), numpy.int64(label)] nll = (-numpy.log(prob + self.eps)).sum() self.sum_metric += nll + self.global_sum_metric += nll self.num_inst += num_examples - + self.global_num_inst += num_examples @register @alias('pearsonr') -@use_np class PearsonCorrelation(EvalMetric): """Computes Pearson correlation. @@ -1574,23 +1430,30 @@ class PearsonCorrelation(EvalMetric): label_names : list of str, or None Name of labels that should be used when updating with update_dict. By default include all labels. + average : str, default 'macro' + Strategy to be used for aggregating across mini-batches. + "macro": average the pearsonr scores for each batch. + "micro": compute a single pearsonr score across all batches. Examples -------- >>> predicts = [mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]])] >>> labels = [mx.nd.array([[1, 0], [0, 1], [0, 1]])] - >>> pr = mx.gluon.metric.PearsonCorrelation() + >>> pr = mx.metric.PearsonCorrelation() >>> pr.update(labels, predicts) >>> print pr.get() ('pearsonr', 0.42163704544016178) """ def __init__(self, name='pearsonr', - output_names=None, label_names=None): + output_names=None, label_names=None, average='macro'): + self.average = average super(PearsonCorrelation, self).__init__( - name, output_names=output_names, label_names=label_names) - self.reset() + name, output_names=output_names, label_names=label_names, + has_global_stats=True) + if self.average == 'micro': + self.reset_micro() - def reset(self): + def reset_micro(self): self._sse_p = 0 self._mean_p = 0 self._sse_l = 0 @@ -1599,8 +1462,13 @@ def reset(self): self._label_nums = 0 self._conv = 0 + def reset(self): self.num_inst = 0 self.sum_metric = 0.0 + self.global_num_inst = 0 + self.global_sum_metric = 0.0 + if self.average == 'micro': + self.reset_micro() def update_variance(self, new_values, *aggregate): #Welford's online algorithm for variance update @@ -1628,26 +1496,34 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for label, pred in zip(labels, preds): check_label_shapes(label, pred, False, True) - label = label.as_np_ndarray().reshape(-1).astype(numpy.float64) - pred = pred.as_np_ndarray().as_in_ctx(label.ctx).reshape(-1).astype(numpy.float64) - - self.num_inst += 1 - self._label_nums, self._mean_l, self._sse_l = \ - self.update_variance(label, self._label_nums, self._mean_l, self._sse_l) - self.update_cov(label, pred) - self._pred_nums, self._mean_p, self._sse_p = \ - self.update_variance(pred, self._pred_nums, self._mean_p, self._sse_p) + label = label.asnumpy().ravel().astype(numpy.float64) + pred = pred.asnumpy().ravel().astype(numpy.float64) + if self.average == 'macro': + pearson_corr = numpy.corrcoef(pred, label)[0, 1] + self.sum_metric += pearson_corr + self.global_sum_metric += pearson_corr + self.num_inst += 1 + self.global_num_inst += 1 + else: + self.global_num_inst += 1 + self.num_inst += 1 + self._label_nums, self._mean_l, self._sse_l = \ + self.update_variance(label, self._label_nums, self._mean_l, self._sse_l) + self.update_cov(label, pred) + self._pred_nums, self._mean_p, self._sse_p = \ + self.update_variance(pred, self._pred_nums, self._mean_p, self._sse_p) def get(self): if self.num_inst == 0: return (self.name, float('nan')) - - n = self._label_nums - pearsonr = self._conv / ((n-1) * numpy.sqrt(self._sse_p / (n - 1)) * numpy.sqrt(self._sse_l / (n - 1))) - return (self.name, float(pearsonr)) + if self.average == 'macro': + return (self.name, self.sum_metric / self.num_inst) + else: + n = self._label_nums + pearsonr = self._conv / ((n-1) * numpy.sqrt(self._sse_p / (n - 1)) * numpy.sqrt(self._sse_l / (n - 1))) + return (self.name, pearsonr) @register -@use_np class PCC(EvalMetric): """PCC is a multiclass equivalent for the Matthews correlation coefficient derived from a discrete solution to the Pearson correlation coefficient. @@ -1691,9 +1567,9 @@ class PCC(EvalMetric): [0]*(false_positives + true_negatives) + [1]*(false_negatives + true_positives) )] - >>> f1 = mx.gluon.metric.F1() + >>> f1 = mx.metric.F1() >>> f1.update(preds = predicts, labels = labels) - >>> pcc = mx.gluon.metric.PCC() + >>> pcc = mx.metric.PCC() >>> pcc.update(preds = predicts, labels = labels) >>> print f1.get() ('f1', 0.95233560306652054) @@ -1701,14 +1577,18 @@ class PCC(EvalMetric): ('pcc', 0.01917751877733392) """ def __init__(self, name='pcc', - output_names=None, label_names=None): + output_names=None, label_names=None, + has_global_stats=True): self.k = 2 super(PCC, self).__init__( - name=name, output_names=output_names, label_names=label_names) + name=name, output_names=output_names, label_names=label_names, + has_global_stats=has_global_stats) def _grow(self, inc): self.lcm = numpy.pad( self.lcm, ((0, inc), (0, inc)), 'constant', constant_values=(0)) + self.gcm = numpy.pad( + self.gcm, ((0, inc), (0, inc)), 'constant', constant_values=(0)) self.k += inc def _calc_mcc(self, cmat): @@ -1719,8 +1599,7 @@ def _calc_mcc(self, cmat): cov_yy = numpy.sum(y * (n - y)) if cov_xx == 0 or cov_yy == 0: return float('nan') - # i = cmat.diagonal() # mxnet.numpy.ndarray.diagonal() is currently not available. - i = cmat[numpy.arange(self.k), numpy.arange(self.k)] + i = cmat.diagonal() cov_xy = numpy.sum(i * n - x * y) return cov_xy / (cov_xx * cov_yy) ** 0.5 @@ -1739,29 +1618,42 @@ def update(self, labels, preds): # update the confusion matrix for label, pred in zip(labels, preds): - label = label.astype('int32', copy=False).as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + label = label.astype('int32', copy=False).asnumpy() + pred = pred.asnumpy() if pred.shape != label.shape: - pred = pred.argmax(axis=1).astype(label, copy=False) + pred = pred.argmax(axis=1) else: pred = pred.astype('int32', copy=False) - n = int(max(pred.max(), label.max())) + n = max(pred.max(), label.max()) if n >= self.k: self._grow(n + 1 - self.k) - bcm = numpy.zeros((self.k, self.k), dtype='float64') + bcm = numpy.zeros((self.k, self.k)) for i, j in zip(pred, label): bcm[i, j] += 1 self.lcm += bcm + self.gcm += bcm + self.num_inst += 1 + self.global_num_inst += 1 @property def sum_metric(self): return self._calc_mcc(self.lcm) * self.num_inst + @property + def global_sum_metric(self): + return self._calc_mcc(self.gcm) * self.global_num_inst + def reset(self): """Resets the internal evaluation result to initial state.""" + self.global_num_inst = 0. + self.gcm = numpy.zeros((self.k, self.k)) + self.reset_local() + + def reset_local(self): + """Resets the local portion of the internal evaluation results to initial state.""" self.num_inst = 0. - self.lcm = numpy.zeros((self.k, self.k), dtype='float64') + self.lcm = numpy.zeros((self.k, self.k)) @register @@ -1782,7 +1674,8 @@ class Loss(EvalMetric): def __init__(self, name='loss', output_names=None, label_names=None): super(Loss, self).__init__( - name, output_names=output_names, label_names=label_names) + name, output_names=output_names, label_names=label_names, + has_global_stats=True) def update(self, _, preds): @@ -1792,7 +1685,9 @@ def update(self, _, preds): for pred in preds: loss = ndarray.sum(pred).asscalar() self.sum_metric += loss + self.global_sum_metric += loss self.num_inst += pred.size + self.global_num_inst += pred.size @register @@ -1814,7 +1709,6 @@ def __init__(self, name='caffe', @register -@use_np class CustomMetric(EvalMetric): """Computes a customized evaluation metric. @@ -1845,7 +1739,7 @@ class CustomMetric(EvalMetric): >>> predicts = [mx.nd.array(np.array([3, -0.5, 2, 7]).reshape(4,1))] >>> labels = [mx.nd.array(np.array([2.5, 0.0, 2, 8]).reshape(4,1))] >>> feval = lambda x, y : (x + y).mean() - >>> eval_metrics = mx.gluon.metric.CustomMetric(feval=feval) + >>> eval_metrics = mx.metric.CustomMetric(feval=feval) >>> eval_metrics.update(labels, predicts) >>> print eval_metrics.get() ('custom()', 6.0) @@ -1859,7 +1753,8 @@ def __init__(self, feval, name=None, allow_extra_outputs=False, super(CustomMetric, self).__init__( name, feval=feval, allow_extra_outputs=allow_extra_outputs, - output_names=output_names, label_names=label_names) + output_names=output_names, label_names=label_names, + has_global_stats=True) self._feval = feval self._allow_extra_outputs = allow_extra_outputs @@ -1878,17 +1773,21 @@ def update(self, labels, preds): labels, preds = check_label_shapes(labels, preds, True) for pred, label in zip(preds, labels): - label = label.as_np_ndarray() - pred = pred.as_np_ndarray().as_in_ctx(label.ctx) + label = label.asnumpy() + pred = pred.asnumpy() reval = self._feval(label, pred) if isinstance(reval, tuple): (sum_metric, num_inst) = reval self.sum_metric += sum_metric + self.global_sum_metric += sum_metric self.num_inst += num_inst + self.global_num_inst += num_inst else: self.sum_metric += reval + self.global_sum_metric += reval self.num_inst += 1 + self.global_num_inst += 1 def get_config(self): raise NotImplementedError("CustomMetric cannot be serialized") @@ -1920,7 +1819,7 @@ def np(numpy_feval, name=None, allow_extra_outputs=False): >>> def custom_metric(label, pred): ... return np.mean(np.abs(label-pred)) ... - >>> metric = mx.gluon.metric.np(custom_metric) + >>> metric = mx.metric.np(custom_metric) """ def feval(label, pred): """Internal eval function.""" diff --git a/python/mxnet/model.py b/python/mxnet/model.py index 7aee2578f2a3..9acabeefcb2d 100644 --- a/python/mxnet/model.py +++ b/python/mxnet/model.py @@ -26,6 +26,8 @@ from . import ndarray as nd from . import symbol as sym +from . import optimizer as opt +from . import metric from . import kvstore as kvs from .context import cpu diff --git a/python/mxnet/module/base_module.py b/python/mxnet/module/base_module.py new file mode 100644 index 000000000000..053a00b3abba --- /dev/null +++ b/python/mxnet/module/base_module.py @@ -0,0 +1,1067 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: disable=fixme, too-many-arguments, too-many-locals, no-else-raise +# pylint: disable=too-many-public-methods, too-many-branches, too-many-lines +"""`BaseModule` defines an API for modules.""" + +import time +import logging +import warnings +import numpy as np + +from .. import metric +from .. import ndarray + +from ..context import cpu +from ..model import BatchEndParam +from ..initializer import Uniform +from ..io import DataDesc, DataIter, DataBatch +from ..base import _as_list + + +def _check_input_names(symbol, names, typename, throw): + """Check that all input names are in symbol's arguments.""" + args = symbol.list_arguments() + for name in names: + if name in args: + continue + candidates = [arg for arg in args if + not arg.endswith('_weight') and + not arg.endswith('_bias') and + not arg.endswith('_gamma') and + not arg.endswith('_beta')] + msg = "\033[91mYou created Module with Module(..., %s_names=%s) but " \ + "input with name '%s' is not found in symbol.list_arguments(). " \ + "Did you mean one of:\n\t%s\033[0m"%( + typename, str(names), name, '\n\t'.join(candidates)) + if throw: + raise ValueError(msg) + else: + warnings.warn(msg) + + +def _check_names_match(data_names, data_shapes, name, throw): + """Check that input names matches input data descriptors.""" + actual = [x[0] for x in data_shapes] + if sorted(data_names) != sorted(actual): + msg = "Data provided by %s_shapes don't match names specified by %s_names (%s vs. %s)"%( + name, name, str(data_shapes), str(data_names)) + if throw: + raise ValueError(msg) + else: + warnings.warn(msg) + + +def _parse_data_desc(data_names, label_names, data_shapes, label_shapes): + """parse data_attrs into DataDesc format and check that names match""" + data_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in data_shapes] + _check_names_match(data_names, data_shapes, 'data', True) + if label_shapes is not None: + label_shapes = [x if isinstance(x, DataDesc) else DataDesc(*x) for x in label_shapes] + _check_names_match(label_names, label_shapes, 'label', False) + else: + _check_names_match(label_names, [], 'label', False) + return data_shapes, label_shapes + + +class BaseModule(object): + """The base class of a module. + + A module represents a computation component. One can think of module as a computation machine. + A module can execute forward and backward passes and update parameters in a model. + We aim to make the APIs easy to use, especially in the case when we need to use the imperative + API to work with multiple modules (e.g. stochastic depth network). + + A module has several states: + + - Initial state: Memory is not allocated yet, so the module is not ready for computation yet. + - Binded: Shapes for inputs, outputs, and parameters are all known, memory has been allocated, + and the module is ready for computation. + - Parameters are initialized: For modules with parameters, doing computation before + initializing the parameters might result in undefined outputs. + - Optimizer is installed: An optimizer can be installed to a module. After this, the parameters + of the module can be updated according to the optimizer after gradients are computed + (forward-backward). + + In order for a module to interact with others, it must be able to report the + following information in its initial state (before binding): + + - `data_names`: list of type string indicating the names of the required input data. + - `output_names`: list of type string indicating the names of the required outputs. + + After binding, a module should be able to report the following richer information: + + - state information + - `binded`: `bool`, indicates whether the memory buffers needed for computation + have been allocated. + - `for_training`: whether the module is bound for training. + - `params_initialized`: `bool`, indicates whether the parameters of this module + have been initialized. + - `optimizer_initialized`: `bool`, indicates whether an optimizer is defined + and initialized. + - `inputs_need_grad`: `bool`, indicates whether gradients with respect to the + input data are needed. Might be useful when implementing composition of modules. + + - input/output information + - `data_shapes`: a list of `(name, shape)`. In theory, since the memory is allocated, + we could directly provide the data arrays. But in the case of data parallelism, + the data arrays might not be of the same shape as viewed from the external world. + - `label_shapes`: a list of `(name, shape)`. This might be `[]` if the module does + not need labels (e.g. it does not contains a loss function at the top), or a module + is not bound for training. + - `output_shapes`: a list of `(name, shape)` for outputs of the module. + + - parameters (for modules with parameters) + - `get_params()`: return a tuple `(arg_params, aux_params)`. Each of those + is a dictionary of name to ``NDArray`` mapping. Those `NDArray` always lives on + CPU. The actual parameters used for computing might live on other devices (GPUs), + this function will retrieve (a copy of) the latest parameters. + - ``set_params(arg_params, aux_params)``: assign parameters to the devices + doing the computation. + - ``init_params(...)``: a more flexible interface to assign or initialize the parameters. + + - setup + - `bind()`: prepare environment for computation. + - `init_optimizer()`: install optimizer for parameter updating. + - `prepare()`: prepare the module based on the current data batch. + + - computation + - `forward(data_batch)`: forward operation. + - `backward(out_grads=None)`: backward operation. + - `update()`: update parameters according to installed optimizer. + - `get_outputs()`: get outputs of the previous forward operation. + - `get_input_grads()`: get the gradients with respect to the inputs computed + in the previous backward operation. + - `update_metric(metric, labels, pre_sliced=False)`: update performance metric + for the previous forward + computed results. + + - other properties (mostly for backward compatibility) + - `symbol`: the underlying symbolic graph for this module (if any) + This property is not necessarily constant. For example, for `BucketingModule`, + this property is simply the *current* symbol being used. For other modules, + this value might not be well defined. + + When those intermediate-level API are implemented properly, the following + high-level API will be automatically available for a module: + + - `fit`: train the module parameters on a data set. + - `predict`: run prediction on a data set and collect outputs. + - `score`: run prediction on a data set and evaluate performance. + + Examples + -------- + >>> # An example of creating a mxnet module. + >>> import mxnet as mx + >>> data = mx.symbol.Variable('data') + >>> fc1 = mx.symbol.FullyConnected(data, name='fc1', num_hidden=128) + >>> act1 = mx.symbol.Activation(fc1, name='relu1', act_type="relu") + >>> fc2 = mx.symbol.FullyConnected(act1, name = 'fc2', num_hidden = 64) + >>> act2 = mx.symbol.Activation(fc2, name='relu2', act_type="relu") + >>> fc3 = mx.symbol.FullyConnected(act2, name='fc3', num_hidden=10) + >>> out = mx.symbol.SoftmaxOutput(fc3, name = 'softmax') + >>> mod = mx.mod.Module(out) + """ + def __init__(self, logger=logging): + self.logger = logger + self.binded = False + self.for_training = False + self.inputs_need_grad = False + self.params_initialized = False + self.optimizer_initialized = False + self._symbol = None + self._total_exec_bytes = 0 + + ################################################################################ + # High Level API + ################################################################################ + def forward_backward(self, data_batch): + """A convenient function that calls both ``forward`` and ``backward``.""" + self.forward(data_batch, is_train=True) + self.backward() + + def score(self, eval_data, eval_metric, num_batch=None, batch_end_callback=None, + score_end_callback=None, + reset=True, epoch=0, sparse_row_id_fn=None): + """Runs prediction on ``eval_data`` and evaluates the performance according to + the given ``eval_metric``. + + Checkout `Module Tutorial `_ + to see an end-to-end use-case. + + Parameters + ---------- + eval_data : DataIter + Evaluation data to run prediction on. + eval_metric : EvalMetric or list of EvalMetrics + Evaluation metric to use. + num_batch : int + Number of batches to run. Defaults to ``None``, indicating run until the `DataIter` + finishes. + batch_end_callback : function + Could also be a list of functions. + reset : bool + Defaults to ``True``. Indicates whether we should reset `eval_data` before starting + evaluating. + epoch : int + Defaults to 0. For compatibility, this will be passed to callbacks (if any). + During training, this will correspond to the training epoch number. + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + + Examples + -------- + >>> # An example of using score for prediction. + >>> # Evaluate accuracy on val_dataiter + >>> metric = mx.metric.Accuracy() + >>> mod.score(val_dataiter, metric) + >>> mod.score(val_dataiter, ['mse', 'acc']) + """ + assert self.binded and self.params_initialized + + if reset: + eval_data.reset() + + if not isinstance(eval_metric, metric.EvalMetric): + eval_metric = metric.create(eval_metric) + + eval_metric.reset() + actual_num_batch = 0 + + for nbatch, eval_batch in enumerate(eval_data): + if num_batch is not None and nbatch == num_batch: + break + self.prepare(eval_batch, sparse_row_id_fn=sparse_row_id_fn) + self.forward(eval_batch, is_train=False) + if isinstance(eval_batch, list): + self.update_metric(eval_metric, [eb.label for eb in eval_batch], pre_sliced=True) + else: + self.update_metric(eval_metric, eval_batch.label) + + if batch_end_callback is not None: + batch_end_params = BatchEndParam(epoch=epoch, + nbatch=nbatch, + eval_metric=eval_metric, + locals=locals()) + for callback in _as_list(batch_end_callback): + callback(batch_end_params) + actual_num_batch += 1 + + if score_end_callback: + params = BatchEndParam(epoch=epoch, + nbatch=actual_num_batch, + eval_metric=eval_metric, + locals=locals()) + for callback in _as_list(score_end_callback): + callback(params) + + return eval_metric.get_name_value() + + def iter_predict(self, eval_data, num_batch=None, reset=True, sparse_row_id_fn=None): + """Iterates over predictions. + + Examples + -------- + >>> for pred, i_batch, batch in module.iter_predict(eval_data): + ... # pred is a list of outputs from the module + ... # i_batch is a integer + ... # batch is the data batch from the data iterator + + Parameters + ---------- + eval_data : DataIter + Evaluation data to run prediction on. + num_batch : int + Default is ``None``, indicating running all the batches in the data iterator. + reset : bool + Default is ``True``, indicating whether we should reset the data iter before start + doing prediction. + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + """ + assert self.binded and self.params_initialized + + if reset: + eval_data.reset() + + for nbatch, eval_batch in enumerate(eval_data): + if num_batch is not None and nbatch == num_batch: + break + self.prepare(eval_batch, sparse_row_id_fn=sparse_row_id_fn) + self.forward(eval_batch, is_train=False) + pad = eval_batch.pad + outputs = [out[0:out.shape[0]-pad] for out in self.get_outputs()] + + yield (outputs, nbatch, eval_batch) + + def predict(self, eval_data, num_batch=None, merge_batches=True, reset=True, + always_output_list=False, sparse_row_id_fn=None): + """Runs prediction and collects the outputs. + + When `merge_batches` is ``True`` (by default), the return value will be a list + ``[out1, out2, out3]``, where each element is formed by concatenating the outputs for + all the mini-batches. When `always_output_list` is ``False`` (as by default), + then in the case of a single output, `out1` is returned instead of ``[out1]``. + + When `merge_batches` is ``False``, the return value will be a nested list like + ``[[out1_batch1, out2_batch1], [out1_batch2], ...]``. This mode is useful because + in some cases (e.g. bucketing), the module does not necessarily produce the same + number of outputs. + + The objects in the results have type `NDArray`. If you need to work with a numpy array, + just call ``.asnumpy()`` on each `NDArray`. + + Parameters + ---------- + eval_data : DataIter or NDArray or numpy array + Evaluation data to run prediction on. + num_batch : int + Defaults to ``None``, indicates running all the batches in the data iterator. + merge_batches : bool + Defaults to ``True``, see above for return values. + reset : bool + Defaults to ``True``, indicates whether we should reset the data iter before + doing prediction. + always_output_list : bool + Defaults to ``False``, see above for return values. + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + + Returns + ------- + list of NDArray or list of list of NDArray + Prediction results. + + Examples + -------- + >>> # An example of using `predict` for prediction. + >>> # Predict on the first 10 batches of val_dataiter + >>> mod.predict(eval_data=val_dataiter, num_batch=10) + """ + assert self.binded and self.params_initialized + + if isinstance(eval_data, (ndarray.NDArray, np.ndarray)): + if isinstance(eval_data, np.ndarray): + eval_data = ndarray.array(eval_data) + self.forward(DataBatch([eval_data])) + return self.get_outputs()[0] + + if not isinstance(eval_data, DataIter): + raise ValueError('eval_data must be of type NDArray or DataIter') + + if reset: + eval_data.reset() + + output_list = [] + + for nbatch, eval_batch in enumerate(eval_data): + if num_batch is not None and nbatch == num_batch: + break + self.prepare(eval_batch, sparse_row_id_fn=sparse_row_id_fn) + self.forward(eval_batch, is_train=False) + pad = eval_batch.pad + outputs = [out[0:out.shape[0]-pad].copy() for out in self.get_outputs()] + + output_list.append(outputs) + + if len(output_list) == 0: + return output_list + + if merge_batches: + num_outputs = len(output_list[0]) + for out in output_list: + assert len(out) == num_outputs, \ + 'Cannot merge batches, as num of outputs is not the same ' + \ + 'in mini-batches. Maybe bucketing is used?' + output_list2 = [ndarray.concatenate([out[i] for out in output_list]) + for i in range(num_outputs)] + + if num_outputs == 1 and not always_output_list: + return output_list2[0] + return output_list2 + + return output_list + + def fit(self, train_data, eval_data=None, eval_metric='acc', + epoch_end_callback=None, batch_end_callback=None, kvstore='local', + optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), + eval_end_callback=None, + eval_batch_end_callback=None, initializer=Uniform(0.01), + arg_params=None, aux_params=None, allow_missing=False, + force_rebind=False, force_init=False, begin_epoch=0, num_epoch=None, + validation_metric=None, monitor=None, sparse_row_id_fn=None): + """Trains the module parameters. + + Checkout `Module Tutorial `_ + to see an end-to-end use-case. + + Parameters + ---------- + train_data : DataIter + Train DataIter. + eval_data : DataIter + If not ``None``, will be used as validation set and the performance + after each epoch will be evaluated. + eval_metric : str or EvalMetric + Defaults to 'accuracy'. The performance measure used to display during training. + Other possible predefined metrics are: + 'ce' (CrossEntropy), 'f1', 'mae', 'mse', 'rmse', 'top_k_accuracy'. + epoch_end_callback : function or list of functions + Each callback will be called with the current `epoch`, `symbol`, `arg_params` + and `aux_params`. + batch_end_callback : function or list of function + Each callback will be called with a `BatchEndParam`. + kvstore : str or KVStore + Defaults to 'local'. + optimizer : str or Optimizer + Defaults to 'sgd'. + optimizer_params : dict + Defaults to ``(('learning_rate', 0.01),)``. The parameters for + the optimizer constructor. + The default value is not a dict, just to avoid pylint warning on dangerous + default values. + eval_end_callback : function or list of function + These will be called at the end of each full evaluation, with the metrics over + the entire evaluation set. + eval_batch_end_callback : function or list of function + These will be called at the end of each mini-batch during evaluation. + initializer : Initializer + The initializer is called to initialize the module parameters when they are + not already initialized. + arg_params : dict + Defaults to ``None``, if not ``None``, should be existing parameters from a trained + model or loaded from a checkpoint (previously saved model). In this case, + the value here will be used to initialize the module parameters, unless they + are already initialized by the user via a call to `init_params` or `fit`. + `arg_params` has a higher priority than `initializer`. + aux_params : dict + Defaults to ``None``. Similar to `arg_params`, except for auxiliary states. + allow_missing : bool + Defaults to ``False``. Indicates whether to allow missing parameters when `arg_params` + and `aux_params` are not ``None``. If this is ``True``, then the missing parameters + will be initialized via the `initializer`. + force_rebind : bool + Defaults to ``False``. Whether to force rebinding the executors if already bound. + force_init : bool + Defaults to ``False``. Indicates whether to force initialization even if the + parameters are already initialized. + begin_epoch : int + Defaults to 0. Indicates the starting epoch. Usually, if resumed from a + checkpoint saved at a previous training phase at epoch N, then this value should be + N+1. + num_epoch : int + Number of epochs for training. + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + + Examples + -------- + >>> # An example of using fit for training. + >>> # Assume training dataIter and validation dataIter are ready + >>> # Assume loading a previously checkpointed model + >>> sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, 3) + >>> mod.fit(train_data=train_dataiter, eval_data=val_dataiter, optimizer='sgd', + ... optimizer_params={'learning_rate':0.01, 'momentum': 0.9}, + ... arg_params=arg_params, aux_params=aux_params, + ... eval_metric='acc', num_epoch=10, begin_epoch=3) + """ + assert num_epoch is not None, 'please specify number of epochs' + + self.bind(data_shapes=train_data.provide_data, label_shapes=train_data.provide_label, + for_training=True, force_rebind=force_rebind) + if monitor is not None: + self.install_monitor(monitor) + self.init_params(initializer=initializer, arg_params=arg_params, aux_params=aux_params, + allow_missing=allow_missing, force_init=force_init) + self.init_optimizer(kvstore=kvstore, optimizer=optimizer, + optimizer_params=optimizer_params) + + if validation_metric is None: + validation_metric = eval_metric + if not isinstance(eval_metric, metric.EvalMetric): + eval_metric = metric.create(eval_metric) + + ################################################################################ + # training loop + ################################################################################ + for epoch in range(begin_epoch, num_epoch): + tic = time.time() + eval_metric.reset() + nbatch = 0 + data_iter = iter(train_data) + end_of_batch = False + next_data_batch = next(data_iter) + while not end_of_batch: + data_batch = next_data_batch + if monitor is not None: + monitor.tic() + self.forward_backward(data_batch) + self.update() + + if isinstance(data_batch, list): + self.update_metric(eval_metric, + [db.label for db in data_batch], + pre_sliced=True) + else: + self.update_metric(eval_metric, data_batch.label) + + try: + # pre fetch next batch + next_data_batch = next(data_iter) + self.prepare(next_data_batch, sparse_row_id_fn=sparse_row_id_fn) + except StopIteration: + end_of_batch = True + + if monitor is not None: + monitor.toc_print() + + if end_of_batch: + eval_name_vals = eval_metric.get_global_name_value() + + if batch_end_callback is not None: + batch_end_params = BatchEndParam(epoch=epoch, nbatch=nbatch, + eval_metric=eval_metric, + locals=locals()) + for callback in _as_list(batch_end_callback): + callback(batch_end_params) + nbatch += 1 + + # one epoch of training is finished + for name, val in eval_name_vals: + self.logger.info('Epoch[%d] Train-%s=%f', epoch, name, val) + toc = time.time() + self.logger.info('Epoch[%d] Time cost=%.3f', epoch, (toc-tic)) + + # sync aux params across devices + arg_params, aux_params = self.get_params() + self.set_params(arg_params, aux_params) + + if epoch_end_callback is not None: + for callback in _as_list(epoch_end_callback): + callback(epoch, self.symbol, arg_params, aux_params) + + #---------------------------------------- + # evaluation on validation set + if eval_data: + res = self.score(eval_data, validation_metric, + score_end_callback=eval_end_callback, + batch_end_callback=eval_batch_end_callback, epoch=epoch) + #TODO: pull this into default + for name, val in res: + self.logger.info('Epoch[%d] Validation-%s=%f', epoch, name, val) + + # end of 1 epoch, reset the data-iter for another epoch + train_data.reset() + + ################################################################################ + # Symbol information + ################################################################################ + @property + def data_names(self): + """A list of names for data required by this module.""" + raise NotImplementedError() + + @property + def output_names(self): + """A list of names for the outputs of this module.""" + raise NotImplementedError() + + ################################################################################ + # Input/Output information + ################################################################################ + @property + def data_shapes(self): + """A list of (name, shape) pairs specifying the data inputs to this module.""" + raise NotImplementedError() + + @property + def label_shapes(self): + """A list of (name, shape) pairs specifying the label inputs to this module. + If this module does not accept labels -- either it is a module without loss + function, or it is not bound for training, then this should return an empty + list ``[]``. + """ + raise NotImplementedError() + + @property + def output_shapes(self): + """A list of (name, shape) pairs specifying the outputs of this module.""" + raise NotImplementedError() + + ################################################################################ + # Parameters of a module + ################################################################################ + def get_params(self): + """Gets parameters, those are potentially copies of the actual parameters used + to do computation on the device. + + Returns + ------- + ``(arg_params, aux_params)`` + A pair of dictionaries each mapping parameter names to NDArray values. + + Examples + -------- + >>> # An example of getting module parameters. + >>> print mod.get_params() + ({'fc2_weight': , 'fc1_weight': , + 'fc3_bias': , 'fc3_weight': , + 'fc2_bias': , 'fc1_bias': }, {}) + """ + raise NotImplementedError() + + def init_params(self, initializer=Uniform(0.01), arg_params=None, aux_params=None, + allow_missing=False, force_init=False, allow_extra=False): + """Initializes the parameters and auxiliary states. + + Parameters + ---------- + initializer : Initializer + Called to initialize parameters if needed. + arg_params : dict + If not ``None``, should be a dictionary of existing `arg_params`. Initialization + will be copied from that. + aux_params : dict + If not ``None``, should be a dictionary of existing `aux_params`. Initialization + will be copied from that. + allow_missing : bool + If ``True``, params could contain missing values, and the initializer will be + called to fill those missing params. + force_init : bool + If ``True``, `force_init` will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. + + Examples + -------- + >>> # An example of initializing module parameters. + >>> mod.init_params() + """ + raise NotImplementedError() + + def set_params(self, arg_params, aux_params, allow_missing=False, force_init=True, + allow_extra=False): + """Assigns parameter and aux state values. + + Parameters + ---------- + arg_params : dict + Dictionary of name to value (`NDArray`) mapping. + aux_params : dict + Dictionary of name to value (`NDArray`) mapping. + allow_missing : bool + If ``True``, params could contain missing values, and the initializer will be + called to fill those missing params. + force_init : bool + If ``True``, will force re-initialize even if already initialized. + allow_extra : boolean, optional + Whether allow extra parameters that are not needed by symbol. + If this is True, no error will be thrown when arg_params or aux_params + contain extra parameters that is not needed by the executor. + + Examples + -------- + >>> # An example of setting module parameters. + >>> sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, n_epoch_load) + >>> mod.set_params(arg_params=arg_params, aux_params=aux_params) + """ + self.init_params(initializer=None, arg_params=arg_params, aux_params=aux_params, + allow_missing=allow_missing, force_init=force_init, + allow_extra=allow_extra) + + def save_params(self, fname): + """Saves model parameters to file. + + Parameters + ---------- + fname : str + Path to output param file. + + Examples + -------- + >>> # An example of saving module parameters. + >>> mod.save_params('myfile') + """ + arg_params, aux_params = self.get_params() + save_dict = {('arg:%s' % k) : v.as_in_context(cpu()) for k, v in arg_params.items()} + save_dict.update({('aux:%s' % k) : v.as_in_context(cpu()) for k, v in aux_params.items()}) + ndarray.save(fname, save_dict) + + def load_params(self, fname): + """Loads model parameters from file. + + Parameters + ---------- + fname : str + Path to input param file. + + Examples + -------- + >>> # An example of loading module parameters. + >>> mod.load_params('myfile') + """ + save_dict = ndarray.load(fname) + arg_params = {} + aux_params = {} + for k, value in save_dict.items(): + arg_type, name = k.split(':', 1) + if arg_type == 'arg': + arg_params[name] = value + elif arg_type == 'aux': + aux_params[name] = value + else: + raise ValueError("Invalid param file " + fname) + self.set_params(arg_params, aux_params) + + def get_states(self, merge_multi_context=True): + """Gets states from all devices + + If `merge_multi_context` is ``True``, returns output of form ``[out1, out2]``. + Otherwise, it returns output of the form + ``[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]``. + All output elements are `NDArray`. + + Parameters + ---------- + merge_multi_context : bool + Defaults to ``True``. In the case when data-parallelism is used, the states + will be collected from multiple devices. A ``True`` value indicates that we + should merge the collected results so that they look like from a single + executor. + + Returns + ------- + A list of ``NDArray`` or a list of list of ``NDArray``. + """ + assert self.binded and self.params_initialized + assert not merge_multi_context + return [] + + def set_states(self, states=None, value=None): + """Sets value for states. Only one of states & value can be specified. + + Parameters + ---------- + states : list of list of NDArray + Source states arrays formatted like + ``[[state1_dev1, state1_dev2], [state2_dev1, state2_dev2]]``. + value : number + A single scalar value for all state arrays. + """ + assert self.binded and self.params_initialized + assert not states and not value + + def install_monitor(self, mon): + """Installs monitor on all executors.""" + raise NotImplementedError() + + ################################################################################ + # Computations + ################################################################################ + # pylint: disable=unused-argument + def prepare(self, data_batch, sparse_row_id_fn=None): + '''Prepares the module for processing a data batch. + + Usually involves switching bucket and reshaping. + For modules that contain `row_sparse` parameters in KVStore, + it prepares the `row_sparse` parameters based on the sparse_row_id_fn. + + When KVStore is used to update parameters for multi-device or multi-machine training, + a copy of the parameters are stored in KVStore. Note that for `row_sparse` parameters, + the `update()` updates the copy of parameters in KVStore, but doesn't broadcast + the updated parameters to all devices / machines. The `prepare` function is used to + broadcast `row_sparse` parameters with the next batch of data. + + Parameters + ---------- + data_batch : DataBatch + The current batch of data for forward computation. + + sparse_row_id_fn : A callback function + The function takes `data_batch` as an input and returns a dict of + str -> NDArray. The resulting dict is used for pulling row_sparse + parameters from the kvstore, where the str key is the name of the param, + and the value is the row id of the param to pull. + ''' + if sparse_row_id_fn is not None: + warnings.warn(UserWarning("sparse_row_id_fn is not invoked for BaseModule.")) + # pylint: enable=unused-argument + + def forward(self, data_batch, is_train=None): + """Forward computation. It supports data batches with different shapes, such as + different batch sizes or different image sizes. + If reshaping of data batch relates to modification of symbol or module, such as + changing image layout ordering or switching from training to predicting, module + rebinding is required. + + Parameters + ---------- + data_batch : DataBatch + Could be anything with similar API implemented. + is_train : bool + Default is ``None``, which means `is_train` takes the value of ``self.for_training``. + + Examples + -------- + >>> import mxnet as mx + >>> from collections import namedtuple + >>> Batch = namedtuple('Batch', ['data']) + >>> data = mx.sym.Variable('data') + >>> out = data * 2 + >>> mod = mx.mod.Module(symbol=out, label_names=None) + >>> mod.bind(data_shapes=[('data', (1, 10))]) + >>> mod.init_params() + >>> data1 = [mx.nd.ones((1, 10))] + >>> mod.forward(Batch(data1)) + >>> print mod.get_outputs()[0].asnumpy() + [[ 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.]] + >>> # Forward with data batch of different shape + >>> data2 = [mx.nd.ones((3, 5))] + >>> mod.forward(Batch(data2)) + >>> print mod.get_outputs()[0].asnumpy() + [[ 2. 2. 2. 2. 2.] + [ 2. 2. 2. 2. 2.] + [ 2. 2. 2. 2. 2.]] + """ + raise NotImplementedError() + + def backward(self, out_grads=None): + """Backward computation. + + Parameters + ---------- + out_grads : NDArray or list of NDArray, optional + Gradient on the outputs to be propagated back. + This parameter is only needed when bind is called + on outputs that are not a loss function. + + Examples + -------- + >>> # An example of backward computation. + >>> mod.backward() + >>> print mod.get_input_grads()[0].asnumpy() + [[[ 1.10182791e-05 5.12257748e-06 4.01927764e-06 8.32566820e-06 + -1.59775993e-06 7.24269375e-06 7.28067835e-06 -1.65902311e-05 + 5.46342608e-06 8.44196393e-07] + ...]] + """ + raise NotImplementedError() + + def get_outputs(self, merge_multi_context=True): + """Gets outputs of the previous forward computation. + + If `merge_multi_context` is ``True``, it is like ``[out1, out2]``. Otherwise, + it returns out put of form ``[[out1_dev1, out1_dev2], [out2_dev1, out2_dev2]]``. + All the output elements have type `NDArray`. When `merge_multi_context` is ``False``, + those `NDArray` instances might live on different devices. + + Parameters + ---------- + merge_multi_context : bool + Defaults to ``True``. In the case when data-parallelism is used, the outputs + will be collected from multiple devices. A ``True`` value indicates that we + should merge the collected results so that they look like from a single + executor. + + Returns + ------- + list of `NDArray` or list of list of `NDArray`. + Output + + Examples + -------- + >>> # An example of getting forward output. + >>> print mod.get_outputs()[0].asnumpy() + [[ 0.09999977 0.10000153 0.10000716 0.10000195 0.09999853 0.09999743 + 0.10000272 0.10000113 0.09999088 0.09999888]] + """ + raise NotImplementedError() + + def get_input_grads(self, merge_multi_context=True): + """Gets the gradients to the inputs, computed in the previous backward computation. + + If `merge_multi_context` is ``True``, it is like ``[grad1, grad2]``. Otherwise, it + is like ``[[grad1_dev1, grad1_dev2], [grad2_dev1, grad2_dev2]]``. All the output + elements have type `NDArray`. When `merge_multi_context` is ``False``, those `NDArray` + instances might live on different devices. + + Parameters + ---------- + merge_multi_context : bool + Defaults to ``True``. In the case when data-parallelism is used, the gradients + will be collected from multiple devices. A ``True`` value indicates that we + should merge the collected results so that they look like from a single + executor. + + Returns + ------- + list of NDArray or list of list of NDArray + Input gradients. + + Examples + -------- + >>> # An example of getting input gradients. + >>> print mod.get_input_grads()[0].asnumpy() + [[[ 1.10182791e-05 5.12257748e-06 4.01927764e-06 8.32566820e-06 + -1.59775993e-06 7.24269375e-06 7.28067835e-06 -1.65902311e-05 + 5.46342608e-06 8.44196393e-07] + ...]] + """ + raise NotImplementedError() + + def update(self): + """Updates parameters according to the installed optimizer and the gradients computed + in the previous forward-backward batch. + + When KVStore is used to update parameters for multi-device or multi-machine training, + a copy of the parameters are stored in KVStore. Note that for `row_sparse` parameters, + this function does update the copy of parameters in KVStore, but doesn't broadcast the + updated parameters to all devices / machines. Please call `prepare` to broadcast + `row_sparse` parameters with the next batch of data. + + Examples + -------- + >>> # An example of updating module parameters. + >>> mod.init_optimizer(kvstore='local', optimizer='sgd', + ... optimizer_params=(('learning_rate', 0.01), )) + >>> mod.backward() + >>> mod.update() + >>> print mod.get_params()[0]['fc3_weight'].asnumpy() + [[ 5.86930104e-03 5.28078526e-03 -8.88729654e-03 -1.08308345e-03 + 6.13054074e-03 4.27560415e-03 1.53817423e-03 4.62131854e-03 + 4.69872449e-03 -2.42400169e-03 9.94111411e-04 1.12386420e-03 + ...]] + """ + raise NotImplementedError() + + def update_metric(self, eval_metric, labels, pre_sliced=False): + """Evaluates and accumulates evaluation metric on outputs of the last forward + computation. + + Parameters + ---------- + eval_metric : EvalMetric + Evaluation metric to use. + labels : list of NDArray if `pre_sliced` parameter is set to `False`, + list of lists of NDArray otherwise. Typically `data_batch.label`. + pre_sliced: bool + Whether the labels are already sliced per device (default: False). + + Examples + -------- + >>> # An example of updating evaluation metric. + >>> mod.forward(data_batch) + >>> mod.update_metric(metric, data_batch.label) + """ + raise NotImplementedError() + + ################################################################################ + # module setup + ################################################################################ + def bind(self, data_shapes, label_shapes=None, for_training=True, + inputs_need_grad=False, force_rebind=False, shared_module=None, + grad_req='write'): + """Binds the symbols to construct executors. This is necessary before one + can perform computation with the module. + + Parameters + ---------- + data_shapes : list of (str, tuple) or DataDesc objects + Typically is ``data_iter.provide_data``. Can also be a list of + (data name, data shape). + label_shapes : list of (str, tuple) or DataDesc objects + Typically is ``data_iter.provide_label``. Can also be a list of + (label name, label shape). + for_training : bool + Default is ``True``. Whether the executors should be bind for training. + inputs_need_grad : bool + Default is ``False``. Whether the gradients to the input data need to be computed. + Typically this is not needed. But this might be needed when implementing composition + of modules. + force_rebind : bool + Default is ``False``. This function does nothing if the executors are already + bound. But with this ``True``, the executors will be forced to rebind. + shared_module : Module + Default is ``None``. This is used in bucketing. When not ``None``, the shared module + essentially corresponds to a different bucket -- a module with different symbol + but with the same sets of parameters (e.g. unrolled RNNs with different lengths). + grad_req : str, list of str, dict of str to str + Requirement for gradient accumulation. Can be 'write', 'add', or 'null' + (default to 'write'). + Can be specified globally (str) or for each argument (list, dict). + + Examples + -------- + >>> # An example of binding symbols. + >>> mod.bind(data_shapes=[('data', (1, 10, 10))]) + >>> # Assume train_iter is already created. + >>> mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) + """ + raise NotImplementedError() + + def init_optimizer(self, kvstore='local', optimizer='sgd', + optimizer_params=(('learning_rate', 0.01),), force_init=False): + """Installs and initializes optimizers, as well as initialize kvstore for + distributed training + + Parameters + ---------- + kvstore : str or KVStore + Defaults to `'local'`. + optimizer : str or Optimizer + Defaults to `'sgd'`. + optimizer_params : dict + Defaults to ``(('learning_rate', 0.01),)``. The default value is not a dictionary, + just to avoid pylint warning of dangerous default values. + force_init : bool + Defaults to ``False``, indicates whether to force re-initializing an optimizer + if it is already installed. + + Examples + -------- + >>> # An example of initializing optimizer. + >>> mod.init_optimizer(optimizer='sgd', optimizer_params=(('learning_rate', 0.005),)) + """ + raise NotImplementedError() + + ################################################################################ + # misc + ################################################################################ + @property + def symbol(self): + """Gets the symbol associated with this module. + + Except for `Module`, for other types of modules (e.g. `BucketingModule`), this + property might not be a constant throughout its life time. Some modules might + not even be associated with any symbols. + """ + return self._symbol diff --git a/tests/nightly/estimator/test_estimator_cnn.py b/tests/nightly/estimator/test_estimator_cnn.py index b3b0d536af24..e6d8b846f614 100644 --- a/tests/nightly/estimator/test_estimator_cnn.py +++ b/tests/nightly/estimator/test_estimator_cnn.py @@ -116,7 +116,7 @@ def test_estimator_cpu(): # Define estimator est = estimator.Estimator(net=net, loss=loss, - train_metrics=mx.gluon.metric.Accuracy(), + train_metrics=mx.metric.Accuracy(), trainer=trainer, context=context) # Call fit() @@ -140,7 +140,7 @@ def test_estimator_gpu(): train_data, test_data = load_data_mnist(batch_size, resize=224) loss = gluon.loss.SoftmaxCrossEntropyLoss() net.hybridize() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) # Define estimator est = estimator.Estimator(net=net, diff --git a/tests/nightly/estimator/test_sentiment_rnn.py b/tests/nightly/estimator/test_sentiment_rnn.py index 69380389d48e..0e4f39c5687f 100644 --- a/tests/nightly/estimator/test_sentiment_rnn.py +++ b/tests/nightly/estimator/test_sentiment_rnn.py @@ -190,11 +190,11 @@ def run(net, train_dataloader, test_dataloader, num_epochs, ctx, lr): trainer = mx.gluon.Trainer(net.collect_params(), 'adam', {'learning_rate': lr}) # Define loss and evaluation metrics loss = gluon.loss.SoftmaxCrossEntropyLoss() - metrics = mx.gluon.metric.CompositeEvalMetric() - acc = mx.gluon.metric.Accuracy() - nested_metrics = mx.gluon.metric.CompositeEvalMetric() - metrics.add([acc, mx.gluon.metric.Loss()]) - nested_metrics.add([metrics, mx.gluon.metric.Accuracy()]) + metrics = mx.metric.CompositeEvalMetric() + acc = mx.metric.Accuracy() + nested_metrics = mx.metric.CompositeEvalMetric() + metrics.add([acc, mx.metric.Loss()]) + nested_metrics.add([metrics, mx.metric.Accuracy()]) # Define estimator est = estimator.Estimator(net=net, loss=loss, train_metrics=nested_metrics, diff --git a/tests/nightly/test_optimizer.py b/tests/nightly/test_optimizer.py new file mode 100644 index 000000000000..0a87368d991e --- /dev/null +++ b/tests/nightly/test_optimizer.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx + +import sys +import os +curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.insert(0, os.path.join(curr_path, '../unittest')) +from common import setup_module, with_seed + +# This script is testing the efficiency of LARS +# We are training LeNet-5 at batch-size 8000 in 10 epochs above 98% accuracy +# Which is not doable with simple SGD + momentum (from what have been tested so far) + +def lenet5(): + """LeNet-5 Symbol""" + #pylint: disable=no-member + data = mx.sym.Variable('data') + conv1 = mx.sym.Convolution(data=data, kernel=(5, 5), num_filter=20) + tanh1 = mx.sym.Activation(data=conv1, act_type="tanh") + pool1 = mx.sym.Pooling(data=tanh1, pool_type="max", + kernel=(2, 2), stride=(2, 2)) + # second conv + conv2 = mx.sym.Convolution(data=pool1, kernel=(5, 5), num_filter=50) + tanh2 = mx.sym.Activation(data=conv2, act_type="tanh") + pool2 = mx.sym.Pooling(data=tanh2, pool_type="max", + kernel=(2, 2), stride=(2, 2)) + # first fullc + flatten = mx.sym.Flatten(data=pool2) + fc1 = mx.sym.FullyConnected(data=flatten, num_hidden=500) + tanh3 = mx.sym.Activation(data=fc1, act_type="tanh") + # second fullc + fc2 = mx.sym.FullyConnected(data=tanh3, num_hidden=10) + # loss + lenet = mx.sym.SoftmaxOutput(data=fc2, name='softmax') + #pylint: enable=no-member + return lenet + +@with_seed() +def test_lars(): + num_epochs = 10 + batch_size = 8000 + mnist = mx.test_utils.get_mnist() + train_iter = mx.io.NDArrayIter(mnist['train_data'], + mnist['train_label'], + batch_size, + shuffle=True) + test_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size) + ctx = mx.gpu(0) + lenet_model = mx.mod.Module(lenet5(), context=ctx) + warmup_epochs = 1 + epoch_it = int(train_iter.num_data / batch_size) + # LARS works best with Polynomial scheduler and warmup + base_lr = 0.01 + optimizer_params={ + 'learning_rate': base_lr, + 'lr_scheduler': mx.lr_scheduler.PolyScheduler(base_lr=base_lr, + max_update=epoch_it * num_epochs, + warmup_steps=epoch_it * warmup_epochs), + 'momentum': 0.9, + 'eta': 14., + } + lenet_model.fit(train_iter, + eval_data=test_iter, + optimizer='lars', + optimizer_params=optimizer_params, + eval_metric='acc', + num_epoch=num_epochs) + + # predict accuracy for lenet + acc = mx.metric.Accuracy() + lenet_model.score(test_iter, acc) + accuracy = acc.get()[1] + assert accuracy > 0.98, "LeNet-5 training accuracy on MNIST was too low" + diff --git a/tests/nightly/test_tlocal_racecondition.py b/tests/nightly/test_tlocal_racecondition.py new file mode 100644 index 000000000000..d43c45937c05 --- /dev/null +++ b/tests/nightly/test_tlocal_racecondition.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +from mxnet import gluon +from mxnet import image +from mxnet import nd +import numpy as np +import logging + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +root_url = ('https://apache-mxnet.s3-accelerate.amazonaws.com/' + 'gluon/dataset/pikachu/') +data_dir = './data/pikachu/' +dataset = {'train.rec': 'e6bcb6ffba1ac04ff8a9b1115e650af56ee969c8', + 'train.idx': 'dcf7318b2602c06428b9988470c731621716c393', + 'val.rec': 'd6c33f799b4d058e82f2cb5bd9a976f69d72d520'} +for k, v in dataset.items(): + gluon.utils.download(root_url+k, data_dir+k, sha1_hash=v) + +T = 1 +devs = [mx.gpu(i) for i in range(4)] +data_shape = 224 * T +batch_size = 20 * len(devs) +rgb_mean = np.array([1,2,3]) + +class_names = ['pikachu'] +num_class = len(class_names) + +def get_iterators(data_shape, batch_size): + train_iter = image.ImageDetIter( + batch_size=batch_size, + data_shape=(3, data_shape, data_shape), + path_imgrec=data_dir+'train.rec', + path_imgidx=data_dir+'train.idx', + shuffle=True, + mean=True, + rand_crop=1, + min_object_covered=0.95, + max_attempts=200) + val_iter = image.ImageDetIter( + batch_size=batch_size, + data_shape=(3, data_shape, data_shape), + path_imgrec=data_dir+'val.rec', + shuffle=False, + mean=True) + return train_iter, val_iter, class_names, num_class + +train_data, test_data, class_names, num_class = get_iterators( + data_shape, batch_size) + + +class MyCustom(mx.operator.CustomOp): + def __init__(self): + super(MyCustom, self).__init__() + def forward(self, is_train, req, in_data, out_data, aux): + self.assign(out_data[0], req[0], 0) + def backward(self, req, out_grad, in_data, out_data, in_grad, aux): + self.assign(in_grad[0], req[0], 0) + self.assign(in_grad[1], req[1], 0) + +@mx.operator.register("MyCustom") +class MyCustomProp(mx.operator.CustomOpProp): + def __init__(self): + super(MyCustomProp, self).__init__(need_top_grad = False) + def list_arguments(self): + return ["data", "label"] + def list_outputs(self): + return ["loss"] + def infer_shape(self, in_shape): + return [in_shape[0], in_shape[1]], [(1, )], [] + def infer_type(self, in_type): + dtype = in_type[0] + return [dtype, dtype], [dtype], [] + def create_operator(self, ctx, shapes, dtypes): + return MyCustom() + +class MyMetric(mx.metric.EvalMetric): + def __init__(self): + super(MyMetric, self).__init__("MyMetric") + self.name = ['empty'] + def update(self, labels, preds): + pass + def get(self): + return self.name, [0] + +if __name__ == '__main__': + x = mx.sym.Variable("data") + label = mx.sym.Variable("label") + x = mx.sym.FullyConnected(data = x, num_hidden = 100) + label = mx.sym.Reshape(data = label, shape = (0, -1)) + sym = mx.sym.Custom(data = x, label = label, op_type = "MyCustom") + model = mx.module.Module(context = devs, symbol = sym, data_names = ('data',), label_names = ('label',)) + model.fit(train_data = train_data, begin_epoch = 0, num_epoch = 20, allow_missing = True, batch_end_callback = mx.callback.Speedometer(batch_size, 5), eval_metric = MyMetric()) diff --git a/tests/python/gpu/test_contrib_amp.py b/tests/python/gpu/test_contrib_amp.py index d7a6e80b8982..86126744a127 100644 --- a/tests/python/gpu/test_contrib_amp.py +++ b/tests/python/gpu/test_contrib_amp.py @@ -103,6 +103,255 @@ def test_amp_coverage(amp_tests): - If you are not sure which list to choose, FP32_FUNCS is the safest option""") +@pytest.mark.skip(reason='Error during waitall(). Tracked in #18099') +@with_seed() +def test_amp_conversion(amp_tests): + def check_amp_convert_symbol(): + x = mx.sym.var("x") + y = mx.sym.var("y") + z = mx.sym.FullyConnected(x, y, num_hidden=10, no_bias=True) + siny = mx.sym.sin(y) + res = z + siny + # Compare symbols with similar computation graphs created using convert_symbol and manually. + res_converted = amp.convert_symbol(res, target_dtype="float16", + target_dtype_ops=["FullyConnected"], + fp32_ops=["sin"]) + + x_fp16 = mx.sym.amp_cast(x, dtype="float16") + y_fp16 = mx.sym.amp_cast(y, dtype="float16") + siny = mx.sym.sin(y) + z = mx.sym.FullyConnected(x_fp16, y_fp16, num_hidden=10, no_bias=True) + amp_casted_z = mx.sym.amp_cast(z, dtype="float32") + res_expected = amp_casted_z + siny + assert same_symbol_structure(res_converted, res_expected), \ + "convert_symbol generating wrong computation graph" + + # convert_symbol called with incorrect inputs + pytest.raises(AssertionError, amp.convert_symbol, res, + target_dtype="float16", target_dtype_ops=["FullyConnected"], + fp32_ops=["elemwise_add"]) + pytest.raises(AssertionError, amp.convert_symbol, res, + target_dtype="float16", target_dtype_ops=["FullyConnected"], + fp32_ops=["Activation"], + conditional_fp32_ops=[('Activation', 'act_type', ['selu'])]) + pytest.raises(AssertionError, amp.convert_symbol, res, + target_dtype="float16", target_dtype_ops=["Activation"], + fp32_ops=["Activation"], + conditional_fp32_ops=[('Activation', 'act_type', ['selu'])]) + pytest.raises(AssertionError, amp.convert_symbol, res, + target_dtype="float16", target_dtype_ops=["FullyConnected"], + fp32_ops=["FullyConnected"]) + + # Test for op in conditional ops with condition not satisfied + x = mx.sym.var("x") + y = mx.sym.var("y") + fc_cond = mx.sym.FullyConnected(x, y, num_hidden=10, no_bias=True) + res_converted = amp.convert_symbol(fc_cond, target_dtype="float16", + target_dtype_ops=[], + fp32_ops=["sin"], + conditional_fp32_ops=[("FullyConnected", "no_bias", ["False"])]) + + res_expected = mx.sym.FullyConnected(x, y, num_hidden=10, no_bias=True) + assert same_symbol_structure(res_converted, res_expected), \ + "convert_symbol generating wrong computation graph when conditional ops is used" + + # Test for op in conditional ops with condition satisfied + res_converted = amp.convert_symbol(fc_cond, target_dtype="float16", target_dtype_ops=[], + fp32_ops=["sin"], + conditional_fp32_ops=[("FullyConnected", "no_bias", ["True"])]) + x_fp32 = mx.sym.amp_cast(x, dtype="float32") + y_fp32 = mx.sym.amp_cast(y, dtype="float32") + res_expected = mx.sym.FullyConnected(x_fp32, y_fp32, num_hidden=10, no_bias=True) + assert same_symbol_structure(res_converted, res_expected), \ + "convert_symbol generating wrong computation graph when conditional ops used with satisfying condition" + + # Test with a real world model, default inputs for convert_symbol + dir_path = os.path.dirname(os.path.realpath(__file__)) + model_path = os.path.join(dir_path, 'model') + if not os.path.isdir(model_path): + os.mkdir(model_path) + + prefix, epoch = download_model("imagenet1k-resnet-18", dst_dir=model_path) + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + inputs = {} + inputs['data'] = mx.nd.ones((1, 3, 224, 224)) + inputs.update(arg_params) + converted_sym = amp.convert_symbol(sym) + exe = converted_sym.simple_bind(mx.gpu(0), data=(1, 3, 224, 224), grad_req='null') + exe.forward(is_train=False, **inputs) + exe.outputs[0].asnumpy() + + inputs2 = {} + inputs2['data'] = mx.nd.ones((1, 3, 224, 224)) + inputs2['fc1_weight'] = inputs['fc1_weight'].astype(np.float16) + inputs2['fc1_bias'] = inputs['fc1_bias'].astype(np.float16) + + # Test with a real world model, tweak inputs for convert_symbol + converted_sym = amp.convert_symbol(sym, target_dtype="float16", + target_dtype_ops=["Convolution"], data_names=["data"], + cast_optional_params=True) + converted_sym2 = amp.convert_symbol(sym, target_dtype="float16", + target_dtype_ops=["Convolution"], data_names=["data"], + cast_optional_params=False) + + exe = converted_sym.simple_bind(mx.gpu(0), data=(1, 3, 224, 224), grad_req='null') + exe2 = converted_sym2.simple_bind(mx.gpu(), data=(1, 3, 224, 224), grad_req='null') + + converted_args = converted_sym.list_arguments() + converted_auxs = converted_sym.list_auxiliary_states() + for i, key in enumerate(exe.arg_arrays): + if converted_args[i] in arg_params: + arg_params[converted_args[i]] = arg_params[converted_args[i]].astype(exe.arg_arrays[i].dtype) + for i, key in enumerate(exe.aux_arrays): + if converted_auxs[i] in aux_params: + aux_params[converted_auxs[i]] = aux_params[converted_auxs[i]].astype(exe.aux_arrays[i].dtype) + + inputs2.update(arg_params) + exe.forward(is_train=False, **inputs2) + exe.outputs[0].wait_to_read() + + inputs['fc1_weight'] = inputs['fc1_weight'].astype(np.float16) + inputs['fc1_bias'] = inputs['fc1_bias'].astype(np.float16) + exe2.forward(is_train=False, **inputs) + exe2.outputs[0].wait_to_read() + + + def check_amp_convert_model(): + # Test with real world model, default inputs for convert_model + dir_path = os.path.dirname(os.path.realpath(__file__)) + model_path = os.path.join(dir_path, 'model') + if not os.path.isdir(model_path): + os.mkdir(model_path) + prefix, epoch = download_model("imagenet1k-resnet-18", dst_dir=model_path) + + sym, arg_params, aux_params = mx.model.load_checkpoint(prefix, epoch) + + # Test with real world model, tweak inputs for convert_model + result_sym, result_arg_params, result_aux_params = amp.convert_model(sym, + arg_params, + aux_params, + target_dtype="float16", + target_dtype_ops=["Convolution"]) + mod = mx.mod.Module(result_sym, data_names=["data"], label_names=["softmax_label"], context=mx.gpu()) + mod.bind(data_shapes=[['data', (1, 3, 224, 224)]], label_shapes=[['softmax_label', (1,)]]) + + mod.set_params(result_arg_params, result_aux_params) + mod.forward(mx.io.DataBatch(data=[mx.nd.ones((1, 3, 224, 224))], + label=[mx.nd.ones((1,))])) + mod.get_outputs()[0].asnumpy() + assert mod._arg_params["stage2_unit1_conv2_weight"].dtype == np.float32 + + # Call convert_model with cast_optional_params set to True + result_sym, result_arg_params, result_aux_params = amp.convert_model(sym, + arg_params, + aux_params, + target_dtype="float16", + target_dtype_ops=["Convolution"], cast_optional_params=True) + mod = mx.mod.Module(result_sym, data_names=["data"], label_names=["softmax_label"], context=mx.gpu()) + mod.bind(data_shapes=[['data', (1, 3, 224, 224)]], label_shapes=[['softmax_label', (1,)]]) + mod.set_params(result_arg_params, result_aux_params) + mod.forward(mx.io.DataBatch(data=[mx.nd.ones((1, 3, 224, 224))], + label=[mx.nd.ones((1,))])) + mod.get_outputs()[0].asnumpy() + assert mod._arg_params["stage2_unit1_conv2_weight"].dtype == np.float16 + + + def check_amp_convert_hybrid_block(): + # Test conversion for hybrid block on CPU + model_cpu = get_model("resnet50_v1") + model_cpu.collect_params().initialize(ctx=mx.cpu()) + model_cpu.hybridize() + model_cpu(mx.nd.random.uniform(0, 1, shape=(1, 3, 224, 224), ctx=mx.cpu())) + converted_model_cpu = amp.convert_hybrid_block(model_cpu) + + # Test with real world model, default inputs for convert_hybrid_block + model = get_model("resnet50_v1") + model.collect_params().initialize(ctx=mx.gpu()) + model.hybridize() + model(mx.nd.zeros((1, 3, 224, 224))) + converted_model = amp.convert_hybrid_block(model) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224), + dtype=np.float32)) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224), + dtype=np.float32)) + + # Test with real world model, tweak inputs for convert_hybrid_block + converted_model = amp.convert_hybrid_block(model, target_dtype="float16", + target_dtype_ops=["Convolution"]) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224), + dtype=np.float32)) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224), + dtype=np.float32)) + + # Check symbolic block + dir_path = os.path.dirname(os.path.realpath(__file__)) + model_path = os.path.join(dir_path, 'model') + if not os.path.isdir(model_path): + os.mkdir(model_path) + prefix, epoch = download_model("imagenet1k-resnet-18", dst_dir=model_path) + net = SymbolBlock.imports(os.path.join(model_path, "imagenet1k-resnet-18-symbol.json"), + input_names=["data", "softmax_label"], + param_file=os.path.join(model_path, "imagenet1k-resnet-18-0000.params")) + net.collect_params().reset_ctx(ctx=mx.gpu()) + net.hybridize() + net(mx.nd.zeros((1, 3, 224, 224)), mx.nd.zeros((1,))) + converted_model = amp.convert_hybrid_block(net) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224)), mx.nd.zeros((1,))) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224)), mx.nd.zeros((1,))) + + # Check symbolic block, tweaked inputs + converted_model = amp.convert_hybrid_block(net, target_dtype="float16", target_dtype_ops=["Convolution"]) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224)), mx.nd.zeros((1, ))) + result = converted_model.forward(mx.nd.zeros((1, 3, 224, 224)), mx.nd.zeros((1, ))) + params = converted_model.collect_params() + assert params["stage2_unit1_conv2_weight"].dtype == np.float32 + + # Pass cast_optional_params as True to convert_hybrid_block + converted_model = amp.convert_hybrid_block(net, target_dtype="float16", target_dtype_ops=["Convolution"], + cast_optional_params=True) + params = converted_model.collect_params() + assert params["stage2_unit1_conv2_weight"].dtype == np.float16 + + + def check_amp_convert_bucketing_module(): + model = train_model(context=mx.current_context()) + result_model = amp.convert_bucketing_module(model) + val_sent = [] + batch_size = 128 + invalid_label = -1 + num_sentence = 1000 + buckets = [5, 10, 20, 30, 40] + len_vocab = 50 + + for _ in range(num_sentence): + len_sentence = randint(6, max(buckets)-1) # leave out the two last buckets empty + val_sentence = [] + for _ in range(len_sentence): + val_sentence.append(randint(1, len_vocab)) + val_sent.append(val_sentence) + + data_val = mx.rnn.BucketSentenceIter(val_sent, batch_size, buckets=buckets, + invalid_label=invalid_label) + result_model.bind(data_val.provide_data, data_val.provide_label, for_training=False) + result_model.score(data_val, mx.metric.Perplexity(invalid_label), + batch_end_callback=mx.callback.Speedometer(batch_size, 1)) + + # AMP conversion with cast_optional_params set to true + # Flaky test when cast_optional_params set to True : https://github.com/apache/incubator-mxnet/issues/16030 + ''' + result_model = amp.convert_bucketing_module(model, cast_optional_params=True) + result_model.bind(data_val.provide_data, data_val.provide_label, for_training=False) + result_model.score(data_val, mx.metric.Perplexity(invalid_label), + batch_end_callback=mx.callback.Speedometer(batch_size, 1)) + ''' + + + with mx.Context(mx.gpu(0)): + check_amp_convert_symbol() + check_amp_convert_model() + check_amp_convert_hybrid_block() + check_amp_convert_bucketing_module() + @with_seed() @pytest.mark.skip(reason='Error during waitall(). Tracked in #18099') @assert_raises_cudnn_not_satisfied(min_version='5.1.10') diff --git a/tests/python/tensorrt/lenet5_train.py b/tests/python/tensorrt/lenet5_train.py new file mode 100644 index 000000000000..441729fe0d56 --- /dev/null +++ b/tests/python/tensorrt/lenet5_train.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import mxnet as mx +import numpy as np +from tempfile import TemporaryDirectory + +def get_iters(mnist, batch_size): + """Get MNIST iterators.""" + train_iter = mx.io.NDArrayIter(mnist['train_data'], + mnist['train_label'], + batch_size, + shuffle=True) + val_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size) + test_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size) + all_test_labels = np.array(mnist['test_label']) + return train_iter, val_iter, test_iter, all_test_labels + +def lenet5(): + """LeNet-5 Symbol""" + #pylint: disable=no-member + data = mx.sym.Variable('data') + data = mx.sym.Cast(data, 'float16') + conv1 = mx.sym.Convolution(data=data, kernel=(5, 5), num_filter=20) + tanh1 = mx.sym.Activation(data=conv1, act_type="tanh") + pool1 = mx.sym.Pooling(data=tanh1, pool_type="max", + kernel=(2, 2), stride=(2, 2)) + # second conv + conv2 = mx.sym.Convolution(data=pool1, kernel=(5, 5), num_filter=50) + tanh2 = mx.sym.Activation(data=conv2, act_type="tanh") + pool2 = mx.sym.Pooling(data=tanh2, pool_type="max", + kernel=(2, 2), stride=(2, 2)) + # first fullc + flatten = mx.sym.Flatten(data=pool2) + fc1 = mx.sym.FullyConnected(data=flatten, num_hidden=500) + tanh3 = mx.sym.Activation(data=fc1, act_type="tanh") + # second fullc + fc2 = mx.sym.FullyConnected(data=tanh3, num_hidden=10) + fc2 = mx.sym.Cast(fc2, 'float32') + # loss + lenet = mx.sym.SoftmaxOutput(data=fc2, name='softmax') + #pylint: enable=no-member + return lenet + + +def train_lenet5(num_epochs, batch_size, train_iter, val_iter, test_iter): + """train LeNet-5 model on MNIST data""" + ctx = mx.gpu(0) + lenet_model = mx.mod.Module(lenet5(), context=ctx) + + lenet_model.fit(train_iter, + eval_data=val_iter, + optimizer='sgd', + optimizer_params={'learning_rate': 0.1, 'momentum': 0.9}, + eval_metric='acc', + batch_end_callback=mx.callback.Speedometer(batch_size, 1), + num_epoch=num_epochs) + + # predict accuracy for lenet + acc = mx.metric.Accuracy() + lenet_model.score(test_iter, acc) + accuracy = acc.get()[1] + assert accuracy > 0.95, "LeNet-5 training accuracy on MNIST was too low" + return lenet_model + + +if __name__ == '__main__': + num_epochs = 10 + batch_size = 128 + model_name = 'lenet5' + model_dir = os.getenv("LENET_MODEL_DIR", "/tmp") + model_file = '%s/%s-symbol.json' % (model_dir, model_name) + params_file = '%s/%s-%04d.params' % (model_dir, model_name, num_epochs) + + if not (os.path.exists(model_file) and os.path.exists(params_file)): + with TemporaryDirectory() as path: + mnist = mx.test_utils.get_mnist(path) + + _, _, _, all_test_labels = get_iters(mnist, batch_size) + + trained_lenet = train_lenet5(num_epochs, batch_size, + *get_iters(mnist, batch_size)[:-1]) + trained_lenet.save_checkpoint(model_name, num_epochs) diff --git a/tests/python/tensorrt/test_cvnets.py b/tests/python/tensorrt/test_cvnets.py new file mode 100644 index 000000000000..99312d76dc7a --- /dev/null +++ b/tests/python/tensorrt/test_cvnets.py @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import gc +import gluoncv +import mxnet as mx +import numpy as np + +from mxnet import gluon +from time import time + +from mxnet.gluon.data.vision import transforms + + +def get_classif_model(model_name, use_tensorrt, ctx=mx.gpu(0), batch_size=128): + mx.contrib.tensorrt.set_use_fp16(False) + h, w = 32, 32 + net = gluoncv.model_zoo.get_model(model_name, pretrained=True) + net.hybridize() + net.forward(mx.nd.zeros((batch_size, 3, h, w))) + net.export(model_name) + _sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, 0) + if use_tensorrt: + sym = _sym.get_backend_symbol('TensorRT') + arg_params, aux_params = mx.contrib.tensorrt.init_tensorrt_params(sym, arg_params, + aux_params) + else: + sym = _sym + executor = sym.simple_bind(ctx=ctx, data=(batch_size, 3, h, w), + softmax_label=(batch_size,), + grad_req='null', force_rebind=True) + executor.copy_params_from(arg_params, aux_params) + return executor + + +def cifar10_infer(model_name, use_tensorrt, num_workers, ctx=mx.gpu(0), batch_size=128): + executor = get_classif_model(model_name, use_tensorrt, ctx, batch_size) + + num_ex = 10000 + all_preds = np.zeros([num_ex, 10]) + + all_label_test = np.zeros(num_ex) + + transform_test = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) + ]) + + data_loader = lambda: gluon.data.DataLoader( + gluon.data.vision.CIFAR10(train=False).transform_first(transform_test), + batch_size=batch_size, shuffle=False, num_workers=num_workers) + + val_data = data_loader() + + for idx, (data, label) in enumerate(val_data): + # Skip last batch if it's undersized. + if data.shape[0] < batch_size: + continue + offset = idx * batch_size + all_label_test[offset:offset + batch_size] = label.asnumpy() + + # warm-up, but don't use result + executor.forward(is_train=False, data=data) + executor.outputs[0].wait_to_read() + + gc.collect() + val_data = data_loader() + example_ct = 0 + start = time() + + # if use_tensorrt: + for idx, (data, label) in enumerate(val_data): + # Skip last batch if it's undersized. + if data.shape[0] < batch_size: + continue + executor.forward(is_train=False, data=data) + preds = executor.outputs[0].asnumpy() + offset = idx * batch_size + all_preds[offset:offset + batch_size, :] = preds[:batch_size] + example_ct += batch_size + + all_preds = np.argmax(all_preds, axis=1) + matches = (all_preds[:example_ct] == all_label_test[:example_ct]).sum() + duration = time() - start + + return duration, 100.0 * matches / example_ct + + +def run_experiment_for(model_name, batch_size, num_workers): + print("\n===========================================") + print("Model: %s" % model_name) + print("===========================================") + print("*** Running inference using pure MXNet ***\n") + mx_duration, mx_pct = cifar10_infer(model_name=model_name, batch_size=batch_size, + num_workers=num_workers, use_tensorrt=False) + print("\nMXNet: time elapsed: %.3fs, accuracy: %.2f%%" % (mx_duration, mx_pct)) + print("\n*** Running inference using MXNet + TensorRT ***\n") + trt_duration, trt_pct = cifar10_infer(model_name=model_name, batch_size=batch_size, + num_workers=num_workers, use_tensorrt=True) + print("TensorRT: time elapsed: %.3fs, accuracy: %.2f%%" % (trt_duration, trt_pct)) + speedup = mx_duration / trt_duration + print("TensorRT speed-up (not counting compilation): %.2fx" % speedup) + + acc_diff = abs(mx_pct - trt_pct) + print("Absolute accuracy difference: %f" % acc_diff) + return speedup, acc_diff + + +def test_tensorrt_on_cifar_resnets(batch_size=32, tolerance=0.1, num_workers=1): + original_use_fp16 = mx.contrib.tensorrt.get_use_fp16() + try: + models = [ + 'cifar_resnet20_v1', + 'cifar_resnet56_v1', + 'cifar_resnet110_v1', + 'cifar_resnet20_v2', + 'cifar_resnet56_v2', + 'cifar_resnet110_v2', + 'cifar_wideresnet16_10', + 'cifar_wideresnet28_10', + 'cifar_wideresnet40_8', + 'cifar_resnext29_16x64d' + ] + + num_models = len(models) + + speedups = np.zeros(num_models, dtype=np.float32) + acc_diffs = np.zeros(num_models, dtype=np.float32) + + test_start = time() + + for idx, model in enumerate(models): + speedup, acc_diff = run_experiment_for(model, batch_size, num_workers) + speedups[idx] = speedup + acc_diffs[idx] = acc_diff + assert acc_diff < tolerance, "Accuracy difference between MXNet and TensorRT > %.2f%% for model %s" % ( + tolerance, model) + + print("Perf and correctness checks run on the following models:") + print(models) + mean_speedup = np.mean(speedups) + std_speedup = np.std(speedups) + print("\nSpeedups:") + print(speedups) + print("Speedup range: [%.2f, %.2f]" % (np.min(speedups), np.max(speedups))) + print("Mean speedup: %.2f" % mean_speedup) + print("St. dev. of speedups: %.2f" % std_speedup) + print("\nAcc. differences: %s" % str(acc_diffs)) + + test_duration = time() - test_start + + print("Test duration: %.2f seconds" % test_duration) + finally: + mx.contrib.tensorrt.set_use_fp16(original_use_fp16) + diff --git a/tests/python/train/test_autograd.py b/tests/python/train/test_autograd.py index f0fdc5ea2576..02a3601eb362 100644 --- a/tests/python/train/test_autograd.py +++ b/tests/python/train/test_autograd.py @@ -55,7 +55,7 @@ def get_net(): batch_size=batch_size, shuffle=True, flat=True, silent=False) def score(net, ctx_list): - metric = gluon.metric.Accuracy() + metric = mx.metric.Accuracy() val_data.reset() for batch in val_data: datas = gluon.utils.split_and_load(batch.data[0], ctx_list, batch_axis=0) @@ -69,7 +69,7 @@ def score(net, ctx_list): def train(net, epoch, ctx_list): net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx_list) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.5}) - metric = gluon.metric.Accuracy() + metric = mx.metric.Accuracy() loss = gluon.loss.SoftmaxCrossEntropyLoss() for i in range(epoch): diff --git a/tests/python/train/test_bucketing.py b/tests/python/train/test_bucketing.py new file mode 100644 index 000000000000..a233e46e0992 --- /dev/null +++ b/tests/python/train/test_bucketing.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import numpy as np +import mxnet as mx +import random +from random import randint +from mxnet.contrib.amp import amp + + +def prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence): + train_sent = [] + val_sent = [] + + for _ in range(num_sentence): + len_sentence = randint(6, max(buckets)-1) # leave out the two last buckets empty + train_sentence = [] + val_sentence = [] + for _ in range(len_sentence): + train_sentence.append(randint(1, len_vocab)) + val_sentence.append(randint(1, len_vocab)) + train_sent.append(train_sentence) + val_sent.append(val_sentence) + + data_train = mx.rnn.BucketSentenceIter(train_sent, batch_size, buckets=buckets, + invalid_label=invalid_label) + data_val = mx.rnn.BucketSentenceIter(val_sent, batch_size, buckets=buckets, + invalid_label=invalid_label) + + return (data_train, data_val) + + +def train_model(context=mx.cpu()): + import logging + head = '%(asctime)-15s %(message)s' + logging.basicConfig(level=logging.DEBUG, format=head) + console = logging.StreamHandler() + console.setLevel(logging.DEBUG) + logging.getLogger('').addHandler(console) + + batch_size = 128 + num_epochs = 5 + num_hidden = 25 + num_embed = 25 + num_layers = 2 + len_vocab = 50 + buckets = [5, 10, 20, 30, 40] + + invalid_label = -1 + num_sentence = 1000 + + data_train, data_val = prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence) + + stack = mx.rnn.SequentialRNNCell() + for i in range(num_layers): + stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i)) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=len_vocab, + output_dim=num_embed, name='embed') + + stack.reset() + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=len_vocab, name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + loss = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + + return loss, ('data',), ('softmax_label',) + + contexts = context + + model = mx.mod.BucketingModule( + sym_gen=sym_gen, + default_bucket_key=data_train.default_bucket_key, + context=contexts) + + logging.info('Begin fit...') + model.fit( + train_data=data_train, + eval_data=data_val, + eval_metric=mx.metric.Perplexity(invalid_label), # Use Perplexity for multiclass classification. + kvstore='device', + optimizer='sgd', + optimizer_params={'learning_rate': 0.01, + 'momentum': 0, + 'wd': 0.00001}, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + num_epoch=num_epochs, + batch_end_callback=mx.callback.Speedometer(batch_size, 50)) + logging.info('Finished fit...') + return model + + +def test_bucket_module(): + # This test forecasts random sequence of words to check bucketing. + # We cannot guarantee the accuracy of such an impossible task, and comments out the following line. + # assert model.score(data_val, mx.metric.MSE())[0][1] < 350, "High mean square error." + model = train_model() + + +if __name__ == "__main__": + test_bucket_module() diff --git a/tests/python/train/test_mlp.py b/tests/python/train/test_mlp.py new file mode 100644 index 000000000000..80885b33f955 --- /dev/null +++ b/tests/python/train/test_mlp.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# pylint: skip-file +import mxnet as mx +import numpy as np +import os, sys +import pickle as pickle +import logging +from mxnet.test_utils import get_mnist_ubyte + + +def test_mlp(tmpdir): + # symbol net + batch_size = 100 + data = mx.symbol.Variable('data') + fc1 = mx.symbol.FullyConnected(data, name='fc1', num_hidden=128) + act1 = mx.symbol.Activation(fc1, name='relu1', act_type="relu") + fc2 = mx.symbol.FullyConnected(act1, name = 'fc2', num_hidden = 64) + act2 = mx.symbol.Activation(fc2, name='relu2', act_type="relu") + fc3 = mx.symbol.FullyConnected(act2, name='fc3', num_hidden=10) + softmax = mx.symbol.SoftmaxOutput(fc3, name = 'sm') + + def accuracy(label, pred): + py = np.argmax(pred, axis=1) + return np.sum(py == label) / float(label.size) + + num_epoch = 4 + prefix = './mlp' + + #check data + path = str(tmpdir) + get_mnist_ubyte(path) + + train_dataiter = mx.io.MNISTIter( + image=os.path.join(path, 'train-images-idx3-ubyte'), + label=os.path.join(path, 'train-labels-idx1-ubyte'), + data_shape=(784,), + label_name='sm_label', + batch_size=batch_size, shuffle=True, flat=True, silent=False, seed=10) + val_dataiter = mx.io.MNISTIter( + image=os.path.join(path, 't10k-images-idx3-ubyte'), + label=os.path.join(path, 't10k-labels-idx1-ubyte'), + data_shape=(784,), + label_name='sm_label', + batch_size=batch_size, shuffle=True, flat=True, silent=False) + # print logging by default + logging.basicConfig(level=logging.DEBUG) + + model = mx.model.FeedForward.create( + softmax, + X=train_dataiter, + eval_data=val_dataiter, + eval_metric=mx.metric.np(accuracy), + epoch_end_callback=mx.callback.do_checkpoint(prefix), + ctx=[mx.cpu(i) for i in range(2)], + num_epoch=num_epoch, + learning_rate=0.1, wd=0.0004, + momentum=0.9) + + logging.info('Finish traning...') + prob = model.predict(val_dataiter) + logging.info('Finish predict...') + val_dataiter.reset() + y = np.concatenate([batch.label[0].asnumpy() for batch in val_dataiter]).astype('int') + py = np.argmax(prob, axis=1) + acc1 = float(np.sum(py == y)) / len(y) + logging.info('final accuracy = %f', acc1) + assert(acc1 > 0.94) + + # predict internal featuremaps + internals = softmax.get_internals() + fc2 = internals['fc2_output'] + mfeat = mx.model.FeedForward(symbol=fc2, + arg_params=model.arg_params, + aux_params=model.aux_params, + allow_extra_params=True) + feat = mfeat.predict(val_dataiter) + assert feat.shape == (10000, 64) + # pickle the model + smodel = pickle.dumps(model) + model2 = pickle.loads(smodel) + prob2 = model2.predict(val_dataiter) + assert np.sum(np.abs(prob - prob2)) == 0 + + # load model from checkpoint + model3 = mx.model.FeedForward.load(prefix, num_epoch) + prob3 = model3.predict(val_dataiter) + assert np.sum(np.abs(prob - prob3)) == 0 + + # save model explicitly + model.save(prefix, 128) + model4 = mx.model.FeedForward.load(prefix, 128) + prob4 = model4.predict(val_dataiter) + assert np.sum(np.abs(prob - prob4)) == 0 + + for i in range(num_epoch): + os.remove('%s-%04d.params' % (prefix, i + 1)) + os.remove('%s-symbol.json' % prefix) + os.remove('%s-0128.params' % prefix) diff --git a/tests/python/train/test_sparse_fm.py b/tests/python/train/test_sparse_fm.py new file mode 100644 index 000000000000..76a2705fe4e5 --- /dev/null +++ b/tests/python/train/test_sparse_fm.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import mxnet.ndarray as nd +from mxnet.test_utils import * +import numpy as np +import os +import sys +CURR_PATH = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.insert(0, os.path.join(CURR_PATH, '../unittest')) +from common import retry + +@retry(5) +def test_factorization_machine_module(verbose=False): + """ Test factorization machine model with sparse operators """ + def check_factorization_machine_module(optimizer=None, num_epochs=None): + print("check_factorization_machine_module( {} )".format(optimizer)) + + def fm(factor_size, feature_dim, init): + x = mx.symbol.Variable("data", stype='csr') + v = mx.symbol.Variable("v", shape=(feature_dim, factor_size), + init=init, stype='row_sparse') + + w1_weight = mx.symbol.var('w1_weight', shape=(feature_dim, 1), + init=init, stype='row_sparse') + w1_bias = mx.symbol.var('w1_bias', shape=(1)) + w1 = mx.symbol.broadcast_add(mx.symbol.dot(x, w1_weight), w1_bias) + + v_s = mx.symbol._internal._square_sum(data=v, axis=1, keepdims=True) + x_s = mx.symbol.square(data=x) + bd_sum = mx.sym.dot(x_s, v_s) + + w2 = mx.symbol.dot(x, v) + w2_squared = 0.5 * mx.symbol.square(data=w2) + + w_all = mx.symbol.Concat(w1, w2_squared, dim=1) + sum1 = mx.symbol.sum(data=w_all, axis=1, keepdims=True) + sum2 = 0.5 * mx.symbol.negative(bd_sum) + model = mx.sym.elemwise_add(sum1, sum2) + + y = mx.symbol.Variable("label") + model = mx.symbol.LinearRegressionOutput(data=model, label=y) + return model + + # model + init = mx.initializer.Normal(sigma=0.01) + factor_size = 4 + feature_dim = 10000 + model = fm(factor_size, feature_dim, init) + + # data iter + num_batches = 5 + batch_size = 64 + num_samples = batch_size * num_batches + # generate some random csr data + csr_nd = rand_ndarray((num_samples, feature_dim), 'csr', 0.1) + label = mx.nd.ones((num_samples,1)) + # the alternative is to use LibSVMIter + train_iter = mx.io.NDArrayIter(data=csr_nd, + label={'label':label}, + batch_size=batch_size, + last_batch_handle='discard') + # create module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label']) + # allocate memory by given the input data and lable shapes + mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) + # initialize parameters by uniform random numbers + mod.init_params(initializer=init) + if optimizer == 'sgd': + # use Sparse SGD with learning rate 0.1 to train + sgd = mx.optimizer.SGD(momentum=0.1, clip_gradient=5.0, learning_rate=0.01, + rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=sgd) + if num_epochs is None: + num_epochs = 10 + expected_accuracy = 0.02 + elif optimizer == 'adam': + # use Sparse Adam to train + adam = mx.optimizer.Adam(clip_gradient=5.0, learning_rate=0.0005, + rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=adam) + if num_epochs is None: + num_epochs = 10 + expected_accuracy = 0.05 + elif optimizer == 'adagrad': + # use Sparse AdaGrad with learning rate 0.1 to train + adagrad = mx.optimizer.AdaGrad(clip_gradient=5.0, learning_rate=0.01, + rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=adagrad) + if num_epochs is None: + num_epochs = 20 + expected_accuracy = 0.09 + else: + raise AssertionError("Unsupported optimizer type '" + optimizer + "' specified") + # use accuracy as the metric + metric = mx.metric.create('MSE') + # train 'num_epochs' epoch + for epoch in range(num_epochs): + train_iter.reset() + metric.reset() + for batch in train_iter: + mod.forward(batch, is_train=True) # compute predictions + mod.update_metric(metric, batch.label) # accumulate prediction accuracy + mod.backward() # compute gradients + mod.update() # update parameters + print('Epoch %d, Training %s' % (epoch, metric.get())) + if num_epochs > 1: + assert(metric.get()[1] < expected_accuracy) + + if verbose is True: + print("============ SGD ==========================") + start = time.clock() + check_factorization_machine_module('sgd') + if verbose is True: + print("Duration: {}".format(time.clock() - start)) + print("============ ADAM ==========================") + start = time.clock() + check_factorization_machine_module('adam') + if verbose is True: + print("Duration: {}".format(time.clock() - start)) + print("============ ADAGRAD ==========================") + start = time.clock() + check_factorization_machine_module('adagrad') + if verbose is True: + print("Duration: {}".format(time.clock() - start)) + +# run as a script +if __name__ == "__main__": + test_factorization_machine_module() diff --git a/tests/python/unittest/test_contrib_svrg_module.py b/tests/python/unittest/test_contrib_svrg_module.py new file mode 100644 index 000000000000..e9509f743f73 --- /dev/null +++ b/tests/python/unittest/test_contrib_svrg_module.py @@ -0,0 +1,307 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import mxnet as mx +import numpy as np +from common import with_seed, assertRaises +from mxnet.contrib.svrg_optimization.svrg_module import SVRGModule +from mxnet.test_utils import * +import unittest + +def setup(): + train_data = np.random.randint(1, 5, [1000, 2]) + weights = np.array([1.0, 2.0]) + train_label = train_data.dot(weights) + + di = mx.io.NDArrayIter(train_data, train_label, batch_size=32, shuffle=True, label_name='lin_reg_label') + X = mx.sym.Variable('data') + Y = mx.symbol.Variable('lin_reg_label') + fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden=1) + lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro") + + mod = SVRGModule( + symbol=lro, + data_names=['data'], + label_names=['lin_reg_label'], update_freq=2) + mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label) + mod.init_params(initializer=mx.init.Uniform(0.01), allow_missing=False, force_init=False, allow_extra=False) + + return di, mod + + +def test_bind_module(): + _, mod = setup() + assert mod.binded == True + assert mod._mod_aux.binded == True + + +def test_module_init(): + _, mod = setup() + assert mod._mod_aux is not None + + +def test_module_initializer(): + def regression_model(m): + x = mx.symbol.var("data", stype='csr') + v = mx.symbol.var("v", shape=(m, 1), init=mx.init.Uniform(scale=.1), + stype='row_sparse') + model = mx.symbol.dot(lhs=x, rhs=v) + y = mx.symbol.Variable("label") + model = mx.symbol.LinearRegressionOutput(data=model, label=y, name="out") + return model + + #shape of the data + n, m = 128, 100 + model = regression_model(m) + + data = mx.nd.zeros(shape=(n, m), stype='csr') + label = mx.nd.zeros((n, 1)) + iterator = mx.io.NDArrayIter(data=data, label={'label': label}, + batch_size=n, last_batch_handle='discard') + + # create module + mod = SVRGModule(symbol=model, data_names=['data'], label_names=['label'], update_freq=2) + mod.bind(data_shapes=iterator.provide_data, label_shapes=iterator.provide_label) + mod.init_params() + v = mod._arg_params['v'] + assert v.stype == 'row_sparse' + assert np.sum(v.asnumpy()) != 0 + + +def test_module_bind(): + x = mx.sym.Variable("data") + net = mx.sym.FullyConnected(x, num_hidden=1) + + mod = SVRGModule(symbol=net, data_names=['data'], label_names=None, update_freq=2) + assertRaises(TypeError, mod.bind, data_shapes=['data', mx.nd.zeros(shape=(2, 1))]) + + mod.bind(data_shapes=[('data', (2, 1))]) + assert mod.binded == True + assert mod._mod_aux.binded == True + + +@unittest.skip("Flaky test https://gitsvrhub.com/apache/incubator-mxnet/issues/12510") +@with_seed() +def test_module_save_load(tmpdir): + import os + + x = mx.sym.Variable("data") + y = mx.sym.Variable("softmax_label") + net = mx.sym.FullyConnected(x, y, num_hidden=1) + + mod = SVRGModule(symbol=net, data_names=['data'], label_names=['softmax_label'], update_freq=2) + mod.bind(data_shapes=[('data', (1, 1))]) + mod.init_params() + mod.init_optimizer(optimizer='sgd', optimizer_params={'learning_rate': 0.1}) + mod.update() + + tmp = str(tmpdir) + tmp_file = os.path.join(tmp, 'svrg_test_output') + mod.save_checkpoint(tmp_file, 0, save_optimizer_states=True) + + mod2 = SVRGModule.load(tmp_file, 0, load_optimizer_states=True, data_names=('data', )) + mod2.bind(data_shapes=[('data', (1, 1))]) + mod2.init_optimizer(optimizer_params={'learning_rate': 0.1}) + assert mod._symbol.tojson() == mod2._symbol.tojson() + + # Multi-device + mod3 = SVRGModule(symbol=net, data_names=['data'], label_names=['softmax_label'], update_freq=3, + context=[mx.cpu(0), mx.cpu(1)]) + mod3.bind(data_shapes=[('data', (10, 10))]) + mod3.init_params() + mod3.init_optimizer(optimizer_params={'learning_rate': 1.0}) + mod3.update() + mod3.save_checkpoint(tmp_file, 0, save_optimizer_states=True) + + mod4 = SVRGModule.load(tmp_file, 0, load_optimizer_states=True, data_names=('data', )) + mod4.bind(data_shapes=[('data', (10, 10))]) + mod4.init_optimizer(optimizer_params={'learning_rate': 1.0}) + assert mod3._symbol.tojson() == mod4._symbol.tojson() + + +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/12510") +@with_seed() +def test_svrgmodule_reshape(): + data = mx.sym.Variable("data") + sym = mx.sym.FullyConnected(data=data, num_hidden=4, name='fc') + + dshape=(3, 4) + mod = SVRGModule(sym, data_names=["data"], label_names=None, context=[mx.cpu(0), mx.cpu(1)], update_freq=2) + mod.bind(data_shapes=[('data', dshape)]) + mod.init_params() + mod._mod_aux.init_params() + mod.init_optimizer(optimizer_params={"learning_rate": 1.0}) + + data_batch = mx.io.DataBatch(data=[mx.nd.ones(dshape)], label=None) + mod.forward(data_batch) + mod.backward([mx.nd.ones(dshape)]) + mod.update() + assert mod.get_outputs()[0].shape == dshape + + dshape = (2, 4) + mod.reshape(data_shapes=[('data', dshape)]) + mod.forward(mx.io.DataBatch(data=[mx.nd.ones(dshape)], + label=None)) + mod.backward([mx.nd.ones(dshape)]) + mod.update() + assert mod.get_outputs()[0].shape == dshape + + +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/12510") +@with_seed() +def test_update_full_grad(): + def create_network(): + train_data = np.random.randint(1, 5, [10, 2]) + weights = np.array([1.0, 2.0]) + train_label = train_data.dot(weights) + + di = mx.io.NDArrayIter(train_data, train_label, batch_size=5, shuffle=True, label_name='lin_reg_label') + X = mx.sym.Variable('data') + Y = mx.symbol.Variable('lin_reg_label') + fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden=1) + lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro") + + mod = SVRGModule( + symbol=lro, + data_names=['data'], + label_names=['lin_reg_label'], update_freq=2) + mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label) + mod.init_params(initializer=mx.init.One(), allow_missing=False, force_init=False, allow_extra=False) + mod.init_optimizer(kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), + force_init=False) + return di, mod + + di, svrg_mod = create_network() + + # Calculates the average of full gradients over number batches + full_grads_weights = mx.nd.zeros(shape=svrg_mod.get_params()[0]['fc1_weight'].shape) + arg, aux = svrg_mod.get_params() + svrg_mod._mod_aux.set_params(arg_params=arg, aux_params=aux) + num_batch = 2 + + for batch in di: + svrg_mod.forward(batch) + svrg_mod.backward() + full_grads_weights = mx.nd.broadcast_add(svrg_mod._exec_group.grad_arrays[0][0], full_grads_weights, axis=0) + full_grads_weights /= num_batch + + di.reset() + svrg_mod.update_full_grads(di) + assert same(full_grads_weights, svrg_mod._param_dict[0]['fc1_weight']) + + +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/12510") +@with_seed() +def test_svrg_with_sgd(): + def create_module_with_sgd(): + train_data = np.random.randint(1, 5, [100, 2]) + weights = np.array([1.0, 2.0]) + train_label = train_data.dot(weights) + + di = mx.io.NDArrayIter(train_data, train_label, batch_size=10, shuffle=True, label_name='lin_reg_label') + X = mx.sym.Variable('data') + Y = mx.symbol.Variable('lin_reg_label') + fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden=1) + lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro") + + reg_mod = mx.mod.Module( + symbol=lro, + data_names=['data'], + label_names=['lin_reg_label']) + reg_mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label) + reg_mod.init_params(initializer=mx.init.One(), allow_missing=False, force_init=False, allow_extra=False) + reg_mod.init_optimizer(kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),)) + + svrg_mod = SVRGModule(symbol=lro, + data_names=['data'], + label_names=['lin_reg_label'], + update_freq=2) + svrg_mod.bind(data_shapes=di.provide_data, label_shapes=di.provide_label) + svrg_mod.init_params(initializer=mx.init.One(), allow_missing=False, force_init=False, allow_extra=False) + svrg_mod.init_optimizer(kvstore='local', optimizer='sgd', optimizer_params=(('learning_rate', 0.01),)) + + return di,reg_mod, svrg_mod + + di, reg_mod, svrg_mod = create_module_with_sgd() + num_epoch = 10 + + # Use metric MSE + metrics = mx.metric.create("mse") + + # Train with SVRGModule + for e in range(num_epoch): + metrics.reset() + if e % svrg_mod.update_freq == 0: + svrg_mod.update_full_grads(di) + di.reset() + for batch in di: + svrg_mod.forward_backward(data_batch=batch) + svrg_mod.update() + svrg_mod.update_metric(metrics, batch.label) + svrg_mse = metrics.get()[1] + + # Train with SGD standard Module + di.reset() + for e in range(num_epoch): + metrics.reset() + di.reset() + for batch in di: + reg_mod.forward_backward(data_batch=batch) + reg_mod.update() + reg_mod.update_metric(metrics, batch.label) + sgd_mse = metrics.get()[1] + + assert svrg_mse < sgd_mse + + +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/12510") +@with_seed() +def test_accumulate_kvstore(): + # Test KVStore behavior when push a list of values + kv = mx.kv.create('local') + kv.init("fc1_weight", mx.nd.zeros(shape=(1, 2))) + kv.init("fc1_weight_full", mx.nd.zeros(shape=(1, 2))) + b = [mx.nd.ones(shape=(1, 2)) for i in range(4)] + a = mx.nd.zeros(shape=(1, 2)) + kv.push("fc1_weight_full", b) + kv.pull("fc1_weight_full", out=a) + assert same(a, [mx.nd.array([4, 4])]) + assert kv.num_workers == 1 + + # Test accumulate in KVStore and allocate gradients + kv_test = mx.kv.create('local') + _, svrg_mod = setup() + svrg_mod.init_optimizer(kvstore=kv_test, optimizer='sgd', optimizer_params=(('learning_rate', 0.01),), + force_init=False) + svrg_mod._accumulate_kvstore("fc1_weight", b) + assert len(svrg_mod._param_dict) == svrg_mod._ctx_len + assert same(svrg_mod._param_dict[0]["fc1_weight"], b[0]) + + +@unittest.skip("Flaky test https://github.com/apache/incubator-mxnet/issues/12510") +@with_seed() +def test_fit(): + di, mod = setup() + num_epoch = 100 + metric = mx.metric.create("mse") + mod.fit(di, eval_metric=metric, optimizer='sgd', optimizer_params=(('learning_rate', 0.025),), num_epoch=num_epoch, + kvstore='local') + + # Estimated MSE for using SGD optimizer of lr = 0.025, SVRG MSE should be smaller + estimated_mse = 1e-5 + assert metric.get()[1] < estimated_mse + diff --git a/tests/python/unittest/test_gluon_batch_processor.py b/tests/python/unittest/test_gluon_batch_processor.py index bff80813bb12..952ed1c4a0da 100644 --- a/tests/python/unittest/test_gluon_batch_processor.py +++ b/tests/python/unittest/test_gluon_batch_processor.py @@ -52,7 +52,7 @@ def test_batch_processor_fit(): num_epochs = 1 ctx = mx.cpu() loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() net.initialize(ctx=ctx) processor = BatchProcessor() trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) @@ -83,7 +83,7 @@ def test_batch_processor_validation(): num_epochs = 1 ctx = mx.cpu() loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() val_loss = gluon.loss.L1Loss() net.initialize(ctx=ctx) processor = BatchProcessor() diff --git a/tests/python/unittest/test_gluon_estimator.py b/tests/python/unittest/test_gluon_estimator.py index 844c8b2b857f..93fceab3ed9e 100644 --- a/tests/python/unittest/test_gluon_estimator.py +++ b/tests/python/unittest/test_gluon_estimator.py @@ -58,7 +58,7 @@ def test_fit(): num_epochs = 1 ctx = mx.cpu() loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) est = Estimator(net=net, @@ -87,7 +87,7 @@ def test_validation(): num_epochs = 1 ctx = mx.cpu() loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() val_loss = gluon.loss.L1Loss() net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) @@ -126,7 +126,7 @@ def test_initializer(): ctx = mx.cpu() loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() # no initializer est = Estimator(net=net, loss=loss, @@ -166,7 +166,7 @@ def test_trainer(): ctx = mx.cpu() loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() net.initialize(ctx=ctx) # input no trainer with warnings.catch_warnings(record=True) as w: @@ -206,7 +206,7 @@ def test_metric(): est.fit(train_data=train_data, epochs=num_epochs) # input list of metrics - metrics = [mx.gluon.metric.Accuracy(), mx.gluon.metric.Accuracy()] + metrics = [mx.metric.Accuracy(), mx.metric.Accuracy()] est = Estimator(net=net, loss=loss, train_metrics=metrics, @@ -227,14 +227,14 @@ def test_metric(): loss=loss, trainer=trainer, context=ctx) - assert isinstance(est.train_metrics[0], mx.gluon.metric.Accuracy) + assert isinstance(est.train_metrics[0], mx.metric.Accuracy) def test_loss(): ''' test with invalid loss ''' net = _get_test_network() ctx = mx.cpu() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) # input invalid loss @@ -250,7 +250,7 @@ def test_context(): ''' test with no context, list of context, invalid context ''' net = _get_test_network() loss = gluon.loss.L2Loss() - metrics = mx.gluon.metric.Accuracy() + metrics = mx.metric.Accuracy() # input no context est = Estimator(net=net, loss=loss, @@ -332,7 +332,7 @@ def test_default_handlers(): net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) - train_acc = mx.gluon.metric.RMSE() + train_acc = mx.metric.RMSE() loss = gluon.loss.L2Loss() est = Estimator(net=net, @@ -359,7 +359,7 @@ def test_default_handlers(): # handler with mixed metrics, some handler use metrics prepared by estimator # some handler use metrics user prepared - logging = LoggingHandler(metrics=[mx.gluon.metric.RMSE("val acc")]) + logging = LoggingHandler(metrics=[mx.metric.RMSE("val acc")]) with pytest.raises(ValueError): est.fit(train_data=train_data, epochs=num_epochs, event_handlers=[logging]) @@ -383,7 +383,7 @@ def test_val_net(): ctx = mx.cpu() loss = gluon.loss.L2Loss() val_loss = gluon.loss.L2Loss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) est = Estimator(net=net, @@ -448,7 +448,7 @@ def test_val_handlers(): net.initialize(ctx=ctx) trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.001}) - train_acc = mx.gluon.metric.RMSE() + train_acc = mx.metric.RMSE() loss = gluon.loss.L2Loss() est = Estimator(net=net, diff --git a/tests/python/unittest/test_gluon_event_handler.py b/tests/python/unittest/test_gluon_event_handler.py index 4cadc9466ed1..a07282cd46dd 100644 --- a/tests/python/unittest/test_gluon_event_handler.py +++ b/tests/python/unittest/test_gluon_event_handler.py @@ -84,7 +84,7 @@ def test_checkpoint_handler(): net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) checkpoint_handler = event_handler.CheckpointHandler(model_dir=tmpdir, model_prefix=model_prefix, @@ -130,7 +130,7 @@ def test_resume_checkpoint(): net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) checkpoint_handler = event_handler.CheckpointHandler(model_dir=tmpdir, model_prefix=model_prefix, @@ -155,7 +155,7 @@ def test_early_stopping(): net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) early_stopping = event_handler.EarlyStoppingHandler(monitor=acc, patience=0, @@ -179,7 +179,7 @@ def test_logging(): net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) est.logger.addHandler(logging.FileHandler(output_dir)) @@ -226,7 +226,7 @@ def epoch_end(self, estimator, *args, **kwargs): test_data = _get_test_data() net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) custom_handler = CustomStopHandler(3, 2) est.fit(test_data, event_handlers=[custom_handler], epochs=3) @@ -249,7 +249,7 @@ def test_logging_interval(): dataloader = _get_test_data(in_size=data_size) num_epochs = 1 ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() logging = LoggingHandler(metrics=[acc], log_interval=log_interval) est = estimator.Estimator(net=net, loss=ce_loss, @@ -273,7 +273,7 @@ def test_logging_interval(): ''' test case #2: log interval is 5 ''' old_stdout = sys.stdout sys.stdout = mystdout = StringIO() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() log_interval = 5 logging = LoggingHandler(metrics=[acc], log_interval=log_interval) est = estimator.Estimator(net=net, @@ -299,7 +299,7 @@ def test_validation_handler_batch_axis(): test_data = _get_test_data() net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) est.fit(test_data, epochs=3) @@ -315,7 +315,7 @@ def test_validation_handler(): net = _get_test_network() ce_loss = loss.SoftmaxCrossEntropyLoss() - acc = mx.gluon.metric.Accuracy() + acc = mx.metric.Accuracy() est = estimator.Estimator(net, loss=ce_loss, train_metrics=acc) val_handler = ValidationHandler(val_data=test_data, eval_fn=est.evaluate, diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py index 5e6c0f798d9e..a07ed440ea69 100644 --- a/tests/python/unittest/test_loss.py +++ b/tests/python/unittest/test_loss.py @@ -65,6 +65,50 @@ def get_net(num_hidden, flatten=True): fc3 = mx.symbol.FullyConnected(act2, name='fc3', num_hidden=num_hidden, flatten=flatten) return fc3 +# tracked at: https://github.com/apache/incubator-mxnet/issues/11692 +@with_seed() +def test_ce_loss(): + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(nclass) + l = mx.symbol.Variable('label') + Loss = gluon.loss.SoftmaxCrossEntropyLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + eval_metric=mx.metric.Loss(), optimizer='adam', + initializer=mx.init.Xavier(magnitude=2)) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + +# tracked at: https://github.com/apache/incubator-mxnet/issues/11691 +@with_seed() +def test_bce_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 20)) + label = mx.nd.array(np.random.randint(2, size=(N,)), dtype='float32') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(1) + l = mx.symbol.Variable('label') + Loss = gluon.loss.SigmoidBinaryCrossEntropyLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + eval_metric=mx.metric.Loss(), optimizer='adam', + initializer=mx.init.Xavier(magnitude=2)) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01 + # Test against npy + data = mx.random.uniform(-5, 5, shape=(10,)) + label = mx.random.uniform(0, 1, shape=(10,)) + mx_bce_loss = Loss(data, label).asnumpy() + prob_npy = 1.0 / (1.0 + np.exp(-data.asnumpy())) + label_npy = label.asnumpy() + npy_bce_loss = - label_npy * np.log(prob_npy) - (1 - label_npy) * np.log(1 - prob_npy) + assert_almost_equal(mx_bce_loss, npy_bce_loss, rtol=1e-4, atol=1e-5) @with_seed() def test_bce_equal_ce2(): @@ -86,6 +130,58 @@ def test_logistic_loss_equal_bce(): assert_almost_equal(loss_binary(data, label), loss_bce(data, label), atol=1e-6) assert_almost_equal(loss_signed(data, 2 * label - 1), loss_bce(data, label), atol=1e-6) +@with_seed() +def test_kl_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.nd.softmax(mx.random.uniform(0, 1, shape=(N, 2))) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = mx.sym.log_softmax(get_net(2)) + l = mx.symbol.Variable('label') + Loss = gluon.loss.KLDivLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + eval_metric=mx.metric.Loss(), optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + + +@with_seed() +def test_l2_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.random.uniform(-1, 1, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) + output = get_net(1) + l = mx.symbol.Variable('label') + Loss = gluon.loss.L2Loss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + + +@with_seed() +def test_l1_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.random.uniform(-1, 1, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) + output = get_net(1) + l = mx.symbol.Variable('label') + Loss = gluon.loss.L1Loss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.1 + @with_seed() def test_ctc_loss(): @@ -114,6 +210,145 @@ def test_ctc_loss(): assert_almost_equal(l, np.array([18.82820702, 16.50581741])) +@with_seed() +def test_ctc_loss_train(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 20, 10)) + label = mx.nd.arange(4, repeat=N).reshape((N, 4)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) + output = get_net(5, False) + l = mx.symbol.Variable('label') + Loss = gluon.loss.CTCLoss(layout='NTC', label_layout='NT') + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 10 + + +@with_seed() +def test_sample_weight_loss(): + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + weight = mx.nd.array([1 for i in range(10)] + [0 for i in range(10)]) + data_iter = mx.io.NDArrayIter(data, {'label': label, 'w': weight}, batch_size=10) + output = get_net(nclass) + l = mx.symbol.Variable('label') + w = mx.symbol.Variable('w') + Loss = gluon.loss.SoftmaxCrossEntropyLoss() + loss = Loss(output, l, w) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'w')) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + eval_metric=mx.metric.Loss(), optimizer='adam') + data_iter = mx.io.NDArrayIter(data[10:], {'label': label, 'w': weight}, batch_size=10) + score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] + assert score > 1 + data_iter = mx.io.NDArrayIter(data[:10], {'label': label, 'w': weight}, batch_size=10) + score = mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] + assert score < 0.05 + + +@with_seed(1234) +def test_saveload(): + nclass = 10 + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, nclass)) + label = mx.nd.array(np.random.randint(0, nclass, size=(N,)), dtype='int32') + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label') + output = get_net(nclass) + l = mx.symbol.Variable('label') + Loss = gluon.loss.SoftmaxCrossEntropyLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + mod.save_checkpoint('test', 100, save_optimizer_states=True) + mod = mx.mod.Module.load('test', 100, load_optimizer_states=True, + data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=100, optimizer_params={'learning_rate': 1.}, + eval_metric=mx.metric.Loss()) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + +@with_seed() +def test_huber_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.random.uniform(-1, 1, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) + output = get_net(1) + l = mx.symbol.Variable('label') + Loss = gluon.loss.HuberLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + + +@with_seed() +def test_hinge_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.nd.sign(mx.random.uniform(-1, 1, shape=(N, 1))) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) + output = get_net(1) + l = mx.symbol.Variable('label') + Loss = gluon.loss.HingeLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.06 + + +@with_seed() +def test_squared_hinge_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + label = mx.nd.sign(mx.random.uniform(-1, 1, shape=(N, 1))) + data_iter = mx.io.NDArrayIter(data, label, batch_size=10, label_name='label', shuffle=True) + output = get_net(1) + l = mx.symbol.Variable('label') + Loss = gluon.loss.SquaredHingeLoss() + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + + +@with_seed() +def test_triplet_loss(): + N = 20 + data = mx.random.uniform(-1, 1, shape=(N, 10)) + pos = mx.random.uniform(-1, 1, shape=(N, 10)) + neg = mx.random.uniform(-1, 1, shape=(N, 10)) + data_iter = mx.io.NDArrayIter(data, {'pos': pos, 'neg': neg}, batch_size=10, + label_name='label', shuffle=True) + output = get_net(10) + pos = mx.symbol.Variable('pos') + neg = mx.symbol.Variable('neg') + Loss = gluon.loss.TripletLoss() + loss = Loss(output, pos, neg) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('pos','neg')) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Xavier(magnitude=2), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + @xfail_when_nonstandard_decimal_separator @with_seed() def test_sdml_loss(): @@ -208,3 +443,51 @@ def test_poisson_nllloss(): loss_compute_full = Loss_compute_full(mx.nd.array(np_pred), mx.nd.array(np_target)) assert_almost_equal(np_compute_full, loss_compute_full.asscalar()) +@with_seed() +def test_poisson_nllloss_mod(): + N = 1000 + data = mx.random.poisson(shape=(N, 2)) + label = mx.random.poisson(lam=4, shape=(N, 1)) + data_iter = mx.io.NDArrayIter(data, label, batch_size=20, label_name='label', shuffle=True) + output = mx.sym.exp(get_net(1)) + l = mx.symbol.Variable('label') + Loss = gluon.loss.PoissonNLLLoss(from_logits=False) + loss = Loss(output, l) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label',)) + mod.fit(data_iter, num_epoch=20, optimizer_params={'learning_rate': 0.01}, + initializer=mx.init.Normal(sigma=0.1), eval_metric=mx.metric.Loss(), + optimizer='adam') + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05 + +@with_seed() +def test_bce_loss_with_pos_weight(): + # Suppose it's a multi-label classification + N = np.random.randint(5, 30) + data = mx.nd.random.uniform(-1, 1, shape=(N, 20)) + label = mx.nd.array(np.random.randint(2, size=(N, 5)), dtype='float32') + pos_weight = mx.nd.random.uniform(0, 10, shape=(1, 5)) + pos_weight = mx.nd.repeat(pos_weight, repeats=N, axis=0) + data_iter = mx.io.NDArrayIter(data, {'label': label, 'pos_w': pos_weight}, batch_size=10, label_name='label') + output = get_net(5) + l = mx.symbol.Variable('label') + pos_w = mx.symbol.Variable('pos_w') + Loss = gluon.loss.SigmoidBinaryCrossEntropyLoss() + loss = Loss(output, l, None, pos_w) + loss = mx.sym.make_loss(loss) + mod = mx.mod.Module(loss, data_names=('data',), label_names=('label', 'pos_w')) + mod.fit(data_iter, num_epoch=200, optimizer_params={'learning_rate': 0.01}, + eval_metric=mx.metric.Loss(), optimizer='adam', + initializer=mx.init.Xavier(magnitude=2)) + assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.01 + # Test against npy + data = mx.nd.random.uniform(-5, 5, shape=(N, 5)) + label = mx.nd.array(np.random.randint(2, size=(N, 5)), dtype='float32') + pos_weight = mx.nd.random.uniform(0, 10, shape=(1, 5)) + mx_bce_loss = Loss(data, label, None, pos_weight).asnumpy() + prob_npy = 1.0 / (1.0 + np.exp(-data.asnumpy())) + label_npy = label.asnumpy() + pos_weight_npy = pos_weight.asnumpy() + npy_bce_loss = (- label_npy * np.log(prob_npy)*pos_weight_npy - (1 - label_npy) * np.log(1 - prob_npy)).mean(axis=1) + assert_almost_equal(mx_bce_loss, npy_bce_loss, rtol=1e-4, atol=1e-5) + diff --git a/tests/python/unittest/test_metric.py b/tests/python/unittest/test_metric.py index 88b9d9cedce2..cc92a59f9a95 100644 --- a/tests/python/unittest/test_metric.py +++ b/tests/python/unittest/test_metric.py @@ -16,7 +16,6 @@ # under the License. import mxnet as mx -from mxnet.test_utils import use_np import numpy as np import scipy from scipy.stats import pearsonr @@ -26,9 +25,9 @@ from copy import deepcopy def check_metric(metric, *args, **kwargs): - metric = mx.gluon.metric.create(metric, *args, **kwargs) + metric = mx.metric.create(metric, *args, **kwargs) str_metric = json.dumps(metric.get_config()) - metric2 = mx.gluon.metric.create(str_metric) + metric2 = mx.metric.create(str_metric) assert metric.get_config() == metric2.get_config() @@ -36,16 +35,93 @@ def test_metrics(): check_metric('acc', axis=0) check_metric('f1') check_metric('mcc') - check_metric('perplexity', axis=-1) + check_metric('perplexity', -1) check_metric('pearsonr') check_metric('pcc') check_metric('nll_loss') check_metric('loss') - composite = mx.gluon.metric.create(['acc', 'f1']) + composite = mx.metric.create(['acc', 'f1']) check_metric(composite) +def _check_global_metric(metric, *args, **kwargs): + def _create_pred_label(): + if use_same_shape: + pred = mx.nd.random.uniform(0, 1, shape=shape) + label = mx.nd.random.uniform(0, 1, shape=shape) + else: + # Make a random prediction + idx = np.random.rand(*shape).argsort(1) + pred = mx.nd.array(1 - 0.1 * idx) + # Label is half 1 and half 0 + # Setting all 0s or all 1s would make either + # MCC or F1 metrics always produce 0 + label = mx.nd.ones(shape[0]) + label[:shape[0] // 2] = 0 + return pred, label + + def _compare_metric_result(m1, m2): + # Compare names + assert m1[0] == m2[0] + # Compare values + if isinstance(m1[1], (list, tuple)): + assert len(m1[1]) == len(m2[1]) + for r1, r2 in zip(m1[1], m2[1]): + assert r1 == r2 or \ + (math.isnan(r1) and + math.isnan(r2)) + else: + assert m1[1] == m2[1] or \ + (math.isnan(m1[1]) and + math.isnan(m2[1])) + + shape = kwargs.pop('shape', (10,10)) + use_same_shape = kwargs.pop('use_same_shape', False) + m1 = mx.metric.create(metric, *args, **kwargs) + m2 = deepcopy(m1) + # check that global stats are not reset when calling + # reset_local() + for i in range(10): + pred, label = _create_pred_label() + m1.update([label], [pred]) + m1.reset_local() + m2.update([label], [pred]) + assert m1.get_global() == m2.get() + + # check that reset_local() properly resets the local state + m1.reset_local() + m2.reset() + pred, label = _create_pred_label() + m1.update([label], [pred]) + m1.reset_local() + pred, label = _create_pred_label() + m1.update([label], [pred]) + m2.update([label], [pred]) + _compare_metric_result(m1.get(), m2.get()) + +@with_seed() +def test_global_metric(): + _check_global_metric('acc') + _check_global_metric('TopKAccuracy', top_k=3) + _check_global_metric('f1', shape=(10,2)) + _check_global_metric('f1', shape=(10,2), average='micro') + _check_global_metric('mcc', shape=(10,2)) + _check_global_metric('mcc', shape=(10,2), average='micro') + _check_global_metric('perplexity', -1) + _check_global_metric('pearsonr', use_same_shape=True) + _check_global_metric('pcc', shape=(10,2)) + _check_global_metric('nll_loss') + _check_global_metric('loss') + _check_global_metric('ce') + _check_global_metric('mae', use_same_shape=True) + _check_global_metric('mse', use_same_shape=True) + _check_global_metric('rmse', use_same_shape=True) + def custom_metric(label, pred): + return np.mean(np.abs(label-pred)) + _check_global_metric(custom_metric, use_same_shape=True) + _check_global_metric(['acc', 'f1'], shape=(10,2)) + def test_nll_loss(): - metric = mx.gluon.metric.create('nll_loss') + metric = mx.metric.create('nll_loss') pred = mx.nd.array([[0.2, 0.3, 0.5], [0.6, 0.1, 0.3]]) label = mx.nd.array([2, 1]) metric.update([label], [pred]) @@ -56,37 +132,36 @@ def test_nll_loss(): def test_acc(): pred = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]]) label = mx.nd.array([0, 1, 1]) - metric = mx.gluon.metric.create('acc') + metric = mx.metric.create('acc') metric.update([label], [pred]) _, acc = metric.get() expected_acc = (np.argmax(pred, axis=1) == label).sum().asscalar() / label.size - np.testing.assert_almost_equal(acc, expected_acc) + assert acc == expected_acc def test_acc_2d_label(): # label maybe provided in 2d arrays in custom data iterator pred = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6], [0.8, 0.2], [0.3, 0.5], [0.6, 0.4]]) label = mx.nd.array([[0, 1, 1], [1, 0, 1]]) - metric = mx.gluon.metric.create('acc') + metric = mx.metric.create('acc') metric.update([label], [pred]) _, acc = metric.get() expected_acc = (np.argmax(pred, axis=1).asnumpy() == label.asnumpy().ravel()).sum() / \ float(label.asnumpy().ravel().size) - np.testing.assert_almost_equal(acc, expected_acc) + assert acc == expected_acc def test_loss_update(): pred = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]]) - metric1 = mx.gluon.metric.create('loss') - metric2 = mx.gluon.metric.create('loss') + metric1 = mx.metric.create('loss') + metric2 = mx.metric.create('loss') metric1.update(None, [pred]) metric2.update(None, pred) _, acc1 = metric1.get() _, acc2 = metric2.get() assert acc1 == acc2 -@xfail_when_nonstandard_decimal_separator -def test_binary_f1(): - microF1 = mx.gluon.metric.create("f1", average="micro") - macroF1 = mx.gluon.metric.F1(average="macro") +def test_f1(): + microF1 = mx.metric.create("f1", average="micro") + macroF1 = mx.metric.F1(average="macro") assert np.isnan(macroF1.get()[1]) assert np.isnan(microF1.get()[1]) @@ -116,7 +191,7 @@ def test_binary_f1(): microF1.update([label11, label12], [pred11, pred12]) macroF1.update([label11, label12], [pred11, pred12]) assert microF1.num_inst == 4 - assert macroF1.num_inst == 4 + assert macroF1.num_inst == 1 # f1 = 2 * tp / (2 * tp + fp + fn) fscore1 = 2. * (1) / (2 * 1 + 1 + 0) np.testing.assert_almost_equal(microF1.get()[1], fscore1) @@ -125,98 +200,29 @@ def test_binary_f1(): microF1.update([label21, label22], [pred21, pred22]) macroF1.update([label21, label22], [pred21, pred22]) assert microF1.num_inst == 6 - assert macroF1.num_inst == 6 + assert macroF1.num_inst == 2 fscore2 = 2. * (1) / (2 * 1 + 0 + 0) fscore_total = 2. * (1 + 1) / (2 * (1 + 1) + (1 + 0) + (0 + 0)) np.testing.assert_almost_equal(microF1.get()[1], fscore_total) - np.testing.assert_almost_equal(macroF1.get()[1], fscore_total) - -def test_multiclass_f1(): - microF1 = mx.gluon.metric.create("f1", class_type="multiclass", average="micro") - macroF1 = mx.gluon.metric.F1(class_type="multiclass", average="macro") - - assert np.isnan(macroF1.get()[1]) - assert np.isnan(microF1.get()[1]) + np.testing.assert_almost_equal(macroF1.get()[1], (fscore1 + fscore2) / 2.) +<<<<<<< HEAD # check one class is zero pred = mx.nd.array([[0.9, 0.1], [0.8, 0.2]]) label = mx.nd.array([0, 0]) macroF1.update([label], [pred]) microF1.update([label], [pred]) - assert macroF1.get()[1] == 0.5 # one class is 1.0, the other is 0. (divided by 0) - assert microF1.get()[1] == 1.0 # globally f1 is 1.0 - macroF1.reset() - microF1.reset() - - # test case from sklearn, here pred is probabilistic distributions instead of predicted labels - pred11 = mx.nd.array([[1, 0, 0], [0, 1, 0]]) - label11 = mx.nd.array([0, 2]) - pred12 = mx.nd.array([[0, 0, 1], [1, 0, 0], [0, 1, 0], [0, 0, 1]]) - label12 = mx.nd.array([1, 0, 0, 1]) - - microF1.update([label11, label12], [pred11, pred12]) - macroF1.update([label11, label12], [pred11, pred12]) - assert microF1.num_inst == 6 - assert macroF1.num_inst == 6 - - # from sklearn.metrics import f1_score - # overall_pred = [0, 1, 2, 0, 1, 2] - # overall_label = [0, 2, 1, 0, 0, 1] - fmacro = 0.26666666666666666 #f1_score(overall_label, overall_pred, average="macro") - fmicro = 0.3333333333333333 #f1_score(overall_label, overall_pred, average="micro") - np.testing.assert_almost_equal(microF1.get()[1], fmicro) - np.testing.assert_almost_equal(macroF1.get()[1], fmacro) - -@xfail_when_nonstandard_decimal_separator -def test_multilabel_f1(): - microF1 = mx.gluon.metric.create("f1", class_type="multilabel", average="micro") - macroF1 = mx.gluon.metric.F1(class_type="multilabel", average="macro") - - assert np.isnan(macroF1.get()[1]) - assert np.isnan(microF1.get()[1]) - - # check one class is zero - pred = mx.nd.array([[0.9, 0.1], - [0.8, 0.2]]) - label = mx.nd.array([[1, 1], [1, 1]]) - macroF1.update([label], [pred]) - microF1.update([label], [pred]) - assert macroF1.get()[1] == 0.5 # one class is 1.0, the other is 0. (divided by 0) - np.testing.assert_almost_equal(microF1.get()[1], 2.0 / 3) - macroF1.reset() - microF1.reset() - - pred11 = mx.nd.array([[0.9, 0.4, 0.3], [0.2, 0.7, 0.8]]) - label11 = mx.nd.array([[1, 0, 1], [0, 0, 1]]) - pred12 = mx.nd.array([[0.6, 0.6, 0.7]]) - label12 = mx.nd.array([[0, 1, 1]]) - - microF1.update([label11, label12], [pred11, pred12]) - macroF1.update([label11, label12], [pred11, pred12]) - assert microF1.num_inst == 3 - assert macroF1.num_inst == 3 - #from sklearn.metrics import f1_score - #overall_pred = [[1, 0, 0], [0, 1, 1], [1, 1, 1]] - #overall_label = [[1, 0, 1], [0, 0, 1], [0, 1, 1]] - fmacro = 0.7111111111111111 #f1_score(overall_label, overall_pred, average="macro") - fmicro = 0.7272727272727272 #f1_score(overall_label, overall_pred, average="micro") - np.testing.assert_almost_equal(microF1.get()[1], fmicro) - np.testing.assert_almost_equal(macroF1.get()[1], fmacro) - -@xfail_when_nonstandard_decimal_separator -def test_mcc(): - microMCC = mx.gluon.metric.create("mcc") - - assert np.isnan(microMCC.get()[1]) - # check divide by zero pred = mx.nd.array([[0.9, 0.1], [0.8, 0.2]]) label = mx.nd.array([0, 0]) microMCC.update([label], [pred]) + macroMCC.update([label], [pred]) assert microMCC.get()[1] == 0.0 + assert macroMCC.get()[1] == 0.0 microMCC.reset() + macroMCC.reset() pred11 = mx.nd.array([[0.1, 0.9], [0.5, 0.5]]) @@ -229,40 +235,51 @@ def test_mcc(): pred22 = mx.nd.array([[0.2, 0.8]]) label22 = mx.nd.array([1]) microMCC.update([label11, label12], [pred11, pred12]) + macroMCC.update([label11, label12], [pred11, pred12]) assert microMCC.num_inst == 4 + assert macroMCC.num_inst == 1 tp1 = 1; fp1 = 0; fn1 = 1; tn1=2 mcc1 = (tp1*tn1 - fp1*fn1) / np.sqrt((tp1+fp1)*(tp1+fn1)*(tn1+fp1)*(tn1+fn1)) np.testing.assert_almost_equal(microMCC.get()[1], mcc1) + np.testing.assert_almost_equal(macroMCC.get()[1], mcc1) microMCC.update([label21, label22], [pred21, pred22]) + macroMCC.update([label21, label22], [pred21, pred22]) assert microMCC.num_inst == 6 + assert macroMCC.num_inst == 2 tp2 = 1; fp2 = 0; fn2 = 0; tn2=1 mcc2 = (tp2*tn2 - fp2*fn2) / np.sqrt((tp2+fp2)*(tp2+fn2)*(tn2+fp2)*(tn2+fn2)) tpT = tp1+tp2; fpT = fp1+fp2; fnT = fn1+fn2; tnT = tn1+tn2; mccT = (tpT*tnT - fpT*fnT) / np.sqrt((tpT+fpT)*(tpT+fnT)*(tnT+fpT)*(tnT+fnT)) np.testing.assert_almost_equal(microMCC.get()[1], mccT) + np.testing.assert_almost_equal(macroMCC.get()[1], .5*(mcc1+mcc2)) def test_perplexity(): pred = mx.nd.array([[0.8, 0.2], [0.2, 0.8], [0, 1.]]) label = mx.nd.array([0, 1, 1]) p = pred.asnumpy()[np.arange(label.size), label.asnumpy().astype('int32')] perplexity_expected = np.exp(-np.log(p).sum()/label.size) - metric = mx.gluon.metric.create('perplexity', axis=-1) + metric = mx.metric.create('perplexity', -1) metric.update([label], [pred]) _, perplexity = metric.get() - np.testing.assert_almost_equal(perplexity, perplexity_expected) + assert perplexity == perplexity_expected def test_pearsonr(): pred1 = mx.nd.array([[0.3, 0.7], [0, 1.], [0.4, 0.6]]) label1 = mx.nd.array([[1, 0], [0, 1], [0, 1]]) pearsonr_expected_np = np.corrcoef(pred1.asnumpy().ravel(), label1.asnumpy().ravel())[0, 1] pearsonr_expected_scipy, _ = pearsonr(pred1.asnumpy().ravel(), label1.asnumpy().ravel()) - micro_pr = mx.gluon.metric.create('pearsonr') + macro_pr = mx.metric.create('pearsonr', average='macro') + micro_pr = mx.metric.create('pearsonr', average='micro') + assert np.isnan(macro_pr.get()[1]) assert np.isnan(micro_pr.get()[1]) + macro_pr.update([label1], [pred1]) micro_pr.update([label1], [pred1]) + np.testing.assert_almost_equal(macro_pr.get()[1], pearsonr_expected_np) + np.testing.assert_almost_equal(macro_pr.get()[1], pearsonr_expected_scipy) np.testing.assert_almost_equal(micro_pr.get()[1], pearsonr_expected_np) np.testing.assert_almost_equal(micro_pr.get()[1], pearsonr_expected_scipy) @@ -275,7 +292,11 @@ def test_pearsonr(): pearsonr_expected_np = np.corrcoef(pred12.asnumpy().ravel(), label12.asnumpy().ravel())[0, 1] pearsonr_expected_scipy, _ = pearsonr(pred12.asnumpy().ravel(), label12.asnumpy().ravel()) + macro_pr.reset() micro_pr.update([label2], [pred2]) + macro_pr.update([label12], [pred12]) + np.testing.assert_almost_equal(macro_pr.get()[1], pearsonr_expected_np) + np.testing.assert_almost_equal(macro_pr.get()[1], pearsonr_expected_scipy) np.testing.assert_almost_equal(micro_pr.get()[1], pearsonr_expected_np) np.testing.assert_almost_equal(micro_pr.get()[1], pearsonr_expected_scipy) @@ -296,18 +317,18 @@ def test_pcc(): [ 7, 3 ], [ 2, 5 ], ]) - met_pcc = mx.gluon.metric.create('pcc') + met_pcc = mx.metric.create('pcc') met_pcc.update(labels, preds) _, pcc = met_pcc.get() # pcc should agree with mcc for binary classification - met_mcc = mx.gluon.metric.create('mcc') + met_mcc = mx.metric.create('mcc') met_mcc.update(labels, preds) _, mcc = met_mcc.get() np.testing.assert_almost_equal(pcc, mcc) # pcc should agree with Pearson for binary classification - met_pear = mx.gluon.metric.create('pearsonr') + met_pear = mx.metric.create('pearsonr') met_pear.update(labels, [p.argmax(axis=1) for p in preds]) _, pear = met_pear.get() np.testing.assert_almost_equal(pcc, pear) @@ -356,7 +377,7 @@ def test_pcc(): # * order # * batch size # * update frequency - labels = [ [ i.reshape(-1) ] for i in labels[0] ] + labels = [ [ i ] for i in labels[0] ] labels.reverse() preds = [ [ i.reshape((1, -1)) ] for i in preds[0] ] preds.reverse() @@ -371,20 +392,19 @@ def test_single_array_input(): pred = mx.nd.array([[1,2,3,4]]) label = pred + 0.1 - mse = mx.gluon.metric.create('mse') + mse = mx.metric.create('mse') mse.update(label, pred) _, mse_res = mse.get() np.testing.assert_almost_equal(mse_res, 0.01) - mae = mx.gluon.metric.create('mae') + mae = mx.metric.create('mae') mae.update(label, pred) mae.get() _, mae_res = mae.get() np.testing.assert_almost_equal(mae_res, 0.1) - rmse = mx.gluon.metric.create('rmse') + rmse = mx.metric.create('rmse') rmse.update(label, pred) rmse.get() _, rmse_res = rmse.get() np.testing.assert_almost_equal(rmse_res, 0.1) - diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py new file mode 100644 index 000000000000..65d86f62baf4 --- /dev/null +++ b/tests/python/unittest/test_module.py @@ -0,0 +1,1031 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import mxnet as mx +import mxnet.ndarray as nd +from mxnet.test_utils import * +import numpy as np +from functools import reduce +from mxnet.module.executor_group import DataParallelExecutorGroup +from common import setup_module, with_seed, assertRaises, teardown_module +from collections import namedtuple +curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) +sys.path.insert(0, os.path.join(curr_path, "../train")) +from test_bucketing import train_model, prepare_bucketing_data + + +@with_seed() +def test_module_dtype(): + dtype = np.float16 + dshape = (3, 8, 7) + + sym = mx.sym.Variable('data') + sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC') + + mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)]) + mod.bind(data_shapes=[mx.io.DataDesc('data', dshape, dtype, layout='TNC')]) + mod.init_params() + mod.forward(mx.io.DataBatch(data=[mx.nd.ones(dshape, dtype=dtype)], + label=None)) + mod.backward([mx.nd.ones(dshape, dtype=dtype)]) + + for x in mod.get_outputs(): + assert x.dtype == dtype + + +def test_module_bind(): + sym = mx.sym.Variable('data') + sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC') + + mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)]) + assertRaises(TypeError, mod.bind, data_shapes=[('data', mx.nd.array([10,10]))]) + assert mod.binded == False + + mod.bind(data_shapes=[('data', (10,10))]) + assert mod.binded == True + + +@with_seed() +def test_module_input_grads(): + a = mx.sym.Variable('a', __layout__='NC') + b = mx.sym.Variable('b', __layout__='NC') + c = mx.sym.Variable('c', __layout__='NC') + + c = a + 2 * b + 3 * c + net = mx.mod.Module(c, data_names=['b', 'c', 'a'], label_names=None, + context=[mx.cpu(0), mx.cpu(1)]) + net.bind(data_shapes=[['b', (5, 5)], ['c', (5, 5)], ['a', (5, 5)]], + label_shapes=None, inputs_need_grad=True) + net.init_params() + + net.forward(data_batch=mx.io.DataBatch(data=[nd.ones((5, 5)), + nd.ones((5, 5)), + nd.ones((5, 5))])) + net.backward(out_grads=[nd.ones((5, 5))]) + input_grads = net.get_input_grads() + b_grad = input_grads[0].asnumpy() + c_grad = input_grads[1].asnumpy() + a_grad = input_grads[2].asnumpy() + assert np.all(a_grad == 1), a_grad + assert np.all(b_grad == 2), b_grad + assert np.all(c_grad == 3), c_grad + + +@with_seed() +def test_module_ctx_group(): + def check_module_ctx_group(ctxs, group2ctxs, grad_ctxs=None): + with mx.AttrScope(ctx_group='dev1'): + a = mx.symbol.Variable('a') + a = a * 2 + with mx.AttrScope(ctx_group='dev2'): + b = mx.symbol.Variable('b') + c = a + b + shape = (2, 5) + mod1 = mx.mod.Module(c, context=ctxs, data_names=['a', 'b'], label_names=None, + group2ctxs=group2ctxs) + mod1.bind(data_shapes=[['a', shape], ['b', shape]], inputs_need_grad=True) + mod1.init_params() + mod1.forward(data_batch=mx.io.DataBatch(data=[mx.nd.ones(shape), mx.nd.ones(shape)]), is_train=True) + mod1.backward([mx.nd.ones(shape)]) + mod1_input_grads = mod1.get_input_grads() + + mod2 = mx.mod.Module(c, context=ctxs, data_names=['a', 'b'], label_names=None) + mod2.bind(data_shapes=[['a', shape], ['b', shape]], inputs_need_grad=True) + mod2.init_params() + mod2.forward(data_batch=mx.io.DataBatch(data=[mx.nd.ones(shape), mx.nd.ones(shape)]), is_train=True) + mod2.backward([mx.nd.ones(shape)]) + mod2_input_grads = mod2.get_input_grads() + + if grad_ctxs is not None: + assert(mod1_input_grads[0].context == grad_ctxs[0]) + assert(mod1_input_grads[1].context == grad_ctxs[1]) + assert(np.all(mod1_input_grads[0].asnumpy() == mod2_input_grads[0].asnumpy())) + assert(np.all(mod1_input_grads[1].asnumpy() == mod2_input_grads[1].asnumpy())) + + check_module_ctx_group([mx.cpu(0)], {'dev1': mx.cpu(1), 'dev2': mx.cpu(2)}, grad_ctxs=[mx.cpu(1), mx.cpu(2)]) + check_module_ctx_group([mx.cpu(0), mx.cpu(1)], + [{'dev1': mx.cpu(2), 'dev2': mx.cpu(3)}, {'dev1': mx.cpu(4), 'dev2': mx.cpu(5)}]) + check_module_ctx_group([mx.cpu(0), mx.cpu(1)], {'dev1': mx.cpu(2), 'dev2': mx.cpu(3)}) + check_module_ctx_group([mx.cpu(0), mx.cpu(1)], {'dev1': mx.cpu(2), 'dev2': [mx.cpu(3)]}) + check_module_ctx_group([mx.cpu(0), mx.cpu(1)], {'dev1':mx.cpu(2), 'dev2':[mx.cpu(3), mx.cpu(3)]}) + check_module_ctx_group([mx.cpu(0), mx.cpu(1)], + {'dev1':[mx.cpu(2), mx.cpu(2)], 'dev2':[mx.cpu(3), mx.cpu(3)]}) + +@with_seed() +def test_bucket_module_ctx_group(): + num_hidden = 10 + batch_size = 5 + def sym_gen(seq_len): + with mx.AttrScope(ctx_group='dev1'): + data = mx.symbol.Variable('data') + weight = mx.symbol.Variable('dev1_weight') + bias = mx.symbol.Variable('dev1_bias') + fc = data + for i in range(seq_len): + fc = mx.symbol.FullyConnected(data=fc, weight=weight, bias=bias, + name='dev1_fc_%d' % i, num_hidden=num_hidden) + with mx.AttrScope(ctx_group='dev2'): + label = mx.symbol.Variable('label') + weight = mx.symbol.Variable('dev2_weight') + bias = mx.symbol.Variable('dev2_bias') + for i in range(seq_len): + fc = mx.symbol.FullyConnected(data=fc, weight=weight, bias=bias, + name='dev2_fc_%d' % i, num_hidden=num_hidden) + sym = mx.symbol.SoftmaxOutput(fc, label, name='softmax') + + return sym, ('data',), ('label',) + + mod = mx.mod.BucketingModule(sym_gen=sym_gen, default_bucket_key=10, context=[mx.cpu(0)], + group2ctxs=[{'dev1': mx.cpu(1), 'dev2': mx.cpu(2)}]) + mod.bind(data_shapes=[['data', (batch_size, num_hidden)]], + label_shapes=[['label', (batch_size,)]], + for_training=True, inputs_need_grad=True) + assert(mod.binded) + +@with_seed() +def test_module_layout(): + sym = mx.sym.Variable('data') + sym = mx.sym.Activation(data=sym, act_type='relu', __layout__='TNC') + + dshape = (3, 8, 7) + mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)]) + mod.bind(data_shapes=[mx.io.DataDesc('data', dshape, layout='TNC')]) + mod.init_params() + mod.forward(mx.io.DataBatch(data=[mx.nd.ones(dshape)], + label=None)) + mod.backward([mx.nd.ones(dshape)]) + assert mod.get_outputs()[0].shape == dshape + + hdshape = (3, 4, 7) + for x in mod.get_outputs(merge_multi_context=False)[0]: + assert x.shape == hdshape + + +@with_seed() +def test_save_load(): + previous_update_on_kvstore = os.getenv('MXNET_UPDATE_ON_KVSTORE', "1") + os.putenv('MXNET_UPDATE_ON_KVSTORE', '1') + def dict_equ(a, b): + assert set(a) == set(b) + for k in a: + assert (a[k].asnumpy() == b[k].asnumpy()).all() + + sym = mx.sym.Variable('data') + sym = mx.sym.FullyConnected(sym, num_hidden=100) + + # single device + mod = mx.mod.Module(sym, ('data',)) + mod.bind(data_shapes=[('data', (10, 10))]) + mod.init_params() + mod.init_optimizer(optimizer_params={'learning_rate':0.1, 'momentum':0.9}) + mod.update() + mod.save_checkpoint('test', 0, save_optimizer_states=True) + + mod2 = mx.mod.Module.load('test', 0, load_optimizer_states=True, data_names=('data',)) + mod2.bind(data_shapes=[('data', (10, 10))]) + mod2.init_optimizer(optimizer_params={'learning_rate':0.1, 'momentum':0.9}) + assert mod._symbol.tojson() == mod2._symbol.tojson() + dict_equ(mod.get_params()[0], mod2.get_params()[0]) + dict_equ(mod._updater.states, mod2._updater.states) + + # multi device + mod = mx.mod.Module(sym, ('data',), context=[mx.cpu(0), mx.cpu(1)]) + mod.bind(data_shapes=[('data', (10, 10))]) + mod.init_params() + mod.init_optimizer(optimizer_params={'learning_rate':0.1, 'momentum':0.9}) + mod.update() + mod.save_checkpoint('test', 0, save_optimizer_states=True) + + mod2 = mx.mod.Module.load('test', 0, load_optimizer_states=True, data_names=('data',)) + mod2.bind(data_shapes=[('data', (10, 10))]) + mod2.init_optimizer(optimizer_params={'learning_rate':0.1, 'momentum':0.9}) + assert mod._symbol.tojson() == mod2._symbol.tojson() + dict_equ(mod.get_params()[0], mod2.get_params()[0]) + dict_equ(mod._kvstore._updater.states, mod2._updater.states) + os.putenv('MXNET_UPDATE_ON_KVSTORE', previous_update_on_kvstore) + + +@with_seed() +def test_bucketing_save_load(): + previous_update_on_kvstore = os.getenv('MXNET_UPDATE_ON_KVSTORE', "1") + os.putenv('MXNET_UPDATE_ON_KVSTORE', '1') + def dict_equ(a, b): + assert set(a) == set(b) + for k in a: + assert (a[k].asnumpy() == b[k].asnumpy()).all() + + + len_vocab = 50 + num_embed = 25 + num_epochs = 5 + batch_size = 128 + num_layers = 2 + num_hidden = 25 + buckets = [5, 10, 20, 30, 40] + invalid_label = -1 + num_sentence=1000 + + stack = mx.rnn.SequentialRNNCell() + for i in range(num_layers): + stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i)) + + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=len_vocab, + output_dim=num_embed, name='embed') + stack.reset() + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=len_vocab, name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + loss = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + + return loss, ('data',), ('softmax_label',) + + model = train_model(context=mx.current_context()) + model.save_checkpoint("test", 0) + data_train, data_val = prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence) + mod2 = mx.mod.BucketingModule.load('test', 0, sym_gen=sym_gen, + default_bucket_key=data_train.default_bucket_key) + + mod2.bind(data_shapes=data_train.provide_data, + label_shapes=data_train.provide_label) + + for bucket_key in model._buckets.keys(): + dict_equ(model._buckets[model._default_bucket_key].get_params()[0], + mod2._buckets[mod2._default_bucket_key].get_params()[0]) + mod2.fit( + train_data=data_train, + eval_data=data_val, + eval_metric=mx.metric.Perplexity(invalid_label), # Use Perplexity for multiclass classification. + kvstore='device', + optimizer='sgd', + optimizer_params={'learning_rate': 0.01, + 'momentum': 0, + 'wd': 0.00001}, + initializer=mx.init.Xavier(factor_type="in", magnitude=2.34), + num_epoch=num_epochs, + batch_end_callback=mx.callback.Speedometer(batch_size, 50)) + os.putenv('MXNET_UPDATE_ON_KVSTORE', previous_update_on_kvstore) + + +@with_seed() +def test_module_reshape(): + data = mx.sym.Variable('data') + sym = mx.sym.FullyConnected(data, num_hidden=20, name='fc') + + dshape = (7, 20) + mod = mx.mod.Module(sym, ('data',), None, context=[mx.cpu(0), mx.cpu(1)]) + mod.bind(data_shapes=[('data', dshape)]) + mod.init_params() + mod.init_optimizer(optimizer_params={'learning_rate': 1}) + + mod.forward(mx.io.DataBatch(data=[mx.nd.ones(dshape)], + label=None)) + mod.backward([mx.nd.ones(dshape)]) + mod.update() + assert mod.get_outputs()[0].shape == dshape + assert (mod.get_params()[0]['fc_bias'].asnumpy() == -1).all() + + dshape = (14, 20) + mod.reshape(data_shapes=[('data', dshape)]) + mod.forward(mx.io.DataBatch(data=[mx.nd.ones(dshape)], + label=None)) + mod.backward([mx.nd.ones(dshape)]) + mod.update() + assert mod.get_outputs()[0].shape == dshape + assert (mod.get_params()[0]['fc_bias'].asnumpy() == -3).all() + + +@with_seed() +def test_module_states(): + stack = mx.rnn.SequentialRNNCell() + for i in range(2): + stack.add(mx.rnn.LSTMCell(num_hidden=20, prefix='lstm_l%d_'%i)) + begin_state = stack.begin_state(func=mx.sym.Variable) + _, states = stack.unroll(10, begin_state=begin_state, inputs=mx.sym.Variable('data')) + + state_names = [i.name for i in begin_state] + mod = mx.mod.Module(mx.sym.Group(states), context=[mx.cpu(0), mx.cpu(1)], + label_names=None, state_names=state_names) + mod.bind(data_shapes=[('data', (5, 10))], label_shapes=None, for_training=False) + mod.init_params() + batch = mx.io.DataBatch(data=[mx.nd.zeros((5, 10))], label=[]) + + mod.set_states(value=1) + mod.forward(batch) + out = mod.get_outputs(merge_multi_context=False) + out1 = mod.get_outputs(merge_multi_context=True) + + mod.set_states(states=out) + mod.forward(batch) + out2 = mod.get_outputs(merge_multi_context=True) + + for x1, x2 in zip(out1, out2): + assert not mx.test_utils.almost_equal(x1.asnumpy(), x2.asnumpy(), rtol=1e-3) + + +@with_seed() +def test_module_switch_bucket(): + vocab_dim = 5000 + num_hidden = 100 + num_embedding = 100 + num_layer = 2 + default_key = 10 + test_key = 5 + batch_size = 32 + contexts = [mx.cpu(0)] + initializer = mx.init.Xavier(factor_type="in", magnitude=2.34) + + #generate symbols for an LSTM network + def sym_gen(seq_len): + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + embed = mx.sym.Embedding(data=data, input_dim=vocab_dim, + output_dim=num_embedding) + stack = mx.rnn.SequentialRNNCell() + for i in range(num_layer): + stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_'%i)) + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=vocab_dim, name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + + return pred, ('data',), ('softmax_label',) + + def create_bucketing_module(key): + model = mx.mod.BucketingModule( + sym_gen = sym_gen, + default_bucket_key = key, + context = contexts) + model.bind([('data', (batch_size, key))], + [('softmax_label', (batch_size, key))], True, False) + model.init_params(initializer=initializer) + return model + #initialize the bucketing module with the default bucket key + bucketing_model = create_bucketing_module(default_key) + #check name + assert bucketing_model.symbol.list_arguments()[1] == "embedding0_weight",\ + "Error in assigning names for args in BucketingModule" + + #switch to test_key + bucketing_model.switch_bucket(test_key, [('data', (batch_size, test_key))], + [('softmax_label', (batch_size, test_key))]) + total_bytes_before = bucketing_model._buckets[default_key]._total_exec_bytes + + #remove test_key and switch again + del bucketing_model._buckets[test_key] + bucketing_model.switch_bucket(test_key, [('data', (batch_size, test_key))], + [('softmax_label', (batch_size, test_key))]) + total_bytes_after = bucketing_model._buckets[default_key]._total_exec_bytes + #the default bucket is expected to reuse the bytes allocated + assert total_bytes_after == total_bytes_before + + +# roywei: Getting rid of fixed seed as flakiness could not be reproduced, +# tracked at: https://github.com/apache/incubator-mxnet/issues/11705 +@with_seed() +def test_module_set_params(): + # data iter + data = mx.nd.array([[0.05, .10]]); + label = mx.nd.array([[.01, 0.99]]); + train_data = mx.io.NDArrayIter(data, label, batch_size=1) + + # symbols + x = mx.symbol.Variable('data') + x = mx.symbol.FullyConnected(name='fc_0', data=x, num_hidden=2) + x = mx.symbol.Activation(name="act_0", data=x, act_type='sigmoid') + x = mx.symbol.FullyConnected(name='fc_1', data=x, num_hidden=2) + x = mx.symbol.Activation(name="act_1", data=x, act_type='sigmoid') + x = mx.symbol.LinearRegressionOutput(data=x, name='softmax', grad_scale=2) + + # create module + mod = mx.mod.Module(x, context=[mx.cpu()]); + mod.bind(train_data.provide_data, label_shapes=train_data.provide_label, + for_training=True) + + arg_params_correct = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]), + 'fc_1_bias' : mx.nd.array([.60, .60])} + + arg_params_missing = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]])} + + arg_params_extra = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]), + 'fc_1_bias' : mx.nd.array([.60, .60]), + 'fc_2_weight': mx.nd.array([.60, .60])} + + arg_params_missing_extra = {'fc_2_weight': mx.nd.array([.60, .60])} + + # test regular set_params + mod.set_params(force_init=True, arg_params=arg_params_correct, aux_params={}) + + # test allow missing + mod.set_params(force_init=True, arg_params=arg_params_missing, aux_params={}, allow_missing=True) + assertRaises(RuntimeError, mod.set_params, + force_init=True, arg_params=arg_params_missing, + aux_params={}, allow_missing=False) + + # test allow extra + mod.set_params(force_init=True, arg_params=arg_params_extra, aux_params={}, allow_missing=True, allow_extra=True) + assertRaises(ValueError, mod.set_params, + force_init=True, arg_params=arg_params_extra, + aux_params={}, allow_missing=True, allow_extra=False) + + # test allow missing + extra, + assertRaises(RuntimeError, mod.set_params, + force_init=True, arg_params=arg_params_missing_extra, + aux_params={}, allow_missing=False, allow_extra=False) + + # test allow missing + extra, this will throw a runtime error + assertRaises(ValueError, mod.set_params, + force_init=True, arg_params=arg_params_missing_extra, + aux_params={}, allow_missing=True, allow_extra=False) + + +@with_seed() +def test_monitor(): + # data iter + data = mx.nd.array([[0.05, .10]]); + label = mx.nd.array([[.01, 0.99]]); + train_data = mx.io.NDArrayIter(data, label, batch_size=1) + + # symbols + x = mx.symbol.Variable('data') + x = mx.symbol.FullyConnected(name='fc_0', data=x, num_hidden=2) + x = mx.symbol.Activation(name="act_0", data=x, act_type='sigmoid') + x = mx.symbol.FullyConnected(name='fc_1', data=x, num_hidden=2) + x = mx.symbol.Activation(name="act_1", data=x, act_type='sigmoid') + x = mx.symbol.LinearRegressionOutput(data=x, name='softmax', grad_scale=2) + + # create monitor + def mean_abs(x): + sum_abs = mx.ndarray.sum(mx.ndarray.abs(x)) + return mx.ndarray.divide(sum_abs, reduce(lambda x, y: x * y, x.shape)) + mon = mx.mon.Monitor(1, stat_func=mean_abs, pattern='.*', sort=True) + + # create module + mod = mx.mod.Module(x, context=[mx.cpu()]); + mod.bind(train_data.provide_data, label_shapes=train_data.provide_label, + for_training=True) + mod.install_monitor(mon) + arg_params = {'fc_0_weight': mx.nd.array([[.15, .20], [.25, .30]]), + 'fc_0_bias' : mx.nd.array([.35, .35]), + 'fc_1_weight': mx.nd.array([[.40, .45], [.50, .55]]), + 'fc_1_bias' : mx.nd.array([.60, .60])} + mod.init_params(arg_params=arg_params) + + data_iter = iter(train_data) + data_batch = next(data_iter) + mon.tic() + mod.forward_backward(data_batch) + res = mon.toc() + keys = ['act_0', 'act_1', 'data', 'fc_0', 'fc_1', 'softmax'] + mon_result_counts = [0, 0, 0, 0, 0, 0] + assert(len(res) == 21) + for n, k, v in res: + for idx, key in enumerate(keys): + if k.startswith(key): + mon_result_counts[idx] += 1 + break + assert(mon_result_counts == [2, 2, 1, 6, 6, 4]) + +@with_seed() +def test_executor_group(): + def get_rnn_sym(num_layers, num_words, num_hidden, num_embed, seq_len, sparse_embedding): + stack = mx.rnn.SequentialRNNCell() + for i in range(num_layers): + stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i)) + data = mx.sym.Variable('data') + label = mx.sym.Variable('softmax_label') + if sparse_embedding: + embed_weight = mx.sym.Variable('embed_weight', stype='row_sparse') + embed = mx.sym.contrib.SparseEmbedding(data=data, input_dim=num_words, + weight=embed_weight, output_dim=num_embed, + name='embed') + else: + embed = mx.sym.Embedding(data=data, input_dim=num_words, + output_dim=num_embed, name='embed') + + stack.reset() + outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True) + + pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden)) + pred = mx.sym.FullyConnected(data=pred, num_hidden=num_words, name='pred') + + label = mx.sym.Reshape(label, shape=(-1,)) + pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax') + return pred + + def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=None, + extra_args=None, check_shared_grad=True): + # Test shared data arrays + for i in range(len(exec_grp_shared.execs)): + # test same shared_data_arrays for two exec groups + shared_data_array1 = exec_grp_shared.shared_data_arrays[i] + shared_data_array2 = exec_grp_created.shared_data_arrays[i] + if extra_args is not None: + assert len(shared_data_array1) == len(extra_args),\ + "exec_grp_shared.shared_data_arrays[%d] should have same number of args as extra_args" + assert len(shared_data_array1) == len(shared_data_array2),\ + "length of shared_data_array of the shared executor group not equal to the created executor group" + for k, v in shared_data_array1.items(): + if extra_args is not None: + assert k in extra_args, "arg %s is not in extra_args" % k + assert k in shared_data_array2,\ + "arg %s of the shared executor group not in the shared_data_array of the created executor group" % k + assert mx.test_utils.same_array(v, shared_data_array2[k]) + + for data_name, array in exec_grp_shared.shared_data_arrays[i].items(): + assert data_name in exec_grp_created.shared_data_arrays[i], \ + "Shared input data '%s' is not in " \ + "shared_data_arrays of created executor group." % (data_name) + assert mx.test_utils.same_array(array, exec_grp_created.shared_data_arrays[i][data_name]), \ + "Shared input data '%s' does not share memory." % (data_name) + + # Test shared argument arrays and gradient arrays + exec_shared = exec_grp_shared.execs[i] + exec_created = exec_grp_created.execs[i] + if shared_arg_names is not None: + # test shared arguments + for arg_name in shared_arg_names: + assert arg_name in exec_created.arg_dict, \ + "Shared argument '%s' is not in arg_dict of created executor group." % (arg_name) + assert mx.test_utils.same_array(exec_shared.arg_dict[arg_name], exec_created.arg_dict[arg_name]), \ + "Shared argument '%s' does not share memory." % (arg_name) + # test shared argument gradients + if check_shared_grad: + for arg_name in shared_arg_names: + assert arg_name in exec_created.grad_dict, \ + "Shared argument gradient '%s' is not in " \ + "grad_dict of created executor group." % (arg_name) + assert mx.test_utils.same_array(exec_shared.grad_dict[arg_name], \ + exec_created.grad_dict[arg_name]), \ + "Shared argument gradient '%s' does not share memory." % (arg_name) + + for arg_name, grad in exec_grp_shared.grad_req.items(): + assert grad == exec_grp_created.grad_req[arg_name], \ + "Gradient requirements for shared argument '%s' are inconsistent. " \ + "Shared executor group requires '%s' while created executor group requires '%s'" \ + %(arg_name, grad, exec_grp_created.grad_req[arg_name]) + + def check_shared_exec_group(sparse_embedding): + # generate an rnn sym with #layers=5 + sym = get_rnn_sym(num_layers=3, num_words=num_words, num_hidden=num_hidden, + num_embed=num_embed, seq_len=max_bucket_size, + sparse_embedding=sparse_embedding) + arg_names1 = sym.list_arguments() + input_names = [name[0] for name in data_shapes] + [name[0] for name in label_shapes] + shared_arg_names = [name for name in arg_names1 if name not in input_names] + exec_group1 = DataParallelExecutorGroup(symbol=sym, contexts=contexts, + workload=workload, data_shapes=data_shapes, + label_shapes=label_shapes, param_names=shared_arg_names, + for_training=True, inputs_need_grad=False) + + # shared_data_arrays should only have input "data" and "softmax_label" arrays + for i in range(len(contexts)): + assert len(exec_group1.shared_data_arrays[i]) == len(input_names),\ + "exec_group1.shared_data_arrays[%d] should have the same number of names as in input_names" % i + for name in input_names: + assert name in exec_group1.shared_data_arrays[i],\ + "arg %s should be in exec_group1.shared_data_arrays[%d]" % (name, i) + + # generate an rnn sym with #layers=5 + sym = get_rnn_sym(num_layers=5, num_words=num_words, num_hidden=num_hidden, + num_embed=num_embed, seq_len=max_bucket_size, + sparse_embedding=sparse_embedding) + arg_names2 = sym.list_arguments() + exec_group2 = DataParallelExecutorGroup(symbol=sym, contexts=contexts, + workload=workload, data_shapes=data_shapes, + label_shapes=label_shapes, param_names=shared_arg_names, + for_training=True, inputs_need_grad=False, + shared_group=exec_group1) + extra_args = [name for name in arg_names2 if name not in shared_arg_names] + check_shared_grad = not sparse_embedding + test_shared_exec_group(exec_grp_shared=exec_group1, exec_grp_created=exec_group2, + shared_arg_names=shared_arg_names, extra_args=extra_args, + check_shared_grad=check_shared_grad) + + contexts = [mx.cpu(0), mx.cpu(1)] + workload = [1] * len(contexts) + batch_size = 32 + max_bucket_size = 80 + num_words = 1000 + num_hidden = 100 + num_embed = 200 + data_shapes = [('data', (batch_size, max_bucket_size))] + label_shapes = [('softmax_label', (batch_size, max_bucket_size))] + sparse_embedding_opt = [True, False] + for opt in sparse_embedding_opt: + check_shared_exec_group(opt) + +@with_seed() +def test_factorization_machine_module(): + """ Test factorization machine model with sparse operators """ + # this unit test is to test the flow, training accuracy is tested in another test + def check_factorization_machine_module(num_epochs=None): + print("check_factorization_machine_module") + + def fm(factor_size, feature_dim, init): + x = mx.symbol.Variable("data", stype='csr') + v = mx.symbol.Variable("v", shape=(feature_dim, factor_size), + init=init, stype='row_sparse') + + w1_weight = mx.symbol.var('w1_weight', shape=(feature_dim, 1), + init=init, stype='row_sparse') + w1_bias = mx.symbol.var('w1_bias', shape=(1)) + w1 = mx.symbol.broadcast_add(mx.symbol.dot(x, w1_weight), w1_bias) + + v_s = mx.symbol._internal._square_sum(data=v, axis=1, keepdims=True) + x_s = mx.symbol.square(data=x) + bd_sum = mx.sym.dot(x_s, v_s) + + w2 = mx.symbol.dot(x, v) + w2_squared = 0.5 * mx.symbol.square(data=w2) + + w_all = mx.symbol.Concat(w1, w2_squared, dim=1) + sum1 = mx.symbol.sum(data=w_all, axis=1, keepdims=True) + sum2 = 0.5 * mx.symbol.negative(bd_sum) + model = mx.sym.elemwise_add(sum1, sum2) + + y = mx.symbol.Variable("label") + model = mx.symbol.LinearRegressionOutput(data=model, label=y) + return model + + # model + init = mx.initializer.Normal(sigma=0.01) + factor_size = 4 + feature_dim = 10000 + model = fm(factor_size, feature_dim, init) + + # data iter + num_batches = 5 + batch_size = 64 + num_samples = batch_size * num_batches + # generate some random csr data + csr_nd = rand_ndarray((num_samples, feature_dim), 'csr', 0.1) + label = mx.nd.ones((num_samples,1)) + # the alternative is to use LibSVMIter + train_iter = mx.io.NDArrayIter(data=csr_nd, + label={'label':label}, + batch_size=batch_size, + last_batch_handle='discard') + # create module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label']) + # allocate memory by given the input data and lable shapes + mod.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label) + # initialize parameters by uniform random numbers + mod.init_params(initializer=init) + + # use Sparse SGD with learning rate 0.1 to train + sgd = mx.optimizer.SGD(momentum=0.1, clip_gradient=5.0, learning_rate=0.01, + rescale_grad=1.0/batch_size) + mod.init_optimizer(optimizer=sgd) + if num_epochs is None: + num_epochs = 50 + expected_accuracy = 0.02 + + # use accuracy as the metric + metric = mx.metric.create('MSE') + # train 'num_epochs' epoch + for epoch in range(num_epochs): + train_iter.reset() + metric.reset() + for batch in train_iter: + mod.forward(batch, is_train=True) # compute predictions + mod.update_metric(metric, batch.label) # accumulate prediction accuracy + mod.backward() # compute gradients + mod.update() # update parameters + print('Epoch %d, Training %s' % (epoch, metric.get())) + if num_epochs > 1: + assert(metric.get()[1] < expected_accuracy) + + check_factorization_machine_module() + +@with_seed() +def test_module_initializer(): + def regression_model(m): + x = mx.symbol.var("data", stype='csr') + v = mx.symbol.var("v", shape=(m, 1), init=mx.init.Uniform(scale=.1), + stype='row_sparse') + model = mx.symbol.dot(lhs=x, rhs=v) + y = mx.symbol.Variable("label") + model = mx.symbol.LinearRegressionOutput(data=model, label=y, name="out") + return model + + n, m = 128, 100 + model = regression_model(m) + + data = mx.nd.zeros(shape=(n, m), stype='csr') + label = mx.nd.zeros((n, 1)) + iterator = mx.io.NDArrayIter(data=data, label={'label':label}, + batch_size=n, last_batch_handle='discard') + + # create module + mod = mx.mod.Module(symbol=model, data_names=['data'], label_names=['label']) + mod.bind(data_shapes=iterator.provide_data, label_shapes=iterator.provide_label) + mod.init_params() + v = mod._arg_params['v'] + assert(v.stype == 'row_sparse') + assert(np.sum(v.asnumpy()) != 0) + +@with_seed() +def test_forward_reshape(): + num_class=10 + data1 = mx.sym.Variable('data1') + data2 = mx.sym.Variable('data2') + conv1 = mx.sym.Convolution(data=data1, kernel=(2, 2), num_filter=2, stride=(2, 2)) + conv2 = mx.sym.Convolution(data=data2, kernel=(3, 3), num_filter=3, stride=(1, 1)) + pooling1 = mx.sym.Pooling(data=conv1, kernel=(2, 2), stride=(1, 1), pool_type="avg") + pooling2 = mx.sym.Pooling(data=conv2, kernel=(2, 2), stride=(1, 1), pool_type="max") + flatten1 = mx.sym.flatten(data=pooling1) + flatten2 = mx.sym.flatten(data=pooling2) + sum = mx.sym.sum(data=flatten1, axis=1) + mx.sym.sum(data=flatten2, axis=1) + fc = mx.sym.FullyConnected(data=sum, num_hidden=num_class) + sym = mx.sym.SoftmaxOutput(data=fc, name='softmax') + + dshape1 = (10, 3, 64, 64) + dshape2 = (10, 3, 32, 32) + lshape = (10,) + + mod = mx.mod.Module(symbol=sym, data_names=['data1', 'data2'], + label_names=['softmax_label']) + mod.bind(data_shapes=[('data1', dshape1), ('data2', dshape2)], + label_shapes=[('softmax_label', lshape)]) + mod.init_params() + mod.init_optimizer(optimizer_params={'learning_rate': 0.01}) + + # Train with original data shapes + data_batch = mx.io.DataBatch(data=[mx.nd.random.uniform(0, 9, dshape1), + mx.nd.random.uniform(5, 15, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + # Train with different batch size + dshape1 = (3, 3, 64, 64) + dshape2 = (3, 3, 32, 32) + lshape = (3,) + data_batch = mx.io.DataBatch(data=[mx.nd.random.uniform(0, 9, dshape1), + mx.nd.random.uniform(5, 15, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + dshape1 = (20, 3, 64, 64) + dshape2 = (20, 3, 32, 32) + lshape = (20,) + data_batch = mx.io.DataBatch(data=[mx.nd.random.uniform(3, 5, dshape1), + mx.nd.random.uniform(10, 25, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + #Train with both different batch size and data shapes + dshape1 = (20, 3, 120, 120) + dshape2 = (20, 3, 32, 64) + lshape = (20,) + data_batch = mx.io.DataBatch(data=[mx.nd.random.uniform(0, 9, dshape1), + mx.nd.random.uniform(5, 15, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + dshape1 = (5, 3, 28, 40) + dshape2 = (5, 3, 24, 16) + lshape = (5,) + data_batch = mx.io.DataBatch(data=[mx.nd.random.uniform(0, 9, dshape1), + mx.nd.random.uniform(15, 25, dshape2)], + label=[mx.nd.ones(lshape)]) + mod.forward(data_batch) + assert mod.get_outputs()[0].shape == tuple([lshape[0], num_class]) + mod.backward() + mod.update() + + #Test score + dataset_shape1 = (30, 3, 30, 30) + dataset_shape2 = (30, 3, 20, 40) + labelset_shape = (30,) + + eval_dataiter = mx.io.NDArrayIter(data=[mx.nd.random.uniform(0, 9, dataset_shape1), + mx.nd.random.uniform(15, 25, dataset_shape2)], + label=[mx.nd.ones(labelset_shape)], + batch_size=5) + assert len(mod.score(eval_data=eval_dataiter, eval_metric='acc')) == 1 + + #Test prediction + dshape1 = (1, 3, 30, 30) + dshape2 = (1, 3, 20, 40) + dataset_shape1 = (10, 3, 30, 30) + dataset_shape2 = (10, 3, 20, 40) + + pred_dataiter = mx.io.NDArrayIter(data=[mx.nd.random.uniform(0, 9, dataset_shape1), + mx.nd.random.uniform(15, 25, dataset_shape2)]) + mod.bind(data_shapes=[('data1', dshape1), ('data2', dshape2)], + for_training=False, force_rebind=True) + assert mod.predict(pred_dataiter).shape == tuple([10, num_class]) + +@with_seed() +def test_forward_types(): + #Test forward with other data batch API + Batch = namedtuple('Batch', ['data']) + data = mx.sym.Variable('data') + out = data * 2 + mod = mx.mod.Module(symbol=out, label_names=None) + mod.bind(data_shapes=[('data', (1, 10))]) + mod.init_params() + data1 = [mx.nd.ones((1, 10))] + mod.forward(Batch(data1)) + assert mod.get_outputs()[0].shape == (1, 10) + data2 = [mx.nd.ones((3, 5))] + mod.forward(Batch(data2)) + assert mod.get_outputs()[0].shape == (3, 5) + + #Test forward with other NDArray and np.ndarray inputs + data = mx.sym.Variable('data') + out = data * 2 + mod = mx.mod.Module(symbol=out, label_names=None) + mod.bind(data_shapes=[('data', (1, 10))]) + mod.init_params() + data1 = mx.nd.ones((1, 10)) + assert mod.predict(data1).shape == (1, 10) + data2 = np.ones((1, 10)) + assert mod.predict(data1).shape == (1, 10) + + +def test_reference_single_batch_during_fit(): + """ + When using C++-based iterators, it's important that only a single batch is referenced at a time. Because C++ + iterators are exposed to the Python code through a C API, there is no concept of reference counting. Hence, + typically C++ iterators will deallocate a batch when next() is called on them. So, we need to make sure the Python + code only references a single batch at a time, otherwise the Python code will attempt to access freed memory, + resulting in either (a) garbage accuracy or (b) a segmentation fault. + """ + current_batch_i = None + + class MockBatch(object): + def __init__(self, i): + self.i = i + + @property + def label(self): + global current_batch_i + assert self.i == current_batch_i + + class MockTrainData(object): + def __init__(self, batches): + self._i = 0 + self._batches = batches + self.provide_data = None + self.provide_label = None + self.reset = lambda: None + + def __iter__(self): + self._i = 0 + return self + + def __next__(self): + global current_batch_i + + if self._i < self._batches: + current_batch_i = self._i + self._i += 1 + return MockBatch(current_batch_i) + raise StopIteration + + def next(self): + return self.__next__() + + mod = mx.mod.BaseModule() + + def empty_fn(*args, **kwargs): + pass + mod.bind = empty_fn + mod.init_params = empty_fn + mod.init_optimizer = empty_fn + mod.forward = empty_fn + mod.backward = empty_fn + mod.update = empty_fn + mod.update_metric = empty_fn + mod.get_params = lambda: (None, None) + + train_data = MockTrainData(batches=2) + mod.fit(train_data, num_epoch=1) + +@with_seed() +def test_bucket_module_grad_req(): + batch_size = 2 + def sym_gen(_): + data = mx.symbol.Variable('data') + weight = mx.symbol.Variable('a', shape=(1,), init=mx.init.One()) + sym = mx.sym.make_loss(mx.sym.broadcast_mul(data, weight)) + return sym, ('data',), None + + mod = mx.mod.BucketingModule(sym_gen=sym_gen, default_bucket_key=10) + mod.bind(data_shapes=[['data', (batch_size, )]], for_training=True, grad_req='write') + mod.init_params() + + mod.forward_backward(mx.io.DataBatch(data=[mx.nd.ones((batch_size,))], + label=None, + provide_data=[mx.io.DataDesc(name='data', shape=(batch_size, ), layout='N')], + bucket_key=10)) + assert(mod._curr_module._exec_group.execs[0].grad_dict['a'].asscalar() == batch_size) + + mod.forward_backward(mx.io.DataBatch(data=[mx.nd.ones((batch_size,))], + label=None, + provide_data=[mx.io.DataDesc(name='data', shape=(batch_size, ), layout='N')], + bucket_key=5)) + assert(mod._curr_module._exec_group.execs[0].grad_dict['a'].asscalar() == batch_size) + + mod = mx.mod.BucketingModule(sym_gen=sym_gen, default_bucket_key=10) + mod.bind(data_shapes=[['data', (batch_size, )]], for_training=True, grad_req='add') + mod.init_params() + + mod.forward_backward(mx.io.DataBatch(data=[mx.nd.ones((batch_size,))], + label=None, + provide_data=[mx.io.DataDesc(name='data', shape=(batch_size,), layout='N')], + bucket_key=10)) + assert(mod._curr_module._exec_group.execs[0].grad_dict['a'].asscalar() == batch_size) + + mod.forward_backward(mx.io.DataBatch(data=[mx.nd.ones((batch_size,))], + label=None, + provide_data=[mx.io.DataDesc(name='data', shape=(batch_size,), layout='N')], + bucket_key=5)) + assert mod._curr_module._grad_req == 'add' + assert(mod._curr_module._exec_group.execs[0].grad_dict['a'].asscalar() == 2 * batch_size) + + +def test_module_update_no_pragram(): + # test module to do update on layers without params + data_shape = (10, 10) + data = mx.sym.Variable('data') + out = mx.sym.Dropout(data, 0.5) + mod = mx.mod.Module(out) + mod.bind(data_shapes=[('data', data_shape)]) + mod.init_params() + mod.init_optimizer() + data_batch = mx.io.DataBatch([nd.ones(data_shape)]) + mod.forward_backward(data_batch) + mod.update() + assert(mod.get_outputs()[0].shape == data_shape) + + +def test_module_init_optimizer(): + def get_module_idx2name(mod): + idx2name = {} + idx2name.update(enumerate(mod._exec_group.param_names)) + return idx2name + + data = mx.sym.Variable('data') + sym = mx.sym.FullyConnected(data, num_hidden=20, name='fc') + batch_size = 8 + opt_params = {'learning_rate': 1, 'rescale_grad': 1.0 / batch_size} + + # Pass an optimizer str + mod1 = mx.mod.Module(sym, ('data',), None, context=mx.cpu(0)) + mod1.bind(data_shapes=[('data', (batch_size, 20))]) + mod1.init_params() + mod1.init_optimizer(optimizer='sgd', optimizer_params=opt_params) + assert mod1._optimizer.idx2name == get_module_idx2name(mod1) + + # Pass an Optimizer object + mod2 = mx.mod.Module(sym, ('data',), None, context=mx.cpu(0)) + mod2.bind(data_shapes=[('data', (batch_size, 20))]) + mod2.init_params() + opt = mx.optimizer.SGD(**opt_params) + mod2.init_optimizer(optimizer=opt) + assert mod2._optimizer.idx2name == get_module_idx2name(mod2) + diff --git a/tools/caffe_converter/test_converter.py b/tools/caffe_converter/test_converter.py new file mode 100644 index 000000000000..49f8bdb167c2 --- /dev/null +++ b/tools/caffe_converter/test_converter.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Test converted models +""" +import os +import argparse +import sys +import logging +import mxnet as mx +from convert_caffe_modelzoo import convert_caffe_model, get_model_meta_info, download_caffe_model +from compare_layers import convert_and_compare_caffe_to_mxnet + +curr_path = os.path.abspath(os.path.dirname(__file__)) +sys.path.append(os.path.join(curr_path, "../../example/image-classification")) +from test_score import download_data # pylint: disable=wrong-import-position +from score import score # pylint: disable=wrong-import-position +logging.basicConfig(level=logging.DEBUG) + +def test_imagenet_model_performance(model_name, val_data, gpus, batch_size): + """test model performance on imagenet """ + logging.info('test performance of model: %s', model_name) + meta_info = get_model_meta_info(model_name) + [model_name, mean] = convert_caffe_model(model_name, meta_info) + sym, arg_params, aux_params = mx.model.load_checkpoint(model_name, 0) + acc = [mx.metric.create('acc'), mx.metric.create('top_k_accuracy', top_k=5)] + if isinstance(mean, str): + mean_args = {'mean_img':mean} + else: + mean_args = {'rgb_mean':','.join([str(i) for i in mean])} + + print(val_data) + gpus_string = '' if gpus[0] == -1 else ','.join([str(i) for i in gpus]) + (speed,) = score(model=(sym, arg_params, aux_params), + data_val=val_data, + label_name='prob_label', + metrics=acc, + gpus=gpus_string, + batch_size=batch_size, + max_num_examples=500, + **mean_args) + logging.info('speed : %f image/sec', speed) + for a in acc: + logging.info(a.get()) + max_performance_diff_allowed = 0.03 + assert acc[0].get()[1] > meta_info['top-1-acc'] - max_performance_diff_allowed + assert acc[1].get()[1] > meta_info['top-5-acc'] - max_performance_diff_allowed + + +def test_model_weights_and_outputs(model_name, image_url, gpu): + """ + Run the layer comparison on one of the known caffe models. + :param model_name: available models are listed in convert_caffe_modelzoo.py + :param image_url: image file or url to run inference on + :param gpu: gpu to use, -1 for cpu + """ + + logging.info('test weights and outputs of model: %s', model_name) + meta_info = get_model_meta_info(model_name) + + (prototxt, caffemodel, mean) = download_caffe_model(model_name, meta_info, dst_dir='./model') + convert_and_compare_caffe_to_mxnet(image_url, gpu, prototxt, caffemodel, mean, + mean_diff_allowed=1e-03, max_diff_allowed=1e-01) + + +def main(): + """Entrypoint for test_converter""" + parser = argparse.ArgumentParser(description='Test Caffe converter') + parser.add_argument('--cpu', action='store_true', help='use cpu?') + parser.add_argument('--image_url', type=str, + default='https://github.com/dmlc/web-data/raw/master/mxnet/doc/'\ + 'tutorials/python/predict_image/cat.jpg', + help='input image to test inference, can be either file path or url') + args = parser.parse_args() + if args.cpu: + gpus = [-1] + default_batch_size = 32 + else: + num_gpus = mx.context.num_gpus() + assert num_gpus, 'At least one GPU is needed to run test_converter in GPU mode' + default_batch_size = 32 * num_gpus + + models = ['bvlc_googlenet', 'vgg-16', 'resnet-50'] + + val = download_data() + for m in models: + test_model_weights_and_outputs(m, args.image_url, gpus[0]) + # Build/testing machines tend to be short on GPU memory + this_batch_size = default_batch_size / 4 if m == 'vgg-16' else default_batch_size + test_imagenet_model_performance(m, val, gpus, this_batch_size) + +if __name__ == '__main__': + main()