import os\n",
- "import numpy as np\n",
- "\n",
- "import tensorflow as tf\n",
- "from tensorflow.python.keras import utils\n",
- "\n",
- "from tensorflow.keras.models import Sequential\n",
- "from tensorflow.keras.layers import Dense, Dropout, Flatten, Activation, Reshape\n",
- "from tensorflow.keras.layers import Convolution2D, MaxPooling2D\n",
- "from tensorflow.keras.layers import BatchNormalization\n",
- "from tensorflow.keras.optimizers import SGD, Adam\n",
- "from tensorflow import keras\n",
- "\n",
- "import mnist_dataset\n",
- "\n",
- "\n",
- "def save_mod(model, mod_path):\n",
- " print('Save to {}'.format(mod_path))\n",
- " tf.saved_model.save(model, mod_path)\n",
- "\n",
- "\n",
- "def load_mod(model_file):\n",
- " model = tf.keras.models.load_model(model_file)\n",
- " print('Load from {}'.format(model_file))\n",
- " return model\n",
- "\n",
- "def save_frezon_pb(model, mod_path):\n",
- " # Convert Keras model to ConcreteFunction\n",
- " full_model = tf.function(lambda x: model(x))\n",
- " concrete_function = full_model.get_concrete_function(\n",
- " x=tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype))\n",
- "\n",
- " # Get frozen ConcreteFunction\n",
- " frozen_model = convert_variables_to_constants_v2(concrete_function)\n",
- "\n",
- " # Generate frozen pb\n",
- " tf.io.write_graph(graph_or_graph_def=frozen_model.graph,\n",
- " logdir=".",\n",
- " name=mod_path,\n",
- " as_text=False)\n",
- "\n",
- "\n",
- "def load_pb(in_model):\n",
- " detection_graph = tf.compat.v1.Graph()\n",
- " with detection_graph.as_default():\n",
- " od_graph_def = tf.compat.v1.GraphDef()\n",
- " with tf.compat.v1.gfile.GFile(in_model, 'rb') as fid:\n",
- " serialized_graph = fid.read()\n",
- " od_graph_def.ParseFromString(serialized_graph)\n",
- " tf.compat.v1.import_graph_def(od_graph_def, name='')\n",
- "\n",
- " return detection_graph\n",
- "\n",
- "def read_data():\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = mnist_dataset.read_data()\n",
- " return x_train, y_train, label_train, x_test, y_test, label_test\n",
- "\n",
- "def create_model(w, c, classes):\n",
- " model = Sequential()\n",
- " model.add(Convolution2D(96, 11, input_shape=(w, w, c), padding='same'))\n",
- " model.add(Activation('relu'))\n",
- " model.add(MaxPooling2D(pool_size=(2, 2)))\n",
- "\n",
- " model.add(Convolution2D(256, 5, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- " model.add(MaxPooling2D(pool_size=(2, 2)))\n",
- "\n",
- " model.add(Convolution2D(384, 3, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Convolution2D(384, 3, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Convolution2D(256, 3, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Convolution2D(256, 7))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Flatten())\n",
- " model.add(Dense(classes))\n",
- " model.add(Activation('softmax'))\n",
- "\n",
- " opt = Adam(learning_rate=0.001)\n",
- " model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
- " return model\n",
- "\n",
- "def train_mod(model, data, epochs=3):\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = data\n",
- " model.fit(x_train, y_train, epochs=epochs, batch_size=600, validation_data=(x_test, y_test), verbose=1)\n",
- " score = model.evaluate(x_test, y_test, verbose=0)\n",
- " print('Test score:', score[0])\n",
- " print('Test accuracy:', score[1])\n",
- "\n",
- "def main():\n",
- " data = read_data()\n",
- "\n",
- " classes = 10\n",
- " w = 28\n",
- " c = 1\n",
- " model = create_model(w ,c, classes)\n",
- " model.summary()\n",
- "\n",
- " epochs = 3\n",
- " train_mod(model, data, epochs)\n",
- " save_mod(model, "alexnet_mnist_fp32_mod")\n",
- "\n",
- "if __name__ == "__main__":\n",
- " main()\n",
- "
\n"
- ],
- "text/latex": [
- "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{os}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{numpy} \\PY{k}{as} \\PY{n+nn}{np}\n",
- "\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{tensorflow} \\PY{k}{as} \\PY{n+nn}{tf}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{python}\\PY{n+nn}{.}\\PY{n+nn}{keras} \\PY{k+kn}{import} \\PY{n}{utils}\n",
- "\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{keras}\\PY{n+nn}{.}\\PY{n+nn}{models} \\PY{k+kn}{import} \\PY{n}{Sequential}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{keras}\\PY{n+nn}{.}\\PY{n+nn}{layers} \\PY{k+kn}{import} \\PY{n}{Dense}\\PY{p}{,} \\PY{n}{Dropout}\\PY{p}{,} \\PY{n}{Flatten}\\PY{p}{,} \\PY{n}{Activation}\\PY{p}{,} \\PY{n}{Reshape}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{keras}\\PY{n+nn}{.}\\PY{n+nn}{layers} \\PY{k+kn}{import} \\PY{n}{Convolution2D}\\PY{p}{,} \\PY{n}{MaxPooling2D}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{keras}\\PY{n+nn}{.}\\PY{n+nn}{layers} \\PY{k+kn}{import} \\PY{n}{BatchNormalization}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{keras}\\PY{n+nn}{.}\\PY{n+nn}{optimizers} \\PY{k+kn}{import} \\PY{n}{SGD}\\PY{p}{,} \\PY{n}{Adam}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{tensorflow} \\PY{k+kn}{import} \\PY{n}{keras}\n",
- "\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{mnist\\PYZus{}dataset}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{save\\PYZus{}mod}\\PY{p}{(}\\PY{n}{model}\\PY{p}{,} \\PY{n}{mod\\PYZus{}path}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{Save to }\\PY{l+s+si}{\\PYZob{}\\PYZcb{}}\\PY{l+s+s1}{\\PYZsq{}}\\PY{o}{.}\\PY{n}{format}\\PY{p}{(}\\PY{n}{mod\\PYZus{}path}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{tf}\\PY{o}{.}\\PY{n}{saved\\PYZus{}model}\\PY{o}{.}\\PY{n}{save}\\PY{p}{(}\\PY{n}{model}\\PY{p}{,} \\PY{n}{mod\\PYZus{}path}\\PY{p}{)}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{load\\PYZus{}mod}\\PY{p}{(}\\PY{n}{model\\PYZus{}file}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{model} \\PY{o}{=} \\PY{n}{tf}\\PY{o}{.}\\PY{n}{keras}\\PY{o}{.}\\PY{n}{models}\\PY{o}{.}\\PY{n}{load\\PYZus{}model}\\PY{p}{(}\\PY{n}{model\\PYZus{}file}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{Load from }\\PY{l+s+si}{\\PYZob{}\\PYZcb{}}\\PY{l+s+s1}{\\PYZsq{}}\\PY{o}{.}\\PY{n}{format}\\PY{p}{(}\\PY{n}{model\\PYZus{}file}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{k}{return} \\PY{n}{model}\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{save\\PYZus{}frezon\\PYZus{}pb}\\PY{p}{(}\\PY{n}{model}\\PY{p}{,} \\PY{n}{mod\\PYZus{}path}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{c+c1}{\\PYZsh{} Convert Keras model to ConcreteFunction}\n",
- " \\PY{n}{full\\PYZus{}model} \\PY{o}{=} \\PY{n}{tf}\\PY{o}{.}\\PY{n}{function}\\PY{p}{(}\\PY{k}{lambda} \\PY{n}{x}\\PY{p}{:} \\PY{n}{model}\\PY{p}{(}\\PY{n}{x}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{concrete\\PYZus{}function} \\PY{o}{=} \\PY{n}{full\\PYZus{}model}\\PY{o}{.}\\PY{n}{get\\PYZus{}concrete\\PYZus{}function}\\PY{p}{(}\n",
- " \\PY{n}{x}\\PY{o}{=}\\PY{n}{tf}\\PY{o}{.}\\PY{n}{TensorSpec}\\PY{p}{(}\\PY{n}{model}\\PY{o}{.}\\PY{n}{inputs}\\PY{p}{[}\\PY{l+m+mi}{0}\\PY{p}{]}\\PY{o}{.}\\PY{n}{shape}\\PY{p}{,} \\PY{n}{model}\\PY{o}{.}\\PY{n}{inputs}\\PY{p}{[}\\PY{l+m+mi}{0}\\PY{p}{]}\\PY{o}{.}\\PY{n}{dtype}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{c+c1}{\\PYZsh{} Get frozen ConcreteFunction}\n",
- " \\PY{n}{frozen\\PYZus{}model} \\PY{o}{=} \\PY{n}{convert\\PYZus{}variables\\PYZus{}to\\PYZus{}constants\\PYZus{}v2}\\PY{p}{(}\\PY{n}{concrete\\PYZus{}function}\\PY{p}{)}\n",
- "\n",
- " \\PY{c+c1}{\\PYZsh{} Generate frozen pb}\n",
- " \\PY{n}{tf}\\PY{o}{.}\\PY{n}{io}\\PY{o}{.}\\PY{n}{write\\PYZus{}graph}\\PY{p}{(}\\PY{n}{graph\\PYZus{}or\\PYZus{}graph\\PYZus{}def}\\PY{o}{=}\\PY{n}{frozen\\PYZus{}model}\\PY{o}{.}\\PY{n}{graph}\\PY{p}{,}\n",
- " \\PY{n}{logdir}\\PY{o}{=}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{.}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{,}\n",
- " \\PY{n}{name}\\PY{o}{=}\\PY{n}{mod\\PYZus{}path}\\PY{p}{,}\n",
- " \\PY{n}{as\\PYZus{}text}\\PY{o}{=}\\PY{k+kc}{False}\\PY{p}{)}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{load\\PYZus{}pb}\\PY{p}{(}\\PY{n}{in\\PYZus{}model}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{detection\\PYZus{}graph} \\PY{o}{=} \\PY{n}{tf}\\PY{o}{.}\\PY{n}{compat}\\PY{o}{.}\\PY{n}{v1}\\PY{o}{.}\\PY{n}{Graph}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{k}{with} \\PY{n}{detection\\PYZus{}graph}\\PY{o}{.}\\PY{n}{as\\PYZus{}default}\\PY{p}{(}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{od\\PYZus{}graph\\PYZus{}def} \\PY{o}{=} \\PY{n}{tf}\\PY{o}{.}\\PY{n}{compat}\\PY{o}{.}\\PY{n}{v1}\\PY{o}{.}\\PY{n}{GraphDef}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{k}{with} \\PY{n}{tf}\\PY{o}{.}\\PY{n}{compat}\\PY{o}{.}\\PY{n}{v1}\\PY{o}{.}\\PY{n}{gfile}\\PY{o}{.}\\PY{n}{GFile}\\PY{p}{(}\\PY{n}{in\\PYZus{}model}\\PY{p}{,} \\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{rb}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)} \\PY{k}{as} \\PY{n}{fid}\\PY{p}{:}\n",
- " \\PY{n}{serialized\\PYZus{}graph} \\PY{o}{=} \\PY{n}{fid}\\PY{o}{.}\\PY{n}{read}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{n}{od\\PYZus{}graph\\PYZus{}def}\\PY{o}{.}\\PY{n}{ParseFromString}\\PY{p}{(}\\PY{n}{serialized\\PYZus{}graph}\\PY{p}{)}\n",
- " \\PY{n}{tf}\\PY{o}{.}\\PY{n}{compat}\\PY{o}{.}\\PY{n}{v1}\\PY{o}{.}\\PY{n}{import\\PYZus{}graph\\PYZus{}def}\\PY{p}{(}\\PY{n}{od\\PYZus{}graph\\PYZus{}def}\\PY{p}{,} \\PY{n}{name}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\n",
- "\n",
- " \\PY{k}{return} \\PY{n}{detection\\PYZus{}graph}\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{read\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{x\\PYZus{}train}\\PY{p}{,} \\PY{n}{y\\PYZus{}train}\\PY{p}{,} \\PY{n}{label\\PYZus{}train}\\PY{p}{,} \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test} \\PY{o}{=} \\PY{n}{mnist\\PYZus{}dataset}\\PY{o}{.}\\PY{n}{read\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{k}{return} \\PY{n}{x\\PYZus{}train}\\PY{p}{,} \\PY{n}{y\\PYZus{}train}\\PY{p}{,} \\PY{n}{label\\PYZus{}train}\\PY{p}{,} \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test}\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{create\\PYZus{}model}\\PY{p}{(}\\PY{n}{w}\\PY{p}{,} \\PY{n}{c}\\PY{p}{,} \\PY{n}{classes}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{model} \\PY{o}{=} \\PY{n}{Sequential}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Convolution2D}\\PY{p}{(}\\PY{l+m+mi}{96}\\PY{p}{,} \\PY{l+m+mi}{11}\\PY{p}{,} \\PY{n}{input\\PYZus{}shape}\\PY{o}{=}\\PY{p}{(}\\PY{n}{w}\\PY{p}{,} \\PY{n}{w}\\PY{p}{,} \\PY{n}{c}\\PY{p}{)}\\PY{p}{,} \\PY{n}{padding}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{same}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{relu}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{MaxPooling2D}\\PY{p}{(}\\PY{n}{pool\\PYZus{}size}\\PY{o}{=}\\PY{p}{(}\\PY{l+m+mi}{2}\\PY{p}{,} \\PY{l+m+mi}{2}\\PY{p}{)}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Convolution2D}\\PY{p}{(}\\PY{l+m+mi}{256}\\PY{p}{,} \\PY{l+m+mi}{5}\\PY{p}{,} \\PY{n}{padding}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{same}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{relu}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{MaxPooling2D}\\PY{p}{(}\\PY{n}{pool\\PYZus{}size}\\PY{o}{=}\\PY{p}{(}\\PY{l+m+mi}{2}\\PY{p}{,} \\PY{l+m+mi}{2}\\PY{p}{)}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Convolution2D}\\PY{p}{(}\\PY{l+m+mi}{384}\\PY{p}{,} \\PY{l+m+mi}{3}\\PY{p}{,} \\PY{n}{padding}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{same}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{relu}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Convolution2D}\\PY{p}{(}\\PY{l+m+mi}{384}\\PY{p}{,} \\PY{l+m+mi}{3}\\PY{p}{,} \\PY{n}{padding}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{same}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{relu}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Convolution2D}\\PY{p}{(}\\PY{l+m+mi}{256}\\PY{p}{,} \\PY{l+m+mi}{3}\\PY{p}{,} \\PY{n}{padding}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{same}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{relu}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Convolution2D}\\PY{p}{(}\\PY{l+m+mi}{256}\\PY{p}{,} \\PY{l+m+mi}{7}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{relu}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Flatten}\\PY{p}{(}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Dense}\\PY{p}{(}\\PY{n}{classes}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{add}\\PY{p}{(}\\PY{n}{Activation}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{softmax}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{opt} \\PY{o}{=} \\PY{n}{Adam}\\PY{p}{(}\\PY{n}{learning\\PYZus{}rate}\\PY{o}{=}\\PY{l+m+mf}{0.001}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{compile}\\PY{p}{(}\\PY{n}{optimizer}\\PY{o}{=}\\PY{n}{opt}\\PY{p}{,} \\PY{n}{loss}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{categorical\\PYZus{}crossentropy}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{metrics}\\PY{o}{=}\\PY{p}{[}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{accuracy}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{]}\\PY{p}{)}\n",
- " \\PY{k}{return} \\PY{n}{model}\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{train\\PYZus{}mod}\\PY{p}{(}\\PY{n}{model}\\PY{p}{,} \\PY{n}{data}\\PY{p}{,} \\PY{n}{epochs}\\PY{o}{=}\\PY{l+m+mi}{3}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{x\\PYZus{}train}\\PY{p}{,} \\PY{n}{y\\PYZus{}train}\\PY{p}{,} \\PY{n}{label\\PYZus{}train}\\PY{p}{,} \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test} \\PY{o}{=} \\PY{n}{data}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{fit}\\PY{p}{(}\\PY{n}{x\\PYZus{}train}\\PY{p}{,} \\PY{n}{y\\PYZus{}train}\\PY{p}{,} \\PY{n}{epochs}\\PY{o}{=}\\PY{n}{epochs}\\PY{p}{,} \\PY{n}{batch\\PYZus{}size}\\PY{o}{=}\\PY{l+m+mi}{600}\\PY{p}{,} \\PY{n}{validation\\PYZus{}data}\\PY{o}{=}\\PY{p}{(}\\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{)}\\PY{p}{,} \\PY{n}{verbose}\\PY{o}{=}\\PY{l+m+mi}{1}\\PY{p}{)}\n",
- " \\PY{n}{score} \\PY{o}{=} \\PY{n}{model}\\PY{o}{.}\\PY{n}{evaluate}\\PY{p}{(}\\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{verbose}\\PY{o}{=}\\PY{l+m+mi}{0}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{Test score:}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{score}\\PY{p}{[}\\PY{l+m+mi}{0}\\PY{p}{]}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{Test accuracy:}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{score}\\PY{p}{[}\\PY{l+m+mi}{1}\\PY{p}{]}\\PY{p}{)}\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{main}\\PY{p}{(}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{data} \\PY{o}{=} \\PY{n}{read\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{classes} \\PY{o}{=} \\PY{l+m+mi}{10}\n",
- " \\PY{n}{w} \\PY{o}{=} \\PY{l+m+mi}{28}\n",
- " \\PY{n}{c} \\PY{o}{=} \\PY{l+m+mi}{1}\n",
- " \\PY{n}{model} \\PY{o}{=} \\PY{n}{create\\PYZus{}model}\\PY{p}{(}\\PY{n}{w} \\PY{p}{,}\\PY{n}{c}\\PY{p}{,} \\PY{n}{classes}\\PY{p}{)}\n",
- " \\PY{n}{model}\\PY{o}{.}\\PY{n}{summary}\\PY{p}{(}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{epochs} \\PY{o}{=} \\PY{l+m+mi}{3}\n",
- " \\PY{n}{train\\PYZus{}mod}\\PY{p}{(}\\PY{n}{model}\\PY{p}{,} \\PY{n}{data}\\PY{p}{,} \\PY{n}{epochs}\\PY{p}{)}\n",
- " \\PY{n}{save\\PYZus{}mod}\\PY{p}{(}\\PY{n}{model}\\PY{p}{,} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{alexnet\\PYZus{}mnist\\PYZus{}fp32\\PYZus{}mod}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{)}\n",
- "\n",
- "\\PY{k}{if} \\PY{n+nv+vm}{\\PYZus{}\\PYZus{}name\\PYZus{}\\PYZus{}} \\PY{o}{==} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{\\PYZus{}\\PYZus{}main\\PYZus{}\\PYZus{}}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{:}\n",
- " \\PY{n}{main}\\PY{p}{(}\\PY{p}{)}\n",
- "\\end{Verbatim}\n"
- ],
- "text/plain": [
- "import os\n",
- "import numpy as np\n",
- "\n",
- "import tensorflow as tf\n",
- "from tensorflow.python.keras import utils\n",
- "\n",
- "from tensorflow.keras.models import Sequential\n",
- "from tensorflow.keras.layers import Dense, Dropout, Flatten, Activation, Reshape\n",
- "from tensorflow.keras.layers import Convolution2D, MaxPooling2D\n",
- "from tensorflow.keras.layers import BatchNormalization\n",
- "from tensorflow.keras.optimizers import SGD, Adam\n",
- "from tensorflow import keras\n",
- "\n",
- "import mnist_dataset\n",
- "\n",
- "\n",
- "def save_mod(model, mod_path):\n",
- " print('Save to {}'.format(mod_path))\n",
- " tf.saved_model.save(model, mod_path)\n",
- "\n",
- "\n",
- "def load_mod(model_file):\n",
- " model = tf.keras.models.load_model(model_file)\n",
- " print('Load from {}'.format(model_file))\n",
- " return model\n",
- "\n",
- "def save_frezon_pb(model, mod_path):\n",
- " # Convert Keras model to ConcreteFunction\n",
- " full_model = tf.function(lambda x: model(x))\n",
- " concrete_function = full_model.get_concrete_function(\n",
- " x=tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype))\n",
- "\n",
- " # Get frozen ConcreteFunction\n",
- " frozen_model = convert_variables_to_constants_v2(concrete_function)\n",
- "\n",
- " # Generate frozen pb\n",
- " tf.io.write_graph(graph_or_graph_def=frozen_model.graph,\n",
- " logdir=\".\",\n",
- " name=mod_path,\n",
- " as_text=False)\n",
- "\n",
- "\n",
- "def load_pb(in_model):\n",
- " detection_graph = tf.compat.v1.Graph()\n",
- " with detection_graph.as_default():\n",
- " od_graph_def = tf.compat.v1.GraphDef()\n",
- " with tf.compat.v1.gfile.GFile(in_model, 'rb') as fid:\n",
- " serialized_graph = fid.read()\n",
- " od_graph_def.ParseFromString(serialized_graph)\n",
- " tf.compat.v1.import_graph_def(od_graph_def, name='')\n",
- "\n",
- " return detection_graph\n",
- "\n",
- "def read_data():\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = mnist_dataset.read_data()\n",
- " return x_train, y_train, label_train, x_test, y_test, label_test\n",
- "\n",
- "def create_model(w, c, classes):\n",
- " model = Sequential()\n",
- " model.add(Convolution2D(96, 11, input_shape=(w, w, c), padding='same'))\n",
- " model.add(Activation('relu'))\n",
- " model.add(MaxPooling2D(pool_size=(2, 2)))\n",
- "\n",
- " model.add(Convolution2D(256, 5, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- " model.add(MaxPooling2D(pool_size=(2, 2)))\n",
- "\n",
- " model.add(Convolution2D(384, 3, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Convolution2D(384, 3, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Convolution2D(256, 3, padding='same'))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Convolution2D(256, 7))\n",
- " model.add(Activation('relu'))\n",
- "\n",
- " model.add(Flatten())\n",
- " model.add(Dense(classes))\n",
- " model.add(Activation('softmax'))\n",
- "\n",
- " opt = Adam(learning_rate=0.001)\n",
- " model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])\n",
- " return model\n",
- "\n",
- "def train_mod(model, data, epochs=3):\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = data\n",
- " model.fit(x_train, y_train, epochs=epochs, batch_size=600, validation_data=(x_test, y_test), verbose=1)\n",
- " score = model.evaluate(x_test, y_test, verbose=0)\n",
- " print('Test score:', score[0])\n",
- " print('Test accuracy:', score[1])\n",
- "\n",
- "def main():\n",
- " data = read_data()\n",
- "\n",
- " classes = 10\n",
- " w = 28\n",
- " c = 1\n",
- " model = create_model(w ,c, classes)\n",
- " model.summary()\n",
- "\n",
- " epochs = 3\n",
- " train_mod(model, data, epochs)\n",
- " save_mod(model, \"alexnet_mnist_fp32_mod\")\n",
- "\n",
- "if __name__ == \"__main__\":\n",
- " main()"
- ]
- },
- "execution_count": 9,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"display.Code('alexnet.py')"
]
@@ -814,225 +261,51 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Define Yaml File\n",
- "\n",
- "We created `quant_config.yaml` to save the necessary parameters for Intel® Neural Compressor (INC).\n",
- "In this case, we only need to change the input/output according to the FP32 model.\n",
+ "### Define Tuning Function\n",
+ "We follow the template to create the tuning function. The function will return a frozen quantized model (INT8 model).\n",
"\n",
- "The input node name is **x**.\n",
+ "The quantization parameters are set by the APIs as following code.\n",
"\n",
- "The output name is **Identity**."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "import neural_compressor as inc\n",
- "print("neural_compressor version {}".format(inc.__version__))\n",
- "\n",
- "import alexnet\n",
- "import math\n",
- "import yaml\n",
- "import mnist_dataset\n",
- "from neural_compressor.quantization import fit\n",
- "from neural_compressor.config import PostTrainingQuantConfig, TuningCriterion, AccuracyCriterion \n",
- "\n",
- "\n",
- "def save_int8_frezon_pb(q_model, path):\n",
- " from tensorflow.python.platform import gfile\n",
- " f = gfile.GFile(path, 'wb')\n",
- " f.write(q_model.graph.as_graph_def().SerializeToString())\n",
- " print("Save to {}".format(path))\n",
- "\n",
- "\n",
- "class Dataloader(object):\n",
- " def __init__(self, batch_size):\n",
- " self.batch_size = batch_size\n",
- "\n",
- " def __iter__(self):\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = mnist_dataset.read_data()\n",
- " batch_nums = math.ceil(len(x_test) / self.batch_size)\n",
- "\n",
- " for i in range(batch_nums - 1):\n",
- " begin = i * self.batch_size\n",
- " end = (i + 1) * self.batch_size\n",
- " yield x_test[begin: end], label_test[begin: end]\n",
- "\n",
- " begin = (batch_nums - 1) * self.batch_size\n",
- " yield x_test[begin:], label_test[begin:]\n",
- "\n",
- "\n",
- "def auto_tune(input_graph_path, config, batch_size): \n",
- " fp32_graph = alexnet.load_pb(input_graph_path)\n",
- " dataloader = Dataloader(batch_size)\n",
- " assert(dataloader)\n",
- " \n",
- " tuning_criterion = TuningCriterion(**config["tuning_criterion"])\n",
- " accuracy_criterion = AccuracyCriterion(**config["accuracy_criterion"])\n",
- " q_model = fit(\n",
- " model=input_graph_path,\n",
- " conf=PostTrainingQuantConfig(**config["quant_config"],\n",
- " tuning_criterion=tuning_criterion,\n",
- " accuracy_criterion=accuracy_criterion,\n",
- " ),\n",
- " calib_dataloader=dataloader,\n",
- " )\n",
- " return q_model\n",
- "\n",
- "\n",
- "batch_size = 200\n",
- "fp32_frezon_pb_file = "fp32_frezon.pb"\n",
- "int8_pb_file = "alexnet_int8_model.pb"\n",
- "\n",
- "with open("quant_config.yaml") as f:\n",
- " config = yaml.safe_load(f.read())\n",
- "config\n",
- "\n",
- "q_model = auto_tune(fp32_frezon_pb_file, config, batch_size)\n",
- "save_int8_frezon_pb(q_model, int8_pb_file)\n",
- "
\n"
- ],
- "text/latex": [
- "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{neural\\PYZus{}compressor} \\PY{k}{as} \\PY{n+nn}{inc}\n",
- "\\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{neural\\PYZus{}compressor version }\\PY{l+s+si}{\\PYZob{}\\PYZcb{}}\\PY{l+s+s2}{\\PYZdq{}}\\PY{o}{.}\\PY{n}{format}\\PY{p}{(}\\PY{n}{inc}\\PY{o}{.}\\PY{n}{\\PYZus{}\\PYZus{}version\\PYZus{}\\PYZus{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{alexnet}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{math}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{yaml}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{mnist\\PYZus{}dataset}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{neural\\PYZus{}compressor}\\PY{n+nn}{.}\\PY{n+nn}{quantization} \\PY{k+kn}{import} \\PY{n}{fit}\n",
- "\\PY{k+kn}{from} \\PY{n+nn}{neural\\PYZus{}compressor}\\PY{n+nn}{.}\\PY{n+nn}{config} \\PY{k+kn}{import} \\PY{n}{PostTrainingQuantConfig}\\PY{p}{,} \\PY{n}{TuningCriterion}\\PY{p}{,} \\PY{n}{AccuracyCriterion} \n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{save\\PYZus{}int8\\PYZus{}frezon\\PYZus{}pb}\\PY{p}{(}\\PY{n}{q\\PYZus{}model}\\PY{p}{,} \\PY{n}{path}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{k+kn}{from} \\PY{n+nn}{tensorflow}\\PY{n+nn}{.}\\PY{n+nn}{python}\\PY{n+nn}{.}\\PY{n+nn}{platform} \\PY{k+kn}{import} \\PY{n}{gfile}\n",
- " \\PY{n}{f} \\PY{o}{=} \\PY{n}{gfile}\\PY{o}{.}\\PY{n}{GFile}\\PY{p}{(}\\PY{n}{path}\\PY{p}{,} \\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{wb}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)}\n",
- " \\PY{n}{f}\\PY{o}{.}\\PY{n}{write}\\PY{p}{(}\\PY{n}{q\\PYZus{}model}\\PY{o}{.}\\PY{n}{graph}\\PY{o}{.}\\PY{n}{as\\PYZus{}graph\\PYZus{}def}\\PY{p}{(}\\PY{p}{)}\\PY{o}{.}\\PY{n}{SerializeToString}\\PY{p}{(}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{Save to }\\PY{l+s+si}{\\PYZob{}\\PYZcb{}}\\PY{l+s+s2}{\\PYZdq{}}\\PY{o}{.}\\PY{n}{format}\\PY{p}{(}\\PY{n}{path}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- "\n",
- "\\PY{k}{class} \\PY{n+nc}{Dataloader}\\PY{p}{(}\\PY{n+nb}{object}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{k}{def} \\PY{n+nf+fm}{\\PYZus{}\\PYZus{}init\\PYZus{}\\PYZus{}}\\PY{p}{(}\\PY{n+nb+bp}{self}\\PY{p}{,} \\PY{n}{batch\\PYZus{}size}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n+nb+bp}{self}\\PY{o}{.}\\PY{n}{batch\\PYZus{}size} \\PY{o}{=} \\PY{n}{batch\\PYZus{}size}\n",
- "\n",
- " \\PY{k}{def} \\PY{n+nf+fm}{\\PYZus{}\\PYZus{}iter\\PYZus{}\\PYZus{}}\\PY{p}{(}\\PY{n+nb+bp}{self}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{x\\PYZus{}train}\\PY{p}{,} \\PY{n}{y\\PYZus{}train}\\PY{p}{,} \\PY{n}{label\\PYZus{}train}\\PY{p}{,} \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test} \\PY{o}{=} \\PY{n}{mnist\\PYZus{}dataset}\\PY{o}{.}\\PY{n}{read\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{n}{batch\\PYZus{}nums} \\PY{o}{=} \\PY{n}{math}\\PY{o}{.}\\PY{n}{ceil}\\PY{p}{(}\\PY{n+nb}{len}\\PY{p}{(}\\PY{n}{x\\PYZus{}test}\\PY{p}{)} \\PY{o}{/} \\PY{n+nb+bp}{self}\\PY{o}{.}\\PY{n}{batch\\PYZus{}size}\\PY{p}{)}\n",
- "\n",
- " \\PY{k}{for} \\PY{n}{i} \\PY{o+ow}{in} \\PY{n+nb}{range}\\PY{p}{(}\\PY{n}{batch\\PYZus{}nums} \\PY{o}{\\PYZhy{}} \\PY{l+m+mi}{1}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{begin} \\PY{o}{=} \\PY{n}{i} \\PY{o}{*} \\PY{n+nb+bp}{self}\\PY{o}{.}\\PY{n}{batch\\PYZus{}size}\n",
- " \\PY{n}{end} \\PY{o}{=} \\PY{p}{(}\\PY{n}{i} \\PY{o}{+} \\PY{l+m+mi}{1}\\PY{p}{)} \\PY{o}{*} \\PY{n+nb+bp}{self}\\PY{o}{.}\\PY{n}{batch\\PYZus{}size}\n",
- " \\PY{k}{yield} \\PY{n}{x\\PYZus{}test}\\PY{p}{[}\\PY{n}{begin}\\PY{p}{:} \\PY{n}{end}\\PY{p}{]}\\PY{p}{,} \\PY{n}{label\\PYZus{}test}\\PY{p}{[}\\PY{n}{begin}\\PY{p}{:} \\PY{n}{end}\\PY{p}{]}\n",
- "\n",
- " \\PY{n}{begin} \\PY{o}{=} \\PY{p}{(}\\PY{n}{batch\\PYZus{}nums} \\PY{o}{\\PYZhy{}} \\PY{l+m+mi}{1}\\PY{p}{)} \\PY{o}{*} \\PY{n+nb+bp}{self}\\PY{o}{.}\\PY{n}{batch\\PYZus{}size}\n",
- " \\PY{k}{yield} \\PY{n}{x\\PYZus{}test}\\PY{p}{[}\\PY{n}{begin}\\PY{p}{:}\\PY{p}{]}\\PY{p}{,} \\PY{n}{label\\PYZus{}test}\\PY{p}{[}\\PY{n}{begin}\\PY{p}{:}\\PY{p}{]}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{auto\\PYZus{}tune}\\PY{p}{(}\\PY{n}{input\\PYZus{}graph\\PYZus{}path}\\PY{p}{,} \\PY{n}{config}\\PY{p}{,} \\PY{n}{batch\\PYZus{}size}\\PY{p}{)}\\PY{p}{:} \n",
- " \\PY{n}{fp32\\PYZus{}graph} \\PY{o}{=} \\PY{n}{alexnet}\\PY{o}{.}\\PY{n}{load\\PYZus{}pb}\\PY{p}{(}\\PY{n}{input\\PYZus{}graph\\PYZus{}path}\\PY{p}{)}\n",
- " \\PY{n}{dataloader} \\PY{o}{=} \\PY{n}{Dataloader}\\PY{p}{(}\\PY{n}{batch\\PYZus{}size}\\PY{p}{)}\n",
- " \\PY{k}{assert}\\PY{p}{(}\\PY{n}{dataloader}\\PY{p}{)}\n",
- " \n",
- " \\PY{n}{tuning\\PYZus{}criterion} \\PY{o}{=} \\PY{n}{TuningCriterion}\\PY{p}{(}\\PY{o}{*}\\PY{o}{*}\\PY{n}{config}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{tuning\\PYZus{}criterion}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]}\\PY{p}{)}\n",
- " \\PY{n}{accuracy\\PYZus{}criterion} \\PY{o}{=} \\PY{n}{AccuracyCriterion}\\PY{p}{(}\\PY{o}{*}\\PY{o}{*}\\PY{n}{config}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{accuracy\\PYZus{}criterion}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]}\\PY{p}{)}\n",
- " \\PY{n}{q\\PYZus{}model} \\PY{o}{=} \\PY{n}{fit}\\PY{p}{(}\n",
- " \\PY{n}{model}\\PY{o}{=}\\PY{n}{input\\PYZus{}graph\\PYZus{}path}\\PY{p}{,}\n",
- " \\PY{n}{conf}\\PY{o}{=}\\PY{n}{PostTrainingQuantConfig}\\PY{p}{(}\\PY{o}{*}\\PY{o}{*}\\PY{n}{config}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{quant\\PYZus{}config}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]}\\PY{p}{,}\n",
- " \\PY{n}{tuning\\PYZus{}criterion}\\PY{o}{=}\\PY{n}{tuning\\PYZus{}criterion}\\PY{p}{,}\n",
- " \\PY{n}{accuracy\\PYZus{}criterion}\\PY{o}{=}\\PY{n}{accuracy\\PYZus{}criterion}\\PY{p}{,}\n",
- " \\PY{p}{)}\\PY{p}{,}\n",
- " \\PY{n}{calib\\PYZus{}dataloader}\\PY{o}{=}\\PY{n}{dataloader}\\PY{p}{,}\n",
- " \\PY{p}{)}\n",
- " \\PY{k}{return} \\PY{n}{q\\PYZus{}model}\n",
- "\n",
- "\n",
- "\\PY{n}{batch\\PYZus{}size} \\PY{o}{=} \\PY{l+m+mi}{200}\n",
- "\\PY{n}{fp32\\PYZus{}frezon\\PYZus{}pb\\PYZus{}file} \\PY{o}{=} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{fp32\\PYZus{}frezon.pb}\\PY{l+s+s2}{\\PYZdq{}}\n",
- "\\PY{n}{int8\\PYZus{}pb\\PYZus{}file} \\PY{o}{=} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{alexnet\\PYZus{}int8\\PYZus{}model.pb}\\PY{l+s+s2}{\\PYZdq{}}\n",
- "\n",
- "\\PY{k}{with} \\PY{n+nb}{open}\\PY{p}{(}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{quant\\PYZus{}config.yaml}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{)} \\PY{k}{as} \\PY{n}{f}\\PY{p}{:}\n",
- " \\PY{n}{config} \\PY{o}{=} \\PY{n}{yaml}\\PY{o}{.}\\PY{n}{safe\\PYZus{}load}\\PY{p}{(}\\PY{n}{f}\\PY{o}{.}\\PY{n}{read}\\PY{p}{(}\\PY{p}{)}\\PY{p}{)}\n",
- "\\PY{n}{config}\n",
- "\n",
- "\\PY{n}{q\\PYZus{}model} \\PY{o}{=} \\PY{n}{auto\\PYZus{}tune}\\PY{p}{(}\\PY{n}{fp32\\PYZus{}frezon\\PYZus{}pb\\PYZus{}file}\\PY{p}{,} \\PY{n}{config}\\PY{p}{,} \\PY{n}{batch\\PYZus{}size}\\PY{p}{)}\n",
- "\\PY{n}{save\\PYZus{}int8\\PYZus{}frezon\\PYZus{}pb}\\PY{p}{(}\\PY{n}{q\\PYZus{}model}\\PY{p}{,} \\PY{n}{int8\\PYZus{}pb\\PYZus{}file}\\PY{p}{)}\n",
- "\\end{Verbatim}\n"
- ],
- "text/plain": [
- "import neural_compressor as inc\n",
- "print(\"neural_compressor version {}\".format(inc.__version__))\n",
- "\n",
- "import alexnet\n",
- "import math\n",
- "import yaml\n",
- "import mnist_dataset\n",
- "from neural_compressor.quantization import fit\n",
- "from neural_compressor.config import PostTrainingQuantConfig, TuningCriterion, AccuracyCriterion \n",
- "\n",
- "\n",
- "def save_int8_frezon_pb(q_model, path):\n",
- " from tensorflow.python.platform import gfile\n",
- " f = gfile.GFile(path, 'wb')\n",
- " f.write(q_model.graph.as_graph_def().SerializeToString())\n",
- " print(\"Save to {}\".format(path))\n",
- "\n",
- "\n",
- "class Dataloader(object):\n",
- " def __init__(self, batch_size):\n",
- " self.batch_size = batch_size\n",
- "\n",
- " def __iter__(self):\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = mnist_dataset.read_data()\n",
- " batch_nums = math.ceil(len(x_test) / self.batch_size)\n",
- "\n",
- " for i in range(batch_nums - 1):\n",
- " begin = i * self.batch_size\n",
- " end = (i + 1) * self.batch_size\n",
- " yield x_test[begin: end], label_test[begin: end]\n",
- "\n",
- " begin = (batch_nums - 1) * self.batch_size\n",
- " yield x_test[begin:], label_test[begin:]\n",
- "\n",
- "\n",
- "def auto_tune(input_graph_path, config, batch_size): \n",
- " fp32_graph = alexnet.load_pb(input_graph_path)\n",
- " dataloader = Dataloader(batch_size)\n",
- " assert(dataloader)\n",
- " \n",
- " tuning_criterion = TuningCriterion(**config[\"tuning_criterion\"])\n",
- " accuracy_criterion = AccuracyCriterion(**config[\"accuracy_criterion\"])\n",
- " q_model = fit(\n",
- " model=input_graph_path,\n",
- " conf=PostTrainingQuantConfig(**config[\"quant_config\"],\n",
- " tuning_criterion=tuning_criterion,\n",
- " accuracy_criterion=accuracy_criterion,\n",
- " ),\n",
- " calib_dataloader=dataloader,\n",
- " )\n",
- " return q_model\n",
- "\n",
- "\n",
- "batch_size = 200\n",
- "fp32_frezon_pb_file = \"fp32_frezon.pb\"\n",
- "int8_pb_file = \"alexnet_int8_model.pb\"\n",
- "\n",
- "with open(\"quant_config.yaml\") as f:\n",
- " config = yaml.safe_load(f.read())\n",
- "config\n",
- "\n",
- "q_model = auto_tune(fp32_frezon_pb_file, config, batch_size)\n",
- "save_int8_frezon_pb(q_model, int8_pb_file)"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
"source": [
"display.Code('inc_quantize_model.py')"
]
@@ -1343,177 +338,11 @@
},
{
"cell_type": "code",
- "execution_count": 14,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "neural_compressor version 2.5.1\n",
- "2024-04-24 19:47:28.802650: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
- "2024-04-24 19:47:28.805195: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n",
- "2024-04-24 19:47:28.834764: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
- "2024-04-24 19:47:28.834786: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
- "2024-04-24 19:47:28.835733: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
- "2024-04-24 19:47:28.841053: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n",
- "2024-04-24 19:47:28.841223: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
- "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
- "2024-04-24 19:47:29.508416: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
- "2024-04-24 19:47:29.820087: I itex/core/wrapper/itex_cpu_wrapper.cc:60] Intel Extension for Tensorflow* AVX512 CPU backend is loaded.\n",
- "2024-04-24 19:47:30 [INFO] Start auto tuning.\n",
- "2024-04-24 19:47:30 [INFO] Quantize model without tuning!\n",
- "2024-04-24 19:47:30 [INFO] Quantize the model with default configuration without evaluating the model. To perform the tuning process, please either provide an eval_func or provide an eval_dataloader an eval_metric.\n",
- "2024-04-24 19:47:30 [INFO] Adaptor has 5 recipes.\n",
- "2024-04-24 19:47:30 [INFO] 0 recipes specified by user.\n",
- "2024-04-24 19:47:30 [INFO] 3 recipes require future tuning.\n",
- "2024-04-24 19:47:32 [INFO] *** Initialize auto tuning\n",
- "2024-04-24 19:47:32 [INFO] {\n",
- "2024-04-24 19:47:32 [INFO] 'PostTrainingQuantConfig': {\n",
- "2024-04-24 19:47:32 [INFO] 'AccuracyCriterion': {\n",
- "2024-04-24 19:47:32 [INFO] 'criterion': 'relative',\n",
- "2024-04-24 19:47:32 [INFO] 'higher_is_better': True,\n",
- "2024-04-24 19:47:32 [INFO] 'tolerable_loss': 0.01,\n",
- "2024-04-24 19:47:32 [INFO] 'absolute': None,\n",
- "2024-04-24 19:47:32 [INFO] 'keys': >,\n",
- "2024-04-24 19:47:32 [INFO] 'relative': 0.01\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'approach': 'post_training_auto_quant',\n",
- "2024-04-24 19:47:32 [INFO] 'backend': 'itex',\n",
- "2024-04-24 19:47:32 [INFO] 'calibration_sampling_size': [\n",
- "2024-04-24 19:47:32 [INFO] 100\n",
- "2024-04-24 19:47:32 [INFO] ],\n",
- "2024-04-24 19:47:32 [INFO] 'device': 'cpu',\n",
- "2024-04-24 19:47:32 [INFO] 'diagnosis': False,\n",
- "2024-04-24 19:47:32 [INFO] 'domain': 'auto',\n",
- "2024-04-24 19:47:32 [INFO] 'example_inputs': 'Not printed here due to large size tensors...',\n",
- "2024-04-24 19:47:32 [INFO] 'excluded_precisions': [\n",
- "2024-04-24 19:47:32 [INFO] ],\n",
- "2024-04-24 19:47:32 [INFO] 'framework': 'tensorflow_itex',\n",
- "2024-04-24 19:47:32 [INFO] 'inputs': 'x',\n",
- "2024-04-24 19:47:32 [INFO] 'model_name': '',\n",
- "2024-04-24 19:47:32 [INFO] 'ni_workload_name': 'quantization',\n",
- "2024-04-24 19:47:32 [INFO] 'op_name_dict': None,\n",
- "2024-04-24 19:47:32 [INFO] 'op_type_dict': None,\n",
- "2024-04-24 19:47:32 [INFO] 'outputs': 'Identity',\n",
- "2024-04-24 19:47:32 [INFO] 'quant_format': 'default',\n",
- "2024-04-24 19:47:32 [INFO] 'quant_level': 'auto',\n",
- "2024-04-24 19:47:32 [INFO] 'recipes': {\n",
- "2024-04-24 19:47:32 [INFO] 'smooth_quant': False,\n",
- "2024-04-24 19:47:32 [INFO] 'smooth_quant_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'layer_wise_quant': False,\n",
- "2024-04-24 19:47:32 [INFO] 'layer_wise_quant_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'fast_bias_correction': False,\n",
- "2024-04-24 19:47:32 [INFO] 'weight_correction': False,\n",
- "2024-04-24 19:47:32 [INFO] 'gemm_to_matmul': True,\n",
- "2024-04-24 19:47:32 [INFO] 'graph_optimization_level': None,\n",
- "2024-04-24 19:47:32 [INFO] 'first_conv_or_matmul_quantization': True,\n",
- "2024-04-24 19:47:32 [INFO] 'last_conv_or_matmul_quantization': True,\n",
- "2024-04-24 19:47:32 [INFO] 'pre_post_process_quantization': True,\n",
- "2024-04-24 19:47:32 [INFO] 'add_qdq_pair_to_weight': False,\n",
- "2024-04-24 19:47:32 [INFO] 'optypes_to_exclude_output_quant': [\n",
- "2024-04-24 19:47:32 [INFO] ],\n",
- "2024-04-24 19:47:32 [INFO] 'dedicated_qdq_pair': False,\n",
- "2024-04-24 19:47:32 [INFO] 'rtn_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'awq_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'gptq_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'teq_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'autoround_args': {\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'scale_propagation_max_pooling': True,\n",
- "2024-04-24 19:47:32 [INFO] 'scale_propagation_concat': True\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'reduce_range': None,\n",
- "2024-04-24 19:47:32 [INFO] 'TuningCriterion': {\n",
- "2024-04-24 19:47:32 [INFO] 'max_trials': 100,\n",
- "2024-04-24 19:47:32 [INFO] 'objective': [\n",
- "2024-04-24 19:47:32 [INFO] 'performance'\n",
- "2024-04-24 19:47:32 [INFO] ],\n",
- "2024-04-24 19:47:32 [INFO] 'strategy': 'basic',\n",
- "2024-04-24 19:47:32 [INFO] 'strategy_kwargs': 'None',\n",
- "2024-04-24 19:47:32 [INFO] 'timeout': 0\n",
- "2024-04-24 19:47:32 [INFO] },\n",
- "2024-04-24 19:47:32 [INFO] 'use_bf16': True\n",
- "2024-04-24 19:47:32 [INFO] }\n",
- "2024-04-24 19:47:32 [INFO] }\n",
- "2024-04-24 19:47:32 [WARNING] [Strategy] Please install `mpi4py` correctly if using distributed tuning; otherwise, ignore this warning.\n",
- "2024-04-24 19:47:32 [WARNING] Node name y specified in yaml doesn't exist in the model.\n",
- "2024-04-24 19:47:32 [WARNING] Found possible input node names: ['x'], output node names: ['Identity'].\n",
- "2024-04-24 19:47:32 [INFO] ConvertLayoutOptimizer elapsed time: 0.06 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass ConvertPlaceholderToConst elapsed time: 2.69 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass SwitchOptimizer elapsed time: 2.93 ms\n",
- "2024-04-24 19:47:32.555803: I tensorflow/core/grappler/devices.cc:66] Number of eligible GPUs (core count >= 8, compute capability >= 0.0): 0\n",
- "2024-04-24 19:47:32.555988: I tensorflow/core/grappler/clusters/single_machine.cc:361] Starting new session\n",
- "2024-04-24 19:47:32 [INFO] Pass GrapplerOptimizer elapsed time: 242.44 ms\n",
- "WARNING:tensorflow:From /intel/oneapi/intelpython/envs/tensorflow/lib/python3.9/site-packages/neural_compressor/adaptor/tf_utils/util.py:399: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
- "Instructions for updating:\n",
- "This API was designed for TensorFlow v1. See https://www.tensorflow.org/guide/migrate for instructions on how to migrate your code to TensorFlow v2.\n",
- "2024-04-24 19:47:32,738 - tensorflow - WARNING - From /intel/oneapi/intelpython/envs/tensorflow/lib/python3.9/site-packages/neural_compressor/adaptor/tf_utils/util.py:399: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
- "Instructions for updating:\n",
- "This API was designed for TensorFlow v1. See https://www.tensorflow.org/guide/migrate for instructions on how to migrate your code to TensorFlow v2.\n",
- "2024-04-24 19:47:32 [INFO] Pass StripUnusedNodesOptimizer elapsed time: 17.31 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass RemoveTrainingNodesOptimizer elapsed time: 3.7 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass SplitSharedInputOptimizer elapsed time: 0.27 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass GraphFoldConstantOptimizer elapsed time: 2.97 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass FuseDecomposedBNOptimizer elapsed time: 5.73 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass FuseColumnWiseMulOptimizer elapsed time: 2.87 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass StripUnusedNodesOptimizer elapsed time: 8.08 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass GraphCseOptimizer elapsed time: 2.8 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass FoldBatchNormNodesOptimizer elapsed time: 2.78 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass RenameBatchNormOptimizer elapsed time: 2.64 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass ConvertLeakyReluOptimizer elapsed time: 2.77 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass ConvertAddToBiasAddOptimizer elapsed time: 2.69 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass FuseTransposeReshapeOptimizer elapsed time: 2.84 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass FuseConvWithMathOptimizer elapsed time: 2.72 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass ExpandDimsOptimizer elapsed time: 2.8 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass FetchWeightFromReshapeOptimizer elapsed time: 2.71 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass MoveSqueezeAfterReluOptimizer elapsed time: 2.79 ms\n",
- "2024-04-24 19:47:32 [WARNING] All replaced equivalent node types are {}\n",
- "2024-04-24 19:47:32 [INFO] Pass StripEquivalentNodesOptimizer elapsed time: 8.33 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass DilatedContraction elapsed time: 2.8 ms\n",
- "2024-04-24 19:47:32 [INFO] Pass Pre Optimization elapsed time: 474.92 ms\n",
- "2024-04-24 19:47:32 [INFO] Do not evaluate the baseline and quantize the model with default configuration.\n",
- "2024-04-24 19:47:32 [INFO] Quantize the model with default config.\n",
- "2024-04-24 19:47:32 [WARNING] Please note that calibration sampling size 100 isn't divisible exactly by batch size 200. So the real sampling size is 200.\n",
- "2024-04-24 19:47:32.967606: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
- "2024-04-24 19:47:32 [WARNING] Please note the 2.15.0 version of TensorFlow is not fully verified! Suggest to use the versions between 1.14.0 and 2.14.0 if meet problem.\n",
- "2024-04-24 19:47:33 [WARNING] Found possible input node names: ['x'], output node names: ['Identity'].\n",
- "Loading data ...\n",
- "Done\n",
- "2024-04-24 19:47:34 [WARNING] Found possible input node names: ['x'], output node names: ['Identity'].\n",
- "2024-04-24 19:47:34 [INFO] Pass GenerateGraphWithQDQPattern elapsed time: 57.31 ms\n",
- "2024-04-24 19:47:34 [WARNING] Found possible input node names: ['x'], output node names: ['Identity'].\n",
- "2024-04-24 19:47:34 [WARNING] Found possible input node names: ['x'], output node names: ['Identity'].\n",
- "2024-04-24 19:47:34 [WARNING] Found possible input node names: ['x'], output node names: ['Identity', 'sequential/conv2d/Conv2D_eightbit_reshape_x', 'sequential/max_pooling2d/MaxPool_eightbit_reshape_sequential/activation/Relu', 'sequential/conv2d_1/Conv2D_eightbit_reshape_sequential/max_pooling2d/MaxPool', 'sequential/max_pooling2d_1/MaxPool_eightbit_reshape_sequential/activation_1/Relu', 'sequential/conv2d_2/Conv2D_eightbit_reshape_sequential/max_pooling2d_1/MaxPool', 'sequential/conv2d_3/Conv2D_eightbit_reshape_sequential/activation_2/Relu', 'sequential/conv2d_4/Conv2D_eightbit_reshape_sequential/activation_3/Relu', 'sequential/conv2d_5/Conv2D_eightbit_reshape_sequential/activation_4/Relu', 'sequential/dense/MatMul_eightbit_reshape_sequential/flatten/Reshape'].\n",
- "2024-04-24 19:47:34 [INFO] Pass StripUnusedNodesOptimizer elapsed time: 10.77 ms\n",
- "2024-04-24 19:47:34 [INFO] Pass ShareQDQForItexYPatternOptimizer elapsed time: 9.13 ms\n",
- "2024-04-24 19:47:34 [INFO] Pass MergeDuplicatedQDQOptimizer elapsed time: 4.54 ms\n",
- "2024-04-24 19:47:34 [INFO] Pass PostHostConstConverter elapsed time: 11.66 ms\n",
- "2024-04-24 19:47:34 [INFO] |******Mixed Precision Statistics******|\n",
- "2024-04-24 19:47:34 [INFO] +-------------+-------+------+---------+\n",
- "2024-04-24 19:47:34 [INFO] | Op Type | Total | INT8 | FP32 |\n",
- "2024-04-24 19:47:34 [INFO] +-------------+-------+------+---------+\n",
- "2024-04-24 19:47:34 [INFO] | MaxPool | 2 | 0 | 2 |\n",
- "2024-04-24 19:47:34 [INFO] | MatMul | 1 | 0 | 1 |\n",
- "2024-04-24 19:47:34 [INFO] | Conv2D | 6 | 0 | 6 |\n",
- "2024-04-24 19:47:34 [INFO] | QuantizeV2 | 16 | 16 | 0 |\n",
- "2024-04-24 19:47:34 [INFO] | Dequantize | 16 | 16 | 0 |\n",
- "2024-04-24 19:47:34 [INFO] +-------------+-------+------+---------+\n",
- "2024-04-24 19:47:34 [INFO] Pass quantize model elapsed time: 1833.94 ms\n",
- "2024-04-24 19:47:34 [INFO] Save tuning history to /code/nc_workspace/2024-04-24_19-47-26/./history.snapshot.\n",
- "2024-04-24 19:47:34 [INFO] [Strategy] Found the model meets accuracy requirements, ending the tuning process.\n",
- "2024-04-24 19:47:34 [INFO] Specified timeout or max trials is reached! Found a quantized model which meet accuracy goal. Exit.\n",
- "2024-04-24 19:47:34 [INFO] Save deploy yaml to /code/nc_workspace/2024-04-24_19-47-26/deploy.yaml\n",
- "Save to alexnet_int8_model.pb\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
"source": [
"!python inc_quantize_model.py"
]
@@ -1540,431 +369,9 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "import tensorflow as tf\n",
- "print("Tensorflow version {}".format(tf.__version__))\n",
- "\n",
- "import numpy as np\n",
- "import time\n",
- "import argparse\n",
- "import os\n",
- "import json\n",
- "\n",
- "\n",
- "import mnist_dataset\n",
- "import alexnet\n",
- "\n",
- "\n",
- "def val_data():\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = mnist_dataset.read_data()\n",
- " return x_test, y_test, label_test\n",
- "\n",
- "\n",
- "def calc_accuracy(predictions, labels):\n",
- " predictions = np.argmax(predictions, axis=1)\n",
- " same = 0\n",
- " for i, x in enumerate(predictions):\n",
- " if x == labels[i]:\n",
- " same += 1\n",
- " if len(predictions) == 0:\n",
- " return 0\n",
- " else:\n",
- " return same / len(predictions)\n",
- "\n",
- "\n",
- "def get_concrete_function(graph_def, inputs, outputs, print_graph=False):\n",
- " def imports_graph_def():\n",
- " tf.compat.v1.import_graph_def(graph_def, name="")\n",
- "\n",
- " wrap_function = tf.compat.v1.wrap_function(imports_graph_def, [])\n",
- " graph = wrap_function.graph\n",
- "\n",
- " return wrap_function.prune(\n",
- " tf.nest.map_structure(graph.as_graph_element, inputs),\n",
- " tf.nest.map_structure(graph.as_graph_element, outputs))\n",
- "\n",
- "\n",
- "def infer_perf_pb(pb_model_file, val_data, inputs=["x:0"], outputs=["Identity:0"]):\n",
- " x_test, y_test, label_test = val_data\n",
- " q_model = alexnet.load_pb(pb_model_file)\n",
- " concrete_function = get_concrete_function(graph_def=q_model.as_graph_def(),\n",
- " inputs=inputs,\n",
- " outputs=outputs,\n",
- " print_graph=True)\n",
- "\n",
- " bt = time.time()\n",
- " _frozen_graph_predictions = concrete_function(x=tf.constant(x_test))\n",
- " et = time.time()\n",
- "\n",
- " accuracy = calc_accuracy(_frozen_graph_predictions[0], label_test)\n",
- " print('accuracy:', accuracy)\n",
- " throughput = x_test.shape[0] / (et - bt)\n",
- " print('max throughput(fps):', throughput)\n",
- "\n",
- " # latency when BS=1\n",
- " times = 1000\n",
- " single_test = x_test[:1]\n",
- "\n",
- " bt = 0\n",
- " warmup = 20\n",
- " for i in range(times):\n",
- " if i == warmup:\n",
- " bt = time.time()\n",
- " _frozen_graph_predictions = concrete_function(x=tf.constant(single_test))\n",
- " et = time.time()\n",
- "\n",
- " latency = (et - bt) * 1000 / (times - warmup)\n",
- " print('latency(ms):', latency)\n",
- "\n",
- " return accuracy, throughput, latency\n",
- "\n",
- "\n",
- "def save_res(result):\n",
- " accuracy, throughput, latency = result\n",
- " res = {}\n",
- " res['accuracy'] = accuracy\n",
- " res['throughput'] = throughput\n",
- " res['latency'] = latency\n",
- "\n",
- " outfile = args.index + ".json"\n",
- " with open(outfile, 'w') as f:\n",
- " json.dump(res, f)\n",
- " print("Save result to {}".format(outfile))\n",
- "\n",
- "parser = argparse.ArgumentParser()\n",
- "parser.add_argument('--index', type=str, help='file name of output', required=True)\n",
- "\n",
- "parser.add_argument('--input-graph', type=str, help='file name for graph', required=True)\n",
- "\n",
- "parser.add_argument('--num-intra-threads', type=str, help='number of threads for an operator', required=False,\n",
- " default="24" )\n",
- "parser.add_argument('--num-inter-threads', type=str, help='number of threads across operators', required=False,\n",
- " default="1")\n",
- "parser.add_argument('--omp-num-threads', type=str, help='number of threads to use', required=False,\n",
- " default="24")\n",
- "\n",
- "args = parser.parse_args()\n",
- "os.environ["KMP_BLOCKTIME"] = "1"\n",
- "os.environ["KMP_SETTINGS"] = "0"\n",
- "os.environ["OMP_NUM_THREADS"] = args.omp_num_threads\n",
- "os.environ["TF_NUM_INTEROP_THREADS"] = args.num_inter_threads\n",
- "os.environ["TF_NUM_INTRAOP_THREADS"] = args.num_intra_threads\n",
- "\n",
- "save_res(infer_perf_pb(args.input_graph, val_data()))\n",
- "
\n"
- ],
- "text/latex": [
- "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{tensorflow} \\PY{k}{as} \\PY{n+nn}{tf}\n",
- "\\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{Tensorflow version }\\PY{l+s+si}{\\PYZob{}\\PYZcb{}}\\PY{l+s+s2}{\\PYZdq{}}\\PY{o}{.}\\PY{n}{format}\\PY{p}{(}\\PY{n}{tf}\\PY{o}{.}\\PY{n}{\\PYZus{}\\PYZus{}version\\PYZus{}\\PYZus{}}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{numpy} \\PY{k}{as} \\PY{n+nn}{np}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{time}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{argparse}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{os}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{json}\n",
- "\n",
- "\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{mnist\\PYZus{}dataset}\n",
- "\\PY{k+kn}{import} \\PY{n+nn}{alexnet}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{val\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{x\\PYZus{}train}\\PY{p}{,} \\PY{n}{y\\PYZus{}train}\\PY{p}{,} \\PY{n}{label\\PYZus{}train}\\PY{p}{,} \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test} \\PY{o}{=} \\PY{n}{mnist\\PYZus{}dataset}\\PY{o}{.}\\PY{n}{read\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{k}{return} \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{calc\\PYZus{}accuracy}\\PY{p}{(}\\PY{n}{predictions}\\PY{p}{,} \\PY{n}{labels}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{predictions} \\PY{o}{=} \\PY{n}{np}\\PY{o}{.}\\PY{n}{argmax}\\PY{p}{(}\\PY{n}{predictions}\\PY{p}{,} \\PY{n}{axis}\\PY{o}{=}\\PY{l+m+mi}{1}\\PY{p}{)}\n",
- " \\PY{n}{same} \\PY{o}{=} \\PY{l+m+mi}{0}\n",
- " \\PY{k}{for} \\PY{n}{i}\\PY{p}{,} \\PY{n}{x} \\PY{o+ow}{in} \\PY{n+nb}{enumerate}\\PY{p}{(}\\PY{n}{predictions}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{k}{if} \\PY{n}{x} \\PY{o}{==} \\PY{n}{labels}\\PY{p}{[}\\PY{n}{i}\\PY{p}{]}\\PY{p}{:}\n",
- " \\PY{n}{same} \\PY{o}{+}\\PY{o}{=} \\PY{l+m+mi}{1}\n",
- " \\PY{k}{if} \\PY{n+nb}{len}\\PY{p}{(}\\PY{n}{predictions}\\PY{p}{)} \\PY{o}{==} \\PY{l+m+mi}{0}\\PY{p}{:}\n",
- " \\PY{k}{return} \\PY{l+m+mi}{0}\n",
- " \\PY{k}{else}\\PY{p}{:}\n",
- " \\PY{k}{return} \\PY{n}{same} \\PY{o}{/} \\PY{n+nb}{len}\\PY{p}{(}\\PY{n}{predictions}\\PY{p}{)}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{get\\PYZus{}concrete\\PYZus{}function}\\PY{p}{(}\\PY{n}{graph\\PYZus{}def}\\PY{p}{,} \\PY{n}{inputs}\\PY{p}{,} \\PY{n}{outputs}\\PY{p}{,} \\PY{n}{print\\PYZus{}graph}\\PY{o}{=}\\PY{k+kc}{False}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{k}{def} \\PY{n+nf}{imports\\PYZus{}graph\\PYZus{}def}\\PY{p}{(}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{tf}\\PY{o}{.}\\PY{n}{compat}\\PY{o}{.}\\PY{n}{v1}\\PY{o}{.}\\PY{n}{import\\PYZus{}graph\\PYZus{}def}\\PY{p}{(}\\PY{n}{graph\\PYZus{}def}\\PY{p}{,} \\PY{n}{name}\\PY{o}{=}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{wrap\\PYZus{}function} \\PY{o}{=} \\PY{n}{tf}\\PY{o}{.}\\PY{n}{compat}\\PY{o}{.}\\PY{n}{v1}\\PY{o}{.}\\PY{n}{wrap\\PYZus{}function}\\PY{p}{(}\\PY{n}{imports\\PYZus{}graph\\PYZus{}def}\\PY{p}{,} \\PY{p}{[}\\PY{p}{]}\\PY{p}{)}\n",
- " \\PY{n}{graph} \\PY{o}{=} \\PY{n}{wrap\\PYZus{}function}\\PY{o}{.}\\PY{n}{graph}\n",
- "\n",
- " \\PY{k}{return} \\PY{n}{wrap\\PYZus{}function}\\PY{o}{.}\\PY{n}{prune}\\PY{p}{(}\n",
- " \\PY{n}{tf}\\PY{o}{.}\\PY{n}{nest}\\PY{o}{.}\\PY{n}{map\\PYZus{}structure}\\PY{p}{(}\\PY{n}{graph}\\PY{o}{.}\\PY{n}{as\\PYZus{}graph\\PYZus{}element}\\PY{p}{,} \\PY{n}{inputs}\\PY{p}{)}\\PY{p}{,}\n",
- " \\PY{n}{tf}\\PY{o}{.}\\PY{n}{nest}\\PY{o}{.}\\PY{n}{map\\PYZus{}structure}\\PY{p}{(}\\PY{n}{graph}\\PY{o}{.}\\PY{n}{as\\PYZus{}graph\\PYZus{}element}\\PY{p}{,} \\PY{n}{outputs}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{infer\\PYZus{}perf\\PYZus{}pb}\\PY{p}{(}\\PY{n}{pb\\PYZus{}model\\PYZus{}file}\\PY{p}{,} \\PY{n}{val\\PYZus{}data}\\PY{p}{,} \\PY{n}{inputs}\\PY{o}{=}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{x:0}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]}\\PY{p}{,} \\PY{n}{outputs}\\PY{o}{=}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{Identity:0}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{x\\PYZus{}test}\\PY{p}{,} \\PY{n}{y\\PYZus{}test}\\PY{p}{,} \\PY{n}{label\\PYZus{}test} \\PY{o}{=} \\PY{n}{val\\PYZus{}data}\n",
- " \\PY{n}{q\\PYZus{}model} \\PY{o}{=} \\PY{n}{alexnet}\\PY{o}{.}\\PY{n}{load\\PYZus{}pb}\\PY{p}{(}\\PY{n}{pb\\PYZus{}model\\PYZus{}file}\\PY{p}{)}\n",
- " \\PY{n}{concrete\\PYZus{}function} \\PY{o}{=} \\PY{n}{get\\PYZus{}concrete\\PYZus{}function}\\PY{p}{(}\\PY{n}{graph\\PYZus{}def}\\PY{o}{=}\\PY{n}{q\\PYZus{}model}\\PY{o}{.}\\PY{n}{as\\PYZus{}graph\\PYZus{}def}\\PY{p}{(}\\PY{p}{)}\\PY{p}{,}\n",
- " \\PY{n}{inputs}\\PY{o}{=}\\PY{n}{inputs}\\PY{p}{,}\n",
- " \\PY{n}{outputs}\\PY{o}{=}\\PY{n}{outputs}\\PY{p}{,}\n",
- " \\PY{n}{print\\PYZus{}graph}\\PY{o}{=}\\PY{k+kc}{True}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{bt} \\PY{o}{=} \\PY{n}{time}\\PY{o}{.}\\PY{n}{time}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{n}{\\PYZus{}frozen\\PYZus{}graph\\PYZus{}predictions} \\PY{o}{=} \\PY{n}{concrete\\PYZus{}function}\\PY{p}{(}\\PY{n}{x}\\PY{o}{=}\\PY{n}{tf}\\PY{o}{.}\\PY{n}{constant}\\PY{p}{(}\\PY{n}{x\\PYZus{}test}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{et} \\PY{o}{=} \\PY{n}{time}\\PY{o}{.}\\PY{n}{time}\\PY{p}{(}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{accuracy} \\PY{o}{=} \\PY{n}{calc\\PYZus{}accuracy}\\PY{p}{(}\\PY{n}{\\PYZus{}frozen\\PYZus{}graph\\PYZus{}predictions}\\PY{p}{[}\\PY{l+m+mi}{0}\\PY{p}{]}\\PY{p}{,} \\PY{n}{label\\PYZus{}test}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{accuracy:}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{accuracy}\\PY{p}{)}\n",
- " \\PY{n}{throughput} \\PY{o}{=} \\PY{n}{x\\PYZus{}test}\\PY{o}{.}\\PY{n}{shape}\\PY{p}{[}\\PY{l+m+mi}{0}\\PY{p}{]} \\PY{o}{/} \\PY{p}{(}\\PY{n}{et} \\PY{o}{\\PYZhy{}} \\PY{n}{bt}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{max throughput(fps):}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{throughput}\\PY{p}{)}\n",
- "\n",
- " \\PY{c+c1}{\\PYZsh{} latency when BS=1}\n",
- " \\PY{n}{times} \\PY{o}{=} \\PY{l+m+mi}{1000}\n",
- " \\PY{n}{single\\PYZus{}test} \\PY{o}{=} \\PY{n}{x\\PYZus{}test}\\PY{p}{[}\\PY{p}{:}\\PY{l+m+mi}{1}\\PY{p}{]}\n",
- "\n",
- " \\PY{n}{bt} \\PY{o}{=} \\PY{l+m+mi}{0}\n",
- " \\PY{n}{warmup} \\PY{o}{=} \\PY{l+m+mi}{20}\n",
- " \\PY{k}{for} \\PY{n}{i} \\PY{o+ow}{in} \\PY{n+nb}{range}\\PY{p}{(}\\PY{n}{times}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{k}{if} \\PY{n}{i} \\PY{o}{==} \\PY{n}{warmup}\\PY{p}{:}\n",
- " \\PY{n}{bt} \\PY{o}{=} \\PY{n}{time}\\PY{o}{.}\\PY{n}{time}\\PY{p}{(}\\PY{p}{)}\n",
- " \\PY{n}{\\PYZus{}frozen\\PYZus{}graph\\PYZus{}predictions} \\PY{o}{=} \\PY{n}{concrete\\PYZus{}function}\\PY{p}{(}\\PY{n}{x}\\PY{o}{=}\\PY{n}{tf}\\PY{o}{.}\\PY{n}{constant}\\PY{p}{(}\\PY{n}{single\\PYZus{}test}\\PY{p}{)}\\PY{p}{)}\n",
- " \\PY{n}{et} \\PY{o}{=} \\PY{n}{time}\\PY{o}{.}\\PY{n}{time}\\PY{p}{(}\\PY{p}{)}\n",
- "\n",
- " \\PY{n}{latency} \\PY{o}{=} \\PY{p}{(}\\PY{n}{et} \\PY{o}{\\PYZhy{}} \\PY{n}{bt}\\PY{p}{)} \\PY{o}{*} \\PY{l+m+mi}{1000} \\PY{o}{/} \\PY{p}{(}\\PY{n}{times} \\PY{o}{\\PYZhy{}} \\PY{n}{warmup}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{latency(ms):}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{latency}\\PY{p}{)}\n",
- "\n",
- " \\PY{k}{return} \\PY{n}{accuracy}\\PY{p}{,} \\PY{n}{throughput}\\PY{p}{,} \\PY{n}{latency}\n",
- "\n",
- "\n",
- "\\PY{k}{def} \\PY{n+nf}{save\\PYZus{}res}\\PY{p}{(}\\PY{n}{result}\\PY{p}{)}\\PY{p}{:}\n",
- " \\PY{n}{accuracy}\\PY{p}{,} \\PY{n}{throughput}\\PY{p}{,} \\PY{n}{latency} \\PY{o}{=} \\PY{n}{result}\n",
- " \\PY{n}{res} \\PY{o}{=} \\PY{p}{\\PYZob{}}\\PY{p}{\\PYZcb{}}\n",
- " \\PY{n}{res}\\PY{p}{[}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{accuracy}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{]} \\PY{o}{=} \\PY{n}{accuracy}\n",
- " \\PY{n}{res}\\PY{p}{[}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{throughput}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{]} \\PY{o}{=} \\PY{n}{throughput}\n",
- " \\PY{n}{res}\\PY{p}{[}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{latency}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{]} \\PY{o}{=} \\PY{n}{latency}\n",
- "\n",
- " \\PY{n}{outfile} \\PY{o}{=} \\PY{n}{args}\\PY{o}{.}\\PY{n}{index} \\PY{o}{+} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{.json}\\PY{l+s+s2}{\\PYZdq{}}\n",
- " \\PY{k}{with} \\PY{n+nb}{open}\\PY{p}{(}\\PY{n}{outfile}\\PY{p}{,} \\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{w}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{)} \\PY{k}{as} \\PY{n}{f}\\PY{p}{:}\n",
- " \\PY{n}{json}\\PY{o}{.}\\PY{n}{dump}\\PY{p}{(}\\PY{n}{res}\\PY{p}{,} \\PY{n}{f}\\PY{p}{)}\n",
- " \\PY{n+nb}{print}\\PY{p}{(}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{Save result to }\\PY{l+s+si}{\\PYZob{}\\PYZcb{}}\\PY{l+s+s2}{\\PYZdq{}}\\PY{o}{.}\\PY{n}{format}\\PY{p}{(}\\PY{n}{outfile}\\PY{p}{)}\\PY{p}{)}\n",
- "\n",
- "\\PY{n}{parser} \\PY{o}{=} \\PY{n}{argparse}\\PY{o}{.}\\PY{n}{ArgumentParser}\\PY{p}{(}\\PY{p}{)}\n",
- "\\PY{n}{parser}\\PY{o}{.}\\PY{n}{add\\PYZus{}argument}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{\\PYZhy{}\\PYZhy{}index}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n+nb}{type}\\PY{o}{=}\\PY{n+nb}{str}\\PY{p}{,} \\PY{n}{help}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{file name of output}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{required}\\PY{o}{=}\\PY{k+kc}{True}\\PY{p}{)}\n",
- "\n",
- "\\PY{n}{parser}\\PY{o}{.}\\PY{n}{add\\PYZus{}argument}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{\\PYZhy{}\\PYZhy{}input\\PYZhy{}graph}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n+nb}{type}\\PY{o}{=}\\PY{n+nb}{str}\\PY{p}{,} \\PY{n}{help}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{file name for graph}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{required}\\PY{o}{=}\\PY{k+kc}{True}\\PY{p}{)}\n",
- "\n",
- "\\PY{n}{parser}\\PY{o}{.}\\PY{n}{add\\PYZus{}argument}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{\\PYZhy{}\\PYZhy{}num\\PYZhy{}intra\\PYZhy{}threads}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n+nb}{type}\\PY{o}{=}\\PY{n+nb}{str}\\PY{p}{,} \\PY{n}{help}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{number of threads for an operator}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{required}\\PY{o}{=}\\PY{k+kc}{False}\\PY{p}{,}\n",
- " \\PY{n}{default}\\PY{o}{=}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{24}\\PY{l+s+s2}{\\PYZdq{}} \\PY{p}{)}\n",
- "\\PY{n}{parser}\\PY{o}{.}\\PY{n}{add\\PYZus{}argument}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{\\PYZhy{}\\PYZhy{}num\\PYZhy{}inter\\PYZhy{}threads}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n+nb}{type}\\PY{o}{=}\\PY{n+nb}{str}\\PY{p}{,} \\PY{n}{help}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{number of threads across operators}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{required}\\PY{o}{=}\\PY{k+kc}{False}\\PY{p}{,}\n",
- " \\PY{n}{default}\\PY{o}{=}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{1}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{)}\n",
- "\\PY{n}{parser}\\PY{o}{.}\\PY{n}{add\\PYZus{}argument}\\PY{p}{(}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{\\PYZhy{}\\PYZhy{}omp\\PYZhy{}num\\PYZhy{}threads}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n+nb}{type}\\PY{o}{=}\\PY{n+nb}{str}\\PY{p}{,} \\PY{n}{help}\\PY{o}{=}\\PY{l+s+s1}{\\PYZsq{}}\\PY{l+s+s1}{number of threads to use}\\PY{l+s+s1}{\\PYZsq{}}\\PY{p}{,} \\PY{n}{required}\\PY{o}{=}\\PY{k+kc}{False}\\PY{p}{,}\n",
- " \\PY{n}{default}\\PY{o}{=}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{24}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{)}\n",
- "\n",
- "\\PY{n}{args} \\PY{o}{=} \\PY{n}{parser}\\PY{o}{.}\\PY{n}{parse\\PYZus{}args}\\PY{p}{(}\\PY{p}{)}\n",
- "\\PY{n}{os}\\PY{o}{.}\\PY{n}{environ}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{KMP\\PYZus{}BLOCKTIME}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]} \\PY{o}{=} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{1}\\PY{l+s+s2}{\\PYZdq{}}\n",
- "\\PY{n}{os}\\PY{o}{.}\\PY{n}{environ}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{KMP\\PYZus{}SETTINGS}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]} \\PY{o}{=} \\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{0}\\PY{l+s+s2}{\\PYZdq{}}\n",
- "\\PY{n}{os}\\PY{o}{.}\\PY{n}{environ}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{OMP\\PYZus{}NUM\\PYZus{}THREADS}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]} \\PY{o}{=} \\PY{n}{args}\\PY{o}{.}\\PY{n}{omp\\PYZus{}num\\PYZus{}threads}\n",
- "\\PY{n}{os}\\PY{o}{.}\\PY{n}{environ}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{TF\\PYZus{}NUM\\PYZus{}INTEROP\\PYZus{}THREADS}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]} \\PY{o}{=} \\PY{n}{args}\\PY{o}{.}\\PY{n}{num\\PYZus{}inter\\PYZus{}threads}\n",
- "\\PY{n}{os}\\PY{o}{.}\\PY{n}{environ}\\PY{p}{[}\\PY{l+s+s2}{\\PYZdq{}}\\PY{l+s+s2}{TF\\PYZus{}NUM\\PYZus{}INTRAOP\\PYZus{}THREADS}\\PY{l+s+s2}{\\PYZdq{}}\\PY{p}{]} \\PY{o}{=} \\PY{n}{args}\\PY{o}{.}\\PY{n}{num\\PYZus{}intra\\PYZus{}threads}\n",
- "\n",
- "\\PY{n}{save\\PYZus{}res}\\PY{p}{(}\\PY{n}{infer\\PYZus{}perf\\PYZus{}pb}\\PY{p}{(}\\PY{n}{args}\\PY{o}{.}\\PY{n}{input\\PYZus{}graph}\\PY{p}{,} \\PY{n}{val\\PYZus{}data}\\PY{p}{(}\\PY{p}{)}\\PY{p}{)}\\PY{p}{)}\n",
- "\\end{Verbatim}\n"
- ],
- "text/plain": [
- "\n",
- "import tensorflow as tf\n",
- "print(\"Tensorflow version {}\".format(tf.__version__))\n",
- "\n",
- "import numpy as np\n",
- "import time\n",
- "import argparse\n",
- "import os\n",
- "import json\n",
- "\n",
- "\n",
- "import mnist_dataset\n",
- "import alexnet\n",
- "\n",
- "\n",
- "def val_data():\n",
- " x_train, y_train, label_train, x_test, y_test, label_test = mnist_dataset.read_data()\n",
- " return x_test, y_test, label_test\n",
- "\n",
- "\n",
- "def calc_accuracy(predictions, labels):\n",
- " predictions = np.argmax(predictions, axis=1)\n",
- " same = 0\n",
- " for i, x in enumerate(predictions):\n",
- " if x == labels[i]:\n",
- " same += 1\n",
- " if len(predictions) == 0:\n",
- " return 0\n",
- " else:\n",
- " return same / len(predictions)\n",
- "\n",
- "\n",
- "def get_concrete_function(graph_def, inputs, outputs, print_graph=False):\n",
- " def imports_graph_def():\n",
- " tf.compat.v1.import_graph_def(graph_def, name=\"\")\n",
- "\n",
- " wrap_function = tf.compat.v1.wrap_function(imports_graph_def, [])\n",
- " graph = wrap_function.graph\n",
- "\n",
- " return wrap_function.prune(\n",
- " tf.nest.map_structure(graph.as_graph_element, inputs),\n",
- " tf.nest.map_structure(graph.as_graph_element, outputs))\n",
- "\n",
- "\n",
- "def infer_perf_pb(pb_model_file, val_data, inputs=[\"x:0\"], outputs=[\"Identity:0\"]):\n",
- " x_test, y_test, label_test = val_data\n",
- " q_model = alexnet.load_pb(pb_model_file)\n",
- " concrete_function = get_concrete_function(graph_def=q_model.as_graph_def(),\n",
- " inputs=inputs,\n",
- " outputs=outputs,\n",
- " print_graph=True)\n",
- "\n",
- " bt = time.time()\n",
- " _frozen_graph_predictions = concrete_function(x=tf.constant(x_test))\n",
- " et = time.time()\n",
- "\n",
- " accuracy = calc_accuracy(_frozen_graph_predictions[0], label_test)\n",
- " print('accuracy:', accuracy)\n",
- " throughput = x_test.shape[0] / (et - bt)\n",
- " print('max throughput(fps):', throughput)\n",
- "\n",
- " # latency when BS=1\n",
- " times = 1000\n",
- " single_test = x_test[:1]\n",
- "\n",
- " bt = 0\n",
- " warmup = 20\n",
- " for i in range(times):\n",
- " if i == warmup:\n",
- " bt = time.time()\n",
- " _frozen_graph_predictions = concrete_function(x=tf.constant(single_test))\n",
- " et = time.time()\n",
- "\n",
- " latency = (et - bt) * 1000 / (times - warmup)\n",
- " print('latency(ms):', latency)\n",
- "\n",
- " return accuracy, throughput, latency\n",
- "\n",
- "\n",
- "def save_res(result):\n",
- " accuracy, throughput, latency = result\n",
- " res = {}\n",
- " res['accuracy'] = accuracy\n",
- " res['throughput'] = throughput\n",
- " res['latency'] = latency\n",
- "\n",
- " outfile = args.index + \".json\"\n",
- " with open(outfile, 'w') as f:\n",
- " json.dump(res, f)\n",
- " print(\"Save result to {}\".format(outfile))\n",
- "\n",
- "parser = argparse.ArgumentParser()\n",
- "parser.add_argument('--index', type=str, help='file name of output', required=True)\n",
- "\n",
- "parser.add_argument('--input-graph', type=str, help='file name for graph', required=True)\n",
- "\n",
- "parser.add_argument('--num-intra-threads', type=str, help='number of threads for an operator', required=False,\n",
- " default=\"24\" )\n",
- "parser.add_argument('--num-inter-threads', type=str, help='number of threads across operators', required=False,\n",
- " default=\"1\")\n",
- "parser.add_argument('--omp-num-threads', type=str, help='number of threads to use', required=False,\n",
- " default=\"24\")\n",
- "\n",
- "args = parser.parse_args()\n",
- "os.environ[\"KMP_BLOCKTIME\"] = \"1\"\n",
- "os.environ[\"KMP_SETTINGS\"] = \"0\"\n",
- "os.environ[\"OMP_NUM_THREADS\"] = args.omp_num_threads\n",
- "os.environ[\"TF_NUM_INTEROP_THREADS\"] = args.num_inter_threads\n",
- "os.environ[\"TF_NUM_INTRAOP_THREADS\"] = args.num_intra_threads\n",
- "\n",
- "save_res(infer_perf_pb(args.input_graph, val_data()))"
- ]
- },
- "execution_count": 15,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"display.Code('profiling_inc.py')"
]
@@ -1978,36 +385,13 @@
},
{
"cell_type": "code",
- "execution_count": 16,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "2024-04-24 19:47:36.427777: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
- "2024-04-24 19:47:36.430221: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n",
- "2024-04-24 19:47:36.460422: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
- "2024-04-24 19:47:36.460443: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
- "2024-04-24 19:47:36.461344: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
- "2024-04-24 19:47:36.466553: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n",
- "2024-04-24 19:47:36.466731: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
- "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
- "2024-04-24 19:47:37.076666: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
- "2024-04-24 19:47:37.453281: I itex/core/wrapper/itex_cpu_wrapper.cc:60] Intel Extension for Tensorflow* AVX512 CPU backend is loaded.\n",
- "Tensorflow version 2.15.0\n",
- "Loading data ...\n",
- "Done\n",
- "2024-04-24 19:47:38.655409: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:117] Plugin optimizer for device_type CPU is enabled.\n",
- "accuracy: 0.9797\n",
- "max throughput(fps): 5728.566484387442\n",
- "latency(ms): 8.566554468505236\n",
- "Save result to 32.json\n"
- ]
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
"source": [
- "!python profiling_inc.py --input-graph=./fp32_frezon.pb --omp-num-threads=4 --num-inter-threads=1 --num-intra-threads=4 --index=32"
+ "!python profiling_inc.py --input-graph=./fp32_frozen.pb --omp-num-threads=4 --num-inter-threads=1 --num-intra-threads=4 --index=32"
]
},
{
@@ -2019,146 +403,22 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {
"scrolled": true
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "2024-04-24 19:47:50.286863: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
- "2024-04-24 19:47:50.289296: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n",
- "2024-04-24 19:47:50.318369: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
- "2024-04-24 19:47:50.318388: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
- "2024-04-24 19:47:50.319257: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
- "2024-04-24 19:47:50.324298: I external/local_tsl/tsl/cuda/cudart_stub.cc:31] Could not find cuda drivers on your machine, GPU will not be used.\n",
- "2024-04-24 19:47:50.324471: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
- "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
- "2024-04-24 19:47:50.932859: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
- "2024-04-24 19:47:51.308169: I itex/core/wrapper/itex_cpu_wrapper.cc:60] Intel Extension for Tensorflow* AVX512 CPU backend is loaded.\n",
- "Tensorflow version 2.15.0\n",
- "Loading data ...\n",
- "Done\n",
- "2024-04-24 19:47:52.629639: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:117] Plugin optimizer for device_type CPU is enabled.\n",
- "accuracy: 0.9797\n",
- "max throughput(fps): 7806.743316356045\n",
- "latency(ms): 8.564782385923424\n",
- "Save result to 8.json\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"!python profiling_inc.py --input-graph=./alexnet_int8_model.pb --omp-num-threads=4 --num-inter-threads=1 --num-intra-threads=4 --index=8"
]
},
{
"cell_type": "code",
- "execution_count": 18,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " \n"
- ]
- },
- {
- "data": {
- "text/html": [
- "{"accuracy": 0.9797, "throughput": 7806.743316356045, "latency": 8.564782385923424}\n",
- "
\n"
- ],
- "text/latex": [
- "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
- "\\PY{p}{\\PYZob{}}\\PY{n+nt}{\\PYZdq{}accuracy\\PYZdq{}}\\PY{p}{:}\\PY{+w}{ }\\PY{l+m+mf}{0.9797}\\PY{p}{,}\\PY{+w}{ }\\PY{n+nt}{\\PYZdq{}throughput\\PYZdq{}}\\PY{p}{:}\\PY{+w}{ }\\PY{l+m+mf}{7806.743316356045}\\PY{p}{,}\\PY{+w}{ }\\PY{n+nt}{\\PYZdq{}latency\\PYZdq{}}\\PY{p}{:}\\PY{+w}{ }\\PY{l+m+mf}{8.564782385923424}\\PY{p}{\\PYZcb{}}\n",
- "\\end{Verbatim}\n"
- ],
- "text/plain": [
- "{\"accuracy\": 0.9797, \"throughput\": 7806.743316356045, \"latency\": 8.564782385923424}"
- ]
- },
- "execution_count": 18,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
"source": [
"display.Code('32.json')\n",
"!echo \" \"\n",
@@ -2174,29 +434,9 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "throughputs [5728.566484387442, 7806.743316356045]\n",
- "latencys [8.566554468505236, 8.564782385923424]\n",
- "accuracys [0.9797, 0.9797]\n"
- ]
- },
- {
- "data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAABRkAAAIbCAYAAABv6ZkDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB02ElEQVR4nOzdeViU9f7/8dewDYgwIIugIi6AoiKYJuKSVlaWmmWdllNpWWl5PNXpWxraoqWhpqfOabHFJT1Wtpl1NCuzNCv3BPd9wy0BFVBg2O7fH/68TxOL4gDj8nxc131dzud+3/e878nmAy8/c4/FMAxDAAAAAAAAAHCe3FzdAAAAAAAAAICLGyEjAAAAAAAAAKcQMgIAAAAAAABwCiEjAAAAAAAAAKcQMgIAAAAAAABwCiEjAAAAAAAAAKcQMgIAAAAAAABwCiEjAAAAAAAAAKcQMgIAAAAAAABwCiEjAAAAAAAAAKcQMgJALWnSpIksFkuZ7W9/+5sk6eTJkxo2bJgaNWokHx8fxcbGasqUKQ7nsNvt+vvf/67g4GD5+vrq5ptv1oEDB8o814IFC5SYmCgfHx8FBwerf//+lfZWXl8Wi0WvvPJKmVrDMHTjjTfKYrFo3rx55/+C4KJRXFysZ599Vk2bNpWPj4+aNWumF198UaWlpRUes2TJknL/Tm3dutWs6dGjR7k1vXv3ro3LAuACNfV+IkknTpzQ3/72N4WHh8vb21uxsbH6+uuva/qScBHIzc3VE088ocjISPn4+Khz585avXq1ub8qPwedUVRUpBdffFHNmzeXt7e34uPj9c033zjUnO1nPwAXH95PKufh6gYA4HKxevVqlZSUmI83btyo6667Tn/5y18kSf/4xz/0448/avbs2WrSpIm+++47DR06VA0aNFC/fv0kSU888YT++9//as6cOQoKCtL//d//qU+fPlq7dq3c3d0lSZ9//rkefvhhvfzyy7rmmmtkGIY2bNhQaW+HDx92eLxw4UI9+OCDuu2228rUvvbaa7JYLE69Fri4TJgwQW+//bZmzpyp1q1ba82aNXrggQdks9n0+OOPV3rstm3b5O/vbz4OCQkx/zx37lwVFhaaj7OyshQfH2/+PwHg0lNT7yeFhYW67rrrFBoaqs8++0yNGjVSenq6/Pz8auxacPF46KGHtHHjRv3nP/9RgwYNNHv2bPXs2VObN29Ww4YNq/Rz0BnPPvusZs+erffee08tW7bUt99+q1tvvVW//vqr2rVrJ+nsP/sBuPjwfnIWBgDAJR5//HGjefPmRmlpqWEYhtG6dWvjxRdfdKi54oorjGeffdYwDMM4ceKE4enpacyZM8fcf/DgQcPNzc345ptvDMMwjKKiIqNhw4bG1KlTneqtX79+xjXXXFNmPDU11WjUqJFx+PBhQ5LxxRdfOPU8uDj07t3bGDRokMNY//79jXvvvbfCY3788UdDknH8+PFzfp5XX33V8PPzM06ePHm+rQK4wNXU+8mUKVOMZs2aGYWFhdXVKi4ReXl5hru7uzF//nyH8fj4eGPUqFHlHlPRz0F/FB4ebrzxxhtljrvnnnsqPObPP/sBuLjwfnJ2fFwaAFygsLBQs2fP1qBBg8xVgV27dtVXX32lgwcPyjAM/fjjj9q+fbtuuOEGSdLatWtVVFSk66+/3jxPgwYN1KZNG/3666+SpN9++00HDx6Um5ub2rVrp/DwcN14443atGnTOff2+++/a8GCBXrwwQcdxvPy8nT33XfrjTfeUFhYmLMvAS4iXbt21eLFi7V9+3ZJUlpamn7++WfddNNNZz32zN/Da6+9Vj/++GOltdOmTdNdd90lX1/faukbwIWnpt5PvvrqKyUlJelvf/ub6tevrzZt2ujll192WPWBy1NxcbFKSkrk7e3tMO7j46Off/65TH1FPwf9md1uP+dzSuX/7Afg4sL7yTlwdcoJAJejjz/+2HB3dzcOHjxojtntdmPAgAGGJMPDw8Pw8vIyZs2aZe7/4IMPDC8vrzLnuu6664zBgwcbhmEYH330kSHJaNy4sfHZZ58Za9asMe6++24jKCjIyMrKOqfeJkyYYAQGBhr5+fkO44MHDzYefPBB87FYyXjZKC0tNZ555hnDYrEYHh4ehsViMV5++eVKj9m6davx7rvvGmvXrjV+/fVX49FHHzUsFouxdOnScutXrlxpSDJWrlxZE5cA4AJRU+8nLVq0MKxWqzFo0CBjzZo1xkcffWTUq1fPGDNmTE1fEi4CSUlJRvfu3Y2DBw8axcXFxn/+8x/DYrEYMTExZWor+jnoz+6++26jVatWxvbt242SkhLju+++M3x8fMr9Wc0wyv/ZD8DFh/eTyhEyAoALXH/99UafPn0cxl555RUjJibG+Oqrr4y0tDTj9ddfN+rWrWssWrTIMIyKQ8aePXsaQ4YMMWskGe+88465v6CgwAgODjbefvvtc+qtRYsWxrBhwxzGvvzySyMqKsrIzc01xwgZLx8fffSR0ahRI+Ojjz4y1q9fb8yaNcuoV6+e8f7771fpPH369DH69u1b7r7Bgwcbbdq0qY52AVzAaur9JDo62oiIiDCKi4vNscmTJxthYWHV1jsuXjt37jSuuuoqQ5Lh7u5uXHnllcY999xjxMbGlqkt7+eg8hw9etTo16+f4ebmZri7uxsxMTHG0KFDDR8fn3Lry/vZD8DFh/eTyvFxaQCoZfv27dP333+vhx56yBzLz8/XyJEj9c9//lN9+/ZV27ZtNWzYMN15552aNGmSJCksLEyFhYU6fvy4w/mOHj2q+vXrS5LCw8MlSa1atTL3W61WNWvWTPv37z9rb8uWLdO2bdscepOkH374Qbt27VJAQIA8PDzk4XH6e8Nuu+029ejRo+ovAi4qTz/9tJ555hndddddiouL03333ad//OMfSklJqdJ5OnXqpB07dpQZz8vL05w5c8r8vQNw6amp95Pw8HDFxMSYX4ImSbGxsTpy5IjDF0zh8tS8eXMtXbpUJ0+eVHp6ulatWqWioiI1bdrUoa6in4PKExISonnz5unUqVPat2+ftm7dqrp165Y5p1T+z34ALk68n1SOkBEAatmMGTMUGhqq3r17m2NFRUUqKiqSm5vj27K7u7tKS0slSe3bt5enp6cWLVpk7j98+LA2btyozp07mzVWq1Xbtm1zOPfevXsVGRl51t6mTZum9u3bKz4+3mH8mWee0fr165WammpukvTqq69qxowZVXsBcNHJy8ur9O/muVq3bp0ZhP/RJ598IrvdrnvvvdepPgFc+Grq/aRLly7auXOnw3m2b9+u8PBweXl5Odc0Lhm+vr4KDw/X8ePH9e2336pfv34O+yv6Oagy3t7eatiwoYqLi/X555+XOadU/s9+AC5uvJ9UwNVLKQHgclJSUmI0btzYGDFiRJl93bt3N1q3bm38+OOPxu7du40ZM2YY3t7exltvvWXWPPLII0ajRo2M77//3vjtt9+Ma665xoiPj3f4eNjjjz9uNGzY0Pj222+NrVu3Gg8++KARGhpqHDt2zKxp0aKFMXfuXIfnz87ONurUqWNMmTLlnK5FfFz6sjFw4ECjYcOGxvz58409e/YYc+fONYKDg43hw4ebNc8884xx3333mY9fffVV44svvjC2b99ubNy40XjmmWcMScbnn39e5vxdu3Y17rzzzlq5FgCuVVPvJ/v37zfq1q1rDBs2zNi2bZsxf/58IzQ01Bg7dmytXh8uTN98842xcOFCY/fu3cZ3331nxMfHGx07dnT4NvKz/Rx03333Gc8884z5eMWKFcbnn39u7Nq1y/jpp5+Ma665xmjatGmZb0Gv7Gc/ABcf3k8qR8gIALXo22+/NSQZ27ZtK7Pv8OHDxv333280aNDA8Pb2Nlq0aGFMnjzZKC0tNWvy8/ONYcOGGfXq1TN8fHyMPn36GPv373c4T2FhofF///d/RmhoqOHn52f07NnT2Lhxo0ONJGPGjBkOY++8847h4+NjnDhx4pyuhZDx8pGTk2M8/vjjRuPGjQ1vb2+jWbNmxqhRowy73W7WDBw40Ojevbv5eMKECUbz5s0Nb29vIzAw0OjatauxYMGCMufetm2bIcn47rvvauNSALhYTb6f/Prrr0ZiYqJhtVqNZs2aGePGjXP4Rzhcvj7++GOjWbNmhpeXlxEWFmb87W9/K/Pzztl+DurevbsxcOBA8/GSJUuM2NhYw2q1GkFBQcZ9991X7pcwVPazH4CLD+8nlbMYhmG4bBklAAAAAAAAgIse92QEAAAAAAAA4BQPVzdQU0pLS3Xo0CH5+fnJYrG4uh0AQCUMw1Bubq4aNGhQ5gsBcBrzGgBcPJjXzo55DQAuDlWZ0y7ZkPHQoUOKiIhwdRsAgCpIT09Xo0aNXN3GBYl5DQAuPsxrFWNeA4CLy7nMaZdsyOjn5yfp9Ivg7+/v4m4AAJXJyclRRESE+d6NspjXAODiwbx2dsxrAHBxqMqcdsmGjGeW3Pv7+zNpAcBFgo9LVYx5DQAuPsxrFWNeA4CLy7nMadwgBAAAAAAAAIBTCBkBAAAAAAAAOIWQEQAAAAAAAIBTCBkBAAAAAAAAOKVKX/xSXFqst1Lf0td7vlZmfqaCfYLVL6qfhrQdIjfL6bzSMAxNSZuiz7Z/ppzCHMUFx2lU4ihFBUaZ5yksKdSkNZO0cM9C2UvsSgxL1KhOoxTmG2bWZNuzNX7VeC1JXyJJ6hHRQ8mJyfL34qbAAAAAAAAAwIWkSiHj9I3T9en2TzWu6zg1D2iuTZmb9Nwvz8nP00/3trrXrJm1eZbGdhmrSP9Ivbv+XQ1eNFj/vfW/8vX0lSRNWDVBSw4s0cSrJirAGqBJayZp2OJh+rjPx3J3c5ckjVg2Qr+f+l1Tek6RJI1ZPkYjl43UG9e+UZ3XDwAAAACXjdUZRzR96wZtOp6pjIJ8vd7lWvVsGGnuNwxDb25ap092b1NOUaHa1gvRc1ckKdoWaNYUlpRoYtoqLdi/W/aSEnWqH67nr+issDq+rrgkAMAFokofl07LSNPVEVfrqkZXqWHdhrq+yfXq3KCzNmVtknR6Qpq9ZbYejntYPSN7KjowWuO6jlNBcYEW7F4gScotzNXcnXP1dIenldQgSbFBsUrplqIdJ3ZoxeEVkqTdJ3brl4O/aEznMUoITVBCaIJGdx6tpQeWak/2nmp+CQAAAADg8pBfXKQWAfX07BVJ5e6funWD3t++Sc9ekaRPet6sYG8fPbj0G50qKjJrXk5dqe8P7tPkpB6afU1v5RUX69GfF6mktLS2LgMAcAGqUsjYLrSdVh5eqb3ZeyVJ245t029Hf1O3ht0kSQdOHlBmfqY6N+hsHuPl7qX2Ye2VlpEmSdqctVnFpcVKavC/SS20TqiiAqKUmpEq6XSY6efpp7Yhbc2a+JB4+Xn6KfVoarm92e125eTkOGwAAAAAgP+5KjxCT8S11/WNmpTZZxiGZu3YpCGx8bq+URPF2AI1vuNVKigp0fz9uyRJuYWFmrtnu4bHd1Tn+g3VKjBIExO7a3v2cS0/eqiWrwYAcCGp0selH2zzoE4WntTN826Wu8VdJUaJHrviMd3U7CZJUlZ+liQpyCfI4bgg7yAdPnVYkpSZnylPN0/ZrLYyNZn5mWZNPZ96ZZ6/nk89ZRVkldtbSkqKxowZU5XLAQAAAAD8fwdO5SqzIF9dwhqaY17u7royJEzrMo/qzuYttel4popKSx1qQn3qKNo/QOsyj6prWCNXtA4AuABUKWT8Zu83mr97viZcNUHNA5pr27FtmrB6gkJ8QtQvqp9ZZ5GlzLHljf2RIcOhprx6wzAqPD45OVlPPvmk+TgnJ0cRERGVPicAAAAA4LTMgnxJUrC3j8N4kLe3Dp06ZdZ4urnJ5mX9U42PeXx57Ha77Ha7+ZhPngHApadKH5eevGayHox7UDc2vVExgTHq27yv7ou9T1M3TJX0vxWMZ1YknpFVkGXuC/YJVlFpkbLt2Q41xwqOOdScWRX5R8cLjivIO6jMuCRZrVb5+/s7bAAAAAAA5xiGZKl8zYgMqdJlJSkpKbLZbObGghAAuPRUKWQsKCmQ258OcXdzl6HTKwwb1W2kYJ9gLT+83NxfVFKktUfWKj4kXpLUKqiVPNw8HGoy8jK088ROJYQkSDp9/8XcolxtyNhg1qzPWK/colwlhCZU6QIBAAAAAGd3ZgXjn1ckHrMXKMjqY9YUlZYqu9DuWFOQr6A/rYD8o+TkZGVnZ5tbenp6NXcPAHC1Kn1cunuj7np3w7sKrxuu5gHNtTVrq2ZtmqVbom+RJFksFt0be6+mrp+qSL9INfZvrPc2vCdvD2/1btZbkuTn5af+Uf01afUkBVgDZPOyafKayYoOiFan8E6SpGYBzdSlYReNXj5azyc9L0kas3yMujfqrqa2ptV4+QAAAAAASWrk66dgbx/9+vtBtQo8/QmywpISrc44ov9r20GS1DowWJ5ubvr194O6MaKZJOlofp525JzQU/FXVnhuq9Uqq9Va4X4AwMWvSiHjyMSRemPdGxq7YqyOFRxTiE+Ibo+5XY/GP2rWDGozSPYSu8auHKsce47iQuL0znXvyNfT16wZ3nG43N3c9dTSp2QvtisxPFFvdH1D7m7uZs2EbhOUsipFQxYNkST1iOihkYkjnb1eAAAAALhsnSoq0v6T/7sf4oGTudpyPEs2L6sa+NbVgOjWenfLekXW9Vekn03vbkmTt7u7+jRuLkny8/JS/6Yxmpi6WgFe3rJ5WfVK2irF2AKVFNrAVZcFALgAWIzKvk3lIpaTkyObzabs7GzuzwgAFzjes8+O1wgALh4X8nv2qqOHNXDJwjLjtzSJUkrHq2QYht7ctE4f796mnMJCtQ0K0XNXJCnGFmjW2kuK9Uraas3fv1v2kmJ1Cm2g59snKbxO3XPu40J+jQAA/1OV92tCRgCAy/GefXa8RgBw8eA9++x4jQDg4lCV9+sqffELAAAAAAAAAPwZISMAAAAAAAAApxAyAgAAAAAAAHBKlb5dGgBw/uJmxrm6hRqzYeAGV7eAc9DkmQWubqFG7R3f29UtAJeVS/k9hfeTi0PsJ9Nd3UKN2XLHIFe3AFxWLuX3E6n23lNYyQgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJzi4eoGAAC41BWXlOq173doXupBZeTaFepv1e1XROjv10TJzc3i6vYAAAAAwGmEjAAA1LC3l+7SByv3afId8YoO9dOGg9l6+tM0+Xl7aFDXpq5uDwAAAACcRsgIAEAN+23/CV3Xqr6uaVlfkhRRr46+Sj2kDQezXdwZAAAAAFQP7skIAEAN69AkUL/szNLujJOSpM2HcrRm3zH1aBFS4TF2u105OTkOGwAAAABcqFjJCABADXu0e3PlFhTr2n8ulbvFohLD0FPXt1C/hIYVHpOSkqIxY8bUYpcAAAAAcP5YyQgAQA377/rDmrfuoP51VzvNf6yrJv8lXu8t263P1h6o8Jjk5GRlZ2ebW3p6ei12DAAAAABVw0pGAABqWMrXW/Roj+a6Ob6BJKllmL8OHs/XW0t26vb2jco9xmq1ymq11mabAAAAAHDeWMkIAEANyy8qkcVicRhzc7PIMFzUEAAAAABUM1YyAgBQw65tWV9v/rBTDQO8FR3qp02HcjTt5z36S4fyVzECAAAAwMWGkBEAgBo2pl9rTf5um56bt0mZJ+2q7++tv3ZsrMeujXZ1awAAAABQLQgZAQCoYXWtHnqhb2u90Le1q1sBAAAAgBrBPRkBAAAAAAAAOIWQEQAAAAAAAIBTCBkBAAAAAAAAOIWQEQAAAAAAAIBTCBkBAAAAAAAAOKVK3y59w2c36NCpQ2XG72xxp57t9KwMw9CUtCn6bPtnyinMUVxwnEYljlJUYJRZW1hSqElrJmnhnoWyl9iVGJaoUZ1GKcw3zKzJtmdr/KrxWpK+RJLUI6KHkhOT5e/lf35XCQAAAAAAAKDGVGkl40d9PtKPd/xobu9e964k6YYmN0iSpm+crlmbZ2lk4kh91PsjBfsEa/CiwTpVdMo8x4RVE7R4/2JNvGqiZvaaqbziPA1bPEwlpSVmzYhlI7T12FZN6TlFU3pO0dZjWzVy2cjquF4AAAAAAAAA1axKIWM973oK9gk2t58O/KQIvwh1qN9BhmFo9pbZejjuYfWM7KnowGiN6zpOBcUFWrB7gSQptzBXc3fO1dMdnlZSgyTFBsUqpVuKdpzYoRWHV0iSdp/YrV8O/qIxnccoITRBCaEJGt15tJYeWKo92Xuq/xUAAAAAAAAA4JTzvidjUUmR5u+er1ujbpXFYtGBkweUmZ+pzg06mzVe7l5qH9ZeaRlpkqTNWZtVXFqspAZJZk1onVBFBUQpNSNVkpSWkSY/Tz+1DWlr1sSHxMvP00+pR1Mr7MdutysnJ8dhAwAAAAAAAFDzzjtkXJy+WLmFueoX1U+SlJWfJUkK8glyqAvyDlJmfqYkKTM/U55unrJZbZXW1POpV+b56vnUU1ZBVoX9pKSkyGazmVtERMT5XhoAAAAAAACAKjjvkPGLHV+oa8OuCq0T6jBukaVMbXljf2TIcKgpr94wjErPkZycrOzsbHNLT0+vtB4AAAAAAABA9TivkPHQyUNacXiF+kf3N8fOrGA8syLxjKyCLHNfsE+wikqLlG3Pdqg5VnDMoebMqsg/Ol5wXEHeQWXGz7BarfL393fYAAAAAAAAANS88woZ5+2cp3re9XRVo6vMsUZ1GynYJ1jLDy83x4pKirT2yFrFh8RLkloFtZKHm4dDTUZehnae2KmEkARJp++/mFuUqw0ZG8ya9RnrlVuUq4TQhPNpFwAAAAAAAEAN8qjqAaVGqebtnKebm98sD7f/HW6xWHRv7L2aun6qIv0i1di/sd7b8J68PbzVu1lvSZKfl5/6R/XXpNWTFGANkM3LpslrJis6IFqdwjtJkpoFNFOXhl00evloPZ/0vCRpzPIx6t6ou5ramlbHNQMAAAAAAACoRlUOGVccWqHDpw7r1qhby+wb1GaQ7CV2jV05Vjn2HMWFxOmd696Rr6evWTO843C5u7nrqaVPyV5sV2J4ot7o+obc3dzNmgndJihlVYqGLBoiSeoR0UMjE0eez/UBAAAAAAAAqGFVDhk7N+ysDQM3lLvPYrFoaMJQDU0YWuHxVnerRiaOrDQ0tFltGt9tfFVbAwAAAAAAAOAC5/3t0gAAAAAAAAAgETICAAAAAAAAcBIhIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcIqHqxsAAOBS12X8Dzp4Ir/M+H2dIvXSLW1c0BEAAOUrLi3VG5vWaf7+XcosyFeIt49uaRKtR1slyM1ikSQZhqE3N63TJ7u3KaeoUG3rhei5K5IUbQt0cfcAAFciZAQAoIZ9NayLSgzDfLz9yEndO22lbooLd2FXAACUNXXren28a6tSOl6laFuANh7L1MjVy+Tn6aUBMa3/f80Gvb99k17u2E1N/Gx6e3OqHlz6jRbeeLt8PT1dfAUAAFfh49IAANSwoLpWhfp5m9virb8rMqiOOjWr5+rWAABwkJqVoWsaNlaPBhFq6OunGyKaqkv9htp4PFPS6VWMs3Zs0pDYeF3fqIlibIEa3/EqFZSUaP7+XS7uHgDgSoSMAADUosLiUs1bd1B3dIiQ5f9/7Kw8drtdOTk5DhsAADWtfXCoVvx+WHtysyVJW09k6bfM39U9vJEk6cCpXGUW5KtLWEPzGC93d10ZEqZ1mUcrPC/zGgBc+vi4NAAAtei7zUeUU1Cs29s3qrQuJSVFY8aMqaWuAAA47aGWbZVbVKTeCz+Xu8WiEsPQE3Ht1btxc0lSZsHpewwHe/s4HBfk7a1Dp05VeF7mNQC49LGSEQCAWvTx6nT1iAlRfX/vSuuSk5OVnZ1tbunp6bXUIQDgcvZ1+h79d98uvdKphz6/rp9SOl6l6ds2at7eHZUeZxhSJQv0mdcA4DLASkYAAGrJgeN5+mVnpt6+t/1Za61Wq6xWay10BQDA/0xKW62HWsapd+NmkqSYgHo6lHdS725Zr1uaRJsrGDML8hXqU8c87pi9QEFWn3LPKTGvAcDlgJWMAADUkk/XHFBQXauuaRnq6lYAAChXfkmx3P60JNHdYlGpYUiSGvn6KdjbR7/+ftDcX1hSotUZR9QumPkNAC5nrGQEAKAWlJYa+mztAd12RSN5uPNvfACAC9PVDSL0zpY0hdepq2hbgDYfz9L72zepf5NoSZLFYtGA6NZ6d8t6Rdb1V6SfTe9uSZO3u7v6/P/7NgIALk+EjAAA1IKfd2bq4Il83dGh8i98AQDAlZ5tl6R/bVyrF3/7VcfsBQr1rqM7mrXQ0FYJZs1DLeNkLynWi78tV05hodoGhWhq917y9fR0XeMAAJcjZAQAoBZcFROiveN7u7oNAAAq5evpqZHtOmlku04V1lgsFg1rc4WGtbmiFjsDAFzo+LwWAAAAAAAAAKcQMp7F6NGjZbFYHLawsDBz/5/3ndleeeUVSdKxY8f097//XS1atFCdOnXUuHFjPfbYY8rOznZ4nu3bt6tfv34KDg6Wv7+/unTpoh9//LHS3u6///4yz9up0//+xXHv3r0V9vfpp59W46sEAAAAAACAyxkflz4HrVu31vfff28+dnd3N/98+PBhh9qFCxfqwQcf1G233SZJOnTokA4dOqRJkyapVatW2rdvnx555BEdOnRIn332mXlc7969FRMTox9++EE+Pj567bXX1KdPH+3atcsh1PyzXr16acaMGeZjLy8v888RERFl+nv33Xc1ceJE3XjjjVV8FQAAAAAAAIDyETKeAw8PjwqDvj+Pf/nll7r66qvVrFkzSVKbNm30+eefm/ubN2+ucePG6d5771VxcbE8PDyUmZmpnTt3avr06Wrbtq0kafz48Xrrrbe0adOmSkNGq9Va4X53d/cy+7744gvdeeedqlu37tkvHAAAAAAAADgHfFz6HOzYsUMNGjRQ06ZNddddd2n37t3l1v3+++9asGCBHnzwwUrPl52dLX9/f3l4nM54g4KCFBsbq1mzZunUqVMqLi7WO++8o/r166t9+/aVnmvJkiUKDQ1VTEyMHn74YR09erTC2rVr1yo1NfWs/QEAAAAAAABVwUrGs0hMTNSsWbMUExOj33//XWPHjlXnzp21adMmBQUFOdTOnDlTfn5+6t+/f4Xny8rK0ksvvaQhQ4aYYxaLRYsWLVK/fv3k5+cnNzc31a9fX998840CAgIqPNeNN96ov/zlL4qMjNSePXv03HPP6ZprrtHatWtltVrL1E+bNk2xsbHq3Llz1V8IAAAAAAAAoAKEjGfxx3sXxsXFKSkpSc2bN9fMmTP15JNPOtROnz5d99xzj7y9vcs9V05Ojnr37q1WrVrphRdeMMcNw9DQoUMVGhqqZcuWycfHR1OnTlWfPn20evVqhYeHl3u+O++80/xzmzZt1KFDB0VGRmrBggVlgs78/Hx9+OGHeu6556r8GgAAAAAAAACV4ePSVeTr66u4uDjt2LHDYXzZsmXatm2bHnrooXKPy83NVa9evVS3bl198cUX8vT0NPf98MMPmj9/vubMmaMuXbroiiuu0FtvvSUfHx/NnDnznHsLDw9XZGRkmd4k6bPPPlNeXp4GDBhwzucDAAAAAAAAzgUhYxXZ7XZt2bKlzOrCadOmqX379oqPjy9zTE5Ojq6//np5eXnpq6++KrPSMS8vT5Lk5ub4n8PNzU2lpaXn3FtWVpbS09PLXfk4bdo03XzzzQoJCTnn8wEAAAAAAADngpDxLJ566iktXbpUe/bs0cqVK3X77bcrJydHAwcONGtycnL06aeflruKMTc3V9dff71OnTqladOmKScnR0eOHNGRI0dUUlIiSUpKSlJgYKAGDhyotLQ0bd++XU8//bT27Nmj3r17m+dq2bKlvvjiC0nSyZMn9dRTT2n58uXau3evlixZor59+yo4OFi33nqrQw87d+7UTz/9VOEqSwAAAAAAAMAZVb4n4++nfterv72qnw/+LHuxXZH+kRrTZYxaB7WWdPr+glPSpuiz7Z8ppzBHccFxGpU4SlGBUeY5CksKNWnNJC3cs1D2ErsSwxI1qtMohfmGmTXZ9myNXzVeS9KXSJJ6RPRQcmKy/L38nbviKjpw4IDuvvtuZWZmKiQkRJ06ddKKFSsUGRlp1syZM0eGYejuu+8uc/zatWu1cuVKSVJUVJTDvj179qhJkyYKDg7WN998o1GjRumaa65RUVGRWrdurS+//NJhZeS2bduUnZ0tSXJ3d9eGDRs0a9YsnThxQuHh4br66qv18ccfy8/Pz+F5pk+froYNG+r666+vttcFAAAAAAAAOKNKIWO2PVsDFg7QlWFXasq1U1TPp57Sc9Pl7/m/4G/6xumatXmWxnYZq0j/SL27/l0NXjRY/731v/L19JUkTVg1QUsOLNHEqyYqwBqgSWsmadjiYfq4z8dyd3OXJI1YNkK/n/pdU3pOkSSNWT5GI5eN1BvXvlFd135O5syZc9aawYMHa/DgweXu69GjhwzDOOs5OnTooG+//bbSmj+ex8fH56z1Z7z88st6+eWXz6kWAAAAAAAAqKoqfVx6+sbpCvMN09iuYxUXEqeGdRuqU3gnRfhHSDodgs3eMlsPxz2snpE9FR0YrXFdx6mguEALdi+QJOUW5mruzrl6usPTSmqQpNigWKV0S9GOEzu04vAKSdLuE7v1y8FfNKbzGCWEJighNEGjO4/W0gNLtSd7TzW/BAAAAAAAAACcUaWQcUn6ErUKaqUnlzyp7h9311/++xd9tv0zc/+BkweUmZ+pzg06m2Ne7l5qH9ZeaRlpkqTNWZtVXFqspAZJZk1onVBFBUQpNSNVkpSWkSY/Tz+1DWlr1sSHxMvP00+pR1PP4zIBAAAAAAAA1JQqfVz6QO4BfbLtEw1oPUAPxz2sDZkbNH7VeHm5e+nm5jcrKz9LkhTkE+RwXJB3kA6fOixJyszPlKebp2xWW5mazPxMs6aeT70yz1/Pp56yCrLK7c1ut8tut5uPc3JyqnJpAAAAAAAAAM5TlVYylqpUsUGxevyKxxUbFKs7Wtyh26Jv08fbPnaos8hS5tjyxv7IkOFQU159Zfc2TElJkc1mM7eIiIizXQ4AAAAAAACAalClkDHEJ0TNA5o7jDWzNdORk0ck/W8F45kViWdkFWSZ+4J9glVUWqRse7ZDzbGCYw41Z1ZF/tHxguMK8g4qMy5JycnJys7ONrf09PSqXBoAAAAAAACA81SlkDEhNEF7s/c6jO3N2avwuuGSpEZ1GynYJ1jLDy839xeVFGntkbWKD4mXJLUKaiUPNw+Hmoy8DO08sVMJIQmSTt9/MbcoVxsyNpg16zPWK7coVwmhCeX2ZrVa5e/v77ABAAAAAAAAqHlVuifjgFYDdN/X9+m99e/phiY3aEPmBn2+43M9n/S8JMliseje2Hs1df1URfpFqrF/Y7234T15e3ird7PekiQ/Lz/1j+qvSasnKcAaIJuXTZPXTFZ0QLQ6hXeSJDULaKYuDbto9PLR5rnHLB+j7o26q6mtaXVePwAAAAAAAAAnVSlkbBPcRq9d/Zpe++01vZ32thr6NdTwK4erT7M+Zs2gNoNkL7Fr7MqxyrHnKC4kTu9c9458PX3NmuEdh8vdzV1PLX1K9mK7EsMT9UbXN+Tu5m7WTOg2QSmrUjRk0RBJUo+IHhqZONLZ6wUAAAAAAABQzaoUMkpS94ju6h7RvcL9FotFQxOGamjC0AprrO5WjUwcWWloaLPaNL7b+Kq2BwAAAAAAAKCWVemejAAAAAAAAADwZ4SMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKYSMAAAAAAAAAJxCyAgAAAAAAADAKR6ubuBCFzczztUt1JgNAze4ugUAAAAAAABcAljJCAAAAAAAAMAphIwAAAAAAAAAnELICAAAAAAAAMAphIwAAAAAAAAAnELICAAAAAAAAMAphIwAAAAAAAAAnELICAAAAAAAAMAphIwAAAAAAAAAnELICAAAAAAAAMApHq5uAACAy8GR7AKNX7hFS7ZnqKCoRE2D62ribW0V18jm6tYAAAAAwGmEjAAA1LDsvCLdNuVXJTUP0vsPdFSQr5f2H8uTvw/TMAAAAIBLA7/dAABQw6Ys3aUGAd6a9Jd4cyyiXh0XdgQAAAAA1YuQEQCAGvb9lt91VXSIhn6wVit3H1N9f2/dlxSpuzs2dnVrAAAAAFAtCBkBAKhh+4/lafbKfXqoa1MN7RGltAMnNPqrTfJyd9Nt7RuVe4zdbpfdbjcf5+Tk1Fa7AAAAAFBlfLs0AAA1zDAMtWngr+G9WqpNQ5vuSTy9inH2yn0VHpOSkiKbzWZuERERtdgxAAAAAFQNISMAADUs1M9b0aF+DmPNQ+vq0In8Co9JTk5Wdna2uaWnp9d0mwAAAABw3vi4NAAANax9ZKB2Z550GNuTcUoNA3wqPMZqtcpqtdZ0awAAAABQLVjJCABADXuwa1Ot239Cb/64U3szT+nL1IP6aNV+DUhq4urWAAAAAKBasJIRAIAaFh8RoHfua6+J32zTvxbvUESgj57v20q3tGvo6tYAAAAAoFoQMgIAUAuuja2va2Pru7oNAAAAAKgRfFwaAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4xaMqxW+lvqUpaVMcxoK8g7TkziWSJMMwNCVtij7b/plyCnMUFxynUYmjFBUYZdYXlhRq0ppJWrhnoewldiWGJWpUp1EK8w0za7Lt2Rq/aryWpJ8+b4+IHkpOTJa/l//5XSUAAAAAAACAGlPllYxRAVH68Y4fzW1uv7nmvukbp2vW5lkamThSH/X+SME+wRq8aLBOFZ0yayasmqDF+xdr4lUTNbPXTOUV52nY4mEqKS0xa0YsG6Gtx7ZqSs8pmtJzirYe26qRy0Y6eakAAAAAAAAAakKVQ0Z3i7uCfYLNrZ53PUmnVzHO3jJbD8c9rJ6RPRUdGK1xXcepoLhAC3YvkCTlFuZq7s65errD00pqkKTYoFildEvRjhM7tOLwCknS7hO79cvBXzSm8xglhCYoITRBozuP1tIDS7Une081XjoAAAAAAACA6lDlkHF/7n5d88k16vV5Lz299Gml56ZLkg6cPKDM/Ex1btDZrPVy91L7sPZKy0iTJG3O2qzi0mIlNUgya0LrhCoqIEqpGamSpLSMNPl5+qltSFuzJj4kXn6efko9mlphX3a7XTk5OQ4bAAAAAAAAgJpXpZAxLjhO47qO09vXva0Xkl5QZn6m7vv6Pp0oOKGs/CxJUpBPkMMxQd5ByszPlCRl5mfK081TNqut0pp6PvXKPHc9n3rKKsiqsLeUlBTZbDZzi4iIqMqlAQAAAAAAADhPVQoZuzXqpusir1NMYIySGiTpzWvflCR9uetLs8YiS5njyhv7I0OGQ0159YZhVHqO5ORkZWdnm1t6enql9QAAAAAAAACqR5U/Lv1HdTzrKDowWvtz9psrGM+sSDwjqyDL3BfsE6yi0iJl27Mdao4VHHOoObMq8o+OFxxXkHdQmfEzrFar/P39HTYAAAAAAAAANc+pkLGwpFC7s3cruE6wGtVtpGCfYC0/vNzcX1RSpLVH1io+JF6S1CqolTzcPBxqMvIytPPETiWEJEg6ff/F3KJcbcjYYNasz1iv3KJcJYQmONMuAAAAAAAAgBrgUZXiSasnqXtEd4X7hutYwTG9u/5dnSo6pX7N+8liseje2Hs1df1URfpFqrF/Y7234T15e3ird7PekiQ/Lz/1j+qvSasnKcAaIJuXTZPXTFZ0QLQ6hXeSJDULaKYuDbto9PLRej7peUnSmOVj1L1RdzW1Na3mywcAAAAAAADgrCqFjL/n/a4RP43Qcftx1bPWU9uQtvrgpg/UoG4DSdKgNoNkL7Fr7MqxyrHnKC4kTu9c9458PX3NcwzvOFzubu56aulTshfblRieqDe6viF3N3ezZkK3CUpZlaIhi4ZIknpE9NDIxJHVcb0AAAAAAAAAqlmVQsZXur9S6X6LxaKhCUM1NGFohTVWd6tGJo6sNDS0WW0a3218VVoDAAAAAAAA4CJO3ZMRAAAAAAAAAKq0khEAAAAAcGn7Pe+UJq9fo5+OHJC9pFhN/Gwa26GrWtcLliQZhqE3N63TJ7u3KaeoUG3rhei5K5IUbQt0cecAAFciZAQAAAAASJKyC+366w8LlBgarne7Xa8gb2/tP5krPy8vs2bq1g16f/smvdyxm5r42fT25lQ9uPQbLbzxdvl6erqwewCAK/FxaQAAAACAJGnq1vUKr+Orlzt2U9ugEDX09VNS/QZqXNdf0ulVjLN2bNKQ2Hhd36iJYmyBGt/xKhWUlGj+/l0u7h4A4EqsZAQAAAAASJJ+PJSuLvUb6olff9DqjCOq71NHdzWP1R3NW0iSDpzKVWZBvrqENTSP8XJ315UhYVqXeVR3Nm9Z7nntdrvsdrv5OCcnp2YvBABQ61jJCAAAAACQJKWfzNWcXVsVWddf7111g+5s3lIvp67QvL07JEmZBfmSpGBvH4fjgry9zX3lSUlJkc1mM7eIiIiauwgAgEsQMgIAAAAAJEmGDLUKDNI/2nZQq8Ag3dm8pf7StIXm7Npa+XGGZLFUvD85OVnZ2dnmlp6eXs2dAwBcjY9LAwAAAAAknV6h2Nw/wGGsmb9N3x3ca+6XTq9oDPWpY9YcsxcoyOq4uvGPrFarrFZrtfcLALhwsJIRAAAAACBJuiK4vvbmZjuM7c3NUYM6dSVJjXz9FOzto19/P2juLywp0eqMI2oXHFqrvQIALiyEjAAAAAAASdLAmNZKyzqqdzanaV9ujubv26VPd2/TX6NiJUkWi0UDolvr3S3rtejAXm3PPq6Rq5fJ291dfRo3d3H3AABX4uPSAAAAAABJUly9EP27y7V6dcNavbU5VY186+qZhET1jfxfgPhQyzjZS4r14m/LlVNYqLZBIZravZd8PT1d2DkAwNUIGQEAAAAApqsbNNbVDRpXuN9isWhYmys0rM0VtdgVAOBCx8elAQAAAAAAADiFkBEAAAAAAACAU/i4NAAANezVRdv1r8U7HMaC61q15tmeLuoIAAAAAKoXISMAALUgpn5dzX4o0XzsbrG4sBsAAAAAqF6EjAAA1AJ3NzeF+nm7ug0AAAAAqBGEjAAA1IK9mafUcdz38vJwU0JEgIbf0FKNg+pUWG+322W3283HOTk5tdEmAAAAAJwXvvgFAIAaltA4QP+8I16zHuyo8f3bKiPXrv5TftXxU4UVHpOSkiKbzWZuERERtdgxAAAAAFQNISMAADXs6hahujEuXC3D/NU1OlgzHrhSkvT5bwcqPCY5OVnZ2dnmlp6eXlvtAgAAAECV8XFpAABqWR0vD7UM89OezFMV1litVlmt1lrsCgAAAADOHysZAQCoZfbiEu08epIvggEAAABwyWAlIwAANWzcgs26Nra+Ggb4KPOkXW/8sFMn7cW6rX1DV7cGAAAAANWCkBEAgBp2OLtAj320TsfzClXP10vtIgL1xdDOahRY8bdLAwAAAMDFhJARAIAa9sZfr3B1CwAAAABQo7gnIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcIpTIePUDVMVNzNOE1ZNMMcMw9BbqW/pmk+uUYfZHfTANw9o5/GdDscVlhTq5ZUvq9ucbur4QUf9ffHfdeTUEYeabHu2kpclK+nDJCV9mKTkZcnKKcxxpl0AAAAAAAAANeC8Q8aNmRv12fbPFBMY4zA+feN0zdo8SyMTR+qj3h8p2CdYgxcN1qmiU2bNhFUTtHj/Yk28aqJm9pqpvOI8DVs8TCWlJWbNiGUjtPXYVk3pOUVTek7R1mNbNXLZyPNtFwAAAAAAAEANOa+QMa8oT88se0YvJL0gfy9/c9wwDM3eMlsPxz2snpE9FR0YrXFdx6mguEALdi+QJOUW5mruzrl6usPTSmqQpNigWKV0S9GOEzu04vAKSdLuE7v1y8FfNKbzGCWEJighNEGjO4/W0gNLtSd7TzVcNgAAAAAAAIDqcl4h47iV49StYTclNUhyGD9w8oAy8zPVuUFnc8zL3Uvtw9orLSNNkrQ5a7OKS4sdjg2tE6qogCilZqRKktIy0uTn6ae2IW3NmviQePl5+in1aGq5PdntduXk5DhsAAAAAAAAAGpelUPGhXsWanPWZj3R/oky+7LysyRJQT5BDuNB3kHKzM+UJGXmZ8rTzVM2q63Smno+9cqcv55PPWUVZJXbV0pKimw2m7lFRERU9dIAAAAAAAAAnIcqhYxHTh3R+FXjNb7beFndrRXWWWQ5p7E/MmQ41JRXbxhGhccnJycrOzvb3NLT0yt9PgAAAAAAAADVw6MqxZuyNulYwTHdOf9Oc6zEKNHa39fqo60f6b+3/FfS6ZWIIXVCzJqsgixzdWOwT7CKSouUbc92WM14rOCYEkITzJozqyL/6HjBcQV5B5UZlySr1SqrteLgEwAAAAAAAEDNqNJKxk7hnTT35rn6tO+n5tY6qLV6N+utT/t+qkZ+jRTsE6zlh5ebxxSVFGntkbWKD4mXJLUKaiUPNw+Hmoy8DO08sVMJIQmSTt9/MbcoVxsyNpg16zPWK7co1wwiAQAAAAAAAFwYqrSS0dfTV9GB0Q5jPh4+CrAGmOP3xt6rqeunKtIvUo39G+u9De/J28NbvZv1liT5efmpf1R/TVo9SQHWANm8bJq8ZrKiA6LVKbyTJKlZQDN1adhFo5eP1vNJz0uSxiwfo+6NuqupranTFw0AAAAAAACg+lQpZDwXg9oMkr3ErrErxyrHnqO4kDi9c9078vX0NWuGdxwudzd3PbX0KdmL7UoMT9QbXd+Qu5u7WTOh2wSlrErRkEVDJEk9InpoZOLI6m4XAAAAAAAAgJOcDhln9Jrh8NhisWhowlANTRha4TFWd6tGJo6sNDS0WW0a3228s+0BAAAAAAAAqGFVuicjAAAAAAAAAPwZISMAAAAAAAAApxAyAgAAAAAAAHAKISMAAAAAAAAApxAyAgAAAAAAAHAKISMAAAAAAAAApxAyAgAAAAAAAHAKISMAAAAAAAAApxAyAgAAAAAAAHAKISMAALXszR93qskzCzTmv5tc3QoAAAAAVAtCRgAAalFa+gl9tGq/Wob5uboVAAAAAKg2hIwAANSSU/ZiPfFxqsb3byubj6er2wEAAACAakPICABALXnuy426ukWoukYHn7XWbrcrJyfHYQMAAACACxUhIwAAteCrtEPadDBHw3u1OKf6lJQU2Ww2c4uIiKjhDgEAAADg/BEyAgBQww6dyNeL/92kV+9MkLen+zkdk5ycrOzsbHNLT0+v4S4BAAAA4Px5uLoBAAAudRsOZivzZKH6vvGzOVZSamjV3mOatXyfto+9Ue5uFodjrFarrFZrbbcKAAAAAOeFkBEAgBrWJSpY3z5xlcPY05+lqXlIXT3SvXmZgBEAAAAALjaEjAAA1LC6Vg+1CPNzGPPxdFdAHc8y4wAAAABwMeKejAAAAAAAAACcwkpGAABc4OMhSa5uAQAAAACqDSsZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADiFkBEAAAAAAACAUwgZAQAAAAAAADjFw9UNAAAAAAAuTO9uSdOrG9bqvuhWGtmukyTJMAy9uWmdPtm9TTlFhWpbL0TPXZGkaFugi7sFALgSKxkBAAAAAGVsOJahT3ZvU4s/hYdTt27Q+9s36dkrkvRJz5sV7O2jB5d+o1NFRS7qFABwISBkBAAAAAA4OFVUpKdXLNWLHbrI38tqjhuGoVk7NmlIbLyub9REMbZAje94lQpKSjR//y4XdgwAcDVCRgAAAACAg5d+W67u4RHqXL+hw/iBU7nKLMhXl7D/jXu5u+vKkDCtyzxa4fnsdrtycnIcNgDApYWQEQAAAABgWrB/tzafyNKTbduX2ZdZkC9JCvb2cRgP8vY295UnJSVFNpvN3CIiIqq3aQCAyxEyAgAAAAAkSYfzTipl3QpNTLxKVvdz/55Qw5Aslor3JycnKzs729zS09OroVsAwIWEb5cGAAAAAEiSNh3PUpa9QLcv+socKzEMrck4og93btHXN94m6fSKxlCfOmbNMXuBgqw+Zc53htVqldVqrXA/AODiR8gIAAAAAJAkJYU20Jc33OowNmrVMjX1t+mhlm0V4eunYG8f/fr7QbUKDJIkFZaUaHXGEf1f2w6uaBkAcIEgZAQAAAAASJJ8PT0VYwt0GPPx8FCAl9UcHxDdWu9uWa/Iuv6K9LPp3S1p8nZ3V5/GzV3RMgDgAkHICAAAAAA4Zw+1jJO9pFgv/rZcOYWFahsUoqnde8nX09PVrQEAXKhKIePHWz/Wx9s/1qGThyRJzQOa65G2j6hbo26SJMMwNCVtij7b/plyCnMUFxynUYmjFBUYZZ6jsKRQk9ZM0sI9C2UvsSsxLFGjOo1SmG+YWZNtz9b4VeO1JH2JJKlHRA8lJybL38vfycsFAAAAAFTFrKtvcnhssVg0rM0VGtbmChd1BAC4EFXp26Xr+9bXE1c8oTm952hO7zlKDEvUYz8+pp3Hd0qSpm+crlmbZ2lk4kh91PsjBfsEa/CiwTpVdMo8x4RVE7R4/2JNvGqiZvaaqbziPA1bPEwlpSVmzYhlI7T12FZN6TlFU3pO0dZjWzVy2chqumQAAAAAAAAA1alKIWOPiB66qtFVamJroia2JnrsisdUx6OO1meul2EYmr1lth6Oe1g9I3sqOjBa47qOU0FxgRbsXiBJyi3M1dydc/V0h6eV1CBJsUGxSumWoh0ndmjF4RWSpN0nduuXg79oTOcxSghNUEJogkZ3Hq2lB5ZqT/ae6n8FAAAAAAAAADilSiHjH5WUlmjhnoXKL85XfEi8Dpw8oMz8THVu0Nms8XL3Uvuw9krLSJMkbc7arOLSYiU1SDJrQuuEKiogSqkZqZKktIw0+Xn6qW1IW7MmPiRefp5+Sj2aer7tAgAAAAAAAKghVf7il+3Ht+ver+9VYUmh6njU0WtXv6bmAc3NADDIJ8ihPsg7SIdPHZYkZeZnytPNUzarrUxNZn6mWVPPp16Z563nU09ZBVkV9mW322W3283HOTk5Vb00AAAAAAAAAOehyisZm/o31Wd9P9MHN32gO1rcoWd/fla7Tuwy91tkKXNMeWN/ZMhwqCmv3jCMSs+RkpIim81mbhEREWe7FAAAAAAAAADVoMoho6e7pxr7N1br4NZ6ov0TiqkXo9lbZpsrGM+sSDwjqyDL3BfsE6yi0iJl27Mdao4VHHOoycovu2LxeMFxBXkHlRk/Izk5WdnZ2eaWnp5e1UsDAAAAAAAAcB7O+56MJkMqLClUo7qNFOwTrOWHl5u7ikqKtPbIWsWHxEuSWgW1koebh0NNRl6Gdp7YqYSQBEmn77+YW5SrDRkbzJr1GeuVW5SrhNCECtuwWq3y9/d32AAAAAAAAADUvCrdk/Ffv/1LXRt2VZhvmE4VndI3e77R6t9Xa0rPKbJYLLo39l5NXT9VkX6RauzfWO9teE/eHt7q3ay3JMnPy0/9o/pr0upJCrAGyOZl0+Q1kxUdEK1O4Z0kSc0CmqlLwy4avXy0nk96XpI0ZvkYdW/UXU1tTav58gEAAAAAAAA4q0ohY1Z+lkYuG6mM/Az5efkpOjBaU3pOMb9RelCbQbKX2DV25Vjl2HMUFxKnd657R76evuY5hnccLnc3dz219CnZi+1KDE/UG13fkLubu1kzodsEpaxK0ZBFQyRJPSJ6aGTiyOq4XgAAAAAAAADVrEoh44tdXqx0v8Vi0dCEoRqaMLTCGqu7VSMTR1YaGtqsNo3vNr4qrQEAAAAAAABwEefvyQgAAAAAAADgskbICAAAAAAAAMAphIwAAAAAAAAAnELICAAAAAAAAMApVfriFwAAUHX/WbFPH6zYpwPH8yVJ0fXr6rFro3V1i1AXdwYAAAAA1YOQEQCAGhbu760RvVoqMqiOJOnz3w5o8Kw1WvBYN8XU93NxdwAAAADgPEJGAABqWM9W9R0eP31DS81esV/r9h8nZAQAAABwSSBkBACgFpWUGlqw4bDyC0t0ReNAV7cDAAAAANWCkBEAgFqw9UiO+r/1q+zFparj5a537muv6EpWMdrtdtntdvNxTk5ObbQJAAAAAOeFb5cGAKAWNAuuq68f66YvhnbWvZ0i9X+fpmnH77kV1qekpMhms5lbRERELXYLAAAAAFVDyAgAQC3w8nBTk2BftW0UoBG9Wio23E/Tf9lbYX1ycrKys7PNLT09vfaaBQAAAIAq4uPSAAC4gGFIhcWlFe63Wq2yWq212BEAAAAAnD9CRgAAatjEb7aqR4tQhdu8daqwWP9NO6QVu7M0c1BHV7cGAAAAANWCkBEAgBqWedKuf3ycqoxcu/y8PdQy3E8zB3VUt+gQV7cGAAAAANWCkBEAgBo28fZ4V7cAAAAAADWKL34BAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BRCRgAAAAAAAABOIWQEAAAAAAAA4BSPqhRP3TBV3+/7Xnuy98jbw1vxIfH6R/t/qKmtqVljGIampE3RZ9s/U05hjuKC4zQqcZSiAqPMmsKSQk1aM0kL9yyUvcSuxLBEjeo0SmG+YWZNtj1b41eN15L0JZKkHhE9lJyYLH8vf+euGAAAAAAAAEC1qtJKxjVH1uiulnfpg5s+0LvXvasSo0RDFg1RXlGeWTN943TN2jxLIxNH6qPeHynYJ1iDFw3WqaJTZs2EVRO0eP9iTbxqomb2mqm84jwNWzxMJaUlZs2IZSO09dhWTek5RVN6TtHWY1s1ctnIarhkAAAAAAAAANWpSiHj29e9rVuiblFUYJRa1Guhl7q8pMOnDmtz1mZJp1cxzt4yWw/HPayekT0VHRitcV3HqaC4QAt2L5Ak5Rbmau7OuXq6w9NKapCk2KBYpXRL0Y4TO7Ti8ApJ0u4Tu/XLwV80pvMYJYQmKCE0QaM7j9bSA0u1J3tPNb8EAAAAAAAAAJzh1D0ZTxaelCTZrDZJ0oGTB5SZn6nODTqbNV7uXmof1l5pGWmSpM1Zm1VcWqykBklmTWidUEUFRCk1I1WSlJaRJj9PP7UNaWvWxIfEy8/TT6lHU8vtxW63Kycnx2EDAAAAAAAAUPPOO2Q0DEOvrH5FV4ReoejAaElSVn6WJCnIJ8ihNsg7SJn5mZKkzPxMebp5msFkRTX1fOqVec56PvWUVZBVbj8pKSmy2WzmFhERcb6XBgAAAAAAAKAKzjtkHLdynLYf364JV00os88iyzmN/ZEhw6GmvHrDMCo8Pjk5WdnZ2eaWnp5e6fMBAAAAAAAAqB7nFTK+vPJlLUlfomk3THP4RugzKxjPrEg8I6sgy9wX7BOsotIiZduzHWqOFRxzqDmzKvKPjhccV5B3UJlxSbJarfL393fYAAAAAAAAANS8KoWMhmFo3IpxWrxvsabdME2N/Bo57G9Ut5GCfYK1/PByc6yopEhrj6xVfEi8JKlVUCt5uHk41GTkZWjniZ1KCEmQdPr+i7lFudqQscGsWZ+xXrlFuUoITajqNQIAAAAAAACoQR5VKR63cpy+3v21/nXNv+Tr6WuuWKzrWVfeHt6yWCy6N/ZeTV0/VZF+kWrs31jvbXhP3h7e6t2styTJz8tP/aP6a9LqSQqwBsjmZdPkNZMVHRCtTuGdJEnNApqpS8MuGr18tJ5Pel6SNGb5GHVv1F1NbU2r8/oBAAAAAAAAOKlKIePH2z6WJA36dpDD+EtdXtItUbec3tdmkOwldo1dOVY59hzFhcTpneveka+nr1k/vONwubu566mlT8lebFdieKLe6PqG3N3czZoJ3SYoZVWKhiwaIknqEdFDIxNHntdFAgAAAAAAAKg5VQoZNwzccNYai8WioQlDNTRhaIU1VnerRiaOrDQ0tFltGt9tfFXaAwDggvTmjzv17aYj2nX0pLw93XVFZKCeubGlmofUdXVrAAAAAFAtzvvbpQEAwLlZueeY7usUqS/+1kX/eTBRJaWGBkxbpbzCYle3BgAAAADVokorGQEAQNXNGtTR4fErt7dV+7Hfa8OBbCU2C3JRVwAAAABQfQgZAQCoZbkFp1cwBtTxqrDGbrfLbrebj3Nycmq8LwAAAAA4X3xcGgCAWmQYhsYu2KwrmwSqRZhfhXUpKSmy2WzmFhERUYtdAgAAAEDVEDICAFCLnv9yk7YcztW/725XaV1ycrKys7PNLT09vZY6BAAAAICq4+PSAADUkhe+3Kjvt/yuT4YkKdzmU2mt1WqV1Wqtpc4AAAAAwDmEjAAA1DDDMPTCV5v07aYjmjM4SRH16ri6JQAAAACoVoSMAADUsOe+3KgvUw/pvQEd5Gt119HcAkmSv7envD3dXdwdAAAAADiPkBEAgBo2e8V+SdJd765wGH/l9rb6Swe+0AUAAADAxY+QEQCAGrZ3fG9XtwAAAAAANYpvlwYAAAAAAADgFEJGAAAAAAAAAE7h49IAAAAAAEnSu1vStOjAPu3OPSFvdw+1CwrV/7W9Uk39bWaNYRh6c9M6fbJ7m3KKCtW2XoieuyJJ0bZAF3YOAHA1VjICAAAAACRJqzOO6K9RsZpzbV9N636Dig1DD/70jfKKi8yaqVs36P3tm/TsFUn6pOfNCvb20YNLv9GpoqJKzgwAuNQRMgIAAAAAJEnvXXWDbm0arWhboFoGBOnlK7vqcN4pbTqeJen0KsZZOzZpSGy8rm/URDG2QI3veJUKSko0f/8uF3cPAHAlQkYAAAAAQLly///qRJuXVZJ04FSuMgvy1SWsoVnj5e6uK0PCtC7zaIXnsdvtysnJcdgAAJcWQkYAAAAAQBmGYWhC2kq1D66vmP9/v8XMgnxJUrC3j0NtkLe3ua88KSkpstls5hYREVFzjQMAXIKQEQAAAABQxku/Lde2E8c1qVOPs9YahmSxVLw/OTlZ2dnZ5paenl59jQIALgh8uzQAAAAAwMHY35brx0Pp+s/VNymsjq85fmYFY2ZBvkJ96pjjx+wFCrL6lDnPGVarVVarteYaBgC4HCsZAQAAAACSTn9E+qXflmvRwX2a0aOXGtX1c9jfyNdPwd4++vX3g+ZYYUmJVmccUbvg0NpuFwBwAWElIwAAAABAkvTib8u1YP9uvdHlWvl6eCojP0+S5OfpJW8PD1ksFg2Ibq13t6xXZF1/RfrZ9O6WNHm7u6tP4+Yu7h4A4EqEjAAAAAAASdKcXVslSQOXLHQYf/nKbrq1abQk6aGWcbKXFOvF35Yrp7BQbYNCNLV7L/l6etZ6vwCACwchIwAAAABAkrTljkFnrbFYLBrW5goNa3NFLXQEALhYcE9GAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFEJGAAAAAAAAAE4hZAQAAAAAAADgFI+qHrDmyBq9v+l9bc7arIz8DL129Wu6tvG15n7DMDQlbYo+2/6ZcgpzFBccp1GJoxQVGGXWFJYUatKaSVq4Z6HsJXYlhiVqVKdRCvMNM2uy7dkav2q8lqQvkST1iOih5MRk+Xv5n//VAgAAAAAAAKh2VV7JmF+cr5jAGI1MHFnu/ukbp2vW5lkamThSH/X+SME+wRq8aLBOFZ0yayasmqDF+xdr4lUTNbPXTOUV52nY4mEqKS0xa0YsG6Gtx7ZqSs8pmtJzirYe26qRy8p/TgAAAAAAAACuU+WQsVujbnrsisfUM7JnmX2GYWj2ltl6OO5h9YzsqejAaI3rOk4FxQVasHuBJCm3MFdzd87V0x2eVlKDJMUGxSqlW4p2nNihFYdXSJJ2n9itXw7+ojGdxyghNEEJoQka3Xm0lh5Yqj3Ze5y8ZAAAAAAAAADVqVrvyXjg5AFl5meqc4PO5piXu5fah7VXWkaaJGlz1mYVlxYrqUGSWRNaJ1RRAVFKzUiVJKVlpMnP009tQ9qaNfEh8fLz9FPq0dRyn9tutysnJ8dhAwAAAAAAAFDzqjVkzMrPkiQF+QQ5jAd5BykzP1OSlJmfKU83T9mstkpr6vnUK3P+ej71lFWQVe5zp6SkyGazmVtERITT1wMAAAAAAADg7Grk26UtspzT2B8ZMhxqyqs3DKPC45OTk5WdnW1u6enpVegYAICatXJ3lh58f7U6jvteTZ5ZoG83HXF1SwAAAABQbao1ZDyzgvHMisQzsgqyzH3BPsEqKi1Stj3boeZYwTGHmjOrIv/oeMFxBXkHlRmXJKvVKn9/f4cNAIALRV5RiWLD/fViv9aubgUAAAAAql21hoyN6jZSsE+wlh9ebo4VlRRp7ZG1ig+JlyS1CmolDzcPh5qMvAztPLFTCSEJkk7ffzG3KFcbMjaYNesz1iu3KFcJoQnV2TIAALXi6haheuqGFurVJtzVrQAAAABAtfOo6gF5RXnan7vffHww96C2Htsqm5dN4XXDdW/svZq6fqoi/SLV2L+x3tvwnrw9vNW7WW9Jkp+Xn/pH9dek1ZMUYA2QzcumyWsmKzogWp3CO0mSmgU0U5eGXTR6+Wg9n/S8JGnM8jHq3qi7mtqaVsd1AwBwQbPb7bLb7eZjvtAMAAAAwIWsyiHjpqxNGvTtIPPxK2tekSTd3Pxmjes6ToPaDJK9xK6xK8cqx56juJA4vXPdO/L19DWPGd5xuNzd3PXU0qdkL7YrMTxRb3R9Q+5u7mbNhG4TlLIqRUMWDZEk9YjooZGJI8/7QgEAuJikpKRozJgxrm4DAAAAAM5JlUPGK8Ou1IaBGyrcb7FYNDRhqIYmDK2wxupu1cjEkZWGhjarTeO7ja9qewAAXBKSk5P15JNPmo9zcnIUERHhwo4AAAAAoGJVDhkBAEDNs1qtslqtrm4DAAAAAM5JtX7xCwAAAAAAAIDLDysZAQCoBafsxdqbdcp8nH4sT5sOZSugjpcaBvi4sDMAAAAAcB4hIwAAtWD9gWzd/d4K8/HYBVskSbdd0UiT74h3VVsAAAAAUC0IGQEAqAVJzYO0d3xvV7cBAAAAADWCezICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcAohIwAAAAAAAACnEDICAAAAAAAAcIqHqxs4mzlb5+j9Te8rIy9DzQOaa0THEWpfv72r2wIAoMr+s3yv3vlpt47m2hVTv66e79NaHZvWc3VbAACclw93btH0bRuUkZ+vKFuAkhMS1SEkzNVtAQBc5IJeyfjNnm80YfUEPRz3sD7t+6na12+vR79/VIdPHnZ1awAAVMl/0w7pxfmbNezqKH39WFdd2aSe7p+xSgdP5Lu6NQAAquzr/bs1PnWlhsTGa+71/dQ+uL6GLPtOh06ddHVrAAAXuaBDxlmbZ6l/VH/dFnObmgU004iOIxTmG6aPt33s6tYAAKiSqT/v0R0dInRXx8aKCvXTC31bK9zmrdkr9rm6NQAAqmzm9o3q3zRGf2nWQs39AzSyXSeF+fhqzq6trm4NAOAiF+zHpYtKirQ5a7MebPOgw3jnBp2VmpFapt5ut8tut5uPs7OzJUk5OTlO9VGSX+LU8RcyZ18bAFXD+8nZjzcMozraueAUFpdq48FsPdq9ucN4t+gQrd13vNxjamJeK7XnnfexFwPmNaB2XcrvKcxrlSssKdGm41l6qGVbh/EuYQ21LutoucfUxLxWknfpfhqAOQ2oXZfy+4nk3HtKVea0CzZkPG4/rhKjREE+QQ7jQd5BysrPKlOfkpKiMWPGlBmPiIiosR4vdrZHba5uAcAlorreT3Jzc2WzXXrvTcfzClVSaijEz8thPMTPqszt9nKPYV6rOttrru4AwKWiut5PLtV57UShXSWGoWBvH4fxIKuPMgvKD5+Z16rG9sAwV7cA4BJSHe8p5zKnXbAhY0UMlZ+cJicn68knnzQfl5aW6tixYwoKCpLFYqmt9s5bTk6OIiIilJ6eLn9/f1e3A+Aid7G9pxiGodzcXDVo0MDVrdQwx/nIMIw/D5mY1wDgtIvx/eSynddkyFLBxMa8BgCnXWzvJ1WZ0y7YkDHQGih3i3uZVYvHCo6VWd0oSVarVVar1WEsICCgJlusEf7+/hfFXzIAF4eL6T3lUlzpcUZgHS+5u1mUkeu4ajHzZKGC61rLPYZ5DQAcXWzvJ5fyvBbgZZW7xVJm1eIxe4GC/rS68QzmNQBwdDG9n5zrnHbBfvGLp7unWgW10vLDyx3Glx9aroSQBNc0BQDAefDycFObhjb9vDPDYfznnZlqHxnooq4AADg/Xu7uah0YpF9/P+Qw/uvvh9QuKNRFXQEAXO2CXckoSQNaDVDyz8lqHdRa8SHx+nT7pzp86rDuaHGHq1sDAKBKHuraVE9+kqq2DQN0RWSAPlyZrkMn8nVPYmNXtwYAQJUNjGmjZ1b9pDaBwUoIDtUnu7bpcN5J3dm8patbAwC4yAUdMvZq2ksn7Cf0dtrbysjPUFRAlN669i01qHvp3dvEarXqhRdeKPMRAgA4H7ynXHj6xjfQibxC/WvxDmXk2hUTVlcz7r9SjQLruLq1GsHfQQDVhfeTC9NNjZvpRKFdb21OVUZBnqJtgXq72/Vq6FvX1a3VCP4eAqgul/L7icU4l++gBgAAAAAAAIAKXLD3ZAQAAAAAAABwcSBkBAAAAAAAAOAUQkYAAAAAAAAATiFkBAAAAAAAAOAUQsZqdv/998tisZTZdu7c6bDP09NTzZo101NPPaVTp05JkrKystSrVy81aNBAVqtVERERGjZsmHJycszzL1myRP369VN4eLh8fX2VkJCgDz74wFWXC6CG3X///brlllvMP1ssFo0fP96hZt68ebJYLA41lW2SVFxcrGeffVZNmzaVj4+PmjVrphdffFGlpaW1en24sDGnAahuzGtwJeY1ANWJOa0sQsYa0KtXLx0+fNhha9q0qcO+3bt3a+zYsXrrrbf01FNPSZLc3NzUr18/ffXVV9q+fbvef/99ff/993rkkUfMc//6669q27atPv/8c61fv16DBg3SgAED9N///tcl1wqgdnl7e2vChAk6fvx4ufv/9a9/Obz3SNKMGTPKjE2YMEFvv/223njjDW3ZskUTJ07UK6+8otdff73WrgUXB+Y0ADWJeQ21jXkNQE1hTpM8XN3ApchqtSosLOys+/7617/qxx9/1Lx58zRlyhQFBgbq0UcfNWsjIyM1dOhQvfLKK+bYyJEjHc732GOP6dtvv9UXX3yhvn371sDVALiQ9OzZUzt37lRKSoomTpxYZr/NZpPNZnMYCwgIKPOetHz5cvXr10+9e/eWJDVp0kQfffSR1qxZU3PN46LEnAagJjGvobYxrwGoKcxprGR0OR8fHxUVFZW779ChQ5o7d666d+9e6Tmys7NVr169mmgPwAXG3d1dL7/8sl5//XUdOHDgvM/TtWtXLV68WNu3b5ckpaWl6eeff9ZNN91UXa3iMsScBqCqmNdwIWNeA1AVzGmEjDVi/vz5qlu3rrn95S9/Kbdu1apV+vDDD3Xttdc6jN99992qU6eOGjZsKH9/f02dOrXC5/rss8+0evVqPfDAA9V6DQAuXLfeeqsSEhL0wgsvnPc5RowYobvvvlstW7aUp6en2rVrpyeeeEJ33313NXaKSwFzGoCaxryG2sS8BqAmXe5zGiFjDbj66quVmppqbv/+97/NfWcmNW9vbyUlJemqq64q87n6V199Vb/99pvmzZunXbt26cknnyz3eZYsWaL7779f7733nlq3bl2j1wTgwjJhwgTNnDlTmzdvPq/jP/74Y82ePVsffvihfvvtN82cOVOTJk3SzJkzq7lTXOyY0wDUBuY11BbmNQA17XKe07gnYw3w9fVVVFRUufuuvvpqTZkyRZ6enmrQoIE8PT3L1ISFhSksLEwtW7ZUUFCQunXrpueee07h4eFmzdKlS9W3b1/985//1IABA2rsWgBcmK666irdcMMNGjlypO6///4qH//000/rmWee0V133SVJiouL0759+5SSkqKBAwdWc7e4mDGnAagNzGuoLcxrAGra5TynETLWssomtfIYhiFJstvt5tiSJUvUp08fTZgwQYMHD672HgFcHMaPH6+EhATFxMRU+di8vDy5uTkuZnd3d1dpaWl1tYfLAHMagOrEvAZXY14DUF0u1zmNkPEC8vXXX+v333/XlVdeqbp162rz5s0aPny4unTpoiZNmkg6PWn17t1bjz/+uG677TYdOXJEkuTl5cUNhYHLTFxcnO65554yH+M5F3379tW4cePUuHFjtW7dWuvWrdM///lPDRo0qAY6xeWIOQ1AVTGv4ULGvAagKi7XOY17Ml5AfHx89N5776lr166KjY3VE088oT59+mj+/Plmzfvvv6+8vDylpKQoPDzc3Pr37+/CzgG4yksvvWT+K3pVvP7667r99ts1dOhQxcbG6qmnntKQIUP00ksv1UCXuBwxpwE4H8xruFAxrwGoqstxTrMY53PFAAAAAAAAAPD/sZIRAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGQEAAAAAAAA4hZARAAAAAAAAgFMIGXFZ2rt3rywWi1JTU13dSoV69OihJ554wqU9bNu2TWFhYcrNzTXH5s2bp6ioKLm7u1dLf/Pnz1e7du1UWlrq9LkAAMjKylJoaKj27t171tqjR48qJCREBw8erPnGAACoJtOmTdP111/v1DneeOMN3XzzzdXUEXAaISMuORaLpdLt/vvvd3WLF5T3339fAQEB5e4bNWqU/va3v8nPz88cGzJkiG6//Xalp6frpZdecvr5+/TpI4vFog8//NDpcwEAzs3999+vW2655byOrWzeuBCkpKSob9++atKkyVlrQ0NDdd999+mFF16o+cYAAOfl119/lbu7u3r16uXqVi4Idrtdzz//vJ577jlzbNGiRYqJiZHNZtPAgQNVWFho7svOzlZMTIz279/vcJ6HH35Yq1ev1s8//1xrvePSR8iIS87hw4fN7bXXXpO/v7/D2L/+9a/zOm9JSclltdruwIED+uqrr/TAAw+YYydPntTRo0d1ww03qEGDBg7hozMeeOABvf7669VyLgDA5Ss/P1/Tpk3TQw89dM7HPPDAA/rggw90/PjxGuwMAHC+pk+frr///e/6+eefywRltamoqMhlz/1Hn3/+uerWratu3bpJkkpLS3XPPffokUce0a+//qpVq1bpvffeM+tHjBihRx55RI0bN3Y4j9Vq1V//+ld+D0O1ImTEJScsLMzcbDabLBZLmbEzdu/erauvvlp16tRRfHy8li9fbu47s1Jj/vz5atWqlaxWq/bt26fjx49rwIABCgwMVJ06dXTjjTdqx44d5nGjR49WQkKCQ0+vvfaaw4qK4uJiPfbYYwoICFBQUJBGjBihgQMHlllVUlpaquHDh6tevXoKCwvT6NGjHfZbLBZNmTJFN954o3x8fNS0aVN9+umn5v4lS5bIYrHoxIkT5lhqaqosFov27t2rJUuW6IEHHlB2dra50vPMc3zyySeKj49Xo0aNzHOdCRWvueYaWSwWLVmyxHyd5s2bp5iYGHl7e+u6665Tenq6+ZxpaWm6+uqr5efnJ39/f7Vv315r1qwx9998881atWqVdu/eXfF/WABArfjnP/+puLg4+fr6KiIiQkOHDtXJkyclqdJ5o7CwUMOHD1fDhg3l6+urxMRELVmyxDzvmfni22+/VWxsrOrWratevXrp8OHDDs8/ffp0tW7dWlarVeHh4Ro2bJgkadCgQerTp49DbXFxscLCwjR9+nRJ0sKFC+Xh4aGkpCSz5vjx47rnnnsUEhIiHx8fRUdHa8aMGeb+uLg4hYWF6Ysvvqi21xAAUD1OnTqlTz75RI8++qj69Omj999/32H/V199pQ4dOsjb21vBwcHq37+/uc9ut2v48OGKiIiQ1WpVdHS0pk2bJqn8Vfnz5s2TxWIxH5/5vW769Olq1qyZrFarDMPQN998o65du5q/y/Xp00e7du1yONeBAwd01113qV69evL19VWHDh20cuVK7d27V25ubg6/C0nS66+/rsjISBmGcdZ5a86cOQ4fc87MzFRGRoaGDh2q1q1b6+abb9bmzZslSb/88ovWrFmjxx9/vNzX9+abb9a8efOUn59/lv8SwLkhZMRlbdSoUXrqqaeUmpqqmJgY3X333SouLjb35+XlKSUlRVOnTtWmTZsUGhqq+++/X2vWrNFXX32l5cuXyzAM3XTTTVX6l60JEybogw8+0IwZM/TLL78oJydH8+bNK1M3c+ZM+fr6auXKlZo4caJefPFFLVq0yKHmueee02233aa0tDTde++9uvvuu7Vly5Zz6qNz585lVns+9dRTkqSffvpJHTp0cKjdtm2bpNP/enb48GF17tzZfJ3GjRunmTNnmtdz1113mcfec889atSokVavXq21a9fqmWeekaenp7k/MjJSoaGhWrZs2bm9gACAGuPm5qZ///vf2rhxo2bOnKkffvhBw4cPl1T5vPHAAw/ol19+0Zw5c7R+/Xr95S9/Ua9evRz+IS4vL0+TJk3Sf/7zH/3000/av3+/ebwkTZkyRX/72980ePBgbdiwQV999ZWioqIkSQ899JC++eYbh1Dy66+/1smTJ3XHHXdIKjt3Safnyc2bN2vhwoXasmWLpkyZouDgYIeajh07MgcBwAXo448/VosWLdSiRQvde++9mjFjhgzDkCQtWLBA/fv3V+/evbVu3TotXrzYYQ4YMGCA5syZo3//+9/asmWL3n77bdWtW7dKz79z50598skn+vzzz837+Z86dUpPPvmkVq9ercWLF8vNzU233nqr+am3kydPqnv37jp06JC++uorpaWlafjw4SotLVWTJk3Us2dPh9BQkmbMmKH7779fFovlrPPWsmXLHK4zJCRE4eHh+u6775Sfn69ly5apbdu2Kiws1KOPPqq3335b7u7u5V5fhw4dVFRUpFWrVlXpdQEqZACXsBkzZhg2m63M+J49ewxJxtSpU82xTZs2GZKMLVu2mMdKMlJTU82a7du3G5KMX375xRzLzMw0fHx8jE8++cQwDMN44YUXjPj4eIfne/XVV43IyEjzcf369Y1XXnnFfFxcXGw0btzY6NevnznWvXt3o2vXrg7nufLKK40RI0aYjyUZjzzyiENNYmKi8eijjxqGYRg//vijIck4fvy4uX/dunWGJGPPnj2Vvkbx8fHGiy++6DB2/PhxQ5Lx448/mmNnXqcVK1aYY1u2bDEkGStXrjQMwzD8/PyM999/v8xz/FG7du2M0aNHV1oDAKgeAwcOdJhzKvPJJ58YQUFB5uPy5o2dO3caFovFOHjwoMP4tddeayQnJ5vHSTJ27txp7n/zzTeN+vXrm48bNGhgjBo1qsJeWrVqZUyYMMF8fMsttxj333+/+bhfv37GoEGDHI7p27ev8cADD1R6jf/4xz+MHj16VFoDAKh9nTt3Nl577TXDMAyjqKjICA4ONhYtWmQYhmEkJSUZ99xzT7nHbdu2zZBk1v5ZeXPZF198YfwxInnhhRcMT09P4+jRo5X2ePToUUOSsWHDBsMwDOOdd94x/Pz8jKysrHLrP/74YyMwMNAoKCgwDMMwUlNTDYvFYv5+Vtm8deb3sZ9++slhfNmyZUaHDh2MJk2aGEOHDjUKCwuNMWPGGE888YSxceNGo3PnzkZMTIzx+uuvlzlnYGDgWX9XA84VKxlxWWvbtq355/DwcEmnv2nyDC8vL4eaLVu2yMPDQ4mJieZYUFCQWrRocc6rB7Ozs/X777+rY8eO5pi7u7vat29faX9nevxjf5IcPhJ25vG59lKZ/Px8eXt7n1Oth4eHw7+mtWzZUgEBAWYfTz75pB566CH17NlT48ePL/NxAkny8fFRXl6e030DAJzz448/6rrrrlPDhg3l5+enAQMGKCsrS6dOnarwmN9++02GYSgmJkZ169Y1t6VLlzq859epU0fNmzc3H/9xXjt69KgOHTqka6+9tsLneeihh8zVH0ePHtWCBQs0aNAgc395c9ejjz6qOXPmKCEhQcOHD9evv/5a5rzMQQBw4dm2bZtWrVplfkLKw8NDd955p3mLjNTU1ArnjNTUVLm7u6t79+5O9RAZGamQkBCHsV27dumvf/2rmjVrJn9/fzVt2lSSzPtFpqamql27dqpXr16557zlllvk4eFh3qZj+vTpuvrqq83ba1U2b535WPOf57quXbtq9erV2rNnj958803t2bNH//nPf/TSSy/pvvvu05AhQ7Rs2TK9+OKLWr9+vcOxzIGoToSMuKz98SO7Z+6/8ccvd/Hx8XG4L4fx/5fm/5lhGGadm5tbmbryPkr9x/NWdO4/9nfmmHP58pk/9vLnc5/rx7qDg4OrdBP8P1/PH8dGjx6tTZs2qXfv3vrhhx/UqlWrMve+OnbsWJkJHABQu/bt26ebbrpJbdq00eeff661a9fqzTfflFT5/FFaWip3d3etXbtWqamp5rZlyxaHL1wrb147M0f5+Pictb8BAwZo9+7dWr58uWbPnq0mTZqYN76Xyp+7brzxRu3bt09PPPGEGWL+8SPaEnMQAFyIpk2bpuLiYjVs2FAeHh7y8PDQlClTNHfuXB0/frzSeeNsc8q5/s7m6+tbZqxv377KysrSe++9p5UrV2rlypWSZH6j89me28vLS/fdd59mzJihwsJCffjhhw7/YFbZvBUUFCSLxVLp72mGYWjw4MGaPHmySktLtW7dOt1+++0KDQ1V9+7dtXTpUod65kBUJ0JGoApatWql4uJicyKRpKysLG3fvl2xsbGSTt8T48iRIw6T1pn7d0iSzWZT/fr1He57UVJSonXr1p1XTytWrCjzuGXLlmYvkhzuX/XHXqTTk1xJSUmZ87Zr1868YfDZFBcXO9y8eNu2bTpx4oTZhyTFxMToH//4h7777jv179/f4T4kBQUF2rVrl9q1a3dOzwcAqBlr1qxRcXGxJk+erE6dOikmJkaHDh1yqClv3mjXrp1KSkp09OhRRUVFOWxhYWHn9Nx+fn5q0qSJFi9eXGFNUFCQbrnlFs2YMUMzZszQAw88UKaP8uaukJAQ3X///Zo9e7Zee+01vfvuuw77N27cyBwEABeQ4uJizZo1S5MnT3b4x6u0tDRFRkbqgw8+UNu2bSucM+Li4lRaWlomUDsjJCREubm5Dqv0//x7UnmysrK0ZcsWPfvss7r22msVGxtbJvBr27atUlNTdezYsQrP89BDD+n777/XW2+9paKiIocvrDnTX3nzlpeXl1q1alXp72nTpk1TUFCQbr75ZnO+PhOgFhUVOczhu3btUkFBAXMgqg0hI1AF0dHR6tevnx5++GH9/PPP5petNGzYUP369ZMk9ejRQxkZGZo4caJ27dqlN998UwsXLnQ4z9///nelpKToyy+/1LZt2/T444/r+PHj5a4GPJtPP/1U06dP1/bt2/XCCy9o1apV5jdxRkVFKSIiQqNHj9b27du1YMECTZ482eH4Jk2a6OTJk1q8eLEyMzPNpfI33HCDli9fXm4A+Weenp76+9//rpUrV+q3337TAw88oE6dOqljx47Kz8/XsGHDtGTJEu3bt0+//PKLVq9ebYay0ulg1Gq1lvnoNwCg5mRnZzv84paamqqQkBAVFxfr9ddf1+7du/Wf//xHb7/9tsNx5c0bMTExuueeezRgwADNnTtXe/bs0erVqzVhwgR9/fXX59zT6NGjNXnyZP373//Wjh079Ntvv+n11193qHnooYc0c+ZMbdmyRQMHDnTYd8MNN2jTpk0Ov/A9//zz+vLLL7Vz505t2rRJ8+fPd5iD8vLytHbtWl1//fVVefkAADVo/vz5On78uB588EG1adPGYbv99ts1bdo0vfDCC/roo4/0wgsvaMuWLdqwYYMmTpwo6fRcNXDgQA0aNEjz5s3Tnj17tGTJEn3yySeSpMTERNWpU0cjR47Uzp079eGHH5b55uryBAYGKigoSO+++6527typH374QU8++aRDzd13362wsDDdcsst+uWXX7R79259/vnnWr58uVkTGxurTp06acSIEbr77rsdVj+ebd664YYb9PPPP5fb39GjRzV27Fj9+9//NvuNjY3Va6+9puXLl2vx4sXml3dKp79EplmzZg63MgGc4qqbQQK14Wxf/LJu3Tpz7M9falLRsceOHTPuu+8+w2azGT4+PsYNN9xgbN++3aHm/7V3PyFRbmEcxx/xOo6FDpoLxxo0EcYBCwOJNiUktRBRyz9MLhpX2sLIIAg3RYgURG0qkyARFxqEDLoQ000LN4VuamDAQCeiCF0kFIoF82txuUNzR73qNNcL9/uBWbxzznvOM2cxL/Nw5jyPHz+Wx+PR/v37dfHiRfX29sYVfvnx44c6OzuVk5Oj3NxcXb9+Xc3NzfL7/bE+VVVVunLlSty49fX1CgQCsWsz06NHj3TmzBllZmaqqKhIIyMjcffMzMzoyJEjcjqdOnnypJ4/fx5X+EWSLl26pAMHDsjMdPPmTUl/FqM5ePCgJicnN12jX9dpdHRUJSUlcjgcOn36tCKRiCRpfX1dfr9fHo9HDodDhYWF6uzs1NraWmyM9vZ2dXR0JKw1ACA1AoGAzCzhFQgEdP/+fbnd7tgzbmhoKKGI2EbPje/fv+vGjRsqLi5WRkaGCgoKdO7cOb1580bS9g7Zl6T+/n55vV5lZGTI7Xbr8uXLce3RaFRFRUWqqanZ8LOdOHFC/f39seuenh75fD5lZWUpLy9P9fX1WlhYiLUPDw/L6/XudAkBAClUW1u76ff83NyczExzc3MaHR1VRUWFHA6H8vPzdf78+Vi/tbU1Xb16VW63Ww6HQ6WlpRoYGIi1B4NBlZaWyul0qra2Vk+ePEko/PL3gp6SND09LZ/Pp8zMTB09elQvX76UmSkYDMb6RCIRNTY2KicnR/v27VNlZWWsKOZfnj59KjPT69ev497/p+dWOBxWVlaWVlZWEmLz+/0JxV1evXqlsrIy5eXl6datW3FtZ8+e1e3btzdYZWB30qRNDpkD8K+JRqPm8/mspaXFenp6tn1fWlqaBYNBa2hoSElcfX19NjY2Zi9evNi0z+DgoHV1ddnKysqu5lheXraysjKbnZ2NHZoMAMBmVldXrbCw0AYGBhL+XmZmNjExYdeuXbNQKBQ7m3grx48ft66uLmttbU1FuAAAbKi3t9eePXtmb9++3fG9LS0tduzYMevu7t71/KFQyKqrq21+ft5cLteuxwF+9cdeBwD8H71//96mpqasqqrK1tfX7eHDh7a4uPif+4HT3t5uX758sa9fv1p2dnZK5lhcXLS+vj4SjACALUWjUfv8+bPdu3fPXC6X1dXVbdivpqbG3r17Zx8/fjSPx7PlmEtLS9bU1GQXLlxIRcgAACT49u2bhcNhe/DgwY42mPzq7t27Nj4+nlQcnz59sqGhIRKM+K3YyQjsgQ8fPpjf77dQKGSSrLy83O7cuWOnTp3a0Tip3sm4HcnuZAQAYDsikYgdPnzYDh06ZIODg1ZdXb3XIQEAsGNtbW02MjJiDQ0NNjw8bOnp6XsdEvDbkGQEAAAAAAAAkBSqSwMAAAAAAABICklGAAAAAAAAAEkhyQgAAAAAAAAgKSQZAQAAAAAAACSFJCMAAAAAAACApJBkBAAAAAAAAJAUkowAAAAAAAAAkkKSEQAAAAAAAEBSSDICAAAAAAAASMpPOCHf/3uaBTMAAAAASUVORK5CYII=",
- "text/plain": [
- "