diff --git a/docs/experiments.rst b/docs/experiments.rst index 37973cdaf7..b02e1c920c 100644 --- a/docs/experiments.rst +++ b/docs/experiments.rst @@ -16,6 +16,7 @@ The following experiments illustrate specific tests using the ``ProgLearn`` pack experiments/recruitment_across_datasets experiments/spiral_exp experiments/spoken_digit_exp + experiments/xor_rxor_bootstrap_exp experiments/xor_rxor_exp experiments/xor_rxor_with_cpd experiments/xor_rxor_with_icp diff --git a/docs/experiments/functions/xor_rxor_bootstrap_fns.py b/docs/experiments/functions/xor_rxor_bootstrap_fns.py new file mode 100644 index 0000000000..428a21cf20 --- /dev/null +++ b/docs/experiments/functions/xor_rxor_bootstrap_fns.py @@ -0,0 +1,184 @@ +import numpy as np +from scipy.spatial import distance +import sklearn.ensemble +from proglearn.sims import generate_gaussian_parity +import random +import math + + +def bootstrap(angle_sweep=range(0, 90, 5), n_samples=100, reps=1000): + """ + Runs getPval many times to perform a bootstrap exeriment. + """ + p_vals = [] + # generate xor + X_xor, y_xor = generate_gaussian_parity(n_samples, angle_params=0) + for angle in angle_sweep: + # print('Processing angle:', angle) + # we can use the same xor as from above but we need a new rxor + # generate rxor with different angles + + X_rxor, y_rxor = generate_gaussian_parity( + n_samples, angle_params=math.radians(angle) + ) + + # we want to pick 70 samples from xor/rxor to train trees so we need to first subset each into arrays with only xor_0/1 and rxor_0/1 + X_xor_0 = X_xor[np.where(y_xor == 0)] + X_xor_1 = X_xor[np.where(y_xor == 1)] + + X_rxor_0 = X_rxor[np.where(y_rxor == 0)] + X_rxor_1 = X_rxor[np.where(y_rxor == 1)] + + # we can concat the first 35 samples from each pair to use to tatal 70 samples for training and 30 for predict proba + X_xor_train = np.concatenate((X_xor_0[0:35], X_xor_1[0:35])) + y_xor_train = np.concatenate((np.zeros(35), np.ones(35))) + + # repeat for rxor + X_rxor_train = np.concatenate((X_rxor_0[0:35], X_rxor_1[0:35])) + y_rxor_train = np.concatenate((np.zeros(35), np.ones(35))) + + # make sure X_rxor_train is the right size everytime, run into errors sometime + while len(X_rxor_train) != 70: + X_rxor, y_rxor = generate_gaussian_parity( + n_samples, angle_params=math.radians(angle) + ) + # we want to pick 70 samples from xor/rxor to train trees so we need to first subset each into arrays with only xor_0/1 and rxor_0/1 + X_xor_0 = X_xor[np.where(y_xor == 0)] + X_xor_1 = X_xor[np.where(y_xor == 1)] + + X_rxor_0 = X_rxor[np.where(y_rxor == 0)] + X_rxor_1 = X_rxor[np.where(y_rxor == 1)] + + # we can concat the first 35 samples from each pair to use to tatal 70 samples for training and 30 for predict proba + X_xor_train = np.concatenate((X_xor_0[0:35], X_xor_1[0:35])) + y_xor_train = np.concatenate((np.zeros(35), np.ones(35))) + + # repeat for rxor + X_rxor_train = np.concatenate((X_rxor_0[0:35], X_rxor_1[0:35])) + y_rxor_train = np.concatenate((np.zeros(35), np.ones(35))) + + # init the rf's + # xor rf + clf_xor = sklearn.ensemble.RandomForestClassifier( + n_estimators=10, min_samples_leaf=int(n_samples / 7) + ) + + # rxor rf + clf_rxor = sklearn.ensemble.RandomForestClassifier( + n_estimators=10, min_samples_leaf=int(n_samples / 7) + ) + + # train rfs + # fit the model using the train data + clf_xor.fit(X_xor_train, y_xor_train) + + # fit rxor model + clf_rxor.fit(X_rxor_train, y_rxor_train) + + # concat the test samples from xor and rxor (30 from each), 60 total test samples + X_xor_rxor_test = np.concatenate( + (X_xor_0[35:], X_rxor_0[35:], X_xor_1[35:], X_rxor_1[35:]) + ) + y_xor_rxor_test = np.concatenate((np.zeros(30), np.ones(30))) + + # predict proba on the new test data with both rfs + # xor rf + xor_rxor_test_xorRF_probas = clf_xor.predict_proba(X_xor_rxor_test) + + # rxor rf + xor_rxor_test_rxorRF_probas = clf_rxor.predict_proba(X_xor_rxor_test) + + # calc the l2 distance between the probas from xor and rxor rfs + d1 = calcL2(xor_rxor_test_xorRF_probas, xor_rxor_test_rxorRF_probas) + + # concat all xor and rxor samples (100+100=200) + X_xor_rxor_all = np.concatenate((X_xor, X_rxor)) + y_xor_rxor_all = np.concatenate((y_xor, y_rxor)) + + # append the pval + p_vals.append( + getPval(X_xor_rxor_all, y_xor_rxor_all, d1, reps, n_samples=n_samples) + ) + + return p_vals + + +def getPval(X_xor_rxor_all, y_xor_rxor_all, d1, reps=1000, n_samples=100): + """ + Shuffles xor and rxor, trains trees, predicts, calculates L2 between probas, and calculates p-val to determine whether the 2 distributions are different. + """ + d1_greater_count = 0 + for i in range(0, reps): + random_idxs = random.sample(range(200), 200) + # subsample 100 samples twice randomly, call one xor and the other rxor + X_xor_new = X_xor_rxor_all[random_idxs[0:100]] + y_xor_new = y_xor_rxor_all[random_idxs[0:100]] + + X_rxor_new = X_xor_rxor_all[random_idxs[100:]] + y_rxor_new = y_xor_rxor_all[random_idxs[100:]] + + # subsample 70 from each and call one xor train and one rxor train + # since we randomly took 100 the pool of 200 samples we should just be able to take the first 70 samples + X_xor_new_train = X_xor_new[0:70] + y_xor_new_train = y_xor_new[0:70] + + X_rxor_new_train = X_rxor_new[0:70] + y_rxor_new_train = y_rxor_new[0:70] + + # train a new forest + # init the rf's + # xor rf + clf_xor_new = sklearn.ensemble.RandomForestClassifier( + n_estimators=10, min_samples_leaf=int(n_samples / 7) + ) + clf_xor_new.fit(X_xor_new_train, y_xor_new_train) + + # rxor rf + clf_rxor_new = sklearn.ensemble.RandomForestClassifier( + n_estimators=10, min_samples_leaf=int(n_samples / 7) + ) + clf_rxor_new.fit(X_rxor_new_train, y_rxor_new_train) + + # take the remaing 30 and call those test + X_xor_new_test = X_xor_new[70:] + y_xor_new_test = y_xor_new[70:] + + X_rxor_new_test = X_rxor_new[70:] + y_rxor_new_test = y_rxor_new[70:] + + # concat our new samples + X_xor_rxor_new_test = np.concatenate((X_xor_new_test, X_rxor_new_test)) + y_xor_rxor_new_test = np.concatenate((y_xor_new_test, y_rxor_new_test)) + + # predict proba using the original xor and rxor rf's and calc l2 + # new xor rf + xor_rxor_new_test_xorRF_probas = clf_xor_new.predict_proba(X_xor_rxor_new_test) + + # new rxor rf + xor_rxor_new_test_rxorRF_probas = clf_rxor_new.predict_proba( + X_xor_rxor_new_test + ) + + # calc l2 for our new data + d2 = calcL2(xor_rxor_new_test_xorRF_probas, xor_rxor_new_test_rxorRF_probas) + + if d1 > d2: + d1_greater_count += 1 + + return 1 - (d1_greater_count / reps) + + +def calcL2(xorRF_probas, rxorRF_probas): + """ + Returns L2 distance between 2 outputs from clf.predict_proba(). + """ + # lists to store % label 0 since we only need one of the probas to calc L2 + xors = [] + rxors = [] + + # iterate through the passed probas to store them in our lists + for xor_proba, rxor_proba in zip(xorRF_probas, rxorRF_probas): + xors.append(xor_proba[0]) + rxors.append(rxor_proba[0]) + + return distance.euclidean(xors, rxors) diff --git a/docs/experiments/xor_rxor_bootstrap_exp.ipynb b/docs/experiments/xor_rxor_bootstrap_exp.ipynb new file mode 100644 index 0000000000..2c2e50592b --- /dev/null +++ b/docs/experiments/xor_rxor_bootstrap_exp.ipynb @@ -0,0 +1,215 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fbec258d-c58d-4218-ae9b-14bc01295e26", + "metadata": { + "tags": [] + }, + "source": [ + "# Gaussian XOR and Gaussian R-XOR Random Forest Bootstrap Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "fa57f1bb", + "metadata": {}, + "source": [ + "## Overview" + ] + }, + { + "cell_type": "markdown", + "id": "d51addd4-5e62-4121-b09a-4b9b3624f359", + "metadata": {}, + "source": [ + "In this experiment, we are interested in learning at which angles RXOR is significantly different from XOR to warrant training a new Random Forest. We will do this for each angle in the angle sweep by:\n", + "1. Generating 100 XOR and 100 RXOR samples and training their respective trees on randomly selected 70 samples from each.\n", + "2. Concatenating the remaing 30 samples from both distributions (60 samples total) and pushing them through both XOR and RXOR random forests to get an array of probabilities for each sample. \n", + "3. Calculate L2 distance between the 2 arrays of probabilities. We will call this d1.\n", + "4. Concatenate ALL XOR and RXOR samples (200 total) and randomly select 70 samples to be XOR_new and 70 samples to be RXOR_new (bootstrap).\n", + "5. Train 2 new trees with XOR_new and RXOR_new.\n", + "6. Use the remaining 60 samples to calculate probabilities from both new trees.\n", + "7. Calculate L2 distance between the new probabilities (d2).\n", + "8. Repeat steps 4-7 1000 times and calculate p-value by 1 - ((# of times d1 > d2)/1000).\n", + "9. This entire experiment is then repeated 100 times to account for randomness.\n", + "\n", + "Finally, we take the mean of the p-values across each 100 tests for each angle and plot." + ] + }, + { + "cell_type": "markdown", + "id": "5f10072f", + "metadata": {}, + "source": [ + "## Running the Experiment" + ] + }, + { + "cell_type": "markdown", + "id": "699c54f3-2eee-45f7-93a2-3fa2ed07852c", + "metadata": {}, + "source": [ + "We will start by importing dependencies and running the experiment outlined above. This will take quite a while." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a76c3187-3074-4eab-b82a-7212693acd1d", + "metadata": {}, + "outputs": [], + "source": [ + "# import\n", + "import time\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "import xor_rxor_bootstrap_fns as fn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f1606f8-4a2c-4dd7-9f9c-ba873f873267", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# set angle sweep\n", + "angle_sweep = range(0, 90, 5)\n", + "# data frame to store p values from each run\n", + "p_val_df = pd.DataFrame()\n", + "\n", + "# time experiment\n", + "start = time.time()\n", + "\n", + "# run the experiment for 100 repetitions, bootstrap each experiment for 1000 reps\n", + "for i in range(100):\n", + " p_val_df[i] = fn.bootstrap(angle_sweep=angle_sweep, n_samples=100, reps=1000)\n", + "end = time.time()\n", + "\n", + "# entire experiment run time\n", + "print(\"\\nThe function took {:.2f} s to compute.\".format(end - start))\n", + "# The function took 26973.70 s to compute." + ] + }, + { + "cell_type": "markdown", + "id": "65ebd653", + "metadata": {}, + "source": [ + "## Visualizing the Results" + ] + }, + { + "cell_type": "markdown", + "id": "68b85968-6853-4c3c-9fc3-d97c3bff8f73", + "metadata": { + "tags": [] + }, + "source": [ + "Next, we compute the mean across each test for each angle. We will use this to the mean p-value for each angle with errors bars for the 25th and 75th percentiles. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "29c30725-cae7-42a0-80f9-ed1336ecde13", + "metadata": {}, + "outputs": [], + "source": [ + "# compute mean across each test for each angle\n", + "p_val_df[\"mean\"] = p_val_df.mean(axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "496fca00-c10b-4396-ad0e-b6ed188f04ab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Text(0.5, 1.0, 'Angle of Rotation vs mean P-Value')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA7F0lEQVR4nO3dd3xddf348df7ZrbZHelK00E3owUKMqqAqAwRHCAgKuIAZChDRb8/HDi+DlSULwgi2wGioCIiiCBb6IDu3XQkXWmSpknTZt7374/PCb1Nb5Kbce654/18PO4jufeenPO5J8l5n896f0RVMcYYk75CQRfAGGNMsCwQGGNMmrNAYIwxac4CgTHGpDkLBMYYk+YsEBhjTJqzQGAOIiKfEZFXfdiviMgDIrJbROYP9v4HSkQuEZF/BV0Ocyi//ibNARYIkpiIvOhdWHOCLksM5gHvB8pU9fiub3r/7B0isldEGkRkiYicE8uO+3qhEJGJIqIiktn5mqr+XlU/EOs+jOP9DTZ7v7caEXlCRMZ02WaciLSLyGFRfv4vIvLT+JXYRGOBIEmJyETg3YAC5wZbmphMADapalMP2/xXVfOBYuBXwKMiUhyHspmBucb7vU3D/e5ui3xTVbcCzwOfinxdRIYBZwMPxaeYpjsWCJLXp4E3gAeBSyPfEJEHReROEfmHiDSKyJuRd2Mi8gERWSMie0TkVyLykoh8PtpBRGSGiDwnInXez3y8uwKJyFgRedLbdr2IfMF7/XPAvcCJ3p3jLT19MFUNA78F8oCp3j6KRORhEdklIptF5GYRCYnITODuiH3Xe9t/UETe9moXlSLynYhDvOx9rfd+5sSutQoROUlEFnjnaIGInBTx3osi8j0Rec07v/8SkRHdnJNVkTUbEcn07pyPEZFcEfmdiNSKSL13nFHd7GeTiHxVRJaKSJOI3Ccio0Tkn14Z/i0iJRHbnyAir3v7XSIip0a8d5lXrkYRqRCRKyLeO1VEqkTkRhGpFpHtInJZT7+vTqpaBzwOHBHl7YfoEgiAi4AVqrpMRL4uIhu8Mq0UkY90cx4Oqc15v4/PRzz/rPf5dovIsyIyIZbypzVVtUcSPoD1wFXAsUAbMCrivQeBOuB4IBP4PfCo994IoAH4qPfel72f/7z3/meAV73v84BK4DJv22OAGuDwbsr0Eu5OPheYA+wCTu+6325+NvK4GcDVQCtQ6r32MPA3oACYCKwFPtfdvoFTgSNxNztHATuBD3vvTcTVpDK7Of4wYDfuwpUJXOw9H+69/yKwAXcHPMR7/qNuPte3gN9HPP8gsNr7/grg78BQ7zMfCxR2s59NuMA/ChgHVANvAUcDOcALwLe9bccBtbi77RCuSa4WGBlRhsMAAU4B9gHHRJy3duC7QJa3j31ASTflepEDfzsjvHL8Nsp2Q4A9wLyI1/4LXOd9fwEw1ivvhUATMCbK7yba7y6yDB/G/W/M9H53NwOvB/3/mugPqxEkIRGZh2tqeUxVF+EuSp/ostkTqjpfVdtxgWCO9/rZuLuwJ7z3bgd2dHOoc3DNOQ+oaruqvoW74zs/SpnG4/oBblLVZlVdjKsFdL0L7MkJ3h19M/BT4JOqWi0iGbiLwzdUtVFVNwE/62nfqvqiqi5T1bCqLgUewV30YvFBYJ2q/tb73I8Aq4EPRWzzgKquVdX9wGMcOL9d/QE4V0SGes8/4b0GLgAPB6aoaoeqLlLVhh7K9X+qulNdU8srwJuq+raqtgB/wQUFgE8CT6vq097nfw5YiPvdo6r/UNUN6rwE/AvXzNipDfiuqrap6tPAXmB6D+W63fu9LQG2Azd03cA7T3/C1WQRkam4wPcH7/0/qeo2r7x/BNbhbmT66grgh6q6yvv7/l9gjtUKemaBIDldCvxLVWu853+gS/MQB1/c9wH53vdjcXf5AKi7jarq5jgTgHd5zQv13j/7JcDoKNuOBepUtTHitc24u9NYvaGqxUAJ8CQHLk4jgGxvfzHtW0TeJSL/8ZqS9gBXevuJxdgux4p2vO7O70FUdT2wCviQFwzO5UAg+C3wLK4vZJuI/EREsnoo186I7/dHed5ZhgnABV1+b/OAMQAicpaIvOE14dXjAkTkuan1LqK9fj7Pl1S1WFXHqeolqrpL3Cisvd7jn952DwEfF5FcXBB/RlWrvTJ9WkQWR5T3CGL/fUWaAPwyYj91uJpPX/4O005m75uYRCIiQ4CPAxki0nkxygGKRWS2qi7pZRfbgbKI/Unk8y4qgZdU9f0xFG0bMExECiKCQTmwNYafPYiq7hWRq4ANInI/sBR3lzoBWBll39FS6P4BuAM4S1WbReQXHLiw9JZyd5t3rEjlwDN9+RwRHsE1L4WAlV5wQFXbgFuAW8R1/j8NrAHu6+dxOlXimme+0PUNcSPMHsfdmf9NVdtE5K+4i+WgUdXf42qika+9IiK1wHm4WsvXvDJNAH4DnI4bMNAhIou7KVPnYIOhuCZOOPjGpBL4gXd8EyOrESSfDwMdwCxcc8QcXHvoK3jV7l78AzhSRD7sdbhdTfQ7fICngGki8ikRyfIex4nroD2IqlYCrwM/9DpBjwI+R5eLQaxUtRbXtPQtVe3ANb/8QEQKvAvHDcDvvM13AmUikh2xiwJcDaVZRI7n4KazXUAYmNzN4Z/2PvcnvM7dC3Hn+6n+fBbgUeADwBc5UBtARE4TkSO9pq8GXLDr6OcxIv0OVwM5Q0QyvN/HqSJShqtZ5eDOQbuInOWVLV4eBn6MG130d++1PFxw3gWuM5voHc6o6i7cDcAnvc/2WVx/R6e7gW+IyOHevopE5AIfPkdKsUCQfC7FtU9vUdUdnQ/c3e8lkaMpovGaky4AfoLrQJyFaz9uibJtI+4icRHuLnkH7p+4u3kLF+M687bh2qy/7bVP99cvgLO9oHIt7m6wAngVd0G939vuBWAFsENEOpvLrgK+KyKNuA7bxyI+1z7gB8BrXhPCCZEH9YLQOcCNuHP0NeCciKa4PlHV7biO0ZOAP0a8NRr4My4IrMJ1tv/ukB30/XiVuLvu/8FdXCuBrwIh73f6Jdz52I0LkE8O9Jh98DCudvVHr28DVV2J6/P5Ly6oHwm81sM+voD7PLXA4bgbELx9/QX3N/qoiDQAy4GzBv9jpBZxTcQmXYlICNdHcImq/ifo8hhj4s9qBGnIazIo9tqL/wfXFvtGwMUyxgTEAkF6OhE35LQGNyTyw97wPmNMGrKmIWOMSXNWIzDGmDSXdPMIRowYoRMnTgy6GMYYk1QWLVpUo6ojo72XdIFg4sSJLFy4MOhiGGNMUhGRrrPl32FNQ8YYk+YsEBhjTJqzQGCMMWnOAoExxqQ5CwTGGJPmLBAYY0yas0BgjDFpzgKBMcakufQKBDt3QsdgrPthjDGpI70CQXU1vP46tByyBosxxqSt9AoEAPX18Mor0NjY66bGGJMO0i8QAOzfD6++Crt2BV0SY4wJXHoGAoD2dnjzTdjcbR4mY4xJC+kbCABUYelSWLnSfW+MMWkovQNBpw0bYOFCG1FkjElLFgg67dhhI4qMMWnJAkGkzhFFDQ1Bl8QYY+LGAkFX+/fDa6/ZiCJjTNqwQBCNjSgyxqQRCwTd6RxRtGKFjSgyxqQ0CwS9qaiwEUXGmJRmgSAWnSOKmpuDLokxxgw6CwSxqq93aSlsRJExJsVYIOiLzhFFtbVBl8QYYwaNBYK+am+HBQugqSnokhhjzKCwQNAfbW0wf777aowxSc4CQX/t3QtvvWVDS40xSc8CwUBUV8OqVUGXwhhjBsQCwUBt2ACVlUGXwhhj+s3XQCAiZ4rIGhFZLyJfj/J+kYj8XUSWiMgKEbnMz/L4ZulS2L076FIYY0y/+BYIRCQDuBM4C5gFXCwis7psdjWwUlVnA6cCPxORbL/K5Jtw2I0k2r8/6JIYY0yf+VkjOB5Yr6oVqtoKPAqc12UbBQpERIB8oA5o97FM/mlpccHAUlEYY5KMn4FgHBDZeF7lvRbpDmAmsA1YBnxZVcNddyQil4vIQhFZuCuR00Pv2QOLFwddCmOM6RM/A4FEea3rWMszgMXAWGAOcIeIFB7yQ6r3qOpcVZ07cuTIwS7n4Nq2DdauDboUxhgTMz8DQRUwPuJ5Ge7OP9JlwBPqrAc2AjN8LFN8rFkD27cHXQpjjImJn4FgATBVRCZ5HcAXAU922WYLcDqAiIwCpgMVPpYpft5+2xLUGWOSgm+BQFXbgWuAZ4FVwGOqukJErhSRK73NvgecJCLLgOeBm1S1xq8yxVVHh0tD0doadEmMMaZHmX7uXFWfBp7u8trdEd9vAz7gZxkCtX+/G0l04okQsrl7xpjEZFcnv9XVwbJlQZfCGGO6ZYEgHrZscUteGmNMArJAEC8rV0Iiz4EwxqQtCwTxogqLFtmCNsaYhGOBIJ5sQRtjTAKyQBBvtqCNMSbBWCAIQnW16zMwxpgEYIEgKBUVsH590KUwxhgLBIFatcqloggfknDVGGPixgJB0Kqq4PXXobk56JIYY9KUBYJEsHs3vPIK1NcHXRJjTBqyQJAompvhtddg69agS2KMSTMWCBJJOOyGlq5aZcNLjTFxY4EgEa1f77KWtifn8s3GmORigSBR7dwJr75qKSmMMb6zQJDIGhtdJ3JNaqzVY4xJTBYIEl1bG7zxBmzcGHRJjDEpygJBMlCF5cth6VKbfGaMGXQWCJLJ5s2udmDrIBtjBpEFgmRTW+v6DRoagi6JMSYWNTWwZInLL1ZTk5A3cr4uXm98sm+fG1F09NEwZkzQpTHGdKetzeUT65pCJjcXCgqgsPDAIz8fQsHcm1sgSFYdHbBwIUyfDtOmBV0aY0w0y5dHzyPW3OwekcvXhkKQl3dwcCgsdEHDZxYIkt2aNVBUBKNGBV0SY0ykHTtcUslYhcNuyHhj48GpZrKyXEAYNQoOO2zwy4n1EaSGvvyxGWP819rqRvkNhrY21ze4ffvg7C8KCwSpYMcOS0dhTCJZuhRaWoIuRcwsEKSCcBi2bQu6FMYYcM06Pt69+8ECQaqw5iFjgtfcDMuWBV2KPrNAkCpqa2H//qBLYUx6W7rUteknGQsEqcRqBcYEZ8sWlzU4CVkgSCW2upkxwdi/H1asCLoU/WaBIJU0NsKePUGXwpj0s3hxUo/cS5tAULO3hW/Or6OpI+iS+Myah4yJr40bk37NkLQJBG9W1PH79Xv56FJhcyr3qW7dausdGxMvTU1ujfEklzaB4INHjeHh00rZ2QofWiK8tDvoEvmkpeXg/CXGGH+ouiahjuRvZkibQAAwb0wuT85WxubAZSuFX1el6M2zNQ8Z47+KCqirC7oUgyKtAgFAeS48cZRy1nD44eYQX1or7Ev+gH4wSzlhjL8aG2H16qBLMWjSLhAADM2AO6YrN00I81QNfGyZUBklU2zS6uhwwcAYM/hU3RoDKbRsrK+BQETOFJE1IrJeRL7ezTanishiEVkhIi/5WZ6DjwtfLIP7ZylVzXDuEuH1+ngdPQ6secgYf6xbl3LDtH0LBCKSAdwJnAXMAi4WkVldtikGfgWcq6qHAxf4VZ7unFYCT85WRmTBp1YI921LkX6DmproC2IYY/pvzx4XCFKMnzWC44H1qlqhqq3Ao8B5Xbb5BPCEqm4BUNVqH8vTrUlD4C9HKacPg+9tDHHjOqE52fsNVG2msTGDKRxOuSahTn4GgnFAZcTzKu+1SNOAEhF5UUQWicino+1IRC4XkYUisnCXT0Mj8zPh7hnKDeVhntglXLBM2JY86cSjs+YhYwbPmjWukzgF+RkIJMprXRtdMoFjgQ8CZwDfFJFDFuBV1XtUda6qzh05cuTgl9QTEvjSePjNjDAbm918gzeTuSmwocE9jDEDs3s3bNgQdCl842cgqALGRzwvA7qunlIFPKOqTapaA7wMzPaxTDF5/3D461FKUQZcskJ4eHsS9xtYrcCYgenocE1CSXsR6J2fgWABMFVEJolINnAR8GSXbf4GvFtEMkVkKPAuICHma08ZCn+drbynGL5VEeKm9UJLMjYNWsoJYwZm1SqXSiKF+RYIVLUduAZ4Fndxf0xVV4jIlSJypbfNKuAZYCkwH7hXVZf7Vaa+KsyEe2cq15Ypj1ULFy4TdiZbv0Fzc9InxDImMLW1Lqlcisv0c+eq+jTwdJfX7u7y/FbgVj/LMRAhgRsnKIfnKzesFc5ZItw9Qzm2MOiS9UFVFfjYt2JMSmprc7mE0kBazizujzOHuyGmQzPgouVJNt9gx46USIxlTNyEw7BgAezbF3RJ4sICQR9Mz3OTz04pcfMNLl8t1CfD8qTt7ZZywpi+ePtt1yyUJiwQ9FFRJvxmhvLNSWFe3A0fXCK8lQxDi230kDGxWbECtnUd4JjaLBD0gwh8biz86UhFgI8vE36T6INzdu1yaxUYY7pXUeEeacYCwQDMKYB/zFFOL4EfbArx+VXC7kRtKrKUE8b0bNu2pF6AfiAsEAxQkZea4juTwrxSDx9cLCxK1Mm81jxkTHS1ta5fIE1ZIBgEIvCZsfD4UUpmyDUV3VUF4URrKtqzJ2VzpRjTb42NboRQCiaTi5UFgkF0ZD48NVs5czj8eHOIz64S6hKtqchqBcYc0NwMb77p5gyksZgCgYiMEpH7ROSf3vNZIvI5f4uWnAoz3epn35sc5vV6OHuxMD+REtdZP4ExTnu7CwL79wddksDFWiN4EJcqYqz3fC1wnQ/lSQki8Kkxbm3k3BBcvFy4szJBmor270+r8dHGRNU5Ycyy8wKxB4IRqvoYEIZ38gjZVNVeHJEPf5+tnD0Cbt0S4tKVQk1r0KXCmoeMWbzYcnBFiDUQNInIcLz1BETkBCCRGjwSVkEm3D5N+d/Dwry5xzUV/TfoM7d9e1p3jJk0t2qVNZF2EWsguAGXQvowEXkNeBi41rdSpRgR+MRol9Y6PwMuWS7cXgkdQTUVtbVZygmTnjZtgvXrgy5Fwokp+6iqviUipwDTcSuPrVHV9O5m74dZXq6imyuEn28J8eJu5SMj3VrJY3PiXJiqKhg7tvftjEkVO3bA8oTJcp9QYgoEUdYSPkZEUNWHfShTSsvPhNumKicVKXdWCd+sCPHNCpiVp7yvBE4fphyZ79Jf+6q6GlpbITvb5wMZkwB274a33krwPDDBiXU9guMivs8FTgfewjURmT4SgY+PggtKlQ37lX/XwfO7hTuq4PaqECOzXC3h9BLl5GIYmuFDITpTTkya5MPOjUkgTU0wf76lYu9BrE1DB/UHiEgR8FtfSpRGRNySmFOGwpVlSl0bvLhbeb5OeKoGHt0ZIieknFTkgsLpw2DMYDYhVVVZIDCpraUF3njD1X6T3Ma9HRQ1tTIsb/Br8f1doWwfMHUwC2JgWBZ8tBQ+Wqq0hmF+g/LvOuH5OvjP7hA3V8DheS4gvK9EOWKgTUj19e5uKS9vsD6CMYmjc8JYki8u09QBd1QK923fy8Uta7nlvCMG/Rix9hH8HW/oKG6k0SzgsUEvjXlHdgjmFcO8YuXbk2BdZxNSnfB/lXB7ZYhSrwnpolHK7IJ+HqiqCqZPH8yiGxM8VVi0yOXXSlKq8FQN/GCTsKNV+FhZFle/d4ovx4q1RvDTiO/bgc2qarOS4kQEpg11j6vKlNo2+I/XhPTkLnhyl/DqXKUkqx87t0BgUtGSJW5ARJJa0wTfrhDeaBAOz1PunB7m2AlDoSDXl+PF2kfwki9HN/0yPAvOL4XzS5U1TXDG4hAPbIcbyvsxImLfPqirg2HDBr+gxgShthYqK4MuRb80tMMvtggPbXeTUb8/OczFoyHD51GEPQYCEWnkQJPQQW8BqqqFvpTKxGx6HpwxTHlwG3x+rEt612dVVRYITOpYty7oEvRZWOGJXfCjTUJtG1w8Cr46oZ+1/H7ocWaxqhaoamGUR4EFgcRxzXiloUP4bX8nC2/bZiknTGrYvdsty5pElu+F85cJX1kXYnyum3T6v1PiFwSgj6OGRKQUN48AAFXdMuglMn12ZD6cWqLct1W4bIz2fd5BWxvs3AljxvhSPmPiJolqA7vb4NbNwiM7XXPvT6eG+ejIOEwmjSLW9QjOFZF1wEbgJWAT8E8fy2X66Noypa7d/VH1i2UkNcmuocHd0CS4DoXfbYfT3hL+uBMuGwMvHKOcXxpMEIDYawTfA04A/q2qR4vIacDF/hXL9NWxhXBCoXLPVuGS0W4dhD6prnZZGTMz3SMj48D3XZ93fm9MIkmC2sCiBvhWhbCiSTihULllsjI9AabxxPrf3KaqtSISEpGQqv5HRH7sa8lMn107XrlkRYg/71Q+2ddWnnC4b1kZRVxA6BogsrLg6KPdV2PiZe9el149QVW3wo83C49XC6Ozlf+bFuacEe7fKBHEGgjqRSQfeBn4vYhU4+YTmARyUhEcXaDctVW4cJSS5eeK1Kpu5mZ7u5vGH2nzZpjiz8QXY6Jaty4hE8p1KDy8HX6+RWgOwxfHKdeMV/L8yB82AD1eKkTkfBHJBc7DpZW4HngG2AB8yP/imb4QcX0FW1uEvwW5+NLGjTYKycTPvn0JudDMxv1w4TLhlo0hji6AZ49WbpqYeEEAeu8svgTYAtwFnIGbO/CQqt6uqrbwbQI6rcSltP5VpQS38E1zsxuSakw8rF+fULWBsMJ92+CsxcLaffCzqWEemqVMHhJ0ybrX2zyCjwBTgOeBLwGVInKXiLwnHoUzfddZK6hoFp4OslawYUOABzdpo7k5oWYRb9oPFy4XvrcxxElF8NzRysdKE6cvoDu9tiKraoNXCzgLOBJYDPyfiCTO2TcHOWM4TBmi3FElhIO6UWposMXBjf/Wr0+IZsiwwv3b4MzFwpomVwu4b6YyKt4rD/ZTzN2JIlICfBS4EBgGPO5XoXyTJiNZQgJXlylr9gn/rguwIFYrMH5qaYEtwc9p3bQfLloufNerBfwrSWoBkXrrLC4QkU+JyNPAKtxKZd8HylX1ujiUb3BNnQoF/c3XnFw+NBLKc91ymIE1n1ZXu2F9xvihoiLQVcfCCg94tYBVTW5m8H0zldFJUguI1FuNYCNwJq6zeLyqXq6qL6gmUM9MX2RkwHHHpUXNIFPgqnHKkr3CK/UBFsRqBcYPbW2waVNgh9/s1QJu2RjiRK8v4PwkqwVE6i0QlKvqJar6d1VtAxCRp+JQLv/k5cExxyTvb6wPPloKY7JdX0FgqqoOnWdgzEBVVLg5LHEWVngwohZw65Qw9ydpLSBSb6OGoq3xNs6nssRPaWlaLMaSHYIrxinzG4Q3g1qoKRwO9M7NpKD2djdXJc62NMPFy4XvbAzxLq8v4IJRqXFP2Z+5p2/HuqGInCkia0RkvYh8vYftjhORDhE5vx/l6Z+pU9Mi2+ZFo2BEVsC1gk2bEmJkh0kRmza5pqE4CSs8tB3OeFtY2QQ/mRLmgZnKmCSvBUTqrbM4V0SuE5E7ROQKEclU1c/GsmMRyQDuBM7CrXF8sYjM6ma7HwPP9r34AzRnTsp3HudmwBfGKq/UC4sbAypEa2tCjfU2SayjI679Tp21gG9XhDi+0NUCPp4itYBIvdUIHgLmAstwF/Sf9WHfxwPrVbVCVVuBR3GpKrq6FjcUNf4LjGZmpkXn8SWjoTgz4FpBRUVwxzapY/Nmd2MRB49Xw5kRtYAHZ6VWLSBSb4Fglqp+UlV/DZwPvLsP+x4HRN4GVtGlf0FExgEfAe7uaUcicrmILBSRhbsGe/WhNOg8zs+Ez45V/l3n/qgDsXdvUuSKNwksHI5bbeCl3fDVdcJsL0dQKtYCIvUWCN5piFPVvnbRRzttXYed/gK4SVV7HAysqveo6lxVnTty5Mg+FiMGadB5fOkYKMhQ7qy0WoFJUpWVLqWEzzbth2vXCNOGwr0zlbEpWguI1Fsa6tki0uB9L8AQ73ksi9dXAeMjnpcBXTORzQUeFRdqRwBni0i7qv41xvIPnqlTob4edvR34d/EVpQJnxoDd1XB+n0wZWgAhaipgT17oKgogIObpKbat/Uy+mlvO3xhlZAh8JuZiZkp1A+9DR/N6LJgfWYfFq9fAEwVkUkikg1cBDzZZf+TVHWiqk4E/gxcFUgQ6HT00ZCfH9jh/fa5sUpOCO7aGmCtwCaYmf6oqnLppn0UVrh+nVCxH+6crozP7f1nUoVvS5d4TUnX4EYDrQIeU9UVInKliFzp13EHJDMTjj8+ZTuPh2e5juO/VkOl/zXs6LZvj0v13qSQONUGflEpPFcn3DxJOanY98MlFD/XsEJVn1bVaap6mKr+wHvtblU9pHNYVT+jqn/2szwxyctzNYMUdflYJUPgrqBGEIXDgUwGMkls+3bfc1Y9Uwu3VwoXlCqfSf3pRYfwNRAkrVGjUrbzeFQOfHwU/LkatgeV+WHz5kDSA5gk5fOi9Kub4Ia1wpx85fuHaUqPDuqOBYLuTJsGo0cHXQpfXDFO6VC4J6i+gra2hEgfbJLAjh1ubQuf1LfB5auF/Az49QzXh5aO0vRjxyhFO4/H58JHSuGRnVATn7k5h9q4MaGWFzQJysfaQLvCNWuEHS1wdxItIuMHCwQ96Zx5nNnbKNvkc1WZ0hKG+7YFVCvYty9lh+qaQbJrlxvS7ZMfbhJe3SN8/zDlmNTONNMrCwS9yc9Pyc7jyUPgnBHw8HZXPQ6EDSU1PVm71rddP17tboI+M8bNGk53FghiMXq06zNIMVeXKU1h4cHtARVg926oC3ItTZOw6up8+9tY0gjfWC+cWKT8v4nWPAkWCGI3fbobTZRCZuTBB4YpD2wXGoMaxGNpJ0w0PtUGqlvhitVCababNJZlV0DAAkHfHHNMynUeX1Om7GkXfhdUc/2OHb7PGDVJpr7e9Q8MspYwfHG1sKfdpY8YlprzRvvFAkFfpGDn8VEF8J5i5b5twv4g1gFXtVqBOZgPI4VU4dsVwqJG4adTlZl5g36IpGaBoK/y82HePBg2LOiSDJprxys1bcJdVcLeIJqItmyJ64pTJoE1NPgymux3O+DRncI1ZcoHRwz67pOeBYL+KCiAk0+G2bODz0tUVgaFveX/69lxhfDuYuX2KuGoN4VzFgvfqRCeqoGd8Zh93NHhZhv7wZbITB7hsC99A2/sgVs2CqeXKDeUW+dwNKnTxhGE8nLXgbxypcuOGE8jRsCsWS6l8+bNsHTpgHb3m5nK/D3KwkZhYQP8cSc8uN3dJ4zPUeYWwtxC5bhCmDIEQoM9/WDjRpg8GUKDdG/S3u6anCoqoKQEDj885fp3kpoqNDa6/oDOR2PjoAfuqma4arUwIRdum6aD/3cbTxn+5cS2QDBQOTlunsH48e5i3OTzEmD5+S4ARI5gGjsWVqxwd9b9lBuC95TAe0rcHVNbGFY2KQsaYFGj8Eo9/GWXu0gXZSrHFrjAMLfA9TPkDvT63dwM27a5Gs5AhMNucfP166HFq85UV7vOx4kT3eivoGtx6UbV/V9EXvQbGgb09xqL/R0ufUSbuhudwmS+2mVmwpFH+rd73/acbkaMgFNPdR1d69cPfpNETo67iJWXH7pmXlYWjBkzqLWSrBDMLnCPz6OowubmA4FhQQO8sNtd/bNFOTIfV2soUE4uhqH9uXnZsKH/gUDVrWC1di3s3x/9/Y0b3TmaNg0mTUrttQeD1NTkFiDqvOjv2RP3JIOq8NX1wqomuH+WMnlIXA8/+HwesWiBYDCFQu5iPW6cqx3U1g58nxkZcNhh7tHTaKXycl+bp0Rg4hD3uGCUqzXUtimLIgLD/dvg1xpixlDlsSP7cQfW0OBWMRvRx968rVthzZrYamNtba72tHnzoTUr0zNVd0FvbXXnMfJrS8uBC38CdPzfvRWeqhFumhDmtJKgSzNAM2b4/ndqgcAP+flw0knuDnXlSveP0lcirrlp+nTIjWGppOHD3VoKfjdNRR4yCz4wHD4w3AWG5g54rk65fp1wxWrhwVn9yOa4YUPsgWDnTli9un/ZKffuhfnzYeRI139QkKbJZhoaXA2q64U98mvn9+3tSZEo8IU6+Mlm4UMjlCvHBV2aARozxi2j6zMLBH4aP/5AZ3JlZew/N3Kku1vt62ig8ePdhTEguRnwoZHQrsr160LcuA5u72sHXXW1u0j3VA2urXWfczBSEOzaBS+95GpUM2ZAdvbA95kMVN05jMPKX/H0nzq4crUwKw9+MiXJ1xYoLIxbnjMLBH7LzoY5cw50Jve00lJhoQsAI0f271jjx7smkoDv2j5SCjtbw/xoc4jR2XDzpD6WZ8MGNzS3qz17YNWqwZ91quqairZtc3dfkyYN3uilRNTWBosW+TJ7N0jP1MK1a4TpQ+G3hytDknnh+awsN3nVx5FCkSwQxMvw4XDKKe4ObN26gzuTc3Pd3WhZ2cA6MHNzobTUNZkE7IpxsKNVuXebMDpb+XxfquhVVe585HgJ4vfudXev233OjtfW5mpvnf0HqbgwUUMDLFiQcmk9ntwF168VjiqAB2cpRcl8ZROBuXNh6NC4HTKZT1fyCYXciJXOzuT6epgyxY2fH6zIX16eEIFABL45Saluhe9vClGaHebcWCs6nUNAx493o4CqquJby2lqchfLESNc/8EAJ+wljK1bYckS34dtxtufq+Fr64S5hXD/TCU/2a9qs2b1fcDEACX7KUtOeXlw4onuDnSwx7SXlro76ZagFiQ+IEPg59OUmhVw4zphRJZyUnGMP7xhgz/DcPuipgZeftkFpMgaSrJRdTWdFMzp9Psd8P82hJhXpPxmZpI3B4FrFZg8Oe6HTeGG0CTgx8SmUGjgk7IGUW7ITeaZNMSl/10V66Cmjo7ESA+h6nIhPf88vPWWq20lQrli1doKb7yRkkHg/m0uCLy3RLl3VgoEgeLi6H1jcWCBIBWVlwddgoMUZbp227wM+MxKYWvwlZW+6+hwTSvz58Nzz8GyZYm/qM6ePa5GU1MTdEkG3a+q4LsbQ5w5XLl7hg58ZnvQcnJc53BAgxSS/fSZaPLzEy476tgceGiWsq8DLl0hwS2PORhaW10fxmuvuZrC6tUuT04iqayEV1+NPss6ianCbVuEn2wOce4I5Y7pSnayX8VCIdc5HMt8Ib+KENiRjb/Gjw+6BIeYnueaibY0w+dXCc2p0Ge5b58bBfbii24+woYNLm9SUMJhWL4cFi9OriasGKjCjzYLv6wULihVbpumZCbzPIFORxwR+I2bBYJUNXZsQi6gc0KR60Be1AjXrRM6En+iauwaGlyn7L//Da+/Hv91Flpa4L//dTmVUoyqSyX9663CJ0crP56iZKRCEJgwwT0ClnhXCjM4MjNdMNiyJeiSHOKcEVDdqnx3Y4hbKuCWyUk+A7QrVTf7ubbW9SWUlroO/FGj/GsD3r0bFi4Mtjbik7DC/9sgPLJT+NxY5eaJKfL3MmyYqw0kAAsEqay8PCEDAcBnx8LOVuXXW4XROcpViTPQaXCFw27FrR07XHAeNcrNS8jPd8OI8/IGHhy2bHEBJ8WaggDa1c0ReGKXcHWZ8pXyFAkCubmuXyBBZrBbIEhlJSXugtNTWosA3TRB2dECP9kcYlR2mI+VBl0in7W3u5FHW7ceeE3EXRQiA0Pn90OH9jzTvLM/wK/V3QLWFobr1gr/qBVuLA9zbeJ1e/VPKATHH59Q81IsEKS68nLXbp2AQgK3TlVq2uCm9W7C2SnJnjK4r1TdyJ79+w/N/RMKuWDQNUB0JuRbuNA1CaWgljBcs0Z4rk74n4lhLk/2LKKRZs92KwsmEAsEqa6szA1vTNBmg+wQ3D1D+fhy4YurhT8e6Ra5Mbjf2d69CVuj80tzh5t8+FK9cMvkMJeOCbpEg2jy5ISa8NkpMRqojH9yclxnZQIryIQHZyolWXDZSmFL6vV3mhjt64DPrhJerocfT0mxINCZXj4BWSBIBwk20ziaUd6Es3Z1E85qk3nCmemzDoXKZrh0pfDGHvj5VOXCVFo8buhQOPbYhF0e1ZqG0kFpqeuQTPChhVOGwn0zlU+sED67UrjXqyWkxKShNNcShm0tsLXz0SxURTzf3gIdCJniZgufHd/km/4aMsR1DvuRW2yQWCBIB53LXq5bF3RJenVsoVvV7IurheMWuArrkJCSnwEFGa4ZKT/y6zuvu2063yuI+Doqm76tkmb6rKmj8wIPVS1Q1SIRF32objv4FxBCGZUN43Lg2AIYNxLG5YQ5pgBm5gX0IQZDbq5LHldc7DqEi4uTYtU7CwTpIkkCAcAZw+GxI5Wle5W9HdDYLuztgIZ22NvhHrv2w952aPSeaw+tnKOzlTOGwxnDlOOLrIYxmNY0ufWBn9998EnNEmVsjrvQn1LiLvLjcqAsF8pyYHQ2ZCV7w3TnRb/zgl9UlFBDQvvC10AgImcCvwQygHtV9Udd3r8EuMl7uhf4oqou8bNMaSsvz62SVlsbdEliMrfQPZye81CEFZo69J0g0RkwGjtgdxu8Ui88uhMe2h6iJFN53zA4c7hycjHJn7UyINtbXPK3P1dDXgZcXaZMG6qU5bgL/chUq4Xl5h58wS8uTtqLfjS+BQIRyQDuBN4PVAELRORJVY0c1L4ROEVVd4vIWcA9wLv8KlPaKy9PmkDQFyHxmoG6+Wv+1BiX9fTleuWZWuGZWvhTdYi8kHJqiQsKp5WQ/CtbxcGedri7Srh/u5sC8dmxLgiUJG7zd/8NHepSQBQVBZoZNB78/NM/HlivqhUAIvIocB7wTiBQ1dcjtn8DSLwBtqlkzBg3EzWeidASxNAMOHO4u+i3huG/e1xQeK4O/lEbIluUecVwxnBXYxieihe2AWgJw2+3wx1VQn278OGRyo3lyvhUvj7Onh33JSOD4mcgGAdURjyvoue7/c8B/4z2hohcDlwOUJ4EQyETVkaGS0SXoikJYpUdcu3Wp5Qo31d4q9EFhWdr4YXdIUIoxxe6oHDGcLeWQroKq1sY/qdbhKoW4d3Fyk0TwhyR6pP+ysvTJgiAv4EgWgth1MZeETkNFwjmRXtfVe/BNRsxd+7cVEpcHH/l5WkfCCJlCBxXCMcVKjdPhBVNyrO1wrN1cMvGELdshKPylTOGK6cWuxEtKdX23YNX6+GHm4QVTcKsPOXhw8K8Jx1SgOTmwuGHB12KuPIzEFQBkWmiyoBtXTcSkaOAe4GzVDX1GrATTXGxy37Z0BB0SRKOCByRD0fkKzdOgA37lGfr4Nla4dbNIW7dDMMylZOKYV6R62xOxaaRFXvdAjCv1AvjcpTbpoY5b2T6BEBmz07ItTz85OenXQBMFZFJwFbgIuATkRuISDnwBPApVV3rY1lMpPJy11dgenTYULhqKFxVpuxoUV7bA6/VC6/Ww1M1brjRhFzl5CKYV6ycWERSd5pWNcPPtgh/3QWFmXDzxDCfHJNmI6vKyhI+JYsffAsEqtouItcAz+KGj96vqitE5Erv/buBbwHDgV+Jm3rdrqpz/SqT8ZSVuYykCZqILhGNzoGPlcLHShVVWL9febXeBYYna+APO0MIyhF5cHKxCwxzCyA3I+iS966+zXUCP7zd1YquGAdfLFOK0uum2A0HTZCFYuJNVJOryX3u3Lm6cOHCoIuR/BYtgm2HtNSZfmgLw9K9rk39tT3C243QpkK2KHML4eRiZV6Ra3ZKlOUVG9vdrN//7Ia7qoTGDhfobijX9O0cP+44GD066FL4RkQWdXejnW4x33QqL7dAMEiyQi41xrGF8GWUpg6Y36DvNCPdujnErUBhhnJCketXGJGljMiCEdkwMgtGZLkhq4M127apwzX1uHQPUOnl9ul8rb79QEQ6rUS5aYIyI5lTOwzU2LEpHQR6Y4EgXY0c6ZJh7d8fdElSTl4GnFbiLrAAu1qV173+hQUNruawLxz9il+c6QUIL0iMyIKRWcrwiOcjstzkuZ2tERd770Jf6T3f3X5w1SMn5Gb9js+FOQVQlhOmLBemDCG9AwC4XEBp2iTUyQJBOhs/HtZaH73fRmbDeSPhvJEHmmH3dbiV2Xa1Qk2b92iFmjZ55/mKve5rY0fv1YRsUcpyYXwOHNV5oY/I7TMiK2EzIAfviCNSKl1Ef1ggSGfl5S4RXZL1E6WCoRlQngHlhww/PfR30RxWL0gcCBiNHS6rauSFPm2Gdw6m0aNhXCqtg9k/FgjS2ZAhbvZk17VyTULJDXkX/BScsxCorCw48sigS5EQ0mmEsInGUnaYdDVrVsonk4uVBYJ0N3p0Qq+cZIwvRo60m6AIFgjSXSjkJpgZky4yM10aCfMOCwTG7oxMepk50/WPmXdYIDAuCV1xcdClMMZ/w4fDxIlBlyLhWCAwjtUKTKrLyLAmoW5YIDBO5xT7oiI309KYVDN9ulu72xzC5hEYJyvLJd3q1NEBzc0uBUV3j46O4MprTF+UlMDkyUGXImFZIDDRZWS4u6ee7qDa2qIHiJ07ob09fmU1piehkGsSshwb3bJAYPovK8s9CgsPfn3/fli2zAUEY4I2bRoUFARdioRmfQRm8A0ZAscfD8ccY/0NJlhFRTBlStClSHhWIzD+GTfOzeBcsQKqqoIujUk3oRDMmWNNQjGwGoHxV3Y2HH00vOtdNonHxNdhhx3abGmiskBg4qO0FE49FSZNCrokJh0UFLi+ARMTaxoy8ZOZ6RYBGTsWliyBvXuDLpFJJLm57g5+MJpypk1zTUMmJhYITPwNGwannOJWR9uwAcLhoEtkgiDixveXlsKoUdaMEyALBCYYoRDMmHGgdlBfH3SJTDxkZ7sLf+fDUqAnBAsEJliFhTBvHlRUwJo1Nls5FRUVHbjrLy62UTwJyAKBCZ6IG+ExZoyrHdTUBF0iMxCZmW7YcOddv60ClvAsEJjEMXQonHgibNkCK1e6FBYmOeTnuzv+0lLXB2QdtUnFAoFJPOXl7qJSV+cS37W0uK+RDwsSwSsocLW4sWMthUOSs0BgElNOjrvIdKczO2q0IBH5CLrPIRRyE+lyc93XnByXi2nvXmhqCr58fZWf7y78dvFPKRYITHKKJTsquCyoLS3Q2tr918jv+zKUVcRd2IcMOfhiH/nIyen+51UPBIWuj5aW2Mvht7y8Axd/G+KZkiwQmNSWmekesS5I0l3gaGtzQx8jL/q5uQNrCxdx/SJDh7q29UhtbYcGh8ZG2LfPBRC/2cU/rVggMCZSXwOHX7Ky3GSrkpKDXw+HXTDorDV09+jPehB5eQfa/IuKBudzmKRggcCYZBIKuXb6/PyetwuHew4UnQ9wS5TaxT+tWSAwJhV1dlJbxlcTAxvsa4wxac4CgTHGpDkLBMYYk+YsEBhjTJqzQGCMMWnO10AgImeKyBoRWS8iX4/yvojI7d77S0XkGD/LY4wx5lC+BQIRyQDuBM4CZgEXi8isLpudBUz1HpcDd/lVHmOMMdH5WSM4HlivqhWq2go8CpzXZZvzgIfVeQMoFpEeMo0ZY4wZbH4GgnFAZcTzKu+1vm6DiFwuIgtFZOGuXbsGvaDGGJPO/JxZHG09uq7ZsmLZBlW9B7gHQER2icjmfpZpBGDLX3XPzk/P7Px0z85NzxLh/Ezo7g0/A0EVMD7ieRmwrR/bHERVR/a3QCKyUFXn9vfnU52dn57Z+emenZueJfr58bNpaAEwVUQmiUg2cBHwZJdtngQ+7Y0eOgHYo6rbfSyTMcaYLnyrEahqu4hcAzwLZAD3q+oKEbnSe/9u4GngbGA9sA+4zK/yGGOMic7X7KOq+jTuYh/52t0R3ytwtZ9l6OKeOB4rGdn56Zmdn+7ZuelZQp8f0XisdmSMMSZhWYoJY4xJcxYIjDEmzaVNIOgt71E6EZHxIvIfEVklIitE5Mve68NE5DkRWed9LeltX6lMRDJE5G0Recp7bufHIyLFIvJnEVnt/R2daOfHEZHrvf+r5SLyiIjkJvq5SYtAEGPeo3TSDtyoqjOBE4CrvfPxdeB5VZ0KPO89T2dfBlZFPLfzc8AvgWdUdQYwG3ee0v78iMg44EvAXFU9Ajdi8iIS/NykRSAgtrxHaUNVt6vqW973jbh/4nG4c/KQt9lDwIcDKWACEJEy4IPAvREv2/kBRKQQeA9wH4CqtqpqPXZ+OmUCQ0QkExiKmySb0OcmXQJBTDmN0pGITASOBt4ERnVO6PO+lgZYtKD9AvgaEI54zc6PMxnYBTzgNZ3dKyJ52PlBVbcCPwW2ANtxk2T/RYKfm3QJBDHlNEo3IpIPPA5cp6oNQZcnUYjIOUC1qi4KuiwJKhM4BrhLVY8Gmkiwpo6geG3/5wGTgLFAnoh8MthS9S5dAkGfcxqlOhHJwgWB36vqE97LOzvTgHtfq4MqX8BOBs4VkU24ZsT3isjvsPPTqQqoUtU3ved/xgUGOz/wPmCjqu5S1TbgCeAkEvzcpEsgiCXvUdoQEcG1765S1Z9HvPUkcKn3/aXA3+JdtkSgqt9Q1TJVnYj7W3lBVT+JnR8AVHUHUCki072XTgdWYucHXJPQCSIy1Ps/Ox3XB5fQ5yZtZhaLyNm4dt/OvEc/CLZEwRGRecArwDIOtIH/D66f4DGgHPcHfYGq1gVSyAQhIqcCX1HVc0RkOHZ+ABCRObiO9GygApcnLISdH0TkFuBC3Oi8t4HPA/kk8LlJm0BgjDEmunRpGjLGGNMNCwTGGJPmLBAYY0yas0BgjDFpzgKBMcakOQsEJm5E5CMioiIyY4D7+YyI3DFIZXpERJaKyPVdXv+OiGwVkcUislJELu5lPxNF5BMxHO+g7URkrojc3v9PcNC+N4nIMu/zvCQiE7zXx4vIRhEZ5j0v8Z5PEJFsEfmFiGzwMmP+zcuz1LnPDu8cLBeRv4tI8WCU1SQWCwQmni4GXsVN0gqciIwGTlLVo1T1tiib3Kaqc3ApA37tzcbuzkSg10DQdTtVXaiqX4q50L07TVWPAl4EbvaOUQncBfzI2+ZHwD2quhn4X6AAmOZlxvwr8IQ3GQpgv6rO8TJp1hHfpWVNnFggMHHh5TU6GfgcEYFARE4VkRcjctv/vvMiJCJne6+9KiK3i7cuQJf9jhSRx0Vkgfc4Oco2uSLygHe3/LaInOa99S+g1LvjfXd3ZVfVdcA+oEScW7075GUicqG32Y+Ad3v7ut67839FRN7yHid1s92pcvB6B3/17ujfEJGjvNe/IyL3e+epQkRiCRz/5eDEirfhZrxeB8wDfiYiQ3ETwa5X1Q7vsz4AtADvjWGfJkX4uni9MRE+jMtfv1ZE6kTkmM5U2Ljsp4fj8j+9BpwsIguBXwPvUdWNIvJIN/v9Je7O/VURKQeeBWZ22eZqAFU90muW+peITAPOBZ7y7vq7JSLHAOtUtVpEPgbMweXgHwEsEJGXcUnXvqKq53g/MxR4v6o2i8hU4BFgbpTtTo041C3A26r6YRF5L/CwdyyAGcBpuLv3NSJyl5fLpjtn4u7u8T57m4h8FXgG+ICqtnrnYkuUhIMLcb+P5yPOQQYuXcJ9PZ0rk5ysRmDi5WJcAje8r5Ft7vNVtUpVw8BiXPPJDKBCVTd623QXCN4H3CEii3H5XApFpKDLNvOA3wKo6mpgMzAthjJfLyJrcKk3vhOxr0dUtUNVdwIvAcdF+dks4Dcisgz4E25BpN5ElvMFYLiIFHnv/UNVW1S1BpewbFQ3+/iPiFTjzssfurx3Fi418hHecyF6Ft7I14d457YWGAY8F8PnMEnGAoHxnbgcPe8F7hWX0fOrwIUR7dAtEZt34Gqq0VKHRxMCTvTaseeo6jhvsZ2DitDPot+mqtNxeWMeFpHcPuzremAnruYwF5eTpzc9pUuPdo6iOQ2YAKwAvvvOjl1uoPfjVqS7XlwGzPXAhCiB8xhcEjnw+gi8fWZjfQQpyQKBiYfzgYdVdYKqTlTV8cBG3B1wd1YDk8UtnAPuYhzNv4BrOp94F7yuXgYu8d6fhkv8tSbWwntpuhfiska+jAtiGSIyErdS13ygEdds06kI2O7Vcj6FS3ZIlO26K+epQE1/1olQ1f3AdcCnvX4HwXUWX6eqW4BbgZ+qahNutayfe00/iMincatqvdBln3twSzB+pZdOc5OELBCYeLgY+EuX1x6nh1E23sXsKuAZEXkVd3e9J8qmXwLmeh2sK4Ero2zzKyDDa6b5I/AZVW2Jsl1PvgvcgEsfvBRYgrtYfs1Ly7wUaBeRJeKGov4KuFRE3sA1QzV5++m6XaTvdH4WXKfypfSTtwrWI7g7+C/g+gI6m3V+BcwQkVOAbwDNwFoRWQdcAHxEo2SjVNW3vc+dEKO+zOCx7KMmYYlIvqru9e5o78R12EYb5mmMGQCrEZhE9gWvo3IFrqnl18EWx5jUZDUCY4xJc1YjMMaYNGeBwBhj0pwFAmOMSXMWCIwxJs1ZIDDGmDT3/wE5djDMFMWASAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "# plot with error bars\n", + "qunatiles = np.nanquantile(p_val_df.iloc[:, :-1], [0.25, 0.75], axis=1)\n", + "plt.fill_between(angle_sweep, qunatiles[0], qunatiles[1], facecolor=\"r\", alpha=0.3)\n", + "plt.plot(angle_sweep, p_val_df[\"mean\"])\n", + "plt.xlabel(\"Angle of Rotation RXOR\")\n", + "plt.ylabel(\"P-Value\")\n", + "plt.title(\"Angle of Rotation vs mean P-Value\")" + ] + }, + { + "cell_type": "markdown", + "id": "cc990f97", + "metadata": {}, + "source": [ + "## Saving our Results" + ] + }, + { + "cell_type": "markdown", + "id": "3aeae5ba-dc05-4454-8bf5-947f6b51cac8", + "metadata": {}, + "source": [ + "Finally, we can write this dataframe to csv to avoid rerunning the experiment." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b21328ff-eb61-405f-9e01-406ecfe7f7e8", + "metadata": {}, + "outputs": [], + "source": [ + "# optional write to csv\n", + "p_val_df.to_csv(\"p_val_df_with_mean.csv\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}