ood_pretrain_only.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import time\n",
    "from pprint import pprint\n",
    "from pathlib import Path\n",
    "from collections import OrderedDict\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar100.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frequently used variables\n",
    "model_name = config.model.name\n",
    "patch_size = config.model.patch_size\n",
    "summary = config.summary\n",
    "\n",
    "# log directories\n",
    "log_root = Path('./logs') / model_name / summary\n",
    "checkpoint_path = log_root / 'checkpoints'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize ViT model and load pretrained weights\n",
    "from ood_with_vit.models.vit import ViT\n",
    "\n",
    "def initialize_vit_model(config, verbose=0):\n",
    "    assert config.model.pretrained, 'only pretrained models are allowed'\n",
    "    if config.model.pretrained:\n",
    "        model = torch.hub.load(\n",
    "            repo_or_dir=config.model.repo,\n",
    "            model=config.model.pretrained_model,\n",
    "            pretrained=True,\n",
    "        )\n",
    "    model = model.to(device=device)\n",
    "    if verbose:\n",
    "        print(model)\n",
    "        \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}\n",
      "{'apple': 0, 'aquarium_fish': 1, 'baby': 2, 'bear': 3, 'beaver': 4, 'bed': 5, 'bee': 6, 'beetle': 7, 'bicycle': 8, 'bottle': 9, 'bowl': 10, 'boy': 11, 'bridge': 12, 'bus': 13, 'butterfly': 14, 'camel': 15, 'can': 16, 'castle': 17, 'caterpillar': 18, 'cattle': 19, 'chair': 20, 'chimpanzee': 21, 'clock': 22, 'cloud': 23, 'cockroach': 24, 'couch': 25, 'crab': 26, 'crocodile': 27, 'cup': 28, 'dinosaur': 29, 'dolphin': 30, 'elephant': 31, 'flatfish': 32, 'forest': 33, 'fox': 34, 'girl': 35, 'hamster': 36, 'house': 37, 'kangaroo': 38, 'keyboard': 39, 'lamp': 40, 'lawn_mower': 41, 'leopard': 42, 'lion': 43, 'lizard': 44, 'lobster': 45, 'man': 46, 'maple_tree': 47, 'motorcycle': 48, 'mountain': 49, 'mouse': 50, 'mushroom': 51, 'oak_tree': 52, 'orange': 53, 'orchid': 54, 'otter': 55, 'palm_tree': 56, 'pear': 57, 'pickup_truck': 58, 'pine_tree': 59, 'plain': 60, 'plate': 61, 'poppy': 62, 'porcupine': 63, 'possum': 64, 'rabbit': 65, 'raccoon': 66, 'ray': 67, 'road': 68, 'rocket': 69, 'rose': 70, 'sea': 71, 'seal': 72, 'shark': 73, 'shrew': 74, 'skunk': 75, 'skyscraper': 76, 'snail': 77, 'snake': 78, 'spider': 79, 'squirrel': 80, 'streetcar': 81, 'sunflower': 82, 'sweet_pepper': 83, 'table': 84, 'tank': 85, 'telephone': 86, 'television': 87, 'tiger': 88, 'tractor': 89, 'train': 90, 'trout': 91, 'tulip': 92, 'turtle': 93, 'wardrobe': 94, 'whale': 95, 'willow_tree': 96, 'wolf': 97, 'woman': 98, 'worm': 99}\n",
      "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}\n"
     ]
    }
   ],
   "source": [
    "from torchvision.datasets import CIFAR10, CIFAR100, SVHN\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "dataset_mean, dataset_std = config.dataset.mean, config.dataset.std\n",
    "dataset_root = config.dataset.root\n",
    "img_size = config.model.img_size\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "\n",
    "cifar10 = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "cifar10_test_dataloader = DataLoader(\n",
    "    dataset=cifar10, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "cifar100 = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "cifar100_test_dataloader = DataLoader(\n",
    "    dataset=cifar100, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.4377, 0.4438, 0.4728], [0.1980, 0.2010, 0.1970]),\n",
    "])      \n",
    "svhn = SVHN(\n",
    "    root=dataset_root, \n",
    "    split='test', \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "svhn_test_dataloader = DataLoader(\n",
    "    dataset=svhn, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "print(cifar10.class_to_idx)\n",
    "print(cifar100.class_to_idx)\n",
    "print(set(svhn.labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set seeds\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "seed = 4321"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from ood_with_vit.utils import compute_penultimate_features\n",
    "from ood_with_vit.visualizer.feature_extractor import FeatureExtractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "model = initialize_vit_model(config)\n",
    "# add hooks for feature extraction\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing cifar10 samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 96.91it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 105.53it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 105.95it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 106.87it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 106.75it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 104.99it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.84it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 101.57it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.77it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 95.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing cifar100 samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 103.60it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.51it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 102.80it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 102.74it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.06it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.60it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.64it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.71it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 104.29it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 102.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing svhn samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 103.32it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.28it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.72it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.83it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.58it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.95it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.47it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.34it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.87it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.06it/s]\n"
     ]
    }
   ],
   "source": [
    "# gather penultimate features\n",
    "set_seed(seed)\n",
    "num_samples = 20\n",
    "num_class = 10\n",
    "\n",
    "print('processing cifar10 samples...')\n",
    "X_cifar10, y_cifar10 = [], []\n",
    "cifar10_img_paths = []\n",
    "cls_indices = random.sample(range(len(cifar10.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(cifar10.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = cifar10[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_cifar10.append(penultimate_features.squeeze().numpy())\n",
    "        y_cifar10.append((cls_idx, cifar10.classes[cls_idx]))\n",
    "        img_path = f'./assets/cifar10/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/cifar10/test/{i:05d}.jpg'\n",
    "        cifar10_img_paths.append(img_path)\n",
    "        \n",
    "print('processing cifar100 samples...')\n",
    "X_cifar100, y_cifar100 = [], []\n",
    "cifar100_img_paths = []\n",
    "cls_indices = random.sample(range(len(cifar100.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(cifar100.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = cifar100[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_cifar100.append(penultimate_features.squeeze().numpy())\n",
    "        y_cifar100.append((cls_idx, cifar100.classes[cls_idx]))\n",
    "        img_path = f'./assets/cifar100/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/cifar100/test/{i:05d}.jpg'\n",
    "        cifar100_img_paths.append(img_path)\n",
    "\n",
    "print('processing svhn samples...')\n",
    "X_svhn, y_svhn = [], []\n",
    "svhn_img_paths = []\n",
    "cls_indices = random.sample(range(len(set(svhn.labels))), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(svhn.labels) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, target = svhn[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_svhn.append(penultimate_features.squeeze().numpy())\n",
    "        y_svhn.append(str(cls_idx))\n",
    "        img_path = f'./assets/svhn/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/svhn/test/{i:05d}.jpg'\n",
    "        svhn_img_paths.append(img_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import HTML, VBox\n",
    "\n",
    "def interactive_plot(partitioned_info, fig, img_width=240, event=\"click\") :\n",
    "    \"\"\"\n",
    "    Make a plot react on hover or click of a data point and update a HTML preview below it.    \n",
    "    \"\"\"\n",
    "    html = HTML(\"\")\n",
    "    fig = go.FigureWidget(data=fig.data, layout=fig.layout)\n",
    "\n",
    "    if event == \"hover\" :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"{img_width}\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_hover(update)\n",
    "    else :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"{img_width}\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_click(update)\n",
    "\n",
    "    return VBox([fig, html])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "146b67193cab45ad9749e0dd3230a74a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/02852.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_cifar10)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_cifar10)\n",
    "df_embed = df_embed.assign(img=cifar10_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_cifar10:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c3da8c9b45a4b3db82661c20abe7468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/03989.jpg'],\n",
       "       …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_cifar100)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_cifar100)\n",
    "df_embed = df_embed.assign(img=cifar100_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_cifar100:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "\n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR100 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ad1346702214ce9af1510b7e20df62c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/svhn/test/06430.jpg'],\n",
       "           …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_svhn)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_svhn)\n",
    "df_embed = df_embed.assign(img=svhn_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_svhn:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT SVHN T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1831ce45e93c489f93814adbcfc80bbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/05878.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_svhn = ['svhn' for _ in y_svhn]\n",
    "X_all = X_cifar10 + X_svhn\n",
    "y_all = y_cifar10 + y_svhn\n",
    "img_all = cifar10_img_paths + svhn_img_paths\n",
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_all)\n",
    "n_samples = int(X_tsne.shape[0] / 2)\n",
    "X_tsne_cifar10, X_tsne_svhn = X_tsne[:n_samples, :], X_tsne[n_samples:, :]\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_all)\n",
    "df_embed = df_embed.assign(img=img_all)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_all:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10, SVHN T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, 64, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52d4baa4ad5f4b1ab276290c92929e4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/47795.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_cifar10)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_cifar10)\n",
    "df_embed = df_embed.assign(img=cifar10_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_cifar10:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bb7cbba2dee49b488f95002d033d530",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/07143.jpg'],\n",
       "       …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_cifar100)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_cifar100)\n",
    "df_embed = df_embed.assign(img=cifar100_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_cifar100:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR100 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "462e691c20ac448faf0b2fe599331604",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/svhn/test/23880.jpg'],\n",
       "           …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_svhn)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_svhn)\n",
    "df_embed = df_embed.assign(img=svhn_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_svhn:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT SVHN PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5ae96b443534f10a5a17774e0a19b41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': \"class=(4, 'deer')<br>x=%{x}<br>y=%{y}<extra></ex…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_all = X_cifar10 + X_svhn\n",
    "y_all = y_cifar10 + y_svhn\n",
    "img_all = cifar10_img_paths + svhn_img_paths\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_all)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_all)\n",
    "df_embed = df_embed.assign(img=img_all)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_all:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    # hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10, SVHN PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7847621dd57f44d7a06564a483c82e52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/04630.jpg'],\n",
       "       …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_all = X_cifar100 + X_svhn\n",
    "y_all = y_cifar100 + y_svhn\n",
    "img_all = cifar100_img_paths + svhn_img_paths\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_all)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_all)\n",
    "df_embed = df_embed.assign(img=img_all)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_all:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10, SVHN T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ood_with_vit.utils import compute_ood_scores\n",
    "from ood_with_vit.utils.ood_metrics import auroc, aupr, fpr_at_95_tpr\n",
    "from ood_with_vit.utils.visualization import plot_roc_curve, plot_precision_recall_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by Mahalanobis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CIFAR10 vs. CIFAR100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar10.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covairance norms: [35.333, 56.15467, 24.710087, 24.760406, 39.9555, 33.052032, 34.579536, 37.751755, 43.906643, 69.12952]\n",
      "covariance norm: 20.314365237275208\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar10 = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "cifar10_train_dataloader = DataLoader(\n",
    "    dataset=cifar10, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, cifar10_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 14.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 14.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 185.77126 id variance: 5098.766\n",
      "ood mean score: 296.23566 ood variance: 13700.672\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=cifar10_test_dataloader,\n",
    "    out_of_dist_dataloader=cifar100_test_dataloader,\n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCNUlEQVR4nO3dd3gUVffA8e8h9N4VpYXeRYgiItKbIurPhgXlNYJ0sL4qisprV6wURQF9FSu2oAhW5FWR3kEEEaRK7zXJ+f1xJ7rElAWyO9nd83mePLs7MztzZnczZ+beO/eKqmKMMSZ25fE7AGOMMf6yRGCMMTHOEoExxsQ4SwTGGBPjLBEYY0yMs0RgjDExzhJBlBORZSLS2u84cgsRuU9EXvNp26+LyCN+bDunicj1IvLlSb7XfpO5jCWCMBKRtSJySET2i8gW78BQNJTbVNX6qjo9lNtIIyIFRORxEfnD289VInKXiEg4tp9BPK1FZEPgNFV9TFVvCdH2REQGichSETkgIhtE5AMRaRiK7Z0sEXlIRN46lXWo6kRV7RjEtv6R/E72Nyki+b3YV3mf71oRGS8iVU90XeZ4lgjC7xJVLQo0Bs4G7vU3nBMnInkzmfUB0A64CCgG9AB6Ay+EIAYRkdz2+30BGAwMAkoDtYBPgItzekNZfAch5+O2JwHdgOuAEsBZwDzcb+6E+Pn55Uqqan9h+gPWAu0DXj8FfB7w+jzgJ2A3sAhoHTCvNDAB2ATsAj4JmNcVWOi97yegUfptAmcAh4DSAfPOBrYD+bzXNwMrvPVPA6oELKtAf2AV8HsG+9YOOAxUSje9GZAC1PBeTwceB2YDe4FP08WU1WcwHXgU+NHblxrAv7yY9wFrgFu9ZYt4y6QC+72/M4CHgLe8Zap6+3UT8If3WQwN2F4h4A3v81gB3A1syOS7rent57lZfP+vA6OAz714ZwHVA+a/AKz3Ppd5QMuAeQ/hDoRvefNvAc4FZnqf1WZgJJA/4D31ga+AncCfwH1AZ+AocMz7TBZ5y5YAxnnr2Qg8AsR583p6n/lzwA5vXk/gB2++ePO2erEtARrgTgKOedvbD0xO/38AxHlx/eZ9JvNI9xvylmvvfZ//mJfF/1dG33Wi913PAL4ABqRbxyLg/7zndQI+v5XA1X4fQ0J2bPI7gFj6S/cPUNH7h3nBe32m9092Ee5KrYP3upw3/3PgPaAUkA9o5U0/2/sHbOb9U93kbadABtv8FugVEM/TwMve80uB1UBdIC9wP/BTwLLq/VOUBgplsG9PAN9nst/r+PsAPd070DTAHaw/DPhnze4zmO79E9f3YsyHO9uujjsYtQIOAk285VuT7sCdycHhVdxB/yzgCFA3cJ+8z7wisDj9+gLW2wdYl833/7q3P+d68U8E3g2YfwNQxpt3B7AFKBgQ9zHgMu+zKQQ0xSXOvN6+rACGeMsXwx3U7wAKeq+bpf8MArb9MfCK952UxyXqtO+sJ5AMDPS2VYjjE0En3AG8pPc91AUqBOzzI1n8H9yF+z+o7b33LKDMify+MlpvFt/1f719LATcCPwYsHw9XFIt4C2zHneikZe/T5rq+X0cCcVfbru0jgWfiMg+3I9sK/CgN/0GYIqqTlHVVFX9CpgLXCQiFYAuQB9V3aWqx1T1e+99vYFXVHWWqqao6hu4g9l5GWz7beBacEUrQHdvGrgD2eOqukJVk4HHgMYiUiXg/Y+r6k5VPZTBusviDjwZ2ezNT/Omqi5V1QPAA8DVIhKX1WcQ8N7XVXWZqiZ7n8PnqvqbOt8DXwItM4kjMw+r6iFVXYQ7IzzLm3418Jj3mW8AXsxiHWWy2P9AH6vqbO8znogrIgRAVd9S1R3evo3AHZBqB7x3pqp+4n02h1R1nqr+7C2/Fncgb+Ut2xXYoqojVPWwqu5T1VkZBSQip+E+4yGqekBVt+LO8LsHLLZJVV/ytpX++z+GSzR1APF+Q8F8FuCubO5X1ZXed7hIVXdksFywn292HvL28RAu+QX+xq8HPlLVI7jPb62qTvD2eQHupOWqHIgh17FEEH6XqWox3NlqHf4+QFYBrhKR3Wl/wAVABaASsFNVd2WwvirAHeneVwlXDJLeh0BzL7FciCs2+V/Ael4IWMdO3BnamQHvX5/Ffm33Ys1IBW9+RutZhzuzL0vWn0GGMYhIFxH5WUR2estfxPFJJxhbAp4fBNIq8M9It72s9n8Hme9/MNtCRO4UkRUissfblxIcvy/p972WiHzmNTzYi0veactXwhW3BKMK7jvYHPC5v4K7Mshw24FU9VtcsdQoYKuIjBWR4kFuO9g4g/18s/PXfqjqPtyVdlrCuxaXnMF9Js3S/RavB07PgRhyHUsEPvHOXl8HnvEmrcedKZcM+Cuiqk9480qLSMkMVrUeeDTd+wqr6jsZbHMX7oz5GlyF27uqqgHruTXdegqp6k+Bq8hil77G/eNUCpwoIs1w/+zfBkwOXKYy7oxyezafwT9iEJECuOT2DHCaqpYEpuASWHbxBmMzrkgoo7jT+waoKCIJJ7MhEWmJq4O4Gijl7cse/t4X+Of+jAF+AWqqanFcWXva8uuBaplsLv161uOuIssGfO7FVbV+Fu85foWqL6pqU1zxSi1ckU+27/O2XT2bZcD9vs4VkYpZLHMAKBzwOqODdvp43gGuFZHmuCK07wLi+j7db7GoqvYNItaIY4nAX88DHUTkLFwl4CUi0klE4kSkoNf8saJ3mf0FMFpESolIPhG50FvHq0AfEWnmtaQpIiIXi0ixTLb5Nq5s9Er+LhYCeBm4V0TqA4hICREJ+jJYVb/GHQw/FJH63j6c5+3XGFVdFbD4DSJST0QKA8OBSaqaktVnkMlm8+OKT7YBySLSBQhs0vgnUEZESgS7H+m8j/tMSonImcCAzBb09m808I4Xc34v/u4ick8Q2yqGK4ffBuQVkWFAdmfVxXCVs/tFpA4QeJD6DKggIkPENest5iVlcJ9L1bRWV97v60tghIgUF5E8IlJdRFoRBBE5x/v95cMdjA/jrjbTtpVZQgJ4DfiPiNT0fr+NRKRM+oW839dXwMci0lRE8nr71EdEbvYWWwh09/4/EnC/8exMwZ39DwfeU9W0uD8DaolID299+bz9rBvEOiOOJQIfqeo2XOXVMFVdj6uwvQ93MFiPO6tK+4564M6cf8HVLQzx1jEX6IW7NN+Fq/DtmcVmk3AtXLZ4ZeJpsXwMPAm86xUzLMXVS5yIK3BnVFNxrUTewrVEGZhuuTdxV0NbcGdhg7wYsvsMjuNd2g/CHbB34a5ykgLm/4I741vjXd5nVFyWleHABuB33BnpJNyZc2YG8XcRyW5ckcflwOQgtjUN97n9iisuO0zWRVEAd+L2eR/uhOC9tBneZ9MBuAT3Oa8C2nizP/Aed4jIfO/5jbjEuhz3WU4i+KKY4t72d3mx78A1RAD3/dfzPv9PMnjvs7jv70tcUhuHq8jNyJW4A/d7uKulpUAC7rsBV99U3YvjYY4/0cmQVx/wEa5V0tsB0/fhTiq641rqbcH9fxTIbp2RSP4uGTAm9ERkOq4lhy93954KEekLdFfVoM6UjYkUdkVgTCZEpIKItPCKSmrjmmJ+7HdcxuQ0u7vOmMzlx7WeiccV9byLqwcwJqpY0ZAxxsQ4KxoyxpgYF3FFQ2XLltWqVav6HYYxxkSUefPmbVfVchnNi7hEULVqVebOnet3GMYYE1FEZF1m86xoyBhjYpwlAmOMiXGWCIwxJsZFXB1BRo4dO8aGDRs4fPiw36GETMGCBalYsSL58uXzOxRjTJSJikSwYcMGihUrRtWqVRF/hscNKVVlx44dbNiwgfj4eL/DMcZEmZAVDYkbVHqriCzNZL6IyIsislpEFotIk5Pd1uHDhylTpkxUJgEAEaFMmTJRfcVjjPFPKOsIXseNj5qZLrheMGviRtkacyobi9YkkCba988Y45+QFQ2p6gwRqZrFIpcC//UGRvlZREqKSIUTGOLOGGMimyocPgwHDsC2be754cOwdaubt24d5MvHgY2/sy1lH1WvSIRzzsnxMPysIziT4/tb3+BN+0ciEJHeuKsGKleuHJbgTlRcXBwNGzYkOTmZ+Ph43nzzTUqWLAnAsmXLGDhwIBs3biQ1NZUbb7yR+++//6+z/C+++IIHHniAgwcPUqBAAdq2bcuIESN83BtjTNBUYd8+2LQJdu2CvXth8WLYvx+WLoVChf6efugQLF8OpUq553v2QGpqlqv/Nh56XQIljsDc2Q3IE2WJIGiqOhYYC5CQkJAre8krVKgQCxcuBOCmm25i1KhRDB06lEOHDtGtWzfGjBlDx44dOXjwIFdccQWjR4+mf//+LF26lAEDBvD5559Tp04dUlJSGDt2rL87Y0ysO3QINmyAP/+ELVtgyRJ3oD961J2979wJixa5A3lKSubrKVjQza9XD0qXhuLF4cwz3XqqV4ciRVxSKFwYSpSAkiWhgBv7Znfx/Nz128u8tuYDapSsznPdXiVPfJvMt3UK/EwEGzl+DNiK3rSI17x5cxYvXgzA22+/TYsWLejY0Y2gWLhwYUaOHEnr1q3p378/Tz31FEOHDqVOnTqAu7Lo2zcqh0U1JndQhe3bYc4cWLAAduyAZcvcwXjPHpcAVq3K+L0VK0LRolCsGDRv7hJGuXJw9tmQJw+cfjrEx7tp8fGQP/9JhZiSmsL5YxqycsdK7j7/bh5q/RCF8mU2cNup8zMRJAEDRORdoBmwJ0fqB4YMAe/MPMc0bgzPPx/UoikpKXzzzTckJiYCrlioadOmxy1TvXp19u/fz969e1m6dCl33HFHzsZrjHEH9eXLYe5cWLHCnd3PmOGSQHoVKrjinYYNoW5d6NzZHfQrVnSvy5d3Z/IhtuPgDkoXKk1cnjgebfsolUpUIuGMhJBvN2SJQETeAVoDZUVkA/AgkA9AVV/GjT16EW6M3YPAv0IVSzgcOnSIxo0bs3HjRurWrUuHDh38DsmY6LRvnzuw794Na9bA77+7opZly1wxzqpVkC+fWy5Q8eJw1llQpQqccYYrrmnZEqpWdWfzPlJVJi6ZyOCpg3mi3RP0atqLy+teHrbth7LV0LXZzFegf45vOMgz95yWVkdw8OBBOnXqxKhRoxg0aBD16tVjxowZxy27Zs0aihYtSvHixalfvz7z5s3jrLPO8iVuY3Kd1FR31r58uWs1s3Yt/Pije/7rr5m/r1Ild7Bv3dqVtTds6Ipqzj3XHeyLFg1P/Cdo/Z719Pm8D1NWTeG8iufRonKL8AehqhH117RpU01v+fLl/5gWbkWKFPnr+fz587Vy5cp67NgxPXjwoMbHx+tXX32lqqoHDx7Uiy++WF988UVVVV20aJFWr15dV65cqaqqKSkpOmbMmAy3kRv205gcd+yY6sKFqg88oFq7tmrx4qquJP/vv9NOU61USbVdO9XevVVfekl18mTVRYtUt2/3ew9O2tuL39ZijxXTwo8W1udnPq/JKckh2xYwVzM5rkZEq6FIc/bZZ9OoUSPeeecdevTowaeffsrAgQPp378/KSkp9OjRgwEDBgDQqFEjnn/+ea699loOHjyIiNC1a1ef98CYENi6FebNc00rZ8+GI0dcZe2mTX8vU7MmXHghtGkDtWtDjRqunL5IEf/iDqFShUrRrGIzxnYdS3wp/7qPibgxixMSEjT9wDQrVqygbt26PkUUPrGynybCpaa6svs5c1xRzrRprhx/y5bjlytZEpo2dRWx9etDt26uOCeKJacm89zM5ziacpShFw4FXKlMOHoOEJF5qpphzbNdERhjTt6RI/DNN/DFF67Z5c6dMGuWm56meHFo0AAGD3bl+E2auLN9nytow23RlkUkJiUyb/M8rq5/9V8JIDd0H2OJwBiTvdRU+O031zJnyRL44w+YPNk1yQwUHw/du7s29jVrQq1artllLjjY+eVI8hEemfEIT/z4BKULleaDqz7girpX5IoEkCZqEkG4Lq/8EmlFeCbC/fmna3//3XeuaGf58n92hdCypSvLb9MGLrjANcc0/7Bq5yqe/PFJrmt4Hc92fJYyhcv4HdI/REUiKFiwIDt27IjarqjVG4+gYMGCfodiolFqqmt7P3OmO/C/997xRTv580OzZnDRRa4it25dKFs2ps/ys7P/6H4+/eVTrm90PQ3KN+CXAb9QrVQ1v8PKVFQkgooVK7Jhwwa2bdvmdyghkzZCmTGnbO1amDLFlel//bWr1E1Tpgy0b+/K8ps1c2f91arZQf8EfPXbV/T+rDfrdq+jSYUm1C1XN1cnAYiSRJAvXz4bucuYzKi6A/6ECfDtt8eX61esCO3aubP8G290rXhirBI3p+w6tIs7v7yT8QvHU6tMLb7v+T11y0VGK7+oSATGmHS+/x4mTnTdIM+c+ff0GjXg1lshIcHdgVusmG8hRpOU1BRajG/Brzt+5d4L7mVYq2EUzBs5RbmWCIyJdEePwqefws8/u2Kfjz46fn7nzq53zL59XZGPyTHbD27/q5O4x9o9RuUSlWlS4aRH3fWNJQJjIk1a2/0FC9xdut9++3ePmnnyuKabTZu6nnirV/c11Gilqry5+E2GTB3CE+2foHfT3lxW5zK/wzpplgiMye1UXWue115zlbx79vw9r0wZV8Rz7bXQqpVrzWNCat3uddz62a1M+20a51c6nwurXOh3SKfMEoExudG+fa5s/6uvICnp7143S5Z0B/yuXaFnT5cIrEVP2Ly1+C36ft4XVeWlLi/R75x+5JHIr1y3RGBMbjFvnivyeeGF4ztiq1ULnnwSevRwA6gY35QrXI4WlVrwStdXqFKyit/h5BhLBMb45fBhd/fu9Onwzjvu7l1w5fodOrjinrZt3UAqxhfHUo4xYuYIjqUc44FWD9CpRic6Vu8YdTeuWiIwJpy2bXNdNrz+umvimZzsptevD489Bpdeal015BILNi8gMSmRBVsW0L1B91zVSVxOs0RgTKitXQujRsGXX7pWPgAFC7py/muvdXfvWpFPrnE4+TDDvx/OUz8+RdnCZfnw6g/5v7r/53dYIWWJwJhQWbAAbr/dFf0AFCoEDzzgmne2bQsFCvgansnY6p2reeanZ7jxrBsZ0XEEpQqV8jukkLNEYExOUXVdNY8fD+PGuRG5ADp2hEcecXfzRmGxQjTYf3Q/H6/4mB5n9aBB+QasHLDS1xHDws0SgTGnQvXvMv8ZM2DzZje9alX4z3/gppvsbt5cbtrqafT+rDfr96wn4YwE6parG1NJACwRGHNyjh6FDz+Ee++FdevctJYt4e673RWAVfjmejsO7uD2L2/nv4v+S52ydfjfv/4XMZ3E5TRLBMaciLVrXbHPuHHu7L98efj3v+G++9yQjCYipHUSt3rnaoa2HMr9F94fUZ3E5TRLBMZk5+hRePddV/b//fduWoUKMHKk68kzr/0bRYptB7ZRpnAZ4vLE8WT7J6lSsgqNT2/sd1i+i/x7o40JlZ9/hl69XOuem25yXT506QKffAIbN0L//pYEIoSqMmHBBGqNrMWr814F4NI6l1oS8Niv2Jj0NmyA3r3hiy/c6+bN3aAtt9xiB/4ItHb3WnpP7s1Xa76iZeWWtIlv43dIuY79qo0BOHbM3fA1YoTr6RNgwABITITGjX0NzZy8Nxe9Sd/P+yIijL5oNLcm3BoVncTlNEsEJrb973+umee330JKiuvPv3VrePRROP98v6Mzp+i0oqdxYZULebnry1QuUdnvcHItSwQm9uzcCe+/Dw8/DFu2uGlnnOFa/yQmQpEi/sZnTtqxlGM89eNTpGgKw1oNo2P1jnSs3tHvsHI9SwQmNixdCh98ALNnuz7+U1Jclw+XXw7PPWc9fEaB+Zvnc/OnN7Poz0Vc1/C6vzqJM9mzRGCi1/79rtz/1Vdh6lQ3LS4OLrrIDePYqpV7bSLaoWOHePj7h3nmp2coV6QcH1/zcUQPG+mHkCYCEekMvADEAa+p6hPp5lcG3gBKesvco6pTQhmTiXKHD7tO3p58En74wXXzXKwY3HEH3Hyz3fEbhdbsWsOzM5+lZ+OePN3h6ZjoJC6nhSwRiEgcMAroAGwA5ohIkqouD1jsfuB9VR0jIvWAKUDVUMVkotjeva7Sd+xY9zxPHqhcGV56Cdq0sXL/KLP3yF4+WvERPRv3pH75+qwauCqqRgwLt1BeEZwLrFbVNQAi8i5wKRCYCBRIuy+/BLAJY06Eqqv4HTLEVfy2agX9+rkRvkrZmWE0mrJqCn0+68PGfRtpdmYz6para0ngFIUyEZwJrA94vQFolm6Zh4AvRWQgUARon9GKRKQ30BugcmVrAhbzVF1Hb+PHw4QJ7gawChVc/z833+x3dCZEth/czm3TbuOtxW9Rr1w9frzqx5jtJC6n+V1ZfC3wuqqOEJHmwJsi0kBVUwMXUtWxwFiAhIQE9SFOk1t88w1ccw3s2OFeN20K99zj7vq1gV6iVloncWt2rWHYhcO4r+V9FMhr33dOCWUi2AgEdsRe0ZsWKBHoDKCqM0WkIFAW2BrCuEwkWr3aHfA//NC9Tkx0XT7XquVvXCak/tz/J+WKlCMuTxzPdHiGKiWr0Oi0Rn6HFXVCea/1HKCmiMSLSH6gO5CUbpk/gHYAIlIXKAhsC2FMJtL8+CNccQXUrOmSQGKiqwt47TVLAlFMVRk3fxy1R9Zm7LyxAFxS+xJLAiESsisCVU0WkQHANFzT0PGqukxEhgNzVTUJuAN4VURuw1Uc91RVK/qJdarwzDOuzH/lSjfttttg4ECIj62Ro2LRml1r6DW5F9/+/i2tqrSifbUMqw5NDgppHYF3T8CUdNOGBTxfDrQIZQwmgqSmwmefuVG/lnuNy/r3dwPAV6vmb2wmLN5Y+Ab9pvQjTuJ4+eKX6dW0l3USFwZ+VxYb4276+vRTePxxmDcP8ueHvn1d1w9WARxTzih2Bm3j2zLm4jFULF7R73BihiUC458jR1xZ/z33uO4gypRxRUL9+0PB2B02MJYcTTnKEz88Qaqm8lDrh+hQvQMdqnfwO6yYY4nAhN/BgzB4sEsC4G4Cu/FG6NED8uXzNzYTNnM2zuHmpJtZunUpPRr1sE7ifGSJwITPsWPwr3+53j+3boVy5VwT0DvuADsAxIyDxw4y7LthPPfzc1QoWoGk7klcUvsSv8OKaZYITOilpMB777mD/kbvVpIPPoArr/Q3LuOL33f9zkuzX6JXk1482f5JShQs4XdIMc+q401oTZ7s+vq//npXKfzuu651kCWBmLLn8B4mLJgAQP3y9Vk9cDUvd33ZkkAuYYnAhM7990O3bq5OYMwYWL/edQ9hxUAx5fNfP6f+6PrcMvkWftn+CwCVSlTK5l0mnKxoyOS8Vatc3z8zZkDLlpCUBCVL+h2VCbNtB7YxZNoQ3l7yNg3KN+Cjaz6iTtk6fodlMmCJwOSc1FQYNcrdBQyuO4iRI60paAxKSU3hggkX8Puu33m49cPcc8E95I/L73dYJhOWCEzOmDXLHfiXLYPGjV1dQO3afkdlwmzL/i2UL1KeuDxxjOg4gqolq9KgfAO/wzLZCLqOQEQKhzIQE6G2bHFNQps3d0VCjz7qBoi3JBBTUjWVV+a+Qq2XavHK3FcA6FqrqyWBCJHtFYGInA+8BhQFKovIWcCtqtov1MGZXEwVHnkEHn7YNQ8tWhRWrICK1i1ArFm9czW9Jvdi+trptI1vS6canfwOyZygYK4IngM6ATsAVHURcGEogzK53KJF0KQJDBvmBoP/4QfYt8+SQAyasGACDcc0ZP7m+bx6yat83eNrqpWyDgIjTVB1BKq6Pt2t3ymhCcfkagcOuLGB07qGePxxd5NYHmuFHKsql6hMp+qdGHXRKM4sfqbf4ZiTFEwiWO8VD6mI5AMGAytCG5bJVVJS4Pnn4dVX3fgAV13luoY+7zy/IzNhdiT5CI//8DipmsrwNsNpV60d7aq18zssc4qCSQR9gBdwg9FvBL4ErH4gVsyeDW3buqsBgEmT3IhhJubM2jCLxKRElm1bxk1n3WSdxEWRYK7pa6vq9ap6mqqWV9UbgLqhDsz47MAB1yNos2aQN6+7M/jQIUsCMejA0QPcPu12mo9rzp4je/js2s94/bLXLQlEkWCuCF4CmgQxzUSL33+H9u1hzRq47jp4+mk44wy/ozI+WbdnHaPnjKZPQh+eaP8ExQsU9zskk8MyTQQi0hw4HygnIrcHzCqOG4PYRKMJE+DWW91dwmPGQJ8+fkdkfLD78G4mLZ/ELU1uoV65eqwetNpGDItiWV0R5MfdO5AXKBYwfS9gXUdGm/37oWdP+PBDKFUKvvsOzjrL76iMDz795VP6ft6XrQe2ckHlC6hTto4lgSiXaSJQ1e+B70XkdVVdF8aYTLi9+y706we7drkWQePGQbFi2b/PRJWtB7Yy6ItBvLfsPRqd1oika5Osk7gYEUwdwUEReRqoD/zVe5iqtg1ZVCY89uyBf/8bXnkF6tZ1g8W0s6aAsSglNYUW41vwx54/eKTNI9zd4m7yxdmwobEimEQwEXgP6IprSnoTsC2UQZkwmDgRbrjBPb/uOncVYL2ExpxN+zZxetHTicsTxwudX6BqyarUK1fP77BMmAXTfLSMqo4Djqnq96p6M2BXA5EqNRUeesglgQIFXOXwxImWBGJMqqYyZs4Y6oysw8tzXwbgopoXWRKIUcFcERzzHjeLyMXAJqB06EIyITNvHvzf/8Eff8C557oBY047ze+oTJj9uuNXek3uxYx1M2hfrT1danTxOyTjs2ASwSMiUgK4A3f/QHFgSCiDMiEwa5brEqJwYXjxRVc5HGetgGPNuPnjGPDFAArmLcj4buPp2bin3Rhmsk8EqvqZ93QP0AZARFqEMiiTg1JT4YEH4KmnXPHPDz/A2Wf7HZXxSdWSVelSowujLhpFhWIV/A7H5BJZ3VAWB1yN62NoqqouFZGuwH1AIcCOJrnd9u3Quzd8/DHEx8PUqVCrlt9RmTA6knyE/8z4DwCPtH3EOokzGcrqimAcUAmYDbwoIpuABOAeVf0kDLGZUzF5Mlx9NRw+DF26uPqAvDYyaSz5af1PJCYl8sv2X7i58c3WSZzJVFZHhgSgkaqmikhBYAtQXVV3hCc0c9JmzoRu3dzzKVNcIjAxY//R/Qz9ZigvzX6JSiUqMfX6qTZqmMlSVs1Hj6pqKoCqHgbWnGgSEJHOIrJSRFaLyD2ZLHO1iCwXkWUi8vaJrN9k4JlnoE0bN3TkTz9ZEohBf+z5g1fmvUL/c/qztO9SSwImW1ldEdQRkcXecwGqe68FUFVtlNWKvTqGUUAHYAMwR0SSVHV5wDI1gXuBFqq6S0TKn8K+mIEDYeRIqFYNvvwSqlf3OyITJrsO7eKD5R/Qu2lv6pWrx5rBazijmPUYa4KTVSI41TEHzgVWq+oaABF5F7gUWB6wTC9glKruAlDVrae4zdh04AB07QrTp0Pjxu5KoFAhv6MyYfLxio/pN6Uf2w5so1WVVtQuW9uSgDkhWXU6d6odzZ0JrA94vQFolm6ZWgAi8iOua+uHVHVq+hWJSG+gN0DlypVPMawoo+p6Cf3tN+sqIsZs2b+FgV8MZNLySTQ+vTGfX/c5tcvW9jssE4H8bkaSF6gJtAYqAjNEpKGq7g5cSFXHAmMBEhISNMwx5m7Dh7sk0L276yrCxISU1BRaTmjJ+j3reaztY9x5/p3WSZw5aaFMBBtxzU/TVPSmBdoAzFLVY8DvIvIrLjHMCWFc0UEVbrsNXngBGjSAN9/0OyITBhv2buCMYmcQlyeOFzu/SHypeOsq2pyyYDqdQ0QKiciJXnPOAWqKSLyI5Ae6A0nplvkEdzWAiJTFFRWtOcHtxJ6jR+HKK10S6NABZsywewSiXKqm8tKsl6gzsg5j5owBoEvNLpYETI7INhGIyCXAQmCq97qxiKQ/oP+DqiYDA4BpwArgfVVdJiLDRcRr5M40YIeILAe+A+6y+xSCcO218NFHbhjJL75wI4qZqPXL9l+4cMKFDJo6iAsqX0DXWl39DslEGVHNushdRObhup2erqpne9OWqGrDMMT3DwkJCTp37lw/Nu0/VbjvPnjiCdeNtBUHRb3X5r/GgCkDKJyvMM93fp4ejXrY3cHmpIjIPFVNyGheUN1Qq+qedD8+q7ANt61boVcv11XEhRfC6NF+R2TCoHqp6lxS+xJGdhnJaUWty3ATGsEkgmUich0Q590ANgj4KbRhmePs3w+tWsEvv0BiIrz8stUJRKnDyYcZ/v1wAB5r9xht4tvQJr6Nz1GZaBdMZfFA3HjFR4C3cd1RDwlhTCbQtm3uPoFffnFdSb/2miWBKPXjHz/S+OXGPP7D42w7sI3sim2NySnBHFHqqOpQYGiogzHp7NvnBpNfswaefBLuusvviEwI7Duyj/u+uY9Rc0ZRpWQVpt0wjY7VO/odlokhwVwRjBCRFSLyHxFpEPKIjLNgAZxzDixZ4oqC7r7b74hMiGzYu4HXFrzGwHMHsqTvEksCJuyCGaGsjYicjhuk5hURKQ68p6qPhDy6WDVzpqsTKFQIPvzQjTNsosqOgzt4f9n79D2nL3XL1WXNoDU2YpjxTVA3lKnqFlV9EeiDu6dgWCiDimmLF7ubxEqWdMNKWhKIKqrKpOWTqDe6HoOmDmLl9pUAlgSMr4K5oayuiDwkIktwg9f/hOsuwuS0OXOgaVM4dMgNKNPQl1s1TIhs3reZK96/gqs+uIpKxSsxt9dc6yTO5ArBVBaPB94DOqnqphDHE7u2boXOnUEEvv4aEjK878NEqLRO4jbu28hT7Z/itua3kTePtf4yuUMwdQTNwxFITPvpJ2jRwj3/+GM3wpiJCuv3rOfM4mcSlyeOUReNIr5UPLXK1PI7LGOOk2nRkIi87z0uEZHFAX9LAkYuM6fqp5+gfXv3/Lnn4LLLfA3H5IyU1BRenPUidUb93UlcpxqdLAmYXCmrK4LB3qP1cBUqkye7QeYLFICFC92NYybirdi2gsSkRGZumEmXGl24pPYlfodkTJYyvSJQ1c3e036qui7wD+gXnvCi2JgxLglUqQKzZlkSiBJj542l8SuN+XXHr7x5+Zt8ft3nVC5ho+qZ3C2Y5qMdMpjWJacDiSnffAP9+kGTJpYEokzN0jW5vM7lLO+/nBsa3WA9hZqIkGnRkIj0xZ35V0tXJ1AM+DHUgUWtV1+F3r3dfQKffQanWY+SkezQsUM8NP0hRIQn2j9hncSZiJRVHcHbwBfA48A9AdP3qerOkEYVrR55BB54wD3/+muoYDcRRbIZ62ZwS9ItrNq5ij5N+6CqdgVgIlJWiUBVda2I9E8/Q0RKWzI4QS+84JJAiRKwbp17NBFp75G93PP1PYyZO4ZqparxzY3f0Da+rd9hGXPSsrsi6ArMww1EE3iqo0C1EMYVXaZOhSFDoGBBVydgSSCibdq3idcXvs7t593O8DbDKZK/iN8hGXNKMk0EqtrVe4wPXzhRaNo0uPRSOP1010TU6gQi0vaD23l/2fv0O6cfdcrW4ffBv9uIYSZqBNPXUAsRKeI9v0FEnhURaw8XjMWL4eKLXS+i335rSSACqSrvLX2PeqPqMWTqEH7d8SuAJQETVYJpPjoGOCgiZwF3AL8BNmp6djZvdr2IpqTAjz9C3bp+R2RO0KZ9m7jsvcvo/mF3qpSswrze8+zOYBOVgun1KllVVUQuBUaq6jgRSQx1YBFt3z7XgdyOHfDdd1C/vt8RmROUkprChRMuZOO+jTzT4RkGnzfYOokzUSuYX/Y+EbkX6AG0FJE8QL7QhhXh7rzTFQtNmACtW/sdjTkB63avo2LxisTliWP0xaOpVqoaNUrX8DssY0IqmKKha3AD19+sqltwYxE8HdKoItm8efD221CnDvTs6Xc0JkgpqSk8O/NZ6o6qy5i5rpO4jtU7WhIwMSHbROAd/CcCJUSkK3BYVf8b8sgi0aFD0LEjHDsGn37qdzQmSEu3LuX88edzx5d30K5aOy6rc5nfIRkTVsG0GroamA1chRu3eJaIXBnqwCLSgw/Czp0wbBjUskrFSPDy3Jdp8koT1uxaw9v/9zZJ3ZOoWNwG4DOxJZg6gqHAOaq6FUBEygFfA5NCGVjEWbXKjSfQoQPcd5/f0ZhspHUHUbdsXa6qfxXPd3qeckXK+R2WMb4IJhHkSUsCnh0EOeh9zEhOhuuvd4/PP+93NCYLB48dZNh3w4iTOJ7s8CStqraiVdVWfodljK+COaBPFZFpItJTRHoCnwNTQhtWhBk2zA08P3w41KvndzQmE9PXTqfRmEaMmDmC/Uf3o6p+h2RMrhDMmMV3icj/ARd4k8aq6sehDSuCTJ0Kjz/ubhgbOtTvaEwG9hzew91f3c3Y+WOpXqo63974rXUVbUyArMYjqAk8A1QHlgB3qurGcAUWEX75Bbp0cV1ITJkCeazELDfavH8zby15izub38nDbR6mcL7CfodkTK6S1ZFrPPAZcAWuB9KXTnTlItJZRFaKyGoRuSeL5a4QERWRhBPdhm/27HEVwwDffw9Vq/oajjnetgPbeGmW+8nWKVuHtYPX8nTHpy0JGJOBrIqGiqnqq97zlSIy/0RWLCJxwCjcUJcbgDkikqSqy9MtVwwYDMw6kfX77oYbYMMGGD0azjnH72iMR1V5Z+k7DPpiEHuP7KVTjU7UKlPLWgQZk4WsrggKisjZItJERJoAhdK9zs65wGpVXaOqR4F3gUszWO4/wJPA4ROO3i9PPeWGmbzlFujb1+9ojGf9nvVc8s4lXP/R9dQoXYMFty6wTuKMCUJWVwSbgWcDXm8JeK1AdkMynQmsD3i9AWgWuICXUCqp6ucicldmKxKR3kBvgMqVfe4Be/dueOghN8zk6NH+xmL+kpyaTOs3WrNl/xae6/QcA88dSFyeOL/DMiYiZDUwTUibVXid1z0L9MxuWVUdC4wFSEhI8K/NX0oKnH++60ri448hn/W957e1u9dSqXgl8ubJyytdX6FaqWpUK2WD5xlzIkLZzGUjUCngdUVvWppiQANguoisBc4DknJ1hXHPnrBihbtvoFMnv6OJacmpyTzz0zPUHVWX0XPclVn7au0tCRhzEkLZwfocoKaIxOMSQHfgurSZqroHKJv2WkSm45qozg1hTCdv4kR46y0oVcoVDRnfLP5zMYlJiczdNJdLa1/KFfWu8DskYyJayBKBqiaLyABgGhAHjFfVZSIyHJirqkmh2naO278fbrsNChSAlStBxO+IYtboOaMZPHUwpQqW4r0r3+Oqelch9n0Yc0qyTQTi/suuB6qp6nBvvOLTVXV2du9V1Smk645CVYdlsmzroCL2w/33w7ZtMHkylLNmiH5I6ySuQfkGdG/Qnec6PUfZwmWzf6MxJluSXX8rIjIGSAXaqmpdESkFfKmqvjSeT0hI0Llzw1h69OefcPrprg+hZcvCt10DwIGjB7j/2/vJmycvT3e08ZCMOVkiMk9VM6yDDaayuJmq9sdr56+qu4D8ORhf7jZtmnscNMjfOGLQN2u+oeGYhjw/63mOpByxTuKMCZFg6giOeXcJK/w1HkFqSKPKTT780D326OFvHDFk9+Hd3PnlnYxbMI6apWsyo+cMWlZp6XdYxkStYK4IXgQ+BsqLyKPAD8BjIY0qt9i/H5KSoGlTKGx91ITLn/v/5N2l7/LvFv9mUZ9FlgSMCbFguqGeKCLzgHaAAJep6oqQR5YbPPige7z3Xn/jiAFpB//B5w2mdtnarB2y1iqDjQmTYCqLM+zTQVX/CElE2QhbZXFKClT0xq7dtMmajIaIqjJxyUQGTx3M/qP7Wdp3KTXL1PQ7LGOiTlaVxcHUEXyOqx8QoCAQD6wE6udYhLnRc8/Bli0wdqwlgRD5Y88f9PmsD1+s/oLmFZszrts4SwLG+CCYoqGGga+9juL6hSyi3GDTJlcsFB8PN9/sdzRRKTk1mdavt2brga282PlF+p3TzzqJM8YnJ3xnsarOF5Fm2S8ZwQYMgKNH4f33Ic4OTjlpza41VClRhbx58vLqJa9SvXR1qpas6ndYxsS0YO4svj3gZR6gCbApZBH5bfVq17Nov36QkHv7v4s0yanJjPhpBA9Of5CnOjzFoGaDaFetnd9hGWMI7oqgWMDzZFydwYehCScXePxx93jrrf7GEUUWbllIYlIi8zfP5/I6l3NVvav8DskYEyDLRODdSFZMVe8MUzz+SkmB8eOhTBlo1MjvaKLCyNkjuW3abZQpVIZJV02ynkKNyYUyTQQiktfrQbRFOAPy1Vtvucf77vM3jiiQ1klco9MacX3D63m207OULlTa77CMMRnI9D4CEZmvqk28TufOBD4ADqTNV9WPwhPi8UJ6H8HZZ8PChbB3LxQrlu3i5p/2H93P0G+Gki8uH890fMbvcIwxnlPtdK4gsAM3RnFX4BLvMbrs3OmSQIsWlgRO0pe/fUmD0Q14afZLHEs5Zp3EGRMhsqojKO+1GFrK3zeUpYm+//DJk92jdSdxwnYd2sXtX97O6wtfp3aZ2sz41wwuqHyB32EZY4KUVSKIA4pyfAJIE32J4LnnoHx5G4v4JGw9sJVJyydx7wX3MqzVMArmLeh3SMaYE5BVItisqsPDFomfFiyARYvggQcgbyiHcY4eW/Zv4Z0l73Bb89tcJ3GD11KmcBm/wzLGnISs6ghip4Odd95xj5dc4m8cEUBVeWPhG9QbVY97v7mXVTtWAVgSMCaCZZUIYue2z5Ur3aPdSZyltbvX0nliZ3p+2pN65eqxsM9C6yTOmCiQaTmIqu4MZyC+SkqCJk2sl9EsJKcm0+aNNmw/uJ1RF42iT0If8kgwjc6MMbmdFYhv2+Yeq1XzN45cavXO1cSXjCdvnryM7zaeaqWqUaVkFb/DMsbkIDulmzLFPV53nb9x5DLHUo7x2P8eo/7o+oyaMwqANvFtLAkYE4XsimDaNPd48cX+xpGLzN88n8SkRBZuWchV9a7imvrX+B2SMSaEYjsRbN/uWgx16QL58/sdTa7w4qwXuX3a7ZQrUo6Prv6Iy+te7ndIxpgQi+1E8Pbb7vGmm/yNIxdI6yTu7NPP5sazbmRExxGUKlTK77CMMWEQ24ngiy/c4xWx2zXyviP7uPebeykQV4ARnUbQskpLWlZp6XdYxpgwiu3K4hUr4PTTY/Zu4qmrp9JgTANGzxmNotZJnDExKjaPgGl27IBWrfyOIux2HNzB7V/ezn8X/Ze6Zevy480/0rxSc7/DMsb4JHYTwS+/wP790Lat35GE3Y5DO/h4xcc8cOEDDG05lAJ5C/gdkjHGRyEtGhKRziKyUkRWi8g9Gcy/XUSWi8hiEflGRMLXSH2UaxsfK91KbN63mWd+egZVpVaZWqwbso7hbYZbEjDGhC4ReOMdjwK6APWAa0WkXrrFFgAJqtoImAQ8Fap4/mHtWvd44YVh26QfVJXxC8ZTd1RdHvjuAVbvXA1gLYKMMX8J5RXBucBqVV2jqkeBd4FLAxdQ1e9U9aD38megYgjjOd4PP0Dt2mHbnB9+3/U7Hd/qSGJSImedfhaL+iyyTuKMMf8QyjqCM4H1Aa83AM2yWD4R+CKjGSLSG+gNULly5ZyJbvduqBm9B8Xk1GTa/rctOw7uYMzFY+jdtLd1EmeMyVCuqCwWkRuABCDDJjyqOhYYC27w+lPeYFq30126nPKqcptVO1ZRrVQ18ubJy4RLJ1C9VHUqlajkd1jGmFwslKeIG4HAI1BFb9pxRKQ9MBTopqpHQhjP3+bPd49Nm4Zlc+FwLOUYj8x4hAZjGjBy9kgAWldtbUnAGJOtUF4RzAFqikg8LgF0B47r4lNEzgZeATqr6tYQxnK8devcY6NGYdtkKM3dNJfEpEQW/7mY7g26c23Da/0OyRgTQUKWCFQ1WUQGANOAOGC8qi4TkeHAXFVNAp4GigIfiBsU5g9V7RaqmP6yebN7PO20kG8q1F74+QVu//J2Ti96Op92/5RutUP/8RljoktI6whUdQowJd20YQHP24dy+5maORMqVYJChXzZfE5I6yQu4YwEEs9O5KkOT1GyYEm/wzLGRKBcUVkcdnPmROz9A3uP7OXfX/2bgnkL8lzn52hRuQUtKrfwOyxjTASLvfaEhw65xzJl/I3jJExZNYX6o+szdv5Y8ubJa53EGWNyROxdESxb5h4vucTfOE7A9oPbGTJ1CBOXTKR+ufpMumoSzSpmdUuGMcYEL/YSQdrQlPHx/sZxAnYd2sXkXyfzYKsHua/lfeSPs9HUjDE5J/YSwd697vGcc/yNIxsb925k4pKJ3HX+XdQsU5N1Q9ZZZbAxJiRir45g2TLXYqhIEb8jyZCq8uq8V6k3uh4PTX+I33b9BmBJwBgTMrGXCNatc4kgF/pt52+0+287en/WmyYVmrC472JqlK7hd1jGmCgXe0VDq1ZBw4Z+R/EPyanJtPtvO3Ye2skrXV/hlia3WCdxxpiwiK1EoApHjuSqK4KV21dSvXR18ubJyxuXvUH10tWpWDx8vXEbY0xsnXLu2eMec0GLoaMpR3l4+sM0HNOQUbPdaGmtqrayJGCMCbvYuiL4zVW8ctZZvoYxe+NsEpMSWbp1Kdc1vI7rG13vazzGmNgWW4ngf/9zjz4OSPP8z89zx5d3UKFoBSZfO5mutbr6FosxxkCsJYJ9+9xjvfRDJ4deWidx5555Lr2a9OLJ9k9SomCJsMdhjDHpxVYi2LgRihWDEuE7AO85vIe7v7qbQvkK8Xzn5zm/0vmcX+n8sG3fGGOyE1uVxT//DBXDVxk7eeVk6o2ux2sLXqNAXAHrJM4YkyvF1hXB5s1QrlzIN7PtwDYGTx3MO0vfoWH5hnxyzSecc2bu7tLCGBO7YuuKYPt2KF8+5JvZc2QPU1ZN4eHWDzO391xLAsaYXC12rghUITUVKlQIyerX71nPW4vf4p4L7qFG6RqsG7LOKoONMREhdq4I0nodbdAgR1ebqqm8PPdl6o+uzyP/e+SvTuIsCRhjIkXsJIL9+91jSkqOrXLVjlW0faMtfT/vy7lnnsuSvkuskzhjTMSJnaKhY8fc45ln5sjqklOT6fBmB3Yf3s24buP4V+N/ISI5sm5jjAmn2EkEycnuMe+p7fKKbSuoWaYmefPk5c3L36R66eqcUeyMHAjQGGP8ETtFQ2lXBPnyndTbjyQf4cHvHqTRy40YOXskAC2rtLQkYIyJeLFzRZBWWXwSieDnDT+TmJTI8m3L6dGoBz0a9cjh4Iwxxj+xkwjSrghOsLJ4xE8juOuru6hYvCJTrptCl5pdQhCcMcb4J3YSQVqHc0F2MZGqqeSRPDSv1Jw+CX14ov0TFC9QPIQBGmOMP2InEWzf7h6z6XBu9+Hd3DHtDgrnK8xLF71kncQZY6Je7FQWHz3qHosVy3SRT375hHqj6vHGojcoVqCYdRJnjIkJsXNFkFY3EBf3j1lbD2xlwJQBfLD8Axqf3pjPrvuMJhWahDlAY4zxhyUCYO+RvXy15isebfsod51/F/niTq6JqTHGRKKYTQR/7PmDNxe9yX0t76NG6Rr8MeQPihXIvNjIGGOiVUjrCESks4isFJHVInJPBvMLiMh73vxZIlI1ZMF4iSA1jzB6zmjqj67PYz889lcncZYEjDGxKmSJQETigFFAF6AecK2IpB8sOBHYpao1gOeAJ0MVDykprCwDrT/qRv8p/WlesTnL+i2zTuKMMTEvlFcE5wKrVXWNqh4F3gUuTbfMpcAb3vNJQDsJUc9tyclH6dQDluxYwYRLJzDthmlULVk1FJsyxpiIEso6gjOB9QGvNwDNMltGVZNFZA9QBtgeuJCI9AZ6A1SuXPmkgslbpx5vLWtN9fsnUKFM1ZNahzHGRKOIuI9AVceqaoKqJpQ72TGHu3Xjgje+syRgjDHphDIRbAQqBbyu6E3LcBkRyQuUAHaEMCZjjDHphDIRzAFqiki8iOQHugNJ6ZZJAm7ynl8JfKt2O68xxoRVyOoIvDL/AcA0IA4Yr6rLRGQ4MFdVk4BxwJsishrYiUsWxhhjwiikN5Sp6hRgSrppwwKeHwauCmUMxhhjshYRlcXGGGNCxxKBMcbEOEsExhgT4ywRGGNMjJNIa60pItuAdSf59rKku2s5Btg+xwbb59hwKvtcRVUzvCM34hLBqRCRuaqa4Hcc4WT7HBtsn2NDqPbZioaMMSbGWSIwxpgYF2uJYKzfAfjA9jk22D7HhpDsc0zVERhjjPmnWLsiMMYYk44lAmOMiXFRmQhEpLOIrBSR1SJyTwbzC4jIe978WSJS1Ycwc1QQ+3y7iCwXkcUi8o2IVPEjzpyU3T4HLHeFiKiIRHxTw2D2WUSu9r7rZSLydrhjzGlB/LYri8h3IrLA+31f5EecOUVExovIVhFZmsl8EZEXvc9jsYg0OeWNqmpU/eG6vP4NqAbkBxYB9dIt0w942XveHXjP77jDsM9tgMLe876xsM/ecsWAGcDPQILfcYfhe64JLABKea/L+x13GPZ5LNDXe14PWOt33Ke4zxcCTYClmcy/CPgCEOA8YNapbjMarwjOBVar6hpVPQq8C1yabplLgTe855OAdiIiYYwxp2W7z6r6naoe9F7+jBsxLpIF8z0D/Ad4EjgczuBCJJh97gWMUtVdAKq6Ncwx5rRg9lmB4t7zEsCmMMaX41R1Bm58lsxcCvxXnZ+BkiJS4VS2GY2J4ExgfcDrDd60DJdR1WRgD1AmLNGFRjD7HCgRd0YRybLdZ++SuZKqfh7OwEIomO+5FlBLRH4UkZ9FpHPYoguNYPb5IeAGEdmAG/9kYHhC882J/r9nK6QD05jcR0RuABKAVn7HEkoikgd4FujpcyjhlhdXPNQad9U3Q0QaqupuP4MKsWuB11V1hIg0x4162EBVU/0OLFJE4xXBRqBSwOuK3rQMlxGRvLjLyR1hiS40gtlnRKQ9MBTopqpHwhRbqGS3z8WABsB0EVmLK0tNivAK42C+5w1AkqoeU9XfgV9xiSFSBbPPicD7AKo6EyiI65wtWgX1/34iojERzAFqiki8iOTHVQYnpVsmCbjJe34l8K16tTARKtt9FpGzgVdwSSDSy40hm31W1T2qWlZVq6pqVVy9SDdVnetPuDkimN/2J7irAUSkLK6oaE0YY8xpwezzH0A7ABGpi0sE28IaZXglATd6rYfOA/ao6uZTWWHUFQ2parKIDACm4VocjFfVZSIyHJirqknAONzl42pcpUx3/yI+dUHu89NAUeADr178D1Xt5lvQpyjIfY4qQe7zNKCjiCwHUoC7VDVir3aD3Oc7gFdF5DZcxXHPSD6xE5F3cMm8rFfv8SCQD0BVX8bVg1wErAYOAv865W1G8OdljDEmB0Rj0ZAxxpgTYInAGGNinCUCY4yJcZYIjDEmxlkiMMaYGGeJwORKIpIiIgsD/qpmsez+HNje6yLyu7et+d4dqie6jtdEpJ73/L5083461Ri99aR9LktFZLKIlMxm+caR3hunCT1rPmpyJRHZr6pFc3rZLNbxOvCZqk4SkY7AM6ra6BTWd8oxZbdeEXkD+FVVH81i+Z64XlcH5HQsJnrYFYGJCCJS1BtHYb6ILBGRf/Q0KiIVRGRGwBlzS296RxGZ6b33AxHJ7gA9A6jhvfd2b11LRWSIN62IiHwuIou86dd406eLSIKIPAEU8uKY6M3b7z2+KyIXB8T8uohcKSJxIvK0iMzx+pi/NYiPZSZeZ2Micq63jwtE5CcRqe3diTscuMaL5Rov9vEiMttbNqMeW02s8bvvbfuzv4z+cHfFLvT+PsbdBV/cm1cWd1dl2hXtfu/xDmCo9zwO199QWdyBvYg3/d/AsAy29zpwpff8KmAW0BRYAhTB3ZW9DDgbuAJ4NeC9JbzH6XhjHqTFFLBMWoyXA294z/PjepEsBPQG7vemFwDmAvEZxLk/YP8+ADp7r4sDeb3n7YEPvec9gZEB738MuMF7XhLXF1ERv79v+/P3L+q6mDBR45CqNk57ISL5gMdE5EIgFXcmfBqwJeA9c4Dx3rKfqOpCEWmFG6zkR69rjfy4M+mMPC0i9+P6qUnE9V/zsaoe8GL4CGgJTAVGiMiTuOKk/53Afn0BvCAiBYDOwAxVPeQVRzUSkSu95UrgOov7Pd37C4nIQm//VwBfBSz/hojUxHWzkC+T7XcEuonInd7rgkBlb10mRlkiMJHieqAc0FRVj4nrUbRg4AKqOsNLFBcDr4vIs8Au4CtVvTaIbdylqpPSXohIu4wWUtVfxY11cBHwiIh8o6rDg9kJVT0sItOBTsA1uIFWwI02NVBVp2WzikOq2lhECuP63+kPvIgbgOc7Vb3cq1ifnsn7BbhCVVcGE6+JDVZHYCJFCWCrlwTaAP8Yc1ncOMx/quqrwGu44f5+BlqISFqZfxERqRXkNv8HXCYihUWkCK5Y538icgZwUFXfwnXml9GYsce8K5OMvIfrKCzt6gLcQb1v2ntEpJa3zQypG21uEHCH/N2VelpXxD0DFt2HKyJLMw0YKN7lkbheaU2Ms0RgIsVEIEFElgA3Ar9ksExrYJGILMCdbb+gqttwB8Z3RGQxrlioTjAbVNX5uLqD2bg6g9dUdQHQEJjtFdE8CDySwdvHAovTKovT+RI3MNDX6oZfBJe4lgPzxQ1a/grZXLF7sSzGDczyFPC4t++B7/sOqJdWWYy7csjnxbbMe21inDUfNcaYGGdXBMYYE+MsERhjTIyzRGCMMTHOEoExxsQ4SwTGGBPjLBEYY0yMs0RgjDEx7v8BAc8h7jBwdb4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.793445925\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAohElEQVR4nO3debxd0/3/8dc782xKShMiEVTSSIJrVhQhlJg1yldbU4sYWvX4qWoNpVX9UlVBY6jWrPgSLVJz1NTckJBEQ8SQBBWRIBIk8fn9sfbtvbluck+Sc+6+55738/E4j7vX2vuc/dkZzueutfZeSxGBmZlVrlZ5B2BmZvlyIjAzq3BOBGZmFc6JwMyswjkRmJlVOCcCM7MK50RgZUPSEZL+UcBxV0v6eVPE1BQkvSFpj2z7XEk35R2TtSxOBFYU2ZfVIkkLJP1H0g2SuhTzHBFxc0TsWcBxP4yIXxbz3DUkhaRPsuucLelSSa1Lca5VIambpMskvZXF+FpW7p53bNZ8ORFYMe0XEV2ALYEq4Oz6B0hq0+RRFd/g7Dp3Ab4NHJ1zPABIagc8AnwdGAZ0A7YH5gLbrMLntYS/KyuAE4EVXUTMBh4ABsJ/f4s+SdKrwKtZ3b6SJkqaL+lpSYNq3i9pA0l3S5ojaa6kK7L670n6Z7YtSb+T9J6kjyS9JKnmfDdIuqDO5x0nabqkDySNkdSzzr6Q9ENJr2axjJKkAq9zOvAUMKTO563KdfWT9GhW976kmyWtuZJ/7ABHAb2BAyNiakR8ERHvRcQvI+L+Ote7cZ2Y/vtnJWlXSbMk/T9J7wJ/kvSypH3rHN8mi3/LrLxddp3zJU2StOsqxG05cyKwopO0AbAP8EKd6gOAbYEBkrYArgd+AKwD/BEYI6l91s3yN+BNoA/QC7itgdPsCewMbAqsARxG+s23fiy7Ab/O9n81+9z6n7cvsDUwKDturwKvczPgG8D0rLyq16Usxp5Af2AD4NxCYqhnD+DBiFiwCu+tsR6wNrAhcDxwK3B4nf17Ae9HxPOSegF/By7I3vMT4C5JPVbj/JYDJwIrpnskzQf+CTwB/KrOvl9HxAcRsYj0BfPHiHguIpZGxJ+Bz4DtSF0YPYEzIuKTiPg0Iv7ZwLkWA12BzQBFxMsR8U4Dxx0BXB8Rz0fEZ8BPge0l9alzzEURMT8i3gIeo85v+MvxvKRPgJeBx4Ers/pVuq6ImB4RD0XEZxExB7iU1O20stYBGvozWBlfAOdksSwCbgGGS+qU7f8OKTkAHAncHxH3Z62Ph4Bq0i8BVkacCKyYDoiINSNiw4g4MfsiqTGzzvaGwOlZd8L8LHlsQPqi3AB4MyKWrOhEEfEocAUwCnhP0mhJ3Ro4tCfpt/Ca9y0gtRx61Tnm3TrbC4EuAJKmZAOuCyR9o84xW2bHfJvUyum8OtclaV1Jt2WDzx8BNwGrMrg7l9TqWR1zIuLTmkLW/fUysF+WDIaTkgOk6z203vXuVIQYrIk5EVhTqTvN7Uzgwixp1Lw6RcSt2b7ehQxURsTlEbEVMIDURXRGA4e9TfrCAkBSZ9JvzrML+PyvR0SX7PVkvX0REXcAzwC/WM3r+hXpz2fziOhG+k27oHGKeh4G9squcXkWAp3qlNert7+h6Yhruof2B6ZmyQHSNd1Y73o7R8RFqxC75ciJwPJwDfBDSdtmg76dJX1LUlfgX6TujYuy+g6Sdqz/AZK2zt7fFvgE+JTUrVHfrcD3JQ2R1J70pftcRLxRpGu5CDhO0nqrcV1dgQXAh1m/e0MJrRA3kr6c75K0maRWktaRdJakmu6aicB3JLWWNIzCuqBuI43JnEBtawBSy2U/SXtln9chG3BefxXjt5w4EViTi4hq4DhS18480mDr97J9S4H9gI2Bt4BZpC6Y+rqRvnjnkbp+5gK/beBcDwM/B+4ifRH3A0YU8VpeAsaR+v5X9brOI3U3fUgafL17FWP5jDRg/G/gIeAjUgLqDjyXHXZqFsd80vjJPQV87jukls8OwO116meSWglnAXNISegM/L1SduSFaczMKpszt5lZhXMiMDOrcE4EZmYVzonAzKzCld2kUt27d48+ffrkHYaZWVmZMGHC+xHR4PQfZZcI+vTpQ3V1dd5hmJmVFUlvLm+fu4bMzCqcE4GZWYVzIjAzq3BlN0ZgZraqFi9ezKxZs/j0008bP7hMdejQgfXXX5+2bdsW/B4nAjOrGLNmzaJr16706dOHAheiKysRwdy5c5k1axZ9+/Yt+H0l6xqSdL3SMoKTl7Nfki5XWkLwxZql78zMSuXTTz9lnXXWaZFJAEAS66yzzkq3eEo5RnADaQHt5dkb2CR7HQ9cVcJYzMwAWmwSqLEq11eyrqGIGFdvOcD69gf+Emn602clrSnpq8tZbnD1/fOf8I9/LFvXsSOcdBJ0a2hhKzOzypDnGEEvll2+cFZW96VEIOl4UquB3r17r9rZnnkGLrigtlwz/famm8LBB6ftBQvgtddgxoy0/6CDVu1cZmbL0bp1azbffHOWLFlC3759ufHGG1lzzTUBmDJlCieffDKzZ8/miy++4KijjuLss8/+72/5DzzwAD//+c9ZuHAh7du3Z7fdduOSSy5Z7ZjK4vbRiBgdEVURUdWjR4NPSDfujDPgiy9qX1OmpPrHH4cTToABA6BrVxgyJCWAgw+GmTNX9IlmZiutY8eOTJw4kcmTJ7P22mszatQoABYtWsTw4cM588wzmTZtGpMmTeLpp5/myiuvBGDy5MmMHDmSm266ialTp1JdXc3GG29clJjyTASzSQt611ifAtaRLborroCbboK+fVOL4Y474MIL075x45o8HDOrHNtvvz2zZ6evvVtuuYUdd9yRPffcE4BOnTpxxRVXcNFFaQnoiy++mJ/97GdsttlmQGpZnHDCCUWJI8+uoTHASEm3AdsCH5ZsfKAhG20ERxwBO+8MRx4Jneqs5/3MM+nnhRemY8ys5TntNJg4sbifOWQIXHZZQYcuXbqURx55hGOOOQZI3UJbbbXVMsf069ePBQsW8NFHHzF58mROP/304sabKVkikHQrsCvQXdIs4BygLUBEXA3cD+xDWtd1IfD9UsXSoA4dUkugIdttBwMHwuTJcNRR0Lt3ShxHH92kIZpZy7No0SKGDBnC7Nmz6d+/P0OHDs07pJLeNXR4I/sDOKlU518tEhx6aEoEN95YW7/nnvD00zBpEpx3HsybBw88AGPGwPDh8J3vQBs/o2dWFgr8zb3YasYIFi5cyF577cWoUaM45ZRTGDBgAOPqdUfPmDGDLl260K1bN77+9a8zYcIEBg8eXPygIqKsXltttVU0mSefjPjd7yKOOioi3Ue07Ev6ct3o0RFLlzZdjGZWsKlTp+YdQnTu3Pm/288//3z07t07Fi9eHAsXLoy+ffvGQw89FBERCxcujG9961tx+eWXR0TEpEmTol+/fjFt2rSIiFi6dGlcddVVDZ6joesEqmM536tlcddQbnbaKfUj/vznsO66cOCBcPvtad9GG8EvfgHV1XDPPbXvOf54aN063Y30xRc5BG1m5WKLLbZg0KBB3HrrrXTs2JF7772XCy64gK997WtsvvnmbL311owcORKAQYMGcdlll3H44YfTv39/Bg4cyIwZM4oSh6LmfvoyUVVVFc1yYZo5c+Cxx+Db3162vndvmDYtjUmYWa5efvll+vfvn3cYJdfQdUqaEBFVDR3vFkGx9OgBhx2WHko777za+rfeSk8w77tvfrGZma2AE0Gxde6cuowi0mBy166p/u9/h+z+XzOz5sSJoJTWXBM++ih1DUH6KcH559dOcWFmTarcusNX1qpcnxNBU9h0U3joodryOedAq1YwenR+MZlVoA4dOjB37twWmwwiW4+gw0qOSXqwuKk9+CDsvXdtefDg1JW0995pLMHMSqaSVyhb0WCxE0FeDjgA7r33y/WvvQYffgh/+AP07Ak/+5kThJmtthUlAj8Gm5e7706Dyb/9LfzmN7X1/fote9waa6SZU83MSsQtguakb990l9HIkWk7m4UQgKuvhr32SmMLq7omg5lVLLcIysXrry9b3nBDePPNtP3DH9bWz5uX7kgyMysC3zXUnL3xRrrN9NJL0wNrNdZaK01jcdFFMGGCb0U1s9XiRFAOfvQjeO89WLKktu6LL+CnP4WqqtRddO65qT4CPvsslzDNrDw5EZST1q3TF/2HH8Lf/pbGEmqcd156WK1VqzSvkZRaC2ZmjfBgcbmbPx/uuy8toAPpVtNFi2r3b7op7L9/GoQeOhS23TYlCTOrKH6OoBLUrIjQKmvkbbst/OtfDR+7YEGaE8nMKobvGqoE0rK/6T/3HPznP+lJ5tdfh9mz4dpr074uXWCPPdKzDB06QL0nEM2ssrhFUEnqthjq6tkT3n4bpk//8gNtZtYieD0CS6SUDGqeTajx9tvp58Ybp2O8sppZRXEiqES9e9eOKXz8MXz+Oey2W+3+1q1h0qT84jOzJuVEUOm6dEljBI88khJCjSFD4LrrcgvLzJqOE4HVatt22aeUjz02dRXtvz+8/35+cZlZSTkR2JdFLLuQzpgxaYqLmjuT/v3v/GIzs6JzIrCG7bFHSggTJ355X//+tUlh6NBlu5TMrOw4EdiKDR6cEsIXX8C778LJJy+7/+GHoX37NDGemZUlJwIrjATrrguXX157x9HUqbX7Tz8dTjkFZs3KL0YzWyVOBLbq+vdPCWGvvVL5D3+ADTbwXEZmZcaJwFbfgw/CO+/A5pvX1klw8MFeK8GsDDgRWHGstx68+OKyU1/ffXea0mL4cPjkk/xiM7MVciKw4tpyy9QKePfd2rr77ksPrt1wQ25hmdnylTQRSBomaZqk6ZLObGD/hpIekfSipMclrV/KeKwJrbtuSggLFsDAganu+99PXUZ+YtmsWSlZIpDUGhgF7A0MAA6XNKDeYf8L/CUiBgHnA78uVTyWk86d4aWX4NZba+uOPRa++lVPbmfWTJSyRbANMD0iZkTE58BtwP71jhkAPJptP9bAfmspRoxILYSnn07ld99Nk9uZWe5KmQh6ATPrlGdldXVNAg7Ktg8Eukpap/4HSTpeUrWk6jlz5pQkWGsi228PixfXlqVll9Y0syaX92DxT4BdJL0A7ALMBpbWPygiRkdEVURU9ejRo6ljtGJr0wZefbW23KmT7yoyy1EpE8FsYIM65fWzuv+KiLcj4qCI2AL4WVY3v4QxWXOx8cbLjhF06ZJaB9Om5ReTWYUqZSIYD2wiqa+kdsAIYEzdAyR1l1QTw0+B60sYjzU3NauhDR1aW7fZZmm5zI8/zi8uswpTskQQEUuAkcBY4GXgjoiYIul8ScOzw3YFpkl6BVgXuLBU8VgzJcE//gGffZbuJgKYMQO6dUv7dt01rZ720kuwZEmuoZq1VF683pqX999Pq6PNnr38/et86X4CM2uEF6+38tG9e5rBtGY95XffTZPZ1d1/0kn5xWfWAjkRWPPVpUt6QnnkyNR1tMceqf7KK1O30aRJ+cZn1kI4EVh5aNcuLZ85Y0Zt3ZAhXh3NrAicCKy89O2bBo132CGV27dPA8lmtsqcCKz8tG4NTzxRWx40CG66Kb94zMqcE4GVpzZt0oDyT36Syv/zP24ZmK0iJwIrb7/9LRx/fNoeNCgNIo8bl29MZmXGicDK3x//CJdeWlveZRe45pr84jErM04E1jL86EdpFtNDDknl44+H3r295oFZAZwIrOXo0AH++tc0ZQXAzJlpYHnhwnzjMmvmnAis5Rk6FObPry137gx/+lNu4Zg1d04E1jKtsQbMm5cmrwM4+mh4++18YzJrppwIrOVac0348EO4MJvUtlcvuOqqXEMya46cCKzlO+us2u0TT4SvfW3ZriOzCudEYJUhIg0kr7UWvPJK+jlkiBOCGU4EVkkOOQQ++AAOOyyVJ01KCUGCO+7INzazHDkRWOW5/XZYsAA2qLOk9re/nRLClCn5xWWWEycCq0ydO8Nbb6Uuo3/+s7Z+4EBYe224804vjWkVw4nAbMcdU0I4+eRUnjcPDj0U2raF6dPzjc2sCTgRmNW4/PI0JcX999fWbbJJmrvIrAVzIjCrS4K9904thDPOSHXjxqVnEN5/P9/YzErEicBseS6+GP7zn7T99tvQoweMH59vTGYl4ERgtiJf+UpqHey+eypvs016mbUgTgRmhXj4YTjnnLQ9fnzqQnr11XxjMisSJwKzQp17Lnz8cW15002XXRDHrEw5EZitjC5dUlfRoYem8umnp9aBWRlzIjBbFXfcAffeW1v2uIGVMScCs1U1fHjtpHU14wYzZ+YaktmqcCIwWx1rrLHsUpi9e6eEsHhxfjGZrSQnArPV1bFjGjcYO7a2rl272mcQzJq5kiYCScMkTZM0XdKZDezvLekxSS9IelHSPqWMx6yk9twzTVFRY7310uR1Zs1cyRKBpNbAKGBvYABwuKQB9Q47G7gjIrYARgBXlioesyYhpdZBzapohx6aWgx1u4/MmplStgi2AaZHxIyI+By4Ddi/3jEBZKuLswbg1cWtZbjwQrjvvrT96adp2muzZqqUiaAXUPcWillZXV3nAkdKmgXcD5zc0AdJOl5StaTqOXPmlCJWs+Lbd99lB42l1H305pv5xWTWgIISgaQdJT0k6RVJMyS9LmlGEc5/OHBDRKwP7APcKOlLMUXE6IioioiqHj16FOG0Zk2kTRv48MP0FDLAQw9Bnz7w+9+nLiSzZqDQFsF1wKXATsDWQFX2c0VmA3XWAmT9rK6uY4A7ACLiGaAD0L3AmMzKQ7duMG0aLF0KO+yQ6k47DVq1ghEjcg3NDApPBB9GxAMR8V5EzK15NfKe8cAmkvpKakcaDB5T75i3gN0BJPUnJQL3/VjL1KoVPPUUvPBCbd3tt9cOMJvlpNBE8Jik30raXtKWNa8VvSEilgAjgbHAy6S7g6ZIOl/S8Oyw04HjJE0CbgW+F+H/EdbCDRmSvviffLK2rlUreO+93EKyyqZCvnclPdZAdUTEbsUPacWqqqqiurq6qU9rVhqLF6eHz2r07g2vvZbGFsyKSNKEiKhqaF9BLYKI+GYDryZPAmYtTtu2qXVw7rmp/NZbqe6YY+Cjj3INzSpHoXcNrSHp0ppbOCVdImmNUgdnVjHOOQeWLIFOnVL5+uvTPEbvvJNvXFYRCh0juB74GDgse30E/KlUQZlVpNat4ZNPYNEiaN8+1fXsWTumYFYihSaCfhFxTvaU8IyIOA/YqJSBmVWsDh3S08jbbZfKkyalweRevZYdYDYrkkITwSJJO9UUJO0ILCpNSGYGwDPPpGcPdt89ld9+G3beOS2CM7exu7fNCldoIjgBGCXpDUlvAlcAPyxdWGYGpJbAww+nrqGXXoKNNkqL4HTvDsOGpW13G9lqKvSuoYkRMRgYBGweEVtExKTShmZmyxg4MN1auv32qTx2bGodtGqVupEWLMg3PitbK7xZWdKREXGTpB/XqwcgIi4tYWxm1pCnn06tgMsvT62E666D556Drl3h2GNh9Oj0tLJZgRprEdTMndt1OS8zy4MEp54K116bksLAgan+2mvTw2jPP59vfFZWCnqyuDnxk8VmyzFpEpxyCowbl8qdO8PNN8Pw4W4h2Oo/WSzpYkndJLWV9IikOZKOLG6YZrZaBg+GJ55I3USQnkk44IA0hlB3CU2zegq9a2jPiPgI2Bd4A9gYOKNUQZnZathmm9RdVNMygPSw2kEH5ReTNWuFJoKaQeVvAX+NiA9LFI+ZFcs3vpFaAjXTVvzf/6X1EKZMyTcua3YKTQR/k/RvYCvgEUk9gE9LF5aZFYWUuojmzoW1104PqQ0cmOpffDHv6KyZKPQ5gjOBHYCqiFgMfMKXF6I3s+Zq7bVTMrj22toWwuDBaaW0MrthxIpvhYlA0m7Zz4OAXYH9s+1hpMRgZuXkmGNSC+Hqq1P5979Pg8lnn51vXJarxla/2AV4FNivgX0B3F30iMys9H7wA9htN+jfP81ndOGF8MYbcNNNeUdmOfBzBGaV7m9/g/3q/K735z/DUUflF4+VRDGeI/iVpDXrlNeSdEGR4jOzPO27L8yZU1v+7nfh+9/32EEFKfSuob0jYn5NISLmAfuUJCIza3rdu6cv/knZXJI33JDGDkaPzjUsaxqFJoLWktrXFCR1BNqv4HgzK0eDBqUZTjfcMJV/8IN0q+kf/pBvXFZShSaCm0nPDxwj6RjgIeDPpQvLzHKz0UZp4HjWLGjXLtWdckpKCP/5T66hWWkU+hzBb4ALgP7Z65cRcXEpAzOznPXqBZ99Bq+/DmutlerWWy/dgmotSqEtAoCXgQcj4ifAk5I8DbVZJejTBz74AH6cLUty/fWezbSFKfSuoeOAO4E/ZlW9gHtKFJOZNUeXXAKvvFJbluC++/KLx4qm0BbBScCOwEcAEfEq8JVSBWVmzdQmm8Ds2Wk2U0hrHey0U+pCsrJVaCL4LCI+rylIakN6stjMKk3PnrBkCdx1Vyo/9RR06ABHHJFvXLbKCk0ET0g6C+goaSjwV8BtQrNKdtBBaZrrnXZK5VtuSU8kf/xxvnHZSis0Efw/YA7wEvAD4H7As1SZVToJnnwyzWzauTPceCN06wYnnph3ZLYSGk0EkloDL0fENRFxaEQckm27a8jMkrXXhrfeSgvfAFx1VUoSp56ab1xWkEYTQUQsBaZJ6t0E8ZhZuVp77TRe8Mkn6XkDgMsvTwnhoovgvffyjc+Wq9CuobWAKdnC9WNqXo29SdIwSdMkTZd0ZgP7fydpYvZ6RdL8lYzfzJqbTp3gnXfSk8lrrpnqfvpTWHdd6NcvjStYs9LYegQ1fr6yH5x1KY0ChgKzgPGSxkTE1JpjIuJHdY4/GdhiZc9jZs1Ur14wbx68/XbqIrrzTpgxI916Ont2uvvImoXGVijrIOk04FBgM+CpiHii5tXIZ28DTI+IGdmtp7ex4uUtDwduLTx0MysLPXvCX/8KixfDLrukul690jKZ1iw01jX0Z6CKdLfQ3sAlK/HZvYCZdcqzsrovkbQh0Je0GlpD+4+XVC2pek7dedPNrHy0aQOPPw7XXZfKNctkvvNOrmFZ44lgQEQcGRF/BA4BvlGiOEYAd2YD018SEaMjoioiqnr06FGiEMysSRx9NEyfDt/8ZloDoWdP+N//zTuqitZYIlhcsxERS1bys2cDG9Qpr5/VNWQE7hYyqxz9+sGjj8I226TyGWd4VtMcNZYIBkv6KHt9DAyq2Zb0USPvHQ9sIqmvpHakL/sv3WkkaTPSXUnPrMoFmFkZe+65tPYBpFlN27SBBQtyDakSrTARRETriOiWvbpGRJs6290aee8SYCQwljSF9R0RMUXS+ZKG1zl0BHCbH1Azq1AbblibDJYuha5d4Ze/zDWkSqNy+/6tqqqK6urqvMMws2KLgN13h8ceS+XBg2HixFxDakkkTYiIqob2rczCNGZmpSOlcYOp2aNGkyalujffzDeuCuBEYGbNS//+sGhRbblPn3SH0eefL/cttnqcCMys+enQIXUV1YwVPP44tG8Pt9+ea1gtlROBmTVfZ5+dBpB33jmVR4yA88/PN6YWyInAzJq3Vq3giSfSraYA55wD220HHzV2B7sVyonAzMrDNtukiesgJYU11khrJi9evOL3WaOcCMysfBx8cBo7uCSb9uy++2CttfKNqQVwIjCz8vPjH9eua/DJJ9C2bb7xlDknAjMrT1JKAv37w5IlqfzWW3lHVZacCMysfHXqlJ4+3nLLVN5wQ6+AtgqcCMysvLVrBxMmwFZbpXLr1h5AXklOBGbWMowfn+4igpQcHnoo33jKiBOBmbUMEtxzDxx0UCrvuSfsumueEZUNJwIzazkkuOsuuOaaVH7iifT8wWef5RtXM+dEYGYtz7HHwnvvpe3x49PcRW3bOiEshxOBmbVMPXrAwoW15SVLUkL4+9/zi6mZciIws5arY8f0JHJE7S2m++4Lv/51vnE1M04EZlYZJkxI3UQAZ52VxhPuvTffmJoJJwIzqxxVVbBgQe2UFAcckBLClCm5hpU3JwIzqyydO6fVzh5+uLZu4EB46qn8YsqZE4GZVabdd192JtOddoKbb843ppw4EZhZZfvxj2sTwJFHwre+BbNn5xtTE3MiMDP7znfgtddgo43g/vth/fVrWwoVwInAzAxSEnjttdpbS3/ykzRnUQVwIjAzq+vMM+Hjj9P24sXQu3e+8TQBJwIzs/q6dKkdJ5g5E4YMadFTWzsRmJk1pGfP9OW/884waVIaN5g/P++oSsKJwMxsedq0gccfh/32S5PYrbUWfPBB3lEVnROBmdmKSDBmDFx2WSqvsw7cdluuIRWbE4GZWSFOPRWOOy5tH344/OUv+cZTRCVNBJKGSZomabqkM5dzzGGSpkqaIumWUsZjZrZaRo+GadPS9ne/C7//fb7xFEnJEoGk1sAoYG9gAHC4pAH1jtkE+CmwY0R8HTitVPGYmRXFppvCuHFp+7TT0hhCmStli2AbYHpEzIiIz4HbgP3rHXMcMCoi5gFExHsljMfMrDi+8Q246aa0/c1vwrPP5hvPaiplIugFzKxTnpXV1bUpsKmkpyQ9K2lYQx8k6XhJ1ZKq58yZU6JwzcxWwhFHwAMPpO3tt4drr803ntWQ92BxG2ATYFfgcOAaSWvWPygiRkdEVURU9ejRo2kjNDNbnmHD4Be/SNvHHQdPPplvPKuolIlgNrBBnfL6WV1ds4AxEbE4Il4HXiElBjOz8nDeeTB2bNreeed0u2mZKWUiGA9sIqmvpHbACGBMvWPuIbUGkNSd1FU0o4QxmZkV3557wt1315Y3Ka/fZ0uWCCJiCTASGAu8DNwREVMknS9peHbYWGCupKnAY8AZETG3VDGZmZXMgQfCokVpe/p0OOywfONZCYqIvGNYKVVVVVFdXZ13GGZmDZsxA/r1S9tnnQUXXphvPBlJEyKiqqF9eQ8Wm5m1LBttBP/6F7RtC7/6FTz6aN4RNcqJwMys2LbeOiUDSGsj39K8J01wIjAzK4UhQ2qTwRFHpKmsmyknAjOzUtl6axg1Km0PGQJLl+YazvI4EZiZldKJJ8KAbJq1Nm3giy/yjacBTgRmZqU2eXLt9tCh+cWxHE4EZmalJsEnn6TtRx+Fu+7KN556nAjMzJpCp061cxEdcgi8+Wa+8dThRGBm1lR22qm2NTB8OCxenG88GScCM7OmdNBBcPXV8OKLUNXgg75NzonAzKyp1ax9/OKLcOSRkPNUP04EZmZNrVUrmDAhbd98M+xff/HGJg4n17ObmVWqLbeEJUvS9n33weuv5xaKE4GZWV5at4YnnkjbgwblFoYTgZlZnnbeGb72NViwILd1j50IzMzyNm5c+lkziNzEnAjMzPL2la/Ubuew8JYTgZlZczB+fPp5wglNfmonAjOz5qCqCg49NLUIzjuvSU/tRGBm1lxce22aoO7cc9PDZk3EicDMrLno1g3Gjk3bgwc32WmdCMzMmpOhQ9OEdAD/+EeTnNKJwMysubnkkvRzr72a5HROBGZmzc3GG8OwYWn7N78p+emcCMzMmqM770w/zzyz5KdyIjAza446d4aePdP255+X9FROBGZmzdV3v5t+nn56SU+jyHlBhJVVVVUV1Tk8gm1m1uSWLoU2bdL2an5XS5oQEQ0uieYWgZlZc9W6NZxxRtp+9tmSncaJwMysOdtvv/SzhD0hTgRmZs3Z9tunn7fdVrJTlDQRSBomaZqk6ZK+dA+UpO9JmiNpYvY6tpTxmJmVnTZtYN114amnSnaKkiUCSa2BUcDewADgcEkDGjj09ogYkr3yWZ7HzKw5O+yw9PO110ry8aVsEWwDTI+IGRHxOXAbsH8Jz2dm1jIdeGD6eemlJfn4UiaCXsDMOuVZWV19B0t6UdKdkjZo6IMkHS+pWlL1nDlzShGrmVnztcsu6e6hAw4oycfnPVh8H9AnIgYBDwF/buigiBgdEVURUdWjR48mDdDMLHetWsHFF6eZSUvx8SX51GQ2UPc3/PWzuv+KiLkR8VlWvBbYqoTxmJlZA0qZCMYDm0jqK6kdMAIYU/cASV+tUxwOvFzCeMzMrAFtSvXBEbFE0khgLNAauD4ipkg6H6iOiDHAKZKGA0uAD4DvlSoeMzNrmOcaMjOrAJ5ryMzMlsuJwMyswjkRmJlVOCcCM7MKV3aDxZLmAG+u4tu7A+8XMZxy4GuuDL7myrA617xhRDT4RG7ZJYLVIal6eaPmLZWvuTL4mitDqa7ZXUNmZhXOicDMrMJVWiIYnXcAOfA1VwZfc2UoyTVX1BiBmZl9WaW1CMzMrB4nAjOzCtciE4GkYZKmSZou6cwG9reXdHu2/zlJfXIIs6gKuOYfS5qarQb3iKQN84izmBq75jrHHSwpJJX9rYaFXLOkw7K/6ymSbmnqGIutgH/bvSU9JumF7N/3PnnEWSySrpf0nqTJy9kvSZdnfx4vStpytU8aES3qRZry+jVgI6AdMAkYUO+YE4Grs+0RwO15x90E1/xNoFO2fUIlXHN2XFdgHPAsUJV33E3w97wJ8AKwVlb+St5xN8E1jwZOyLYHAG/kHfdqXvPOwJbA5OXs3wd4ABCwHfDc6p6zJbYItgGmR8SMiPgcuA3Yv94x+1O7LOadwO6S1IQxFluj1xwRj0XEwqz4LGnFuHJWyN8zwC+B3wCfNmVwJVLINR8HjIqIeQAR8V4Tx1hshVxzAN2y7TWAt5swvqKLiHGk9VmWZ3/gL5E8C6xZb5GvldYSE0EvYGad8qysrsFjImIJ8CGwTpNEVxqFXHNdx5B+oyhnjV5z1mTeICL+3pSBlVAhf8+bAptKekrSs5KGNVl0pVHINZ8LHClpFnA/cHLThJablf3/3qiSrVBmzZOkI4EqYJe8YyklSa2AS6m8Ve/akLqHdiW1+sZJ2jwi5ucZVIkdDtwQEZdI2h64UdLAiPgi78DKRUtsEcwGNqhTXj+ra/AYSW1Izcm5TRJdaRRyzUjaA/gZMDwiPmui2EqlsWvuCgwEHpf0BqkvdUyZDxgX8vc8CxgTEYsj4nXgFVJiKFeFXPMxwB0AEfEM0IE0OVtLVdD/95XREhPBeGATSX0ltSMNBo+pd8wY4LvZ9iHAo5GNwpSpRq9Z0hbAH0lJoNz7jaGRa46IDyOie0T0iYg+pHGR4RFRzuucFvJv+x5SawBJ3UldRTOaMMZiK+Sa3wJ2B5DUn5QI5jRplE1rDHBUdvfQdsCHEfHO6nxgi+saioglkkYCY0l3HFwfEVMknQ9UR8QY4DpS83E6aVBmRH4Rr74Cr/m3QBfgr9m4+FsRMTy3oFdTgdfcohR4zWOBPSVNBZYCZ0RE2bZ2C7zm04FrJP2INHD8vXL+xU7SraRk3j0b9zgHaAsQEVeTxkH2AaYDC4Hvr/Y5y/jPy8zMiqAldg2ZmdlKcCIwM6twTgRmZhXOicDMrMI5EZiZVTgnArMGSFoqaaKkyZLuk7RmkT//jew+fyQtKOZnm60sJwKzhi2KiCERMZD0rMlJeQdkVipOBGaNe4ZsUi9J/SQ9KGmCpCclbZbVryvp/yRNyl47ZPX3ZMdOkXR8jtdgtlwt7slis2KS1Jo0fcF1WdVo4IcR8aqkbYErgd2Ay4EnIuLA7D1dsuOPjogPJHUExku6q5yf9LWWyYnArGEdJU0ktQReBh6S1AXYgdppOgDaZz93A44CiIilpKnNAU6RdGC2vQFpAjgnAmtWnAjMGrYoIoZI6kSa5+Yk4AZgfkQMKeQDJO0K7AFsHxELJT1OmhDNrFnxGIHZCmSrup1CmthsIfC6pEPhv2vHDs4OfYS0BCiSWktagzS9+bwsCWxGmgrbrNlxIjBrRES8ALxIWgDlCOAYSZOAKdQum3gq8E1JLwETSGvnPgi0kfQycBFpKmyzZsezj5qZVTi3CMzMKpwTgZlZhXMiMDOrcE4EZmYVzonAzKzCORGYmVU4JwIzswr3/wFHHmUibqBEUgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.8042694946122347\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.7116\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CIFAR100 vs. CIFAR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar100.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  3.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covairance norms: [263.46378, 77.92307, 62.325638, 80.66709, 75.18534, 100.40797, 66.52207, 70.2855, 67.92541, 131.20055, 72.79468, 67.23096, 94.89134, 131.36543, 52.57952, 68.92815, 108.78612, 156.35555, 66.387245, 76.153435, 186.54181, 92.1108, 116.830666, 200.74992, 104.38893, 76.71766, 56.18107, 72.12981, 113.84088, 65.256004, 110.993095, 112.384674, 47.549763, 142.16095, 106.667244, 64.68393, 77.938354, 140.21097, 84.57663, 141.1265, 86.61233, 136.75073, 84.28827, 108.02133, 53.766808, 52.737347, 69.79799, 181.3637, 102.13129, 159.1394, 52.997185, 76.635635, 310.8027, 237.88107, 91.407074, 57.405407, 108.757965, 91.88912, 135.49464, 150.76845, 203.73753, 135.37445, 93.17572, 79.47289, 59.076694, 58.19088, 54.70186, 70.325676, 101.08861, 110.29933, 89.110725, 157.89583, 53.995583, 88.88261, 72.54234, 112.906395, 84.69865, 60.59433, 66.72352, 67.59271, 68.5114, 120.50207, 88.786995, 108.55526, 70.414055, 76.757904, 144.27342, 125.93828, 71.104095, 90.18949, 85.55628, 59.082256, 84.97329, 63.88701, 172.62997, 116.16797, 184.04216, 92.89649, 67.44909, 111.20053]\n",
      "covariance norm: 17.99201274070443\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar100 = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "cifar100_train_dataloader = DataLoader(\n",
    "    dataset=cifar100, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, cifar100_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:08<00:00, 12.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 13.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 282.28717 id variance: 13077.243\n",
      "ood mean score: 362.8451 ood variance: 16439.283\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=cifar100_test_dataloader,\n",
    "    out_of_dist_dataloader=cifar10_test_dataloader,\n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABC2UlEQVR4nO3dd3gUVffA8e8hdKSDitKRXkSIBZAmVaXoiyiKKIogICi296di5UXFgiJVUBREVBRFURDEgigqvTdBQLpApIeScn5/3ImuMWWBbCabPZ/n2Se7s7MzZ3Y3c3bunTlXVBVjjDGRK4ffARhjjPGXJQJjjIlwlgiMMSbCWSIwxpgIZ4nAGGMinCUCY4yJcJYIsjkRWSMizfyOI6sQkcdE5E2f1j1BRAb7se6MJiJdReSrM3ytfSezGEsEmUhEtorIcRE5KiJ7vB3DOaFcp6rWVNW5oVxHEhHJIyLPi8g2bzs3isjDIiKZsf4U4mkmIjsCp6nqc6p6V4jWJyJyr4isFpFjIrJDRD4SkdqhWN+ZEpGnReTds1mGqk5W1dZBrOtfye9Mv5MiktuLfaP3/m4VkbdEpPzpLsv8kyWCzNdeVc8B6gKXAI/6G87pE5GcqTz1EdACuAYoCHQDegGvhSAGEZGs9v19DbgPuBcoBlQBPgWuzegVpfEZhJyP654KdABuAQoDFwNLcN+50+Ln+5clqardMukGbAVaBjx+EZgR8PgK4CfgILACaBbwXDHgbWAXcAD4NOC5dsBy73U/AXWSrxO4ADgOFAt47hJgP5DLe3wnsM5b/mygXMC8CtwDbAS2pLBtLYATQJlk0y8HEoCLvMdzgeeBhcBh4LNkMaX1HswFngXme9tyEXCHF/MRYDNwtzdvAW+eROCod7sAeBp415unvLddtwPbvPdiYMD68gETvfdjHfBfYEcqn21lbzsvS+PznwCMAmZ48S4AKgU8/xqw3XtflgCNA557GrcjfNd7/i7gMuBn773aDYwEcge8piYwB/gT+AN4DGgLnALivPdkhTdvYWC8t5ydwGAgynuuu/eevwrEeM91B370nhfvub1ebKuAWrgfAXHe+o4Cnyf/PwCivLh+896TJST7DnnztfQ+z389l8b/V0qfdQ/vs54HfAn0S7aMFcB/vPvVAt6/DcCNfu9DQrZv8juASLol+wco7f3DvOY9vtD7J7sGd6TWyntc0nt+BjAFKArkApp60y/x/gEv9/6pbvfWkyeFdX4L9AyI5yXgde9+R2ATUB3ICTwO/BQwr3r/FMWAfCls2xDg+1S2+3f+3kHP9XY0tXA7648D/lnTew/mev/ENb0Yc+F+bVfC7YyaArFAPW/+ZiTbcaeyc3gDt9O/GDgJVA/cJu89Lw2sTL68gOX2Bn5P5/Of4G3PZV78k4EPAp6/FSjuPfcgsAfIGxB3HHCd997kA+rjEmdOb1vWAQO8+QviduoPAnm9x5cnfw8C1j0NGOt9JufiEnXSZ9YdiAf6e+vKxz8TQRvcDryI9zlUB0oFbPPgNP4PHsb9H1T1XnsxUPx0vl8pLTeNz/odbxvzAbcB8wPmr4FLqnm8ebbjfmjk5O8fTTX83o+E4pbVDq0jwacicgT3JdsLPOVNvxWYqaozVTVRVecAi4FrRKQUcDXQW1UPqGqcqn7vva4XMFZVF6hqgqpOxO3Mrkhh3e8BN4NrWgG6eNPA7cieV9V1qhoPPAfUFZFyAa9/XlX/VNXjKSy7BG7Hk5Ld3vNJJqnqalU9BjwB3CgiUWm9BwGvnaCqa1Q13nsfZqjqb+p8D3wFNE4ljtQ8o6rHVXUF7hfhxd70G4HnvPd8BzA8jWUUT2P7A01T1YXeezwZ10QIgKq+q6ox3rYNxe2Qqga89mdV/dR7b46r6hJV/cWbfytuR97Um7cdsEdVh6rqCVU9oqoLUgpIRM7DvccDVPWYqu7F/cLvEjDbLlUd4a0r+ecfh0s01QDxvkPBvBfgjmweV9UN3me4QlVjUpgv2Pc3PU9723gcl/wCv+NdgU9U9STu/duqqm9727wM96OlcwbEkOVYIsh816lqQdyv1Wr8vYMsB3QWkYNJN+BKoBRQBvhTVQ+ksLxywIPJXlcG1wyS3MdAAy+xNME1m/wQsJzXApbxJ+4X2oUBr9+exnbt92JNSSnv+ZSW8zvul30J0n4PUoxBRK4WkV9E5E9v/mv4Z9IJxp6A+7FAUgf+BcnWl9b2x5D69gezLkTkIRFZJyKHvG0pzD+3Jfm2VxGRL7wTDw7jknfS/GVwzS3BKIf7DHYHvO9jcUcGKa47kKp+i2uWGgXsFZFxIlIoyHUHG2ew7296/toOVT2CO9JOSng345IzuPfk8mTfxa7A+RkQQ5ZjicAn3q/XCcDL3qTtuF/KRQJuBVR1iPdcMREpksKitgPPJntdflV9P4V1HsD9Yr4J1+H2gapqwHLuTracfKr6U+Ai0tikr3H/OGUCJ4rI5bh/9m8DJgfOUxb3i3J/Ou/Bv2IQkTy45PYycJ6qFgFm4hJYevEGYzeuSSiluJP7BigtItFnsiIRaYzrg7gRKOptyyH+3hb49/aMAdYDlVW1EK6tPWn+7UDFVFaXfDnbcUeRJQLe90KqWjON1/xzgarDVbU+rnmlCq7JJ93XeeuulM484L5fl4lI6TTmOQbkD3ic0k47eTzvAzeLSANcE9p3AXF9n+y7eI6q9gki1rBjicBfw4BWInIxrhOwvYi0EZEoEcnrnf5Y2jvM/hIYLSJFRSSXiDTxlvEG0FtELvfOpCkgIteKSMFU1vkerm30Bv5uFgJ4HXhURGoCiEhhEQn6MFhVv8btDD8WkZreNlzhbdcYVd0YMPutIlJDRPIDg4CpqpqQ1nuQympz45pP9gHxInI1EHhK4x9AcREpHOx2JPMh7j0pKiIXAv1Sm9HbvtHA+17Mub34u4jII0GsqyCuHX4fkFNEngTS+1VdENc5e1REqgGBO6kvgFIiMkDcab0FvaQM7n0pn3TWlff9+goYKiKFRCSHiFQSkaYEQUQu9b5/uXA74xO4o82kdaWWkADeBP4nIpW9728dESmefCbv+zUHmCYi9UUkp7dNvUXkTm+25UAX7/8jGvcdT89M3K//QcAUVU2K+wugioh085aXy9vO6kEsM+xYIvCRqu7DdV49qarbcR22j+F2Bttxv6qSPqNuuF/O63F9CwO8ZSwGeuIOzQ/gOny7p7Ha6bgzXPZ4beJJsUwDXgA+8JoZVuP6JU5HJ9wvqlm4s0TexZ2J0j/ZfJNwR0N7cL/C7vViSO89+Afv0P5e3A77AO4oZ3rA8+txv/g2e4f3KTWXpWUQsAPYgvtFOhX3yzk19/J3E8lBXJPH9cDnQaxrNu59+xXXXHaCtJuiAB7CbfMR3A+CKUlPeO9NK6A97n3eCDT3nv7I+xsjIku9+7fhEuta3Hs5leCbYgp56z/gxR6DOxEB3Odfw3v/P03hta/gPr+vcEltPK4jNyU34HbcU3BHS6uBaNxnA66/qZIXxzP884dOirz+gE9wZyW9FzD9CO5HRRfcmXp7cP8fedJbZjiSv1sGjAk9EZmLO5PDl6t7z4aI9AG6qGpQv5SNCRd2RGBMKkSklIg08ppKquJOxZzmd1zGZDS7us6Y1OXGnT1TAdfU8wGuH8CYbMWahowxJsJZ05AxxkS4sGsaKlGihJYvX97vMIwxJqwsWbJkv6qWTOm5sEsE5cuXZ/HixX6HYYwxYUVEfk/tOWsaMsaYCGeJwBhjIpwlAmOMiXBh10eQkri4OHbs2MGJEyf8DiVk8ubNS+nSpcmVK5ffoRhjsplskQh27NhBwYIFKV++POLP8LghparExMSwY8cOKlSo4Hc4xphsJmRNQ+IGld4rIqtTeV5EZLiIbBKRlSJS70zXdeLECYoXL54tkwCAiFC8ePFsfcRjjPFPKPsIJuDGR03N1bgqmJVxo2yNOZuVZdckkCS7b58xxj8haxpS1XkiUj6NWToC73gDo/wiIkVEpNRpDHFnjDHhRxUOHIDDh+HgQfjzT9iyBXLmhLg4dzt5EnbsgGLFICGBY4kn2JdwhPLtu8Gll2Z4SH72EVzIP+ut7/Cm/SsRiEgv3FEDZcuWzZTgTldUVBS1a9cmPj6eChUqMGnSJIoUKQLAmjVr6N+/Pzt37iQxMZHbbruNxx9//K9f+V9++SVPPPEEsbGx5MmTh6uuuoqhQ4f6uDXGmDMWH+927OvXw6pVsGEDHD8OCxbA7t1uR38avq0APdtD4ZOweHE1cmSzRBA0VR0HjAOIjo7OklXy8uXLx/LlywG4/fbbGTVqFAMHDuT48eN06NCBMWPG0Lp1a2JjY+nUqROjR4/mnnvuYfXq1fTr148ZM2ZQrVo1EhISGDdunL8bY4xJWVyc28Hv2gW//+5uv/0GmzfDmjUQFQXHjkFi4j9flysXNGwIlSpBmTJQrRrkywcXXAA5crhpJUu6+bzbwcRYHv7pad5c/hYXFbuIV9u/SY7yoRkKw89EsJN/jgFb2psW9ho0aMDKlSsBeO+992jUqBGtW7sRFPPnz8/IkSNp1qwZ99xzDy+++CIDBw6kWrVqgDuy6NMnWw6Lakz4OHAAvvnG7eA3bYLPP4c9e/49n4jbiZcrBx06uCRQpw5UqAA1akCpUnD++ZA372mtPiExgYZjGrMhZgP/bfhfnm72NPlypTZw29nzMxFMB/qJyAfA5cChDOkfGDAAvF/mGaZuXRg2LKhZExIS+Oabb+jRowfgmoXq16//j3kqVarE0aNHOXz4MKtXr+bBBx/M2HiNMafn+HGYPdvt/EeO/Odz55zjdvT588MVV7j9QZUqUKuWSwK5c2dYGDGxMRTLV4yoHFE8e9WzlClchugLojNs+akJWSIQkfeBZkAJEdkBPAXkAlDV13Fjj16DG2M3FrgjVLFkhuPHj1O3bl127txJ9erVadWqld8hGWNSs3kzTJ8OY8e6Zp7Dh930qCi47DIoXBhuusn9yi+ZYsHODKWqTF41mftm3ceQFkPoWb8n11e/PuTrTRLKs4ZuTud5Be7J8BUH+cs9oyX1EcTGxtKmTRtGjRrFvffeS40aNZg3b94/5t28eTPnnHMOhQoVombNmixZsoSLL77Yl7iNiQgxMfDRR/DjjzB1qjsrB9xZOUWLQqdO0KKF+3uazThna/uh7fSe0ZuZG2dyRekraFS2UaauH3CZKJxu9evX1+TWrl37r2mZrUCBAn/dX7p0qZYtW1bj4uI0NjZWK1SooHPmzFFV1djYWL322mt1+PDhqqq6YsUKrVSpkm7YsEFVVRMSEnTMmDEpriMrbKcxYeHoUdXFi1VfeEG1USPVHDlU3YmbquXKqT7xhOqSJaqJib6G+d7K97TgcwU1/7P5ddjPwzQ+IT5k6wIWayr71bA4ayjcXHLJJdSpU4f333+fbt268dlnn9G/f3/uueceEhIS6NatG/369QOgTp06DBs2jJtvvpnY2FhEhHbt2vm8BcaEoSNHXIvAvHnw9df/fK5TJ+jf3525k4XqdRXNV5TLS1/OuHbjqFDUv/IxYTdmcXR0tCYfmGbdunVUr17dp4gyT6RspzFB27kTJk92zT6B+4UbboCWLaFJE3eqZha5Mj8+MZ5Xf36VUwmnGNhkIOBaZTKjcoCILFHVFHue7YjAGBM+EhJg/nyYNs3t/Hd6Z5znyQP/+Q/07Alt2mSZHX+gFXtW0GN6D5bsXsKNNW/8KwFkhfIxlgiMMVnb7t3wyy8wY4ZLAH/+6aZXqADXXAODBkG9elly5w9wMv4kg+cNZsj8IRTLV4yPOn9Ep+qdskQCSJJtEkFmHV75Jdya8Iw5K1u3wocfwrvvujINSS65xPUDtG2bKad1ZoSNf27khfkvcEvtW3il9SsUz1/c75D+JVskgrx58xITE5NtS1GrNx5B3kw+rc2YTLVjB7z+OrzwgqvXk6R7d3c+f5MmUDzr7URTcvTUUT5b/xld63Sl1rm1WN9vPRWLVvQ7rFRli0RQunRpduzYwb59+/wOJWSSRigzJltRhS+/hCFD4Icf3LTy5V1H7w03QLNmrv0/jMz5bQ69vujF7wd/p16pelQvWT1LJwHIJokgV65cNnKXMeFCFRYtcm37y5a5K3sB2rWDJ55wV/aGoQPHD/DQVw/x1vK3qFK8Ct93/57qJcPjLL9skQiMMWFgyxaYOBE++MCVZgZ3Xv+jj7pO34pZ+1dzWhISE2j0ViN+jfmVR698lCebPknenOHTlGuJwBgTGqdOwVtvwZIl8M477jHAxRfDK69At25QooS/MZ6l/bH7/yoS91yL5yhbuCz1Sp3xqLu+sURgjMlYBw/Cyy/Dm2/CH3+4adHRrlLn889D1aq+hpcRVJVJKycxYNYAhrQcQq/6vbiu2nV+h3XGLBEYY87egQPujJ9PPvn7Ct/ChV2Bt+uuc1U9s4nfD/7O3V/czezfZtOwTEOalGvid0hnzRKBMebMHT7s6vc/9ZQ75TN3brjzTlfCuVWrLHuR15l6d+W79JnRB1VlxNUj6HtpX3JIDr/DOmuWCIwxp++771zH7+efuyt9GzSARx6B9u2z3c4/UMn8JWlUphFj242lXJFyfoeTYSwRGGOCc+IETJoEb7zhTv8EN1LXlCnuvP9sKC4hjqE/DyUuIY4nmj5Bm4va0LpS62x34aolAmNM6nbvhtGj3RHATz+5awBy54abb4bHH3fj8mZTy3Yvo8f0Hizbs4wutbpkqSJxGc0SgTHm33btgqFD3WmeSa6+Gvr2ddU9s1BN/4x2Iv4Eg74fxIvzX6RE/hJ8fOPH/Kf6f/wOK6QsERhjnIQEd9bP00/D2rVu2lVXuZ1/x46QMzJ2F5v+3MTLP73MbRffxtDWQymar6jfIYVcZHyyxpjUbdgAzz4Lc+fC9u1w3nnuzJ9HHoHKlf2OLlMcPXWUaeum0e3ibtQ6txYb+m3wdcSwzGaJwJhItHUrDB7savzv2eOm5c7trgS+6SbIn9/X8DLT7E2z6fVFL7Yf2k70BdFUL1k9opIAQPifAGuMCd6aNdCjhxvUZfx4lwRuvBFWrICTJ+GOOyImCcTExnD7p7fTdnJb8ufKzw93/BA2ReIymh0RGJPd7d/vrvB9+WX47Tc3rV49GDUKLr88W5/3n5qkInGb/tzEwMYDebzJ42FVJC6jWSIwJruKjXVlnYcPd1f9ligBXbu6ej9lyvgdnS/2HdtH8fzFicoRxQstX6BckXLUPb+u32H5zpqGjMlOtm1zv/yrVoUCBdzpn7VquWsA9u51Qz9GYBJQVd5e9jZVRlbhjSVvANCxWkdLAh47IjAmO1i2zF3gNXOme1yxohvopXdvV+s/Apt/kmw9uJVen/dizuY5NC7bmOYVmvsdUpZjicCYcJWYCLNmwXvvufP/jx+HBx90Hb41a/odXZYwacUk+szog4gw+prR3B19d7YoEpfRLBEYE27i4uDtt+GxxyAmxl3l27mze2wJ4B/OO+c8mpRrwuvtXqds4bJ+h5NlWSIwJlwcOwZPPvl32Ydy5dzjnj0hXz5/Y8si4hLieHH+iyRoAk82fZLWlVrTulJrv8PK8iwRGJPV/fGH6wAeM8YlgwoVXBPQ3XdHTNmHYCzdvZQ7P7uTFX+s4Jbat/xVJM6kz75FxmRV69e7Ub+GD3dVP1u3hvvvd0XfbAf3l+Nxx3nm+2d4+aeXKVmgJNNumhbWw0b6IaSJQETaAq8BUcCbqjok2fNlgYlAEW+eR1R1ZihjMibLW78e/vMfWLfOPW7YEF54Aa680t+4sqjNBzbzys+v0L1ud15q9VJEFInLaCHrPheRKGAUcDVQA7hZRJIXL38c+FBVLwG6AKNDFY8xWZoqzJ7tTvmsXt0lge7d3fi/8+dbEkjm8MnDTFg+AYCa59ZkY/+NvNnhTUsCZyiURwSXAZtUdTOAiHwAdATWBsyjQCHvfmFgVwjjMSbrUYX333dXAG/e7Kb17QsPPACVKvkbWxY1c+NMen/Rm51HdnL5hZdTvWT1bDVspB9CeULthcD2gMc7vGmBngZuFZEdwEygf0oLEpFeIrJYRBbv27cvFLEak/m+/db9+u/a1RV8e+MN2LfP1QCyJPAv+2P3021aN65971oK5inI/DvnR2yRuIzmd2fxzcAEVR0qIg2ASSJSS1UTA2dS1XHAOIDo6Gj1IU5jMs6hQ3Dtta7J55xz4NVX3RXAeSO36Fl6korEbT6wmSebPMljjR8jT848foeVbYQyEewEAoualPamBeoBtAVQ1Z9FJC9QAtgbwriM8ceKFfDBB+62dSu0auVq/5x7rt+RZVl/HP2DkgVKEpUjipdbvUy5IuWoc14dv8PKdkLZNLQIqCwiFUQkN64zeHqyebYBLQBEpDqQF7C2H5O9bNkCt90GdevCkCGu+NuUKfDVV5YEUqGqjF86nqojqzJuyTgA2ldtb0kgREJ2RKCq8SLSD5iNOzX0LVVdIyKDgMWqOh14EHhDRO7HdRx3V1Vr+jHZw5o1cNdd8Msv7nHDhvDcc9CkiV0HkIbNBzbT8/OefLvlW5qWa0rLii39DinbC2kfgXdNwMxk054MuL8WaBTKGIzJdBMnwujRsGiRu/K3fXt46SVXGtqkaeLyifSd2ZcoieL1a1+nZ/2eViQuE/jdWWxM9pGY6EpBP/+8Gwvg//7PnQoagfX/z9QFBS/gqgpXMebaMZQuVNrvcCKGJQJjMsIff0CHDrBwIURHu/LQxYv7HVWWdyrhFEN+HEKiJvJ0s6dpVakVrSq18jusiGPHXMacjYQEGDkSqlVzSeCuu9xfSwLpWrRzEfXH1eepuU+x+cBmrHvQP3ZEYMyZ+uUXdzHY5s3uArDPP7dSEEGIjYvlye+e5NVfXqXUOaWY3mU67au29zusiGZHBMacroQEVwKiQQPXJDRiBGzcaEkgSFsObGHEwhH0rNeTNX3XWBLIAuyIwJhgxcXB+PHQp497fNVV7nqAEiX8jSsMHDpxiE/WfcIdl9xBzXNrsqn/JsoUtk70rMKOCIxJjyoMGwaFCv2dBO68E77+2pJAEGb8OoOao2ty1+d3sX7/egBLAlmMJQJj0rJrlzsb6P774cQJN1LYsWPuyMAuCkvTvmP76PpJV9q9346i+Yryc4+fqVaimt9hmRRY05AxqRk3Dh56CI4cgXvucUcFNjRkUBISE7jy7SvZcmALzzR7hkeufITcUbn9Dsukwr7VxqRk4kQ3JnDRovDjj9DILoAPxp6jezi3wLlE5YhiaOuhlC9Snlrn1vI7LJOOoJuGRCR/KAMxJks4etSVhOjeHXLlchVDLQmkK1ETGbt4LFVGVGHs4rEAtKvSzpJAmEg3EYhIQxFZC6z3Hl8sIjakpMl+Jk2CCy6AL75wHcNbt1p5iCBs+nMTLd5pQe8Zvbn0wktpc1Ebv0MypymYI4JXgTZADICqrgCahDIoYzLVhAlQpIgrFX3sGMyZ4waPueACvyPL8t5e9ja1x9Rm6e6lvNH+Db7u9jUVi1b0OyxzmoJqGlLV7ckmJYQgFmMylyrcey/ccYfb8ffoAbt3Q0srexyssoXL0qZSG9b2Xctd9e5C7EyqsBRMZ/F2EWkIqIjkAu4D1oU2LGNCKD7enRH05JMQEwM33eROBy1QwO/IsryT8Sd5/sfnSdREBjUfRIuKLWhRsYXfYZmzFMwRQW/gHtzA8zuBukDfEMZkTOh8/DGcf747HVQEBg+G99+3JBCEBTsWUH9cfZ75/hm2HdpmReKykWCOCKqqatfACSLSCJgfmpCMCYHYWHj0URg+HEqWhLFjoWdPuygsCMdOHeOJ755g2C/DuLDQhXxx8xdcW+Vav8MyGSiYRDACqBfENGOynh073GAx06bB4cPQsePfncMmKL8f+p3Ri0bTO7o3Q1oOoVCeQn6HZDJYqolARBoADYGSIvJAwFOFcGMQG5N1xcXBM8/As8+6x1ddBQMGuGsETLoOnjjI1LVTuaveXdQoWYNN926yEcOysbSOCHID53jzFAyYfhi4IZRBGXNWFi50O/y9e6FuXdcc1Lix31GFjc/Wf0afGX3Ye2wvV5a9kmolqlkSyOZSTQSq+j3wvYhMUNXfMzEmY87M8uWuH2DWLPf4wQddkTgTlL3H9nLvl/cyZc0U6pxXh+k3T7cicREimD6CWBF5CagJ5E2aqKpXhSwqY07HyZPwwgvw1FPuccOG8MYbUKOGv3GFkYTEBBq91Yhth7YxuPlg/tvov+SKyuV3WCaTBJMIJgNTgHa4U0lvB/aFMihjgrZ+PVx3HWzY4Jp/3nkHypf3O6qwsevILs4/53yickTxWtvXKF+kPDVKWgKNNMFcR1BcVccDcar6vareCdjRgPHfvHlQvbpLAm+/Dd9+a0kgSImayJhFY6g2shqvL34dgGsqX2NJIEIFkwjivL+7ReRaEbkEKBbCmIxJW0KCawpq1sxVCJ0wwVULtbECgvJrzK80n9icvjP7cnnpy7n6oqv9Dsn4LJj/nMEiUhh4EHf9QCFgQCiDMiZV+/e7awF++smdEvruu1CqlN9RhY3xS8fT78t+5M2Zl7c6vEX3ut2tPpBJPxGo6hfe3UNAc/jrymJjMs+ff8Jrr8Grr7oRw+6/350RlMNGWz0d5YuU5+qLrmbUNaMoVdASqHHSuqAsCrgRV2NolqquFpF2wGNAPuCSzAnRRLxVq6BpUzhwAOrXd81CLazQWTBOxp/kf/P+B8DgqwZbkTiTorSOCMYDZYCFwHAR2QVEA4+o6qeZEJsx8PTT8PrrLgnMmgVtbNCTYP20/Sd6TO/B+v3rubPunaiqNQOZFKWVCKKBOqqaKCJ5gT1AJVWNyZzQTESbP99VBk26OGzqVEsCQTp66igDvxnIiIUjKFO4DLO6zrJRw0ya0mpgPaWqiQCqegLYfLpJQETaisgGEdkkIo+kMs+NIrJWRNaIyHuns3yTDSUNEHPllS4JPPggnDoFnTr5HVnY2HZoG2OXjOWeS+9hdZ/VlgRMutI6IqgmIiu9+wJU8h4LoKpaJ60Fe30Mo4BWwA5gkYhMV9W1AfNUBh4FGqnqARE59yy2xYSzxEQ3UMxrr7kB5K+8Ej74AC680O/IwsKB4wf4aO1H9Krfixola7D5vs1cUNCG2jTBSSsRVD/LZV8GbFLVzQAi8gHQEVgbME9PYJSqHgBQ1b1nuU4Tjg4cgJtvhtmzoVo1GDnSOoNPw7R10+g7sy/7ju2jabmmVC1R1ZKAOS1pFZ0720JzFwKBYx3vAC5PNk8VABGZjytt/bSqzkq+IBHpBfQCKFu27FmGZbKUH390Q0Xu2gVPPOFKR1uHZlD2HN1D/y/7M3XtVOqeX5cZt8ygaomqfodlwpDfl2LmBCoDzYDSwDwRqa2qBwNnUtVxwDiA6OhoGx8vu5g9Gzp0cGMIT5vmagaZoCQkJtD47cZsP7Sd5656jocaPmRF4swZC2Ui2Ik7/TRJaW9aoB3AAlWNA7aIyK+4xLAohHEZvyUmwsMPwyuvuPGDv/vONQmZdO04vIMLCl5AVI4ohrcdToWiFaxUtDlrQV2WKSL5ROR0jzkXAZVFpIKI5Aa6ANOTzfMp7mgAESmBayrafJrrMeFkxw6oWdMlgc6dYeVKSwJBSNRERiwYQbWR1RizaAwAV1e+2pKAyRDpJgIRaQ8sB2Z5j+uKSPId+r+oajzQD5gNrAM+VNU1IjJIRDp4s80GYkRkLfAd8LBdp5CNjR0LZcq4aqHPPw8ffugGkjdpWr9/PU3ebsK9s+7lyrJX0q5KO79DMtmMqKbd5C4iS3Blp+eq6iXetFWqWjsT4vuX6OhoXbx4sR+rNmcqIQEeecTVBsqf35WLvjz5eQMmJW8ufZN+M/uRP1d+hrUdRrc63ezqYHNGRGSJqkan9FwwfQRxqnoo2ZfPOmxNcA4fhnbt4IcfXNXQSZOgYMH0X2cAqFS0Eu2rtmfk1SM575zz/A7HZFPBJII1InILEOVdAHYv8FNowzLZQkLC30ng7rtdzSCTphPxJxj0/SAAnmvxHM0rNKd5heY+R2Wyu2A6i/vjxis+CbyHK0c9IIQxmXCnCoMGuf6AH36Aa66xJBCE+dvmU/f1ujz/4/PsO7aP9JptjckowRwRVFPVgcDAUAdjsoFjx6BrV/jsMyhc2A0hefvtfkeVpR05eYTHvnmMUYtGUa5IOWbfOpvWlVr7HZaJIMEkgqEicj4wFZiiqqtDHJMJV7//DrVquVpBnTvDlCl2lXAQdhzewZvL3qT/Zf15tsWznJP7HL9DMhEm3aYhVW2OG5lsHzBWRFaJyOMhj8yEl4ULoU4dlwSeesqdGmpJIFUxsTF/XQ9QvWR1Nt+7mdeufs2SgPFFUBeUqeoeVR0O9MZdU/BkKIMyYWbkyL9PB/3ySzeYjEmRqjJ17VRqjK7BvbPuZcP+DQA2bKTxVTAXlFUXkadFZBVu8PqfcOUiTKQ7edJdG9C/vysXPXs2tG3rd1RZ1u4ju+n0YSc6f9SZMoXKsLjnYisSZ7KEYPoI3gKmAG1UdVeI4zHh4uuvoXt32LkTLrnEjSBWsaLfUWVZSUXidh7ZyYstX+T+BveTM4ffNR+NcdL9Jqpqg8wIxISJLVtcwbiPP4bixV2HcKdOEBXld2RZ0vZD27mw0IVE5Yhi1DWjqFC0AlWKV/E7LGP+IdWmIRH50Pu7SkRWBtxWBYxcZiLJ/v2uQNznn8P998PGjXDjjZYEUpCQmMDwBcOpNurvInFtLmpjScBkSWkdEdzn/bUKVwYWLHADyMTFuaOB66/3O6Isa92+dfSY3oOfd/zM1RddTfuq7f0OyZg0pXpEoKq7vbt9VfX3wBvQN3PCM1nCnDlwxRWuhPRHH1kSSMO4JeOoO7Yuv8b8yqTrJzHjlhmULWyj6pmsLZjTR1ulMO3qjA7EZEGJie6MoGuuceWiV61y/QEmVZWLVeb6atez9p613FrnVqsUasJCqk1DItIH98u/YrI+gYLA/FAHZnx25IgbRnLuXNcv8NVXrnaQ+Yfjccd5eu7TiAhDWg6xInEmLKV1RPAe0B43qlj7gFt9Vb01E2IzfomJgUaNXBJ47DFYu9aSQArm/T6Pi1+/mBd/epFDJw5ZkTgTttLqLFZV3Soi9yR/QkSKqeqfIYzL+OXQIVcvaM8eGDcOevb0O6Is5/DJwzzy9SOMWTyGikUr8s1t33BVhav8DsuYM5ZWIngPd8bQEtxANIGNnQrY1UPZzW+/uf6APXvgf/+zJJCKXUd2MWH5BB644gEGNR9EgdwF/A7JmLOSaiJQ1Xbe3wqZF47xzYYNUK+eG0tg8mS45Ra/I8pS9sfu58M1H9L30r5UK1GNLfdtsRHDTLYRTK2hRiJSwLt/q4i8IiJ2Plx2cuIE3HADxMbCjBmWBAKoKlNWT6HGqBoMmDWAX2N+BbAkYLKVYE4fHQPEisjFwIPAb8CkkEZlMs/27VC7NqxeDcOGQXM74yXJriO7uG7KdXT5uAvlipRjSa8ldmWwyZaCqXoVr6oqIh2Bkao6XkR6hDowkwn273djCBw8CBMnwm23+R1RlpGQmECTt5uw88hOXm71MvddcZ8ViTPZVjDf7CMi8ijQDWgsIjmAXKENy4Tcp5+66qGHDsEzz1gS8Px+8HdKFypNVI4oRl87mopFK3JRsYv8DsuYkAqmaegm3MD1d6rqHtxYBC+FNCoTWt98A126uLpBU6fCkzbOUEJiAq/8/ArVR1VnzGJXJK51pdaWBExECKYM9R4RmQxcKiLtgIWq+k7oQzMhsXgxdOzoSkj/9BOUK+d3RL5bvXc1Pab3YOHOhbSr0o7rql3nd0jGZKpgzhq6EVgIdAZuBBaIyA2hDsyEwIoVcOmlcOqUOyqwJMDri1+n3th6bD6wmff+8x7Tu0yndCEbgM9ElmD6CAYCl6rqXgARKQl8DUwNZWAmg6lCt26QO/ff9YMimKoiIlQvUZ3ONTszrM0wShYo6XdYxvgimESQIykJeGIIctB7k0UcOwYXX+yuHB46FBpE7qBzsXGxPPndk0RJFC+0eoGm5ZvStHxTv8MyxlfB7NBnichsEekuIt2BGcDM0IZlMszGja520G+/uQRw//1+R+SbuVvnUmdMHYb+PJSjp45akThjPMF0Fj8sIv8BrvQmjVPVaaENy2SILVsgOtpdMfzhh9C5s98R+eLQiUP8d85/Gbd0HJWKVuLb2761UtHGBEhrPILKwMtAJWAV8JCq7syswMxZ2r3bdQyfOOFOEe3Y0e+IfLP76G7eXfUuDzV4iGeaP0P+XPn9DsmYLCWtpqG3gC+ATrgKpCNOd+Ei0lZENojIJhF5JI35OomIikj06a7DpODXX90ZQTEx8MUXEZkE9h3bx4gF7itbrUQ1tt63lZdav2RJwJgUpNU0VFBV3/DubxCRpaezYBGJAkbhhrrcASwSkemqujbZfAWB+4AFp7N8k4oVK6BlS3ex2KRJ0CqlkUazL1Xl/dXvc++X93L45GHaXNSGKsWr2BlBxqQhrSOCvCJyiYjUE5F6QL5kj9NzGbBJVTer6ingAyCln6b/A14ATpx29Oafli+HunXdWUJz5sCtkTWQ3PZD22n/fnu6ftKVi4pdxLK7l1mROGOCkNYRwW7glYDHewIeK5DekEwXAtsDHu8ALg+cwUsoZVR1hog8nNqCRKQX0AugbFmrgJ2iL7+E229393/5xRWTiyDxifE0m9iMPUf38GqbV+l/WX+ickT5HZYxYSGtgWlCelqFV7zuFaB7evOq6jhgHEB0dLSd85fc4sVuZLESJeCddyIqCWw9uJUyhcqQM0dOxrYbS8WiFalY1AbPM+Z0hPLCsJ1A4Ijnpb1pSQoCtYC5IrIVuAKYbh3Gp2nfPmjbFgoVgh9/dFcPR4D4xHhe/ullqo+qzuhFowFoWbGlJQFjzkAoC6wvAiqLSAVcAugC/DX0laoeAkokPRaRubhTVBeHMKbs5ehRaNECDhyAr76CqlX9jihTrPxjJT2m92DxrsV0rNqRTjU6+R2SMWEtZIlAVeNFpB8wG4gC3lLVNSIyCFisqtNDte6IsH+/OyNo1SrXHNSihd8RZYrRi0Zz36z7KJq3KFNumELnGp0REb/DMiaspZsIxP2XdQUqquogb7zi81V1YXqvVdWZJCtHoaopFr9X1WZBRWxg2zZo1sxdOTxgQEQ0ByUViat1bi261OrCq21epUT+Eum/0BiTrmCOCEYDibizhAYBR4CPgUtDGJdJzcqVcOWVcOQITJkCN97od0QhdezUMR7/9nFy5sjJS61fokm5JjQp18TvsIzJVoLpLL5cVe/BO89fVQ8AuUMalUnZ5s2uiuiRI/D119k+CXyz+Rtqj6nNsAXDOJlw0orEGRMiwRwRxHlXCSv8NR5BYkijMv8WGwvXXefuv/JKtu4TOHjiIA999RDjl42ncrHKzOs+j8blGvsdljHZVjBHBMOBacC5IvIs8CPwXEijMv+0fTs0bOg6hgcPzvalpP84+gcfrP6A/2v0f6zovcKSgDEhFkwZ6skisgRoAQhwnaquC3lkxomJcaWk9+2DceOgZ0+/IwqJpJ3/fVfcR9USVdk6YKt1BhuTSYI5a6gsEAt8HjhNVbeFMjCDG17yjjtg71749NNsWUVUVZm8ajL3zbqPo6eOck3la6hcvLIlAWMyUTB9BDNw/QMC5AUqABuAmiGMyxw9CpUquSTQr1+2TALbDm2j9xe9+XLTlzQo3YDxHcZTuXhlv8MyJuIE0zRUO/CxVyiub8giMu5isZYtXRJ44AF46SW/I8pw8YnxNJvQjL3H9jK87XD6XtrXisQZ45PTvrJYVZeKyOXpz2nO2E03uXEFhg+H/v39jiZDbT6wmXKFy5EzR07eaP8GlYpVonyR8n6HZUxEC6aP4IGAhzmAesCukEUUyVRdZ/C338JDD2WrJBCfGM/Qn4by1NyneLHVi9x7+b20qJh9T4E1JpwEc0RQMOB+PK7P4OPQhBPhPvsMxo93/QGDB/sdTYZZvmc5Pab3YOnupVxf7Xo61+jsd0jGmABpJgLvQrKCqvpQJsUTuWJi4PrrXTnpyZMhTx6/I8oQIxeO5P7Z91M8X3Gmdp5qlUKNyYJSvaBMRHKqagLQKBPjiUzLl7sicgAjR0KBAn5GkyGSykHUOa8OXWt3Ze09ay0JGJNFpXVEsBDXH7BcRKYDHwHHkp5U1U9CHFtkePVVd2ZQzpzZoojc0VNHGfjNQHJF5eLl1i9bkThjwkAwJSbyAjG46qPtgPbeX3O2Jk92SaBJE/j997BPAl/99hW1RtdixMIRxCXEWZE4Y8JEWkcE53pnDK3m7wvKkth/+NkaPx7uugvKlXMDz+fP73dEZ+zA8QM88NUDTFg+garFqzLvjnlcWfZKv8MyxgQprUQQBZzDPxNAEksEZ2PBAujTBwoWhJkzwzoJAOw9tpepa6fy6JWP8mTTJ8mbM6/fIRljTkNaiWC3qg7KtEgixfbtbojJc891g82XL+93RGdkz9E9vL/qfe5vcL8rEnffVornL+53WMaYM5BWIrCBYDPab79BmzZuYJk5c8IyCagq76x4h/tn309sXCztqrSjcvHKlgSMCWNpdRbbZZ8ZKTER6td3yWDyZLg8/Kp0bD24lbaT29L9s+7UKFmD5b2XW5E4Y7KBVI8IVPXPzAwkW1OFDh3g0CEYOBBuucXviE5bfGI8zSc2Z3/sfkZdM4re0b3JIcGcdGaMyepOu+icOU2JidC5M8yYAZ06hV3piE1/bqJCkQrkzJGTtzq8RcWiFSlXpJzfYRljMpD9pAu1AQPgk0/cKGNTpvgdTdDiEuJ47ofnqDm6JqMWjQKgeYXmlgSMyYbsiCCUXnwRRoxwF4zNnQsSHv3vS3cvpcf0Hizfs5zONTpzU82b/A7JGBNClghCZcgQePRRqFzZXTAWJklg+ILhPDD7AUoWKMknN37C9dWv9zskY0yIWSIIhZ9/dkmgVCk3wEy+fH5HlC5VRUS45PxLuO3i2xjaeihF8xX1OyxjTCawRJDR9u2Dtm0hb1745pssnwSOnDzCo988Sp6oPAxtM5TG5RrTuFxjv8MyxmQi6yzOaF27wuHDrk+genW/o0nTrE2zqDWmFqMXjUZRKxJnTISyI4KM9Pjj7orhgQOz9AVjMbExPPDVA7yz4h2ql6jO/Dvn06BMA7/DMsb4xBJBRvnuO3j2Wbj2WnjqKb+jSVPM8RimrZvGE02eYGDjgeTJmT1GQzPGnJmQNg2JSFsR2SAim0TkkRSef0BE1orIShH5RkTC8yT1gwehRw83zOT770OuXH5H9C+7j+zm5Z9eRlWpUrwKvw/4nUHNB1kSMMaELhF44x2PAq4GagA3i0iNZLMtA6JVtQ4wFXgxVPGEjKoba3jLFhgzxpWWzkJUlbeWvUX1UdV54rsn2PTnJgA7I8gY85dQHhFcBmxS1c2qegr4AOgYOIOqfqeqsd7DX4DSIYwnNF54wXUMN2qU5WoIbTmwhdbvtqbH9B5cfP7FrOi9worEGWP+JZR9BBcC2wMe7wDS6kHtAXyZ0hMi0gvoBVC2bNmMiu/svfaau16gWDHXR5CFxCfGc9U7VxETG8OYa8fQq34vKxJnjElRlugsFpFbgWigaUrPq+o4YBxAdHR01jjHccECV0eoaFF3vUAW6RfYGLORikUrkjNHTt7u+DaVilaiTOEyfodljMnCQvkTcScQuAcq7U37BxFpCQwEOqjqyRDGk7Fat3Z/ly2DunV9DQVckbjB8wZTa0wtRi4cCUCz8s0sCRhj0hXKI4JFQGURqYBLAF2AfzSii8glwFigraruDWEsGWv8eHfRWJcubvB5ny3etZge03uw8o+VdKnVhZtr3+x3SMaYMBKyRKCq8SLSD5gNRAFvqeoaERkELFbV6cBLwDnAR+KKsm1T1Q6hiilDbNoE/fpB2bLw7rt+R8Nrv7zGA189wPnnnM9nXT6jQ9Ws/fYZY7KekPYRqOpMYGayaU8G3G8ZyvWHxIABcOIEfPEFREX5FkZSkbjoC6LpcUkPXmz1IkXyFvEtHmNM+MoSncVho3t3N9LYww9D7dq+hHD45GH+b87/kTdnXl5t+yqNyjaiUdlGvsRijMke7HzCYG3cCBMnQrNmrpSED2ZunEnN0TUZt3QcOXPktCJxxpgMYUcEwUhIgObN3f0xYzL9VNH9sfsZMGsAk1dNpmbJmkztPJXLS2fdonbGmPBiiSAYr70GO3fCnXdCtWqZvvoDxw/w+a+f81TTp3is8WPkjsqd6TEYY7IvCbfmhejoaF28eHHmrTAx0Q03GRsLu3Zl2pCTOw/vZPKqyTzc8GFEhIMnDlpnsDHmjInIElWNTuk5OyJIzzvvwObN8PrrmZIEVJU3l77JQ3MeIi4hjv9U/w8XFbvIkoAxJmSsszgtq1bBXXdBmTJwxx0hX91vf/5Gi3da0OuLXtQrVY+VfVZyUbGLQr5eY0xksyOC1Ki6YScTEmDaNMgd2nb5+MR4WrzTgj+P/8nYdmO5q95dViTOGJMpLBGkZtgwd0Tw2GNQv37IVrNh/wYqFatEzhw5mXjdRCoVq0TpQuFXjdsYE77sJ2dKVq2CBx6A/PndOMQhcCrhFM/MfYbaY2ozauEoAJqWb2pJwBiT6eyIILm4OLj7bnf/u+8gX74MX8XCnQvpMb0Hq/eu5pbat9C1TtcMX4cxxgTLEkFyt9wCP/8Mo0fDZZdl+OKH/TKMB796kFLnlOLzmz+nXZV2Gb4OY4w5HZYIAs2fD1OnQufO0KdPhi46qUjcZRdeRs96PXmh5QsUzls4Q9dhjDFnwi4oC1SliqsptH8/FC+eIYs8dOIQ/53zX/LlysewtsMyZJnGGHO60rqgzDqLkyxY4JJA+/YZlgQ+3/A5NUbX4M1lb5InKo8ViTPGZEnWNATumoF+/dz9YcPOenH7ju3jvln38f7q96l9bm0+velTLr3w0rNerjHGhIIlAoARI2DxYnj5ZahY8awXd+jkIWZunMkzzZ7hkSsfsSJxxpgszfoI4uOhUiWIiYGDByHnmeXG7Ye28+7Kd3nkykcQEQ6dOGSdwcaYLMP6CNIyZw5s2wb/+98ZJYFETeT1xa9Tc3RNBv8wmN8O/AZgScAYEzYsEYwY4f7efvtpv3RjzEaumngVfWb04bILL2NVn1VWJM4YE3Yiu49g2jT48ktXRqJYsdN6aXxiPK0mteLgiYOM7zCeO+regWTSWAXGGJORIjsRvPmm+/vQQ0G/ZN2+dVQuXpmcOXIy6fpJVCpWiQsKXhCiAI0xJvQit2lo3TqYORN69YLC6bfnn4w/yVPfPUWd1+swcuFIABqXa2xJwBgT9iL3iOCxx9zfRx5Jd9ZfdvxCj+k9WLtvLd3qdKNbnW4hDs4YYzJPZCaCpUvh00+hZ0+oUCHNWYf+NJSH5zxM6UKlmXnLTK6ufHXmxGiMMZkk8hKBKgwaBDlywPPPpzpboiaSQ3LQoEwDekf3ZkjLIRTKUygTAzXGmMwReYnggw/gs8/g4YdTrCl08MRBHpz9IPlz5WfENSNoWKYhDcs09CFQY4zJHJHXWdy3L5QrB0OG/OupT9d/So1RNZi4YiIF8xS0InHGmIgQWUcEGza4MhLXXuuahjx7j+2l38x+fLT2I+qeX5cvbvmCeqXq+RenMcZkoshKBElXEfft+4/Jh08eZs7mOTx71bM83PBhckXl8iE4Y4zxR2Qlgpkz3d+GDdl2aBuTVkziscaPcVGxi9g2YBsF8xT0Nz5jjPFBSPsIRKStiGwQkU0i8q8T9kUkj4hM8Z5fICLlQxkPW7eSGF2f0YtGU3N0TZ778bm/isRZEjDGRKqQJQIRiQJGAVcDNYCbRaRGstl6AAdU9SLgVeCFUMXD1q1sKKY0u3Yv98y8hwalG7Cm7xorEmeMiXihPCK4DNikqptV9RTwAdAx2TwdgYne/alACwlR5bb4zz+jTTdYlesAb3d8m9m3zqZ8kfKhWJUxxoSVUPYRXAhsD3i8A7g8tXlUNV5EDgHFgf2BM4lIL6AXQNmyZc8omJxly/PuN3Wp9MwMShWy+kDGGJMkLK4jUNVxqhqtqtElS5Y8s4V07MiVny6zJGCMMcmEMhHsBMoEPC7tTUtxHhHJCRQGYkIYkzHGmGRCmQgWAZVFpIKI5Aa6ANOTzTMdSBoa7AbgW7XLeY0xJlOFrI/Aa/PvB8wGooC3VHWNiAwCFqvqdGA8MElENgF/4pKFMcaYTBTSC8pUdSYwM9m0JwPunwA6hzIGY4wxaQuLzmJjjDGhY4nAGGMinCUCY4yJcJYIjDEmwkm4na0pIvuA38/w5SVIdtVyBLBtjgy2zZHhbLa5nKqmeEVu2CWCsyEii1U12u84MpNtc2SwbY4ModpmaxoyxpgIZ4nAGGMiXKQlgnF+B+AD2+bIYNscGUKyzRHVR2CMMebfIu2IwBhjTDKWCIwxJsJly0QgIm1FZIOIbBKRR1J4Po+ITPGeXyAi5X0IM0MFsc0PiMhaEVkpIt+ISDk/4sxI6W1zwHydRERFJOxPNQxmm0XkRu+zXiMi72V2jBktiO92WRH5TkSWed/va/yIM6OIyFsisldEVqfyvIjIcO/9WCki9c56paqarW64kte/ARWB3MAKoEayefoCr3v3uwBT/I47E7a5OZDfu98nErbZm68gMA/4BYj2O+5M+JwrA8uAot7jc/2OOxO2eRzQx7tfA9jqd9xnuc1NgHrA6lSevwb4EhDgCmDB2a4zOx4RXAZsUtXNqnoK+ADomGyejsBE7/5UoIWISCbGmNHS3WZV/U5VY72Hv+BGjAtnwXzOAP8DXgBOZGZwIRLMNvcERqnqAQBV3ZvJMWa0YLZZgULe/cLArkyML8Op6jzc+Cyp6Qi8o84vQBERKXU268yOieBCYHvA4x3etBTnUdV44BBQPFOiC41gtjlQD9wvinCW7jZ7h8xlVHVGZgYWQsF8zlWAKiIyX0R+EZG2mRZdaASzzU8Dt4rIDtz4J/0zJzTfnO7/e7pCOjCNyXpE5FYgGmjqdyyhJCI5gFeA7j6Hktly4pqHmuGO+uaJSG1VPehnUCF2MzBBVYeKSAPcqIe1VDXR78DCRXY8ItgJlAl4XNqbluI8IpITdzgZkynRhUYw24yItAQGAh1U9WQmxRYq6W1zQaAWMFdEtuLaUqeHeYdxMJ/zDmC6qsap6hbgV1xiCFfBbHMP4EMAVf0ZyIsrzpZdBfX/fjqyYyJYBFQWkQoikhvXGTw92TzTgdu9+zcA36rXCxOm0t1mEbkEGItLAuHebgzpbLOqHlLVEqpaXlXL4/pFOqjqYn/CzRDBfLc/xR0NICIlcE1FmzMxxowWzDZvA1oAiEh1XCLYl6lRZq7pwG3e2UNXAIdUdffZLDDbNQ2paryI9ANm4844eEtV14jIIGCxqk4HxuMOHzfhOmW6+Bfx2Qtym18CzgE+8vrFt6lqB9+CPktBbnO2EuQ2zwZai8haIAF4WFXD9mg3yG1+EHhDRO7HdRx3D+cfdiLyPi6Zl/D6PZ4CcgGo6uu4fpBrgE1ALHDHWa8zjN8vY4wxGSA7Ng0ZY4w5DZYIjDEmwlkiMMaYCGeJwBhjIpwlAmOMiXCWCEyWJCIJIrI84FY+jXmPZsD6JojIFm9dS70rVE93GW+KSA3v/mPJnvvpbGP0lpP0vqwWkc9FpEg689cN92qcJvTs9FGTJYnIUVU9J6PnTWMZE4AvVHWqiLQGXlbVOmexvLOOKb3lishE4FdVfTaN+bvjqq72y+hYTPZhRwQmLIjIOd44CktFZJWI/KvSqIiUEpF5Ab+YG3vTW4vIz95rPxKR9HbQ84CLvNc+4C1rtYgM8KYVEJEZIrLCm36TN32uiESLyBAgnxfHZO+5o97fD0Tk2oCYJ4jIDSISJSIvicgir8b83UG8LT/jFRsTkcu8bVwmIj+JSFXvStxBwE1eLDd5sb8lIgu9eVOq2Goijd+1t+1mt5RuuKtil3u3abir4At5z5XAXVWZdER71Pv7IDDQux+FqzdUArdjL+BN/z/gyRTWNwG4wbvfGVgA1AdWAQVwV2WvAS4BOgFvBLy2sPd3Lt6YB0kxBcyTFOP1wETvfm5cFcl8QC/gcW96HmAxUCGFOI8GbN9HQFvvcSEgp3e/JfCxd787MDLg9c8Bt3r3i+BqERXw+/O2m7+3bFdiwmQbx1W1btIDEckFPCciTYBE3C/h84A9Aa9ZBLzlzfupqi4Xkaa4wUrme6U1cuN+SafkJRF5HFenpgeufs00VT3mxfAJ0BiYBQwVkRdwzUk/nMZ2fQm8JiJ5gLbAPFU97jVH1RGRG7z5CuOKxW1J9vp8IrLc2/51wJyA+SeKSGVcmYVcqay/NdBBRB7yHucFynrLMhHKEoEJF12BkkB9VY0TV1E0b+AMqjrPSxTXAhNE5BXgADBHVW8OYh0Pq+rUpAci0iKlmVT1V3FjHVwDDBaRb1R1UDAboaonRGQu0Aa4CTfQCrjRpvqr6ux0FnFcVeuKSH5c/Z17gOG4AXi+U9XrvY71uam8XoBOqrohmHhNZLA+AhMuCgN7vSTQHPjXmMvixmH+Q1XfAN7EDff3C9BIRJLa/AuISJUg1/kDcJ2I5BeRArhmnR9E5AIgVlXfxRXzS2nM2DjvyCQlU3CFwpKOLsDt1PskvUZEqnjrTJG60ebuBR6Uv0upJ5Ui7h4w6xFcE1mS2UB/8Q6PxFWlNRHOEoEJF5OBaBFZBdwGrE9hnmbAChFZhvu1/Zqq7sPtGN8XkZW4ZqFqwaxQVZfi+g4W4voM3lTVZUBtYKHXRPMUMDiFl48DViZ1FifzFW5goK/VDb8ILnGtBZaKG7R8LOkcsXuxrMQNzPIi8Ly37YGv+w6okdRZjDtyyOXFtsZ7bCKcnT5qjDERzo4IjDEmwlkiMMaYCGeJwBhjIpwlAmOMiXCWCIwxJsJZIjDGmAhnicAYYyLc/wOVZW2w5iLPbQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.684721245\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAouUlEQVR4nO3dd7xU1bn/8c+XJiBgAaJRUBQrCjY0lms0xl7QmCbRJCZETVFvotfEaGISNWqKNeK1R6NR4jXRoGL82Y3ReMVGwIrYDt4ooIBIUfD5/bFmnDmHA2dO2TNnzv6+X695zey918w8+5R5Zq2111qKCMzMLL+61ToAMzOrLScCM7OccyIwM8s5JwIzs5xzIjAzyzknAjOznHMisLoh6XBJ/6+CcpdK+mk1YqoGSa9K2rPw+OeSrq91TNa1OBFYhyh8WC2StEDSW5KukdSvI98jIv4YEXtXUO7bEXFGR753kaSQ9H7hPGdKOk9S9yzeqy0kDZB0gaTXCzG+XNgeVOvYrPNyIrCOdFBE9AO2BUYDP2laQFKPqkfV8bYqnOduwJeBb9Y4HgAk9QLuBbYA9gUGADsBc4Ad2vB6XeF3ZRVwIrAOFxEzgTuBLeHjb9Hfk/QS8FJh34GSnpY0V9IjkkYVny9pqKS/SJolaY6kiwv7j5T0cOGxJJ0v6W1J8yX9S1Lx/a6RdGbZ6x0labqkdyRNlLRO2bGQ9G1JLxViGS9JFZ7ndOAfwNZlr9eW8xou6b7CvtmS/ihp9Vb+2AG+BqwHfC4ino2IjyLi7Yg4IyImlZ3vRmUxffyzkrS7pAZJP5L0b+D3kp6TdGBZ+R6F+LctbO9YOM+5kp6RtHsb4rYacyKwDidpKLA/8FTZ7kOATwEjJG0DXA0cAwwELgMmSlql0MxyO/AaMAxYF5jQzNvsDXwa2ARYDfgS6Ztv01j2AM4uHP9k4XWbvt6BwPbAqEK5fSo8z82AXYHphe22npcKMa4DbA4MBX5eSQxN7An8LSIWtOG5RWsDawLrA0cDNwJjy47vA8yOiCclrQvcAZxZeM5/AX+WNLgd72814ERgHelWSXOBh4EHgbPKjp0dEe9ExCLSB8xlEfFYRCyLiGuBJcCOpCaMdYCTIuL9iFgcEQ83814fAv2BzQBFxHMR8X/NlDscuDoinoyIJcCPgZ0kDSsrc05EzI2I14H7KfuGvwJPSnofeA54ALiksL9N5xUR0yPi7ohYEhGzgPNIzU6tNRBo7mfQGh8BPyvEsgi4ARgjqW/h+FdIyQHgCGBSREwq1D7uBiaTvgRYHXEisI50SESsHhHrR8R3Cx8kRW+UPV4fOLHQnDC3kDyGkj4ohwKvRcTSlb1RRNwHXAyMB96WdLmkAc0UXYf0Lbz4vAWkmsO6ZWX+XfZ4IdAPQNK0QofrAkm7lpXZtlDmy6RazqrtOS9Ja0maUOh8ng9cD7Slc3cOqdbTHrMiYnFxo9D89RxwUCEZjCElB0jn+8Um5/sfHRCDVZkTgVVL+TS3bwC/LCSN4q1vRNxYOLZeJR2VEXFRRGwHjCA1EZ3UTLE3SR9YAEhalfTNeWYFr79FRPQr3P7e5FhExE3Ao8Bp7Tyvs0g/n5ERMYD0Tbuifoom7gH2KZzjiiwE+pZtr93keHPTERebhw4Gni0kB0jndF2T8101Is5pQ+xWQ04EVgtXAN+W9KlCp++qkg6Q1B/4X1LzxjmF/b0l7dL0BSRtX3h+T+B9YDGpWaOpG4FvSNpa0iqkD93HIuLVDjqXc4CjJK3djvPqDywA5hXa3ZtLaJW4jvTh/GdJm0nqJmmgpFMkFZtrnga+Iqm7pH2prAlqAqlP5juUagOQai4HSdqn8Hq9Cx3OQ9oYv9WIE4FVXURMBo4iNe28S+psPbJwbBlwELAR8DrQQGqCaWoA6YP3XVLTzxzgN8281z3AT4E/kz6IhwOHdeC5/At4iNT239bz+gWpuWkeqfP1L22MZQmpw/h54G5gPikBDQIeKxT7z0Icc0n9J7dW8Lr/R6r57Az8qWz/G6RawinALFISOgl/rtQdeWEaM7N8c+Y2M8s5JwIzs5xzIjAzyzknAjOznKu7SaUGDRoUw4YNq3UYZmZ15YknnpgdEc1O/1F3iWDYsGFMnjy51mGYmdUVSa+t6JibhszMcs6JwMws55wIzMxyru76CMzM2urDDz+koaGBxYsXt1y4TvXu3ZshQ4bQs2fPip/jRGBmudHQ0ED//v0ZNmwYFS5EV1cigjlz5tDQ0MAGG2xQ8fMyaxqSdLXSMoJTV3Bcki5SWkJwSnHpOzOzrCxevJiBAwd2ySQAIImBAwe2usaTZR/BNaQFtFdkP2Djwu1o4L8zjMXMDKDLJoGitpxfZokgIh4C3llJkYOBPxQW+PgnsLqk7FY2evhhuPhi8GyrZmaN1PKqoXVpvHxhA42XD/yYpKMlTZY0edasWW17twsvhOOOg9dfb9vzzcw6QPfu3dl6663ZcsstOeigg5g7d+7Hx6ZNm8Yee+zBpptuysYbb8wZZ5xB+VIBd955J6NHj2bEiBFss802nHjiiR0SU11cPhoRl0fE6IgYPXhwsyOkW3bQQel+2bKOC8zMrJX69OnD008/zdSpU1lzzTUZP348AIsWLWLMmDGcfPLJvPDCCzzzzDM88sgjXHLJJQBMnTqVY489luuvv55nn32WyZMns9FGG3VITLVMBDNJC3oXDaGCdWTNzLqKnXbaiZkz08feDTfcwC677MLee+8NQN++fbn44os555y0BPSvf/1rTj31VDbbbDMg1Sy+853vdEgctbx8dCJwrKQJwKeAeYUl8czMsvf978PTT3fsa269NVxwQUVFly1bxr333su4ceOA1Cy03XbbNSozfPhwFixYwPz585k6dWqHNQU1lVkikHQjsDswSFID8DOgJ0BEXApMAvYnreu6EPhGVrGYmXUWixYtYuutt2bmzJlsvvnm7LXXXrUOKbtEEBFjWzgewPeyen8zs5Wq8Jt7Ryv2ESxcuJB99tmH8ePHc/zxxzNixAgeeuihRmVnzJhBv379GDBgAFtssQVPPPEEW221VYfHVBedxWZmXU3fvn256KKLOPfcc1m6dCmHH344Dz/8MPfccw+Qag7HH388P/zhDwE46aSTOOuss3jxxRcB+Oijj7j00ks7JBYnAjOzGtlmm20YNWoUN954I3369OGvf/0rZ555JptuuikjR45k++2359hjjwVg1KhRXHDBBYwdO5bNN9+cLbfckhkzZnRIHJ5ryMysihYsWNBo+7bbbvv48ciRI3nggQdW+NwDDzyQAw88sMNjco3AzCznnAjMzHLOicDMciW6+HxjbTk/JwIzy43evXszZ86cLpsMiusR9O7du1XPc2exmeXGkCFDaGhooM2TV9aB4gplreFEYGa50bNnz1at3JUXbhoyM8s5JwIzs5xzIjAzyzknAjOznHMiMDPLOScCM7OccyIwM8s5JwIzs5xzIjAzyzknAjOznHMiMDPLOScCM7OccyIwM8s5JwIzs5xzIjAzyzknAjOznHMiMDPLOScCM7OccyIwM8s5JwIzs5xzIjAzyzknAjOznMs0EUjaV9ILkqZLOrmZ4+tLulfSFEkPSBqSZTxmZra8zBKBpO7AeGA/YAQwVtKIJsV+C/whIkYBpwNnZxWPmZk1L8sawQ7A9IiYEREfABOAg5uUGQHcV3h8fzPHzcwsY1kmgnWBN8q2Gwr7yj0DHFp4/Dmgv6SBTV9I0tGSJkuaPGvWrEyCNTPLq1p3Fv8XsJukp4DdgJnAsqaFIuLyiBgdEaMHDx5c7RjNzLq0Hhm+9kxgaNn2kMK+j0XEmxRqBJL6AZ+PiLkZxmRmZk1kWSN4HNhY0gaSegGHARPLC0gaJKkYw4+BqzOMx8zMmpFZIoiIpcCxwF3Ac8BNETFN0umSxhSK7Q68IOlFYC3gl1nFY2ZmzcuyaYiImARMarLvtLLHNwM3ZxmDmZmtXK07i2tv1ix4/vlaR2FmVjOZ1gg6tdmzoX9/GDoUliyBiFpHZGZWE/mtEQweDF/+ckoCALfcUtt4zMxqJJ+JYOHCdP/Xv5b2HXooLFtuCIOZWZeXz0QwZ07z+xctqm4cZmadQP4SwXvvwe67N39MqmooZmadQf4Swfnnw4wZtY7CzKzTyF8imDp1+X1rrpnuZ85c/piZWReXv0Tw1FPL7zvllHTvpiEzy6H8JYKPPlp+39tvp/tNNoEDDmi+jJlZF5W/RFDuj3+Ed95JA8uKJk2CvfaqXUxmZlWW70Twla/AGmvAuHGN9993X/Plzcy6oHwngqJ+/WodgZlZzTgRQGoaevnlxlcUzZ5du3jMzKoov5POFa8UKtpww8bbgwenTmNfSWRmXVx+awS/rGANnHfeyT4OM7May28iWJHyUcdXXFG7OMzMqsSJoKkNNoCbC4umuZ/AzHLAiaA522+f7ldfvaZhmJlVQz4TwYABKz9eHGB24YXZx2JmVmP5SQTlS1HusMPKy66xRrp305CZ5UB+EkH5/EGVNvmstlomoZiZdSb5GUdQXiPo1avl8muuCQMHZhePmVknkZ9EUF4j6Nmz5fLvvONxBGaWC/lpGmptjaDo6KPhxRc7Ph4zs04in4lg//0rf94VV8Cmm3Z8PGZmnUR+EkF509Ahh7Rc/i9/ySwUM7POJD+JoLxGUIkdd2y8PX9+x8ViZtaJ5CcRtHb5yU9+EubMKW37UlIz66LykwhaWyOAdAnp6aeXtm+6qePiMTPrJDJNBJL2lfSCpOmSTm7m+HqS7pf0lKQpklrRi9tKbV2Q/rDDSo+//OWOicXMrBPJLBFI6g6MB/YDRgBjJY1oUuwnwE0RsQ1wGHBJVvG0qUYAsPHGcOmlpW0JFi3qmJjMzDqBLGsEOwDTI2JGRHwATAAOblImgOIMcKsBb2YWTVsTAcAxxzTe/slPUkK44Yb2xWRm1glkmQjWBd4o224o7Cv3c+AISQ3AJOC45l5I0tGSJkuaPGvWrLZF09amoaJXXy09Pu+8dH/44XDtte17XTOzGqsoEUjaRdLdkl6UNEPSK5JmtPzMFo0FromIIcD+wHWSlospIi6PiNERMXrw4MFte6dijeCEE9r2/PXXhwsuWH7/kUfCsmVte00zs06g0hrBVcB5wH8A2wOjC/crMxMYWrY9pLCv3DjgJoCIeBToDQyqMKbWKdYI2rMY/fHHlx4fcUTpcY8eKUlIMHZs21/fzKwGKk0E8yLizoh4OyLmFG8tPOdxYGNJG0jqReoMntikzOvAZwEkbU5KBG1s+2lBsUbQrR2tYVKaiG7+fLjuOjj33NKxH/wg3U+YkMo1NLT9fczMqqjST8X7Jf1G0k6Sti3eVvaEiFgKHAvcBTxHujpomqTTJY0pFDsROErSM8CNwJER7enVXYmOqBFAWrSmuILZypqZhg5tf7+EmVkVVDoN9acK96PL9gWwx8qeFBGTSJ3A5ftOK3v8LLBLhTG0TzG/tDcRNDV/flr68qKL4AtfgPXWg6VL07Hu3dt3tZKZWRVUlAgi4jNZB5K5jmgaak7//o0/7D/8EP74x1IfggSbbQb/+EcaqWxm1slUetXQapLOK17CKelcSfU1+U5HNQ1V4vDD4etfL20//3xa7UyCz37WTUZm1qlU+vX4auA94EuF23zg91kFlYmsagQrctVVze+/777UZLR4cXXiMDNrQaWfisMj4meFUcIzIuIXwIZZBtbhqlkjgFL/wMKF8K1vLb885i9+UZ04zMxaUGkiWCTpP4obknYB6mvCnWrXCIr69EmrnH3wQYph553T/nPOqW4cZmYrUOlVQ98Bri30Cwh4Bzgyq6AyUe0awYr8/e+ptgCw1lrw1lu1jcfMcq+ir8cR8XREbAWMAkZGxDYR8Uy2oXWwrC4fba1u3UojlN9+G047beXlzcwyttIagaQjIuJ6SSc02Q9ARJyXYWwdq1ZNQ8258EJ4//3UoXzGGfDQQ/DAA7WOysxyqqVPxVUL9/1XcKsfnaVpqOjKK0uPH3wwJai2TqhnZtYOK60RRMRlhfv6v8SlM9UIiiJKiSkCZs9O21OmwMiRtY3NzHKj0gFlv5Y0QFJPSfdKmiXpiJaf2Yl0thpBUXPjCUaNarz9/vvVicXMcqnSr8d7R8R84EDgVWAj4KSsgspEZ6wRAKyySootAnr1Ku2XSrd+/dJ9ceyB1z8wsw5U6adisQnpAOB/ImJeRvFkp7PWCMotWZJGHq/Iz3+e4u/Ro3Ofh5nVlUoTwe2Snge2A+6VNBiorzkSOsvloy35zGdSQrj00rQmcsSK5yZ67bXqxmZmXVKl4whOBnYGRkfEh8D7LL8QfedWL4kAUhPRMceUVjuTUnL43e/g1lthu+3S/mHD0rHttoM5La0TZGbWvJbGEewREfdJOrRsX3mRv2QVWIerp0TQnF694Nhj0+M990z9BkVPPgmDmqzwedVV8NWvLj/HkZlZEy3VCHYr3B/UzO3ADOPKTr0mgnKrrpo6jGfPTgviNGfcuMadz2ZmK9DSOIKfFe6/UZ1wMtTVVgrr1i2tcXDccem2eDF87Wvw5ptpEZyihQuhb9/axWlmnV6l4wjOkrR62fYaks7MLKosdYUaQXN694abboKHH05Jb9dd0/5VV03n/N57tY3PzDqtSq8a2i8i5hY3IuJdYP9MIspKV6sRtOTBBxtvDxiQEsKFF8L558O8+rsC2MyyUWki6C5pleKGpD7AKisp33l11RpBU1K67PTDDxvv//734YQTYPXVYccdaxGZmXUylSaCP5LGD4yTNA64G7g2u7AykLcaAZQGn33wAdx+Owwf3vj4Y4+VRi8fdFBtYjSzmqt0HMGvgDOBzQu3MyLi11kGlpm81AjK9ewJBxwA06eXprO48MLGZW6/vZQUXnihNnGaWU20ZuKd54C/RcR/AX+XVF/TUOexRrAyxx+ffiZLly7fRLTZZvlMmGY5VelVQ0cBNwOXFXatC9yaUUzZqPcBZVnp3h0efTT9fF5/vfExCW65xUnUrIurtEbwPWAXYD5ARLwEfCKroDLlRLBiQ4emD/3f/76079BD05iFJUtqF5eZZarSRLAkIj4obkjqAdTX10R/q63ckUcuf7VR794piXptBLMup9JE8KCkU4A+kvYC/ge4LbuwMuQaQWV69EjJs+nCOcW1EQ49tPnnmVndqTQR/AiYBfwLOAaYBPwkq6Ay4RpB2xQXzpk0qfH+W24pXWXUtG/BzOpKi4lAUnfguYi4IiK+GBFfKDyuz09W1wjaZr/9Speennde42Prr59+rscdV5vYzKxdWkwEEbEMeEHSelWIJzt1mrc6pR/8IA1SmzkTPlF2zcDFF5dqCY8+Wrv4zKxVKm0aWgOYVli4fmLx1tKTJO0r6QVJ0yWd3Mzx8yU9Xbi9KGluK+NvPdcIOkbPnrDOOvDWW2lK7F/9qvHxnXde+bKbZtZprHQa6jI/be0LF5qUxgN7AQ3A45ImRsSzxTIR8YOy8scB27T2fSrmGkF2unWDH/4w3SLSZagzZ8JnP5vmO3LyNevUWlqhrDfwbWAjUkfxVRGxtMLX3gGYHhEzCq81gbS85bMrKD8W+FmFr916HlBWHRI0NJR+zt2aqXROmZLWSFhrrcYrrZlZTbTUNHQtMJqUBPYDzm3Fa68LvFG23VDYtxxJ6wMbAM22JUg6WtJkSZNnzZrVihCafbH2Pd8q89ZbKz42ahRstBH0759WUvNaCWY11VIiGBERR0TEZcAXgF0ziuMw4OZCx/RyIuLyiBgdEaMHDx7ctndw01B1feIT6We+bFm6LVmSBql9+tONy119dVor4ckn07KbDz9cm3jNcqylPoKPh5dGxFK17tv0TGBo2faQwr7mHEaaxiJ7rhFUV7FpqLh+cvmCObfcUhqYtt12jZ83b15KEGaWuZZqBFtJml+4vQeMKj6WNL+F5z4ObCxpA0m9SB/2y11pJGkz0lVJ2V5v6BpB5/O5z6XO5OastlrpUtTvf3/lTU1m1i4rTQQR0T0iBhRu/SOiR9njlX5dK3QqHwvcRZrC+qaImCbpdEljyooeBkyo2gA11wg6F6k0UK3YlNTUhRfC2munshMmrDh5mFmbtGY9glaLiEkRsUlEDI+IXxb2nRYRE8vK/DwilhtjkEEwmb+FdYBu3UpJYdEi+GmTK5fHjk1TZ0tw1lnpCiX/bs3aJdNE0Cm5RlA/eveG008vJYZ99218/NRT05iFbt3S77VfvzRJ3pw5tYnXrE7lJxH4W2P9u/PONLXF3XfDgQcuf/z996FPHxg0KCWGG2+E2bOrH6dZnclPIihyjaC+9ewJe+4Jt93WuG/h5ZeXL/uVr8Dgwel3vvPO8Pe/N98HYZZz+UkErhF0bRtuWEoKH30Ev/hF4+OPPprGMPTokUY1z5uX9i9dCm++Wf14zTqR/CUC1wi6PglOO61xYjjrrNTnAKkTevXVU7mePWHddUuXqkpwwgkpQZjlRH4SQZETQf5I8OMfpwRQvvbyKqs0X/7881OCKCaGN95ovpxZF5GfROCmIYM0wrlYU1i8ePkxDI88svxz1luvcY1Bgr/8pfqxm2UkP4mgyDUCW5Fu3WCnnVJSmDcvXZ30xS82X/bzn2+cGDbaaPn1nc3qRH4SgWsE1hoDBqSrk266qXGtYenStCBPUy+/nC5d7d8fjjoKXnut+jGbtVF+EkGRawTWHt27p0V3ypPDK6+UJtdbsACuvBKGDWtcYyjvmzDrZPKTCFwjsKwMG5b6F95/HyZOTH0KTfXuDSedVPXQzCqRn0RQ5BqBZaVvXzjooNQsFJEW3CmvCfz2t+nv7/HHYeHC2sVp1kR+EoFrBFZt/fqVrlJ6+unS/h12gFVXLTUbXXwxzG9pVnez7OQvEbhGYLWw1VZpYNv++8PBBzc+dtxxaf2FVVeFXXapTXyWa/lJBEVOBFYrEtxxB9x6a/piUpxAr2jhwjSOQUqJY/jwtBaDp8CwjOUnEbhpyDqb4gR6xauPivMfAUyZAjNmpNXZyqfAmLjcIn9m7ZafRFDkGoF1VgMGpIQwcya88w5cdBFsvXXjMgcfnP6G+/SBww6DU07xjKrWbvlJBK4RWL1YZx1YY43Ud/DUU6XpL848s1Rm8WL405/g7LPTjKoSbLaZB7JZm+QnERS5RmD1qFu3tCJbcTbVV16BI45oXOaFF0oD2f71r5qEafUpP4nANQLrKqT0gX/ddY1HOJ9+eqnMqFGp3G9+U7MwrX7kJxEUuUZgXdVPf5oSwk9+Utr3wx+WOpoPPthfiKxZ+UkE/gewvDjjjPT33nRK7YkTUxOTBHvtBQ8+mC5h9f9G7uUvEbhGYHlRnFK7OGtquXvugd13T4vzFJPDVlulS1Ytd/KTCIqcCCyPundvvADPHXekRFFuypQ0iE2Cl16qTZxWE/lJBK7+miXduqWpLh55pJQcXnst1RCKNtmk1Ldw7701C9WqIz+JoMg1ArPlrbce3H9/qi2cfXbjY3vumf5vHnzQX6i6qPwkAv8Bm7WsWzc4+eRSTeH660vHdt+91J/w1ls1C9E6Xn4SQZFrBGaVO/zwNIr5yisb71977fS/NHw4/PKXtYnNOkx+EoFrBGZts8oqMG5cqZZw6aWlaS1mzEjjFor9CcXZUidPTiOdrS7kJxEUuUZg1j7HHAMffpimupg5M112WlScLXX77dPcR8UEsd12MHdurSK2FmSaCCTtK+kFSdMlnbyCMl+S9KykaZJuyCwY1wjMOt4666TV1yLgzjth5Mi0MtvXvta43JNPpon0pNTcNGVKTcK15mWWCCR1B8YD+wEjgLGSRjQpszHwY2CXiNgC+H5W8ZS9aeZvYZZL++6bPuDfew+uvbbUlLRkCRx9dKncDTekWoQE224L3/pWumKpoSHVNPylreqyrBHsAEyPiBkR8QEwAWiyRh9HAeMj4l2AiHg7s2j8x2VWG716wWWXpaakBx+Er361dOypp+Cqq2CPPWDo0FS2Wzd4/PHaxZtDWSaCdYE3yrYbCvvKbQJsIukfkv4pad/mXkjS0ZImS5o8a9astkXjKSbMakuCT38a/vCHUm3hpZfgqKPgu9+FIUNKZXfYIZU/5ZS0hKdlqtadxT2AjYHdgbHAFZJWb1ooIi6PiNERMXrw4MHte0cnArPOY6ON4PLLYfx4eOONlByuvhpWXz0dP/tsWHXV9H/73/+dJsmzDpdlIpgJDC3bHlLYV64BmBgRH0bEK8CLpMTQ8dw0ZFYfvvENePfd1Gew226l/d/9brqUVUr9EW+8seLXsFbJMhE8DmwsaQNJvYDDgKYrb99Kqg0gaRCpqSjb6Q9dIzCrD+uuCw88kJqGbrmlcVK46640Lcb++8Pvf58GvVmbZZYIImIpcCxwF/AccFNETJN0uqQxhWJ3AXMkPQvcD5wUEXMyCiiTlzWzjPXpA4cckpJCRLqy6NZbU6fynXfCN7+Zynz962lcg7Vapn0EETEpIjaJiOER8cvCvtMiYmLhcUTECRExIiJGRsSELOMBXCMwq3c9eqTV1pYtg6lT0xVHkDqhhwxJ/+MjRriW0Aq17iyuHtcIzLqeLbZI02QvWZJGNRc991yqJUjw8MM1C69e5CcRFLlGYNb19OoF55+fvvB98AEcemjp2K67pv/7kSNhwYLaxdiJ5ScRuEZglg89e8Kf/5z+5x96KDUlQWpG6t8/TbP94Ye1jbGTyV8icI3ALD923bU0bUVx/qNf/SrVICR4553axtdJ5CcRFDkRmOXTtdfCb38Lm25a2jdwYGpG+utfaxdXJ5CfROCmITM78UR4/nlYuhSuuAIOOCCNUTjkENhxx7SOcw7lJxEUuUZgZt27p1lPb78dpk2DbbaBxx6DXXZJnxFjx6aRzTmRv0RgZlZuxIi0XsLUqWlQGsCECWk21I02ysVKa04EZmaQxiRcc00aqHbmmWnfyy+nldZ23DFdidRF5ScRuI/AzCrRrRucemr6zPjb39Kym489Bl/4QmnpzQMOSP0MXUR+EkGR+wjMrFL77AP/+7/w6qvwox+V9k+alMYrrL126meoc/lLBGZmrbX++nDOOaWRy8UxCW+9BQcdBKutVtejlvOTCNw0ZGYdoWfP0prMxctN589Po5altP5ynclPIihy05CZdZSddkp9BSecUNq3xx6w4YYweHBKEHUgf4nAzKwjde8O556bagg335z2vfIKzJ6dmowOOADmzattjC1wIjAz6yif/3xKCMuWwQ47pH2TJqU1mM87r9NeaZSfROA+AjOrlm7d0iWn8+aVEsKJJ6b+hauvrm1szchPIihyH4GZVcuAASkhzJ0Le++d9o0bB337wsUXd5ovqPlLBGZm1bbaanDXXTBlCuy/PyxaBMcdl2oOr79e6+hylAg6SeY1sxwbORLuuAPeew+GD0/71l8/TWMxe3bNwspPIihy05CZ1Vq/fjB9eloHYa210sR2gwfDgQfWJJz8JQIzs85izBj497/TqGVItQUJZs6sahhOBGZmtfajH8HChaX1lYcMSRPeVUl+EoH7CMysM+vTJ62v/J//mbb32w9eeqkqb52fRFDkPgIz68wuuADuvjs93mSTqnyJzV8iMDPr7PbcE7797fT4gAMyf7v8JAI3DZlZPbnoonR/551p6usM5ScRFLlpyMzqQc+eVasV5C8RmJnVi0suSQPP7rkn06uInAjMzDorqbT4zZgxmb1NfhKB+wjMrB594hOw447p0tKMlsPMNBFI2lfSC5KmSzq5meNHSpol6enC7VtZxlN408zfwsysQ+23X7p/5plMXj6zRCCpOzAe2A8YAYyVNKKZon+KiK0LtyuzisfMrG595jPp/h//yOTls6wR7ABMj4gZEfEBMAE4OMP3W7nevdN99+41C8HMrE1GjEifXf/+dyYv3yOTV03WBd4o224APtVMuc9L+jTwIvCDiHijaQFJRwNHA6y33npti+aqq+B3v4Pddmvb883MamXgQJg1Ky10k4FadxbfBgyLiFHA3cC1zRWKiMsjYnREjB48eHDb3mmtteDMM9NCEGZm9WaNNTJr0cjyU3EmMLRse0hh38ciYk5ELClsXglsl2E8ZmbWjCwTwePAxpI2kNQLOAyYWF5A0ifLNscAz2UYj5mZNSOzPoKIWCrpWOAuoDtwdURMk3Q6MDkiJgLHSxoDLAXeAY7MKh4zM2ueos4GWo0ePTomT55c6zDMzOqKpCciYnRzx9xzamaWc04EZmY550RgZpZzTgRmZjlXd53FkmYBr7Xx6YOA2R0YTj3wOeeDzzkf2nPO60dEsyNy6y4RtIekySvqNe+qfM754HPOh6zO2U1DZmY550RgZpZzeUsEl9c6gBrwOeeDzzkfMjnnXPURmJnZ8vJWIzAzsyacCMzMcq5LJgJJ+0p6QdJ0SSc3c3wVSX8qHH9M0rAahNmhKjjnEyQ9K2mKpHslrV+LODtSS+dcVu7zkkJS3V9qWMk5S/pS4Xc9TdIN1Y6xo1Xwt72epPslPVX4+96/FnF2FElXS3pb0tQVHJekiwo/jymStm33m0ZEl7qRprx+GdgQ6AU8A4xoUua7wKWFx4cBf6p13FU4588AfQuPv5OHcy6U6w88BPwTGF3ruKvwe94YeApYo7D9iVrHXYVzvhz4TuHxCODVWsfdznP+NLAtMHUFx/cH7gQE7Ag81t737Io1gh2A6RExIyI+ACYABzcpczClZTFvBj4rSVWMsaO1eM4RcX9ELCxs/pO0Ylw9q+T3DHAG8CtgcTWDy0gl53wUMD4i3gWIiLerHGNHq+ScAygu5rsa8GYV4+twEfEQaX2WFTkY+EMk/wRWb7LIV6t1xUSwLvBG2XZDYV+zZSJiKTAPGFiV6LJRyTmXG0f6RlHPWjznQpV5aETcUc3AMlTJ73kTYBNJ/5D0T0n7Vi26bFRyzj8HjpDUAEwCjqtOaDXT2v/3FmW2Qpl1TpKOAEYDu9U6lixJ6gacR/5WvetBah7anVTre0jSyIiYW8ugMjYWuCYizpW0E3CdpC0j4qNaB1YvumKNYCYwtGx7SGFfs2Uk9SBVJ+dUJbpsVHLOSNoTOBUYExFLqhRbVlo65/7AlsADkl4ltaVOrPMO40p+zw3AxIj4MCJeAV4kJYZ6Vck5jwNuAoiIR4HepMnZuqqK/t9boysmgseBjSVtIKkXqTN4YpMyE4GvFx5/AbgvCr0wdarFc5a0DXAZKQnUe7sxtHDOETEvIgZFxLCIGEbqFxkTEfW8zmklf9u3kmoDSBpEaiqaUcUYO1ol5/w68FkASZuTEsGsqkZZXROBrxWuHtoRmBcR/9eeF+xyTUMRsVTSscBdpCsOro6IaZJOByZHxETgKlL1cTqpU+aw2kXcfhWe82+AfsD/FPrFX4+IMTULup0qPOcupcJzvgvYW9KzwDLgpIio29puhed8InCFpB+QOo6PrOcvdpJuJCXzQYV+j58BPQEi4lJSP8j+wHRgIfCNdr9nHf+8zMysA3TFpiEzM2sFJwIzs5xzIjAzyzknAjOznHMiMDPLOScCs2ZIWibpaUlTJd0mafUOfv1XC9f5I2lBR762WWs5EZg1b1FEbB0RW5LGmnyv1gGZZcWJwKxlj1KY1EvScEl/k/SEpL9L2qywfy1Jt0h6pnDbubD/1kLZaZKOruE5mK1QlxtZbNaRJHUnTV9wVWHX5cC3I+IlSZ8CLgH2AC4CHoyIzxWe069Q/psR8Y6kPsDjkv5czyN9rWtyIjBrXh9JT5NqAs8Bd0vqB+xMaZoOgFUK93sAXwOIiGWkqc0Bjpf0ucLjoaQJ4JwIrFNxIjBr3qKI2FpSX9I8N98DrgHmRsTWlbyApN2BPYGdImKhpAdIE6KZdSruIzBbicKqbseTJjZbCLwi6Yvw8dqxWxWK3ktaAhRJ3SWtRpre/N1CEtiMNBW2WafjRGDWgoh4CphCWgDlcGCcpGeAaZSWTfxP4DOS/gU8QVo7929AD0nPAeeQpsI263Q8+6iZWc65RmBmlnNOBGZmOedEYGaWc04EZmY550RgZpZzTgRmZjnnRGBmlnP/H8rsEYAlZDXoAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.6653673709559564\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.801\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CIFAR10 vs. SVHN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar10.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  4.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covairance norms: [35.333, 56.15467, 24.710087, 24.760406, 39.9555, 33.052032, 34.579536, 37.751755, 43.906643, 69.12952]\n",
      "covariance norm: 20.314365237275208\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar10_train = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "cifar10_train_dataloader = DataLoader(\n",
    "    dataset=cifar10_train, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, cifar10_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:14<00:00, 17.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000 10000\n",
      "id mean score: 185.77122 id variance: 5098.7666\n",
      "ood mean score: 198.51315 ood variance: 7935.3193\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=cifar10_test_dataloader,\n",
    "    out_of_dist_dataloader=svhn_test_dataloader,\n",
    ")\n",
    "print(len(id_ood_scores), len(ood_ood_scores))\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, airplane: 1219, mean: 200.83541870117188, var: 5953.31494140625\n",
      "1, automobile: 982, mean: 180.28318786621094, var: 5549.2197265625\n",
      "2, bird: 1458, mean: 203.9126739501953, var: 4917.78173828125\n",
      "3, cat: 1712, mean: 218.58245849609375, var: 5362.91015625\n",
      "4, deer: 620, mean: 152.279052734375, var: 2285.1669921875\n",
      "5, dog: 791, mean: 162.62451171875, var: 2515.4228515625\n",
      "6, frog: 810, mean: 178.7567901611328, var: 4185.275390625\n",
      "7, horse: 767, mean: 164.74615478515625, var: 4294.2294921875\n",
      "8, ship: 916, mean: 177.46817016601562, var: 4754.640625\n",
      "9, truck: 725, mean: 148.37918090820312, var: 3145.055908203125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:14<00:00, 17.99it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, airplane: 3865, mean: 277.4150695800781, var: 6065.61767578125\n",
      "1, automobile: 12, mean: 330.41131591796875, var: 5214.02587890625\n",
      "2, bird: 5620, mean: 122.05317687988281, var: 2861.732666015625\n",
      "3, cat: 16308, mean: 205.9731903076172, var: 6601.2509765625\n",
      "4, deer: 3, mean: 85.57693481445312, var: 152.88162231445312\n",
      "5, dog: 46, mean: 155.63922119140625, var: 1258.68798828125\n",
      "6, frog: 149, mean: 241.08758544921875, var: 8427.986328125\n",
      "7, horse: 1, mean: 146.46051025390625, var: 0.0\n",
      "8, ship: 28, mean: 277.72357177734375, var: 6558.03076171875\n"
     ]
    }
   ],
   "source": [
    "mahalanobis.compute_dataset_ood_score(cifar10_test_dataloader)\n",
    "classes = set(mahalanobis.closest_classes)\n",
    "for c in classes:\n",
    "    tmd = np.array(mahalanobis.total_mahalanobis_distances)\n",
    "    cc = np.array(mahalanobis.closest_classes)\n",
    "    c_tmd = tmd[list(np.where(cc == c)[0])]\n",
    "    n_samples = mahalanobis.closest_classes.count(c)\n",
    "    print(f'{c}, {cifar10.classes[c]}: {n_samples}, mean: {c_tmd.mean()}, var: {c_tmd.var()}')\n",
    "\n",
    "mahalanobis.compute_dataset_ood_score(svhn_test_dataloader)\n",
    "classes = set(mahalanobis.closest_classes)\n",
    "for c in classes:\n",
    "    tmd = np.array(mahalanobis.total_mahalanobis_distances)\n",
    "    cc = np.array(mahalanobis.closest_classes)\n",
    "    c_tmd = tmd[list(np.where(cc == c)[0])]\n",
    "    n_samples = mahalanobis.closest_classes.count(c)\n",
    "    print(f'{c}, {cifar10.classes[c]}: {n_samples}, mean: {c_tmd.mean()}, var: {c_tmd.var()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABB/0lEQVR4nO3dd3gU1dfA8e8hQSnSwUbvEIqgUUSkS1MQG4piQaMIiFgQRVFURMSCIr0IglJEUTAKihXxFem9SBFBqnSkl+S8f9zJzzWmbEgmm3I+z5MnuzvtzGw5c++duVdUFWOMMdlXjlAHYIwxJrQsERhjTDZnicAYY7I5SwTGGJPNWSIwxphszhKBMcZkc5YIsjgRWSMijUIdR0YhIs+JyHsh2vZ4EekXim2nNRHpICLfnOOy9pnMYCwRpCMR2SIiJ0TkqIjs9n4YLvBzm6paTVXn+LmNOCJyvoi8JiJ/evu5UUR6ioikx/YTiKeRiGwPfE1V+6vqgz5tT0Sku4isFpFjIrJdRD4RkRp+bO9cichLIjIxNetQ1Umq2jyIbf0n+Z3rZ1JEzvNi3+gd3y0iMk5EyqR0XebfLBGkvzaqegFQC6gNPBvacFJORMITmfQJ0BS4HsgH3AN0At71IQYRkYz2+X0XeAzoDhQGKgEzgBvSekNJvAe+C+G2pwE3AncBBYDLgCW4z1yKhPL4ZUiqan/p9AdsAa4LeP4GMDPg+dXAPOAQsAJoFDCtMPA+sBM4CMwImNYaWO4tNw+oGX+bwKXACaBwwLTawD4gp/f8AWCdt/7ZQOmAeRV4BNgI/JHAvjUFTgIl471eB4gBKnjP5wCvAQuBv4HP48WU1DGYA7wK/OLtSwXgfi/mI8Bm4GFv3rzePLHAUe/vUuAlYKI3Txlvv+4D/vSORe+A7eUGJnjHYx3wNLA9kfe2orefVyXx/o8HhgEzvXgXAOUDpr8LbPOOyxKgfsC0l3A/hBO96Q8CVwG/esdqFzAUOC9gmWrAt8AB4C/gOaAlcBo44x2TFd68BYCx3np2AP2AMG9aR++YvwPs96Z1BP7Pmy7etD1ebKuA6riTgDPe9o4CX8T/HgBhXly/e8dkCfE+Q95813nv53+mJfH9Sui9jvLe67nAV0C3eOtYAdziPa4ScPzWA7eH+jfEt9+mUAeQnf7ifQFKeF+Yd73nxb0v2fW4kloz73kxb/pMYCpQCMgJNPRer+19Aet4X6r7vO2cn8A2fwAeCojnTWCk97gtsAmoCoQDzwPzAuZV70tRGMidwL4NAH5KZL+38s8P9Bzvh6Y67sf604Ava3LHYI73Ja7mxZgTd7ZdHvdj1BA4Dlzuzd+IeD/cifw4jMH96F8GnAKqBu6Td8xLACvjry9gvZ2Brcm8/+O9/bnKi38S8FHA9LuBIt60HsBuIFdA3GeAm7xjkxu4Apc4w719WQc87s2fD/ej3gPI5T2vE/8YBGx7OjDKe08uxCXquPesI3AWeNTbVm7+nQha4H7AC3rvQ1XgkoB97pfE96An7ntQ2Vv2MqBISj5fCa03iff6A28fcwP3Ar8EzB+BS6rne/Nsw51ohPPPSVNEqH9H/PjLaEXr7GCGiBzBfcj2AC96r98NzFLVWaoaq6rfAouB60XkEqAV0FlVD6rqGVX9yVuuEzBKVReoaoyqTsD9mF2dwLYnA3eCq1oB2nuvgfshe01V16nqWaA/UEtESgcs/5qqHlDVEwmsuyjuhychu7zpcT5U1dWqegx4AbhdRMKSOgYBy45X1TWqetY7DjNV9Xd1fgK+AeonEkdiXlbVE6q6AndGeJn3+u1Af++YbwcGJ7GOIknsf6DpqrrQO8aTcFWEAKjqRFXd7+3bQNwPUuWAZX9V1RnesTmhqktUdb43/xbcD3lDb97WwG5VHaiqJ1X1iKouSCggEbkId4wfV9VjqroHd4bfPmC2nao6xNtW/Pf/DC7RVAHE+wwFcyzAlWyeV9X13nu4QlX3JzBfsMc3OS95+3gCl/wCP+MdgM9U9RTu+G1R1fe9fV6GO2lplwYxZDiWCNLfTaqaD3e2WoV/fiBLA+1E5FDcH3AtcAlQEjigqgcTWF9poEe85UriqkHi+xSo6yWWBrhqk58D1vNuwDoO4M7Qigcsvy2J/drnxZqQS7zpCa1nK+7MvihJH4MEYxCRViIyX0QOePNfz7+TTjB2Bzw+DsQ14F8ab3tJ7f9+Et//YLaFiDwlIutE5LC3LwX4977E3/dKIvKld+HB37jkHTd/SVx1SzBK496DXQHHfRSuZJDgtgOp6g+4aqlhwB4RGS0i+YPcdrBxBnt8k/O//VDVI7iSdlzCuxOXnMEdkzrxPosdgIvTIIYMxxJBiHhnr+OBt7yXtuHOlAsG/OVV1QHetMIiUjCBVW0DXo23XB5VnZLANg/izpjvwDW4faSqGrCeh+OtJ7eqzgtcRRK79B3ui1My8EURqYP7sv8Q8HLgPKVwZ5T7kjkG/4lBRM7HJbe3gItUtSAwC5fAkos3GLtwVUIJxR3f90AJEYk8lw2JSH1cG8TtQCFvXw7zz77Af/dnBPAbUFFV8+Pq2uPm3waUS2Rz8dezDVeKLBpw3POrarUklvn3ClUHq+oVuOqVSrgqn2SX87ZdPpl5wH2+rhKREknMcwzIE/A8oR/t+PFMAe4Ukbq4KrQfA+L6Kd5n8QJV7RJErJmOJYLQGgQ0E5HLcI2AbUSkhYiEiUgu7/LHEl4x+ytguIgUEpGcItLAW8cYoLOI1PGupMkrIjeISL5EtjkZVzd6G/9UCwGMBJ4VkWoAIlJARIIuBqvqd7gfw09FpJq3D1d7+zVCVTcGzH63iESISB6gLzBNVWOSOgaJbPY8XPXJXuCsiLQCAi9p/AsoIiIFgt2PeD7GHZNCIlIc6JbYjN7+DQemeDGf58XfXkR6BbGtfLh6+L1AuIj0AZI7q86Ha5w9KiJVgMAfqS+BS0TkcXGX9ebzkjK441Im7qor7/P1DTBQRPKLSA4RKS8iDQmCiFzpff5y4n6MT+JKm3HbSiwhAbwHvCIiFb3Pb00RKRJ/Ju/z9S0wXUSuEJFwb586i8gD3mzLgfbe9yMS9xlPzizc2X9fYKqqxsX9JVBJRO7x1pfT28+qQawz07FEEEKquhfXeNVHVbfhGmyfw/0YbMOdVcW9R/fgzpx/w7UtPO6tYzHwEK5ofhDX4Nsxic1G465w2e3VicfFMh14HfjIq2ZYjWuXSIlbcWdUX+OuEpmIuxLl0XjzfYgrDe3GnYV192JI7hj8i1e07477wT6IK+VEB0z/DXfGt9kr3idUXZaUvsB24A/cGek03JlzYrrzTxXJIVyVx83AF0FsazbuuG3AVZedJOmqKICncPt8BHdCMDVugndsmgFtcMd5I9DYm/yJ93+/iCz1Ht+LS6xrccdyGsFXxeT3tn/Qi30/7kIEcO9/hHf8ZySw7Nu49+8bXFIbi2vITchtuB/uqbjS0mogEvfegGtvKu/F8TL/PtFJkNce8BnuqqTJAa8fwZ1UtMddqbcb9/04P7l1ZkbyT82AMf4TkTm4KzlCcndvaohIF6C9qgZ1pmxMZmElAmMSISKXiEg9r6qkMu5SzOmhjsuYtGZ31xmTuPNwV8+UxVX1fIRrBzAmS7GqIWOMyeasasgYY7K5TFc1VLRoUS1TpkyowzDGmExlyZIl+1S1WELTMl0iKFOmDIsXLw51GMYYk6mIyNbEplnVkDHGZHOWCIwxJpuzRGCMMdlcpmsjSMiZM2fYvn07J0+eDHUovsmVKxclSpQgZ86coQ7FGJPFZIlEsH37dvLly0eZMmWQ0AyP6ytVZf/+/Wzfvp2yZcuGOhxjTBbjW9WQuEGl94jI6kSmi4gMFpFNIrJSRC4/122dPHmSIkWKZMkkACAiFClSJEuXeIwxoeNnG8F43PioiWmF6wWzIm6UrRGp2VhWTQJxsvr+GWNCx7dEoKpzcaNcJaYt8IE3PN18oKC4kbOMMcYEOLZpHVtefhyWL/dl/aG8aqg4/+5vfTv/Hhbxf0Skk4gsFpHFe/fuTZfgUiosLIxatWpRvXp12rRpw6FDh/43bc2aNTRp0oTKlStTsWJFXnnlFQL7ePrqq6+IjIwkIiKC2rVr06NHjxDsgTEmI/ph8qvUHBrBLbveJfbXeckvcA4yxeWjqjpaVSNVNbJYsQTvkA653Llzs3z5clavXk3hwoUZNmwYACdOnODGG2+kV69erF+/nhUrVjBv3jyGD3edWK5evZpu3boxceJE1q5dy+LFi6lQoUIod8UYkwEcOnmIh967kaYbnydHeE7euWciObp09WVboUwEO/j3GLAlvNcyvbp167Jjh9uVyZMnU69ePZo3dyMo5smTh6FDhzJggBuG94033qB3795UqVIFcCWLLl2y5LCoxpggxcTGcM3oOozb9gVPr8rPyic20LBeB9+2F8rLR6OBbiLyEVAHOOyNnZo6jz+e9vVotWrBoEFBzRoTE8P3339PVFQU4KqFrrjiin/NU758eY4ePcrff//N6tWrrSrIGAPA/uP7KZy7MGE5wnh1Xm5K/hpO5PgZULyMr9v18/LRKcCvQGUR2S4iUd5A0529WWYBm3Fj7I4B/CnzpJMTJ05Qq1YtLr74Yv766y+aNWsW6pCMMZmEqjJx5UQqDa3Ee0vfg48/5uapK4i8vzc0bpz8ClLJtxKBqt6ZzHQFHknzDQd55p7W4toIjh8/TosWLRg2bBjdu3cnIiKCuXPn/mvezZs3c8EFF5A/f36qVavGkiVLuOyyy0IStzEmtLYd3kbnmZ2ZtXEWV5e4mnpL/oLOD0PFitCrV7rEkCkaizOTPHnyMHjwYAYOHMjZs2fp0KED//d//8d3330HuJJD9+7defrppwHo2bMn/fv3Z8OGDQDExsYycuTIkMVvjEk/U1ZNodrwaszZModBLQbxf6VfJqJzH6hXDxYtgly50iUOSwQ+qF27NjVr1mTKlCnkzp2bzz//nH79+lG5cmVq1KjBlVdeSbdu3QCoWbMmgwYN4s4776Rq1apUr16dzZs3h3gPjDHpoVDuQtQpUYfVXVbz2EU3EnbTLVCpEkyfDgUKpFscmW7M4sjISI0/MM26deuoWrVqiCJKP9llP43Jqs7GnuWdX9/hdMxpejfoDbj2ATlwAK67DtavhxUrXLVQGhORJaoamdC0LNHpnDHGZHQrdq8gKjqKJbuWcHu1210CEEEOHYImTWDdOpg40ZckkBxLBMYY46NTZ0/Rb24/BvwygMK5C/NJu0+4teqtrv+w06ehbVtYuRJmzHCPQyDLJIK47JpVZbYqPGOMs/HARl7/5XXuqnEXbzd/myJ5irgJqtClC/z8MwwfHrIkAFmksThXrlzs378/y/5Yxo1HkCudriAwxqTO0dNHmbRyEgDVL6zOb91+Y8JNE/5JArGx0KkTjBsHDz/sEkIIZYkSQYkSJdi+fTsZtUO6tBA3QpkxJmP79vdv6fRlJ7Ye2srll1xO1WJVKVeo3D8znDkD7dvDZ59B164weHDogvVkiUSQM2dOG7nLGBNSB08c5KlvnmLc8nFUKlKJnzr+RNViCVzl9+STLgk89xz06wcZoEo7SyQCY4wJpZjYGOqNq8eG/Rt49tpn6dOwD7nC41Xlqro7hYcOddVBr74ammATYInAGGPO0b7j+/7XSVz/pv0pVaAUl1+SyKi7zz0Hb7zhqoWGDEnfQJORJRqLjTEmPakqH6z4gEpDvE7igJuq3JR4EvjqKxgwAG6+GSZPhpw50zHa5FkiMMaYFNh6aCutJrXivhn3UbVYVRqUbpD0Al9+6S4NrVoVxo7NEG0C8VnVkDHGBGniyol0mdkFVWVIqyF0vbIrOSSJ8+lvv4XbboPy5eHHH6FQofQLNgUsERhjTJCK5SlGvZL1GNV6FKULlk565nnz4KabXBKYMwcy6DC7YInAGGMSdSbmDAN/HciZmDO80PAFWlRoQfPyzZPvxWDcOHjkEShSBL77LkMnAbA2AmOMSdCyXcuo814dnv3+WdbuW/u/nguSTAJnzkBUlPurVs2VBC65JH0CTgVLBMYYE+Dk2ZM89/1zXDnmSnYe2cmnt3/KlFunJF8K2LMHbrnFlQYefRR+/RUqVEifoFPJqoaMMSbApgObeGveW9x72b0MbD6QQrmDaOD96Sdo0waOHYP+/eHZZ/0PNA1ZIjDGZHtHTx9l+rrp3HPZPVS/sDrru62nbKEgu62ZMQPuvBMuvRR++AEiExz7JUOzRGCMydZmb5pNpy87se3wNiIvjaRqsarBJ4HoaHd5aMWKrj3goot8jdUv1kZgjMmW9h/fz30z7qPlpJbkyZmHn+//OeFO4hKi6rqLuOkmqF7dVQ1l0iQAViIwxmRDcZ3EbTqwid71e/N8g+f/20lcYrZudd1Hz5oFN9wAU6dC3rz+BuwzSwTGmGxj77G9FMlThLAcYbx+3euULliaWhfXCn4F337rrgw6dQpeegmefx7CwvwKN91Y1ZAxJstTVd5f9j6VhlZizJIxALSt0jZlSWDQIGjZ0t0XsGQJvPhilkgCYCUCY0wWt+XQFjp90YlvN39L/VL1aVy2ccpWcOaMO/vv3x+uuAK+/x4KFPAl1lCxRGCMybI+XPEhXWZ2QUQYfv1wHo58OOlO4uLbs8f1HDp/vhtHYPRoyJfPv4BDxBKBMSbLuuiCi2hQugEjW4+kVIFSwS+o6hqBH3kEDh923Uc/8IB/gYaYJQJjTJZxJuYMb/zyBjEaQ5+GfWhevjnNyzdP2UpiYqB7dxg+HCpXhs8/h2uv9SfgDMISgTEmS1i6aykPfP4AK/5awV017kJVk+8fKL5Vq6BjR1i61HUcN3IkhGf9n8msv4fGmCztxJkTvPzTy7w17y2K5S3G9Dumc1OVm1K+ovnzoXVrVyKYMAHuuSdDjibmB18vHxWRliKyXkQ2iUivBKaXEpEfRWSZiKwUkev9jMcYk/VsPriZt399m461OrK269pzSwKDBkH9+m4s4Xnz4N57s00SAB8TgYiEAcOAVkAEcKeIRMSb7XngY1WtDbQHhvsVjzEm6/j71N+MXz4egGoXVmPjoxt578b3guspNNCGDa7X0CeegLp1XZVQ1SC7mchC/CwRXAVsUtXNqnoa+AhoG28eBfJ7jwsAO32MxxiTBczaOIvqw6sTFR3Fur3rAJIfNjIh770HNWu6u4VffNHdH5AJBpHxg59tBMWBbQHPtwN14s3zEvCNiDwK5AWuS2hFItIJ6ARQqlQKLgEzxmQZ+47v44nZTzBx5UQiikXwS7tfgu8kLtDff8ODD8Inn7irgSZPhpIl0z7gTCTUXUzcCYxX1RLA9cCHIv+920NVR6tqpKpGFsvgY38aY9JeXCdxH63+iD4N+rC001KuLnF1ylf03Xdw2WUuCfTu7UoB2TwJgL8lgh1A4BEu4b0WKApoCaCqv4pILqAosMfHuIwxmcRfR/+iWN5ihOUI461mb1G6YGlqXlQz5Svau9e1A0yaBBdf7HoObdUq7QPOpPwsESwCKopIWRE5D9cYHB1vnj+BpgAiUhXIBez1MSZjTCagqoxdOpbKQyszesloANpUbpPyJLB3LzzzDJQvD1OmQK9eroHYksC/+FYiUNWzItINmA2EAeNUdY2I9AUWq2o00AMYIyJP4BqOO6qq+hWTMSbj23xwMw998RA//PEDDUs35LpyCTYdJk3V3QvQowccOOD6C3rxRahdO+0DzgJ8vaFMVWcBs+K91ifg8Vqgnp8xGGMyjwnLJ9B1VlfCJIyRN4zkoSseSlkncQDffANPPeXuEq5UCb780l0aahIV6sZiY4z5n0vzXUqTsk1Y+8jalPcUunIlNGsGLVq4juImTIB16ywJBMG6mDDGhMzpmNMM+L8BxGosLzV6iWblm9GsfLOUreTECTdewNtvuy6i+/VzDcN58vgSc1ZkicAYExKLdizigegHWL1nNffUvOfcOonbts0NHbl4Mdx8MwwdCpde6k/AWZhVDRlj0tXxM8d56punuHrs1Rw8cZDo9tF8cPMHKUsCqjBjBlx1FaxeDR98AJ99ZkngHFkiMMakqz8O/sGQhUN46PKHWNN1DW0qt0nZChYsgKuvdiWAQoVcJ3H33ONPsNmEJQJjjO8OnzzM+8veB1wncZse3cTI1iMpkCsFY/+eOAHPPQf16sHvv8OAAbBsmV0SmgasjcAY46uZG2by8JcPs+voLuqWrEuVolUoWSAF3TrExsKPP0K3bvDbb3DXXTBkCBQu7F/Q2YyVCIwxvth7bC8dPutA6ymtKZS7EL9G/UqVolVStpI1a+CKK+C66+DgQTds5KRJlgTSmJUIjDFpLiY2hmvfv5Y/Dv7By41epte1vTgv7LzgV3D0qLsM9PXXIXdu12V0+/aQN69/QWdjlgiMMWlm99HdXJj3QsJyhDGw+UDKFCxD9QurB7+Co0dh7Fh49VXXT9Add8DLL7tB5I1vgq4aEhG7O8MYk6BYjWXU4lFUGlKJUYtHAdC6Uuvgk8DJkzBwIBQvDo8/DqVLu3aBjz6yJJAOkk0EInKNiKwFfvOeXyYiNqSkMQaATQc20fSDpnSe2Zkri19JiwotUraCn3927QBPPeX+//gjLFwIjRr5Eq/5r2Cqht4BWuB1Ia2qK0Skga9RGWMyhfeXvU/XWV05L+w8xrQZQ1TtqOBvDFu+HHr2dIPFXHQRfPqpuzcgGw0an1EE1Uagqtvivbkx/oRjjMlMShUoRYvyLRh2/TCK5y8e3EJHj7o2gNdfhwsucGMEPP20uznMhEQwiWCbiFwDqIjkBB4D1vkbljEmIzp19hSv/d9rxGosfRv3pWm5pjQt1zS4hf/8Ezp1cncCHzni7gcYPBiKFPE3aJOsYBqLOwOP4Aaj3wHUArr6GJMxJgNasH0BV4y+gpd/epk/D/9J0GNIbdgAUVFQoYKrBipf3rULTJpkSSCDCKZEUFlVOwS+ICL1gF/8CckYk5EcO32MF358gUHzB1E8f3G+vPNLbqh0Q/ILbtzouoYeNQpy5IAOHdyA8ZUq+R+0SZFgSgRDgnzNGJMFbT28leGLhtM5sjNruq5JPgkcOQJPPul+8MeOhYcegj/+cAPFWBLIkBItEYhIXeAaoJiIPBkwKT9uDGJjTBZ16OQhpq2dxoOXP0hEsQg2dd9Eifwlkl7o5El4913XEHzkCNx7L7z2mnUNnQkkVTV0HnCBN0++gNf/Bm7zMyhjTOh8/tvndJnZhT3H9nBtqWupUrRK0kngxAkYNszdELZ7N7Rs6QaKv/rq9AvapEqiiUBVfwJ+EpHxqro1HWMyxoTAnmN76P5Vd6aumUrNi2oSfWd08p3Effgh9OjhuoOoX99VBbVqZfcCZDLBNBYfF5E3gWpArrgXVbWJb1EZY9JVTGwM9cbV48/Df9KvcT+ervc0OcNyJjzz6dPwzjtuhLD5893dwOPGQevW6RqzSTvBJIJJwFSgNe5S0vuAvX4GZYxJHzuP7OTiCy4mLEcY77Z8lzIFyxBRLCLhmVVh4kTXCdzvv0OBAvDGG26g+HDrvzIzC+aqoSKqOhY4o6o/qeoDgJUGjMnEYjWWEYtGUGVoFUYuHgnA9RWvTzgJnDnjqoAiI10DcEwMTJvmxgfo2dOSQBYQzDt4xvu/S0RuAHYCNiqEMZnUhv0beOiLh5i7dS7XlbuOVhVaJTxjbKy7B+DVV2HHDndD2Jgx8MAD7r4Ak2UEkwj6iUgBoAfu/oH8wON+BmWM8cfYpWPp9lU3coXnYtyN4+hYq2PCncTNmuV6A123DurWheHDXRuAJYAsKdlEoKpfeg8PA43hf3cWG2MymTIFy9CqQiuGXT+MS/Jd8u+Jqq4foFdegdmzoWRJdxXQ/ffbVUBZXFI3lIUBt+P6GPpaVVeLSGvgOSA3UDt9QjTGnKtTZ0/xytxXAOjXpF/CncSdOgXR0TBgACxd6oaGfP55eO4599hkeUmVCMYCJYGFwGAR2QlEAr1UdUY6xGaMSYV52+YRFR3Fb/t+44FaD6Cq/64Gio2F8eOhf393FdDFF8OgQdCxo7siyGQbSSWCSKCmqsaKSC5gN1BeVfenT2jGmHNx9PRRen/fmyELh1CyQEm+7vD1v0cNO3HCdQUxZgxs3ux6A50xA66/HnImcu+AydKSavk5raqxAKp6Etic0iQgIi1FZL2IbBKRXonMc7uIrBWRNSIyOSXrN8b815+H/2TUklE8cuUjrO6y+p8ksGuXGwOgTBl49lk3PvCHH8L69dC2rSWBbCypEkEVEVnpPRagvPdcAFXVmkmt2GtjGAY0A7YDi0QkWlXXBsxTEXgWqKeqB0XkwlTsizHZ1sETB/lk7Sd0uqITEcUi2PzYZi7N53X2tmYN9OvnBoIPC4Mrr4TJk6FpkAPKmCwvqURQNZXrvgrYpKqbAUTkI6AtsDZgnoeAYap6EEBV96Rym8ZkO9PXTafrrK7sPbaXhqUbUrloZZcEtm+Hl15y3T+cf74bHObJJyEikTuHTbaVVKdzqe1orjiwLeD5dqBOvHkqAYjIL7iurV9S1a/jr0hEOgGdAEqVKpXKsIzJGnYf3c2jXz3KtLXTqHVxLWbeNZPKRSrBokUwciRMnQrHjsEjj7iEULRoqEM2GVSo7w0PByoCjYASwFwRqaGqhwJnUtXRwGiAyMjIIMfHMybriomNof779dl2eBv9m/TnqWueIufuPdCmDcycCeedB7fc4i4BrVEj1OGaDM7PRLADd/lpnBLea4G2AwtU9Qzwh4hswCWGRT7GZUymtf3v7Vya71LCcoQxuOVgyhYqS5XNf8ODnVy9vyr06uU6grvQmtxMcIK6X1xEcotI5RSuexFQUUTKish5QHsgOt48M3ClAUSkKK6qaHMKt2NMlhersQxZMIQqQ6swYtEIOHCAVh8vpUq9tlCnjqsGuu8+WLXKjQpmScCkQLIlAhFpA7yFG7GsrIjUAvqq6o1JLaeqZ0WkGzAbV/8/TlXXiEhfYLGqRnvTmovIWiAG6Gn3KRjzb7/t+40Hox/kl22/0KJ0U1pH/wYDi7uhIevXd20A994LBQuGOlSTSYlq0lXuIrIE1+30HFWt7b22SlVDUvEYGRmpixcvDsWmjUl37y19j26zupEnPDeDTjTgnhHzkL374Lbb4IUXoGaSV3Eb8z8iskRVIxOaFlQ31Kp6OF4PhdZga0w6KK8FaXOsOENH7+CiA9FQvTp8Nh2uvTbUoZksJJhEsEZE7gLCvBvAugPz/A3LmOzp5NmT9P2pLxw+TP9fctF4xAganzrl7gh+5BEbEN74IphE8CjQGzgFTMbV6/fzMyhjsqNftswl6uO7WX9iGw8uAZ2VA7mjvRsaskKFUIdnsrBgEkEVVe2NSwbGmDR2ZP9OnhvXgWHH5lD6EMz+MS/Nb+oBQx6A0qVDHZ7JBoK5fHSgiKwTkVdEpLrvERmTXezZAwMGsP2a6rx3eA6Pbi7GqoghNP9puysFWBIw6SSYEcoai8jFuEFqRolIfmCqqlr1kDHnYu9e9g8ewMdfv02XxVC1Xj0213+CS5rfYiOBmZAI6oYyVd2tqoOBzsByoI+fQRmTJa1ahd5yM9MaX0TEybfpfoOw/vuP4f/+j0ta3GpJwIRMsolARKqKyEsisgo3eP08XHcRxpjkHD4MQ4ZAw4bsuqYmt+b+gnbtlJLFq7K48zIqN2kX6giNCaqxeBwwFWihqjt9jseYrCEmxg38/vLLsHMnMRXKUb9HIXaEn+CNxq/xRN0nCM8R6j4fjXGCaSOomx6BGJMlqMK0afD447BzJ9vqRlB80kTCGjVm2KbZlC1UlkpFKoU6SmP+JdGqIRH52Pu/SkRWBvytChi5zBgDLgHMnQuRkXD77cQULsTgIXdTpfUWRuRxYzG1qNDCkoDJkJIqETzm/W+dHoEYkynFxLhxf/v1g99/h4svZt2QPkTl+oZfd0ykVYVWtKncJtRRGpOkREsEqrrLe9hVVbcG/gFd0yc8YzKwBQugSRO4/34ID4fhwxn9SS9qHRrAhgMb+fDmD5l510xKFbBR9UzGFszlo80SeK1VWgdiTKaxYAG0bOn6/Vm2DAYNcuMAdOlCxeI1ubnKzax9ZC1317wbsUtCTSaQVBtBF++S0crx2gj+AKyNwGQvcY3AjRq5BLB4MfTvz4mtv/NMxE56/fQCAI3LNuaj2z7iwrw2MIzJPJJqI5gMfAW8BvQKeP2Iqh7wNSpjMoozZ+Czz+Cdd1xJ4JJLYMAA6NyZuYdW8ODEemw8sJHOV3RGVa0EYDKlpBKBquoWEXkk/gQRKWzJwGRZqu6M/913YdIk91qpUjByJDz4IH+fPUav73oxYvEIyhUqx/f3fk+Tsk1CG7MxqZBciaA1sAQ3EE3gqY4C5XyMy5j0FxsLs2fDq6/CL79AzpzQtCncfbf7C3dfl50HdzJ++XievPpJ+jbuS97z8oY4cGNSJ9FEoKqtvf9l0y8cY0Lg9GmIjoZnnoHNm+Gii6B/f3joIShaFIB9x/fx8bKP6XplV6oUrcIfj/3BRRdcFOLAjUkbwfQ1VE9E8nqP7xaRt0XEroczmd/u3e7sv2xZaNfOlQgGDHD3Azz7LBQtiqoydfVUIoZF8PjXj7Nh/wYASwImSwnm8tERwHERuQzoAfwOfOhrVMb4RRVWrIAbb3QNv88/D1WqwBdfwMaNrlSQ11X17Dyyk5um3kT7T9tTumBplnRaYncGmywpmF6vzqqqikhbYKiqjhWRKL8DMyZNxcbC119Dt27wxx+uvr9dO+jTxw0IH09MbAwN3m/AjiM7eKvZWzx29WPWSZzJsoL5ZB8RkWeBe4D6IpIDyOlvWMakkZ073dU+770Hu3ZBiRKuOui++6B48f/MvvXQVkrkL0FYjjCG3zCccoXKUaGwjRdssrZgqobuwA1c/4Cq7saNRfCmr1EZk1rbtsETT7j6/1deccM+TpoEGzbAc8/9JwnExMbw9q9vU3VYVUYsHgFA8/LNLQmYbCGYbqh3i8gk4EoRaQ0sVNUP/A/NmHNw5Ai8+Sa8/rq7Guiqq2DcOKhWLdFFVu9ZTVR0FAt3LKR1pdbcVOWm9IvXmAwgmKuGbgcWAu1w4xYvEJHb/A7MmBTZscOVAIoXdyWAZs1g/Xp3N3ASSWDk4pFcPupyNh/czORbJhPdPpoS+W0APpO9BNNG0Bu4UlX3AIhIMeA7YJqfgRkTlB07XPcP777rGoRvvx26d4e6SY+nFNcdRNWiVWlXrR2DWgyiWN5i6RS0MRlLMIkgR1wS8OwnyEHvjfHFqVOuA7hRo+Dnn91r7du7RuBySd/wfvzMcfr82IcwCeP1Zq/TsExDGpZpmA5BG5NxBZMIvhaR2cAU7/kdwCz/QjImEUePujP/QYNg3z73o//SS64UULVqsovP2TKHB6Mf5PeDv9M1sqt1EmeMJ5jG4p4icgtwrffSaFWd7m9YxgT46y+XAEaMgEOH4JprYMwYd1NYjuQLp4dPHubpb59m9NLRlC9Unh/u/YHGZRv7H7cxmUSiiUBEKgJvAeWBVcBTqrojvQIz2VxsLPz4I7z/Pnz8sesO+oYboEcPaJyyH/FdR3cxcdVEnqr7FC83fpk8OfP4FLQxmVNSp1PjgC+BW3E9kA5J6cpFpKWIrBeRTSLSK4n5bhURFZHIlG7DZDHHjsGQIRARAdddB9Onu/r/pUvhyy+DTgJ7j+1lyAL3ka1StApbHtvCm83ftCRgTAKSqhrKp6pjvMfrRWRpSlYsImHAMNxQl9uBRSISrapr482XD3gMWJCS9ZssJq4B+Jln3JVAVaq4xuC774Y8wf94qypTVk+h+1fd+fvU37So0IJKRSrZFUHGJCGpEkEuEaktIpeLyOVA7njPk3MVsElVN6vqaeAjoG0C870CvA6cTHH0JvOLjYXPP4eKFd2PfsGCMGcOrF0LnTqlKAlsO7yNNlPa0OGzDlQoXIFlDy+zTuKMCUJSJYJdwNsBz3cHPFcguSGZigPbAp5vB+oEzuAllJKqOlNEeia2IhHpBHQCKFXKesDOEmJi4NNP4bXXYPlyNwLYF19Aq1YQFpbi1Z2NPUujCY3YfXQ377R4h0evepSwHClfjzHZUVID0/h6WYXXed3bQMfk5lXV0cBogMjISPUzLuOzmBh39c8bb7j+gEqWhOHD4f77IVeuFK9uy6EtlMxfkvAc4YxqPYpyhcpRrpANnmdMSvh5Y9gOoGTA8xLea3HyAdWBOSKyBbgaiLYG4ywqJgZGj3ZdPj/6qOsFdMoU1wlcly4pTgJnY8/y1ry3qDqsKsMXDQfgunLXWRIw5hz42cH6IqCiiJTFJYD2wF1xE1X1MFA07rmIzMFdorrYx5hMejtwAN5+GyZPduMA1KzpLgm9996g7gFIyMq/VhIVHcXinYtpW7ktt0bcmsZBG5O9+JYIVPWsiHQDZgNhwDhVXSMifYHFqhrt17ZNBrBnj+v1s39/1yNo6dLw0UfuLuBU3M07fNFwHvv6MQrlKsTU26bSLqKd3R1sTColmwjEfcs6AOVUta83XvHFqrowuWVVdRbxuqNQ1T6JzNsoqIhNxnbsmKv/f+cdlwAaNXKPa9VK1WrjuoOofmF12ldvzzst3qFonqLJL2iMSVYwJYLhQCzuKqG+wBHgU+BKH+Mymc3x467Tt2HD4PBhaNPG9QN0eTBXGifu2OljPP/D84TnCOfN5m/SoHQDGpRukDYxG2OA4BqL66jqI3jX+avqQeA8X6MymcfJk24QmBIlXDVQgwbw008QHZ3qJPD95u+pMaIGgxYM4lTMKVTtgjFj/BBMieCMd5ewwv/GI4j1NSqT8R07Bm+95S793LMHWraEXr2gYeq7dD508hBPffMUY5eNpWLhisztOJf6peunQdDGmIQEkwgGA9OBC0XkVeA24HlfozIZl6rrDO7+++HPP6F5c9cRXLNmqWoEDvTX0b/4aPVHPFPvGV5s+CK5c+ZOk/UaYxIWTDfUk0RkCdAUEOAmVV3ne2Qm45k/H556Cn75xQ0J+fXX0KJFmqw67sf/sasfo3LRymx5fIs1BhuTToIZs7gUcBz4AogGjnmvmexi92649VY3/OPGjTBwoOsLKA2SgKoyceVEIoZH8PR3T7Nx/0YASwLGpKNgqoZm4toHBMgFlAXWA4mPCG6yhh07oF8/+OADNx7Aiy+6AeILFEiT1f95+E86f9mZrzZ9Rd0SdRl741gqFqmYJus2xgQvmKqhGoHPvY7iuvoWkQm9jRvhzTdh/HjXJtCuHbzwQlDDQQbrbOxZGo1vxJ5jexjccjBdr+xqncQZEyIpvrNYVZeKSJ3k5zSZSkyMu+RzyBDXGAxwxx3wyiuui+g0svngZkoXKE14jnDGtBlD+cLlKVOwTJqt3xiTcsHcWfxkwNMcwOXATt8iMunv22/dpZ9Ll0LevNCnjxsLoHjxNNvE2dizDJw3kBfnvMgbzd6ge53uNC3XNM3Wb4w5d8GUCPIFPD6LazP41J9wTLqaPx+efx6+/94NADN0qEsAOXOm6WaW715OVHQUS3ct5eYqN9Muol2art8YkzpJJgLvRrJ8qvpUOsVj/HbgAIwZA2PHuraAwoXdVUBdukDutL9ef+jCoTwx+wmK5C7CtHbTrKdQYzKgRBOBiIR7PYjWS8+AjE9U4cMP4bHH4NAhuOYa6NYN7rsvza4C+vfmXCdxNS+qSYcaHXi7xdsUzl04zbdjjEm9pEoEC3HtActFJBr4BDgWN1FVP/M5NpNWFiyAJ5+EefOgWjWYNcvdE+CDo6eP0vv73uQMy8lbzd+yTuKMyQSC6XQuF7Af1/toa6CN999kdLt2wU03wdVXu2qg1193DcI+JYFvfv+G6sOrM2ThEM7EnLFO4ozJJJIqEVzoXTG0mn9uKItj3/CM7uOPXb3/4cPQsyf07u1LFRDAwRMHefKbJxm/fDyVi1Rm7v1zubbUtb5syxiT9pJKBGHABfw7AcSxRJBRLVrkrgT65ht3A9icOVCjRrKLpcaeY3uYtnYaz177LH0a9iFXeMoHoTfGhE5SiWCXqvZNt0hM6hw+7O4FGD0a8ueH115z7QLn+TN0xO6ju5myagpP1H3CdRL32BaK5Cniy7aMMf5KKhHYQLCZxbffQlQUbNsGd9/tBosvVsyXTakqH6z4gCdmP8HxM8dpXak1FYtUtCRgTCaWVGOx3faZ0e3fD+3buzEBwsJcddCHH/qWBLYc2kLLSS3p+HlHIopFsLzzcuskzpgsINESgaoeSM9ATAotXgz33gvr1kHXrjBgAOTLl/xy5+hs7FkaT2jMvuP7GHb9MDpHdiaHBHPRmTEmo0txp3MmxE6edD/8778PBQu6UkCzZr5tbtOBTZQtWJbwHOGMu3Ec5QqVo3TB0r5tzxiT/uyULjNZtw7q1HFJoHt3+P1335LAmZgz9P+5P9WGV2PYomEANC7b2JKAMVmQlQgyg9hY1yHcs89CeDhMnQq33+7b5pbuWkpUdBTLdy+nXUQ77qh2h2/bMsaEniWCjG7pUnjoIfc/IgJmzoQyZXzb3OAFg3ly9pMUy1uMz27/jJur3uzbtowxGYNVDWVUqjB5MjRoAH/84e4PWLXKtyQQ1x1E7Ytrc+9l97K261pLAsZkE1YiyIhOnoTbbnNn/7VquZHDSpb0ZVNHTh3h2e+f5fyw8xnYYiD1S9enfun6vmzLGJMxWYkgI1GFn35yncTNnAlPP+16DvUpCXy96Wuqj6jO8EXDUdQ6iTMmm7ISQUaxejXcdZer/rn4Ypgyxd0s5oP9x/fz5DdP8sGKD6hatCq/PPALdUv60yOpMSbjs0SQEezcCW3bwubN8O670LGj6y/IJ/tP7Gf6uum80OAFetfvzfnh5/u2LWNMxudr1ZCItBSR9SKySUR6JTD9SRFZKyIrReR7Ecl+F6n/+KMbH2DnTvj6a3d/gA9JYNeRXbw17y1UlUpFKrH18a30bdzXkoAxxr9E4I13PAxoBUQAd4pIRLzZlgGRqloTmAa84Vc8GdJrr0GTJu7x7NnQokWab0JVGbdsHFWHVeWFH19g04FNABTKXSjNt2WMyZz8LBFcBWxS1c2qehr4CGgbOIOq/qiqx72n84ESPsaTsbz8Mjz3HFx3nWsfaJD2wzn+cfAPmk9sTlR0FJddfBkrOq+wTuKMMf/hZxtBcWBbwPPtQJ0k5o8Cvkpogoh0AjoBlCpVKq3iC52hQ+Gll1y7wJQpkDt3mm/ibOxZmnzQhP3H9zPihhF0uqKTdRJnjElQhmgsFpG7gUigYULTVXU0MBogMjIy817jqAovvACvvgqNG/uSBDbu30i5QuUIzxHO+23fp3yh8pQs4M/lp8aYrMHPU8QdQOAvUAnvtX8RkeuA3sCNqnrKx3hC6/Rp11XEq69C69bw1VdpmgTOxJyh39x+VB9RnaELhwLQqEwjSwLGmGT5WSJYBFQUkbK4BNAeuCtwBhGpDYwCWqrqHh9jCa1Nm6BdO1i+HB57DAYOdAPJpJHFOxcTFR3Fyr9W0r56e+6scWeardsYk/X5lghU9ayIdANmA2HAOFVdIyJ9gcWqGg28CVwAfCIiAH+q6o1+xRQS0dFw//2uRDBxInTokKarf3f+uzz5zZNcfMHFfN7+c26snLUOnzHGf762EajqLGBWvNf6BDy+zs/th9y0aa676EqV4LPPXO+haURVEREiL40kqnYUbzR7g4K5CqbZ+o0x2UeGaCzOco4cgT59YPBgqFEDfv45zW4S+/vU3zzz7TPkCs/FOy3foV6petQrVS9N1m2MyZ7sesK0tmkT1K4Ngwa5aqC5c9MsCczaOItqw6sxeulownOEWydxxpg0YSWCtPTzz3D33bBvn7tTuHnzNFntvuP7ePzrx5m0ahLVilVjWrtp1CmR1C0ZxhgTPCsRpJWJE6FRIzesZHR0miUBgIMnDvLFhi94seGLLH14qSUBY0yashJBWhg4EJ56Cq68Er75BgoWTPUqd/y9g0mrJtHzmp5ULFKRrY9vtcZgY4wvrESQWpMnuyRwww3wyy+pTgKqypglY4gYHsFLc17i94O/A1gSMMb4xhJBakRHwz33uMbhqVMhZ85Ure73A7/T9IOmdPqyE5dfcjkru6ykQuEKaRSsMcYkzKqGztWvv8Itt0DVqvD995A3b6pWdzb2LE0/aMqBEwcY1XoUD17+oHUSZ4xJF5YIzsW6da4x+MIL3WAyhc69b//1+9ZTvnB5wnOEM+GmCZQvXJ4S+bNPb9zGmNCzU86UihtWMjzcDTRf4tx+tE/HnOblOS9TY0QNhi0cBkDDMg0tCRhj0p2VCFLi779dddDOna730IrnNsjLwh0LiYqOYvWe1dxV4y461Ezb/oeMMSYlLBEE6+RJaNoUli1zVwrVr39Oqxk0fxA9vunBJRdcwhd3fkHrSq3TOFBjjEkZSwTBUIVHHoHFi2H8eNeRXIpX4TqJu6r4VTx0+UO8ft3rFMhVIO1jNcaYFLJEEIz+/WHcOIiKgvvuS9Gih08e5ulvnyZ3ztwMajmIa0pewzUlr/EpUGOMSTlrLE7OZ5/B88/DrbfCyJEpWvSL9V8QMTyC95a9x/lh51snccaYDMlKBEnZuBE6dnTjCEya5K4UCsLeY3t57OvHmLJ6CjUurMGMO2ZwZfEr/Y3VGGPOkSWCxOzdC02awNmz8OmncP75QS96+NRhZm2cxcuNXqbXtb04L+w8HwM1xpjUsUSQkFOn3BjDu3e7G8aqVEl2kW2HtzFx5UR6XduLCoUrsPXxrdYYbIzJFKyNICH9+7ubxYYMcZeMJiFWYxm5eCTVhlej38/9/tdJnCUBY0xmYYkgvsmToW9fuPlmePjhJGfduH8jTSY0ocvMLlxV/CpWdVllncQZYzIdqxoKtHw5dOsGkZGuN1GRRGc9G3uWZh8249DJQ4y9cSz317ofSWJ+Y4zJqCwRxDl4EJo1c43CH3yQaJfS6/auo2KRioTnCOfDmz+kfOHyXJrv0nQO1hhj0o5VDcXp2dONNTxpkutaOp5TZ0/x4o8vUnNkTYYuHApA/dL1LQkYYzI9KxGAu2ls7Fi4/353yWg887fPJyo6irV713JPzXu4p+Y9IQjSGGP8YYlg0ybo1Alq1HBXCcUzcN5Aen7bkxL5SzDrrlm0qtgqBEEaY4x/snciiI2FDh3cfQOTJv1rlLFYjSWH5KBuybp0juzMgOsGkP/8/CEM1hhj/JG9E8Grr8LChTBmjCsRAIdOHqLH7B7kyZmHIdcPsU7ijDFZXvZtLP7pJ3jxRdeldFQUADN+m0HEsAgmrJhAvvPzWSdxxphsIXuWCA4ehIcecsNMjh7NnuN76TarG5+s/YRaF9fiy7u+5PJLLg91lMYYky6yZyJ48EHXSDx9OhQowN8HNvHt5m95tcmr9LymJznDEr6HwBhjsqLslwhmzoTPPuPP5x7hw0KreU5vpELhCvz5+J/kOz9fqKMzxph052sbgYi0FJH1IrJJRHolMP18EZnqTV8gImX8jIedO4m9vyPDWxahWt4J9P+//v/rJM6SgDEmu/ItEYhIGDAMaAVEAHeKSES82aKAg6paAXgHeN2veFBlfc8HaNRmH49cvZ+6Jeuypusa6yTOGJPt+VkiuArYpKqbVfU08BHQNt48bYEJ3uNpQFPxqee2sxPep8WFs1lVKhfvt32f2XfPpkzBMn5syhhjMhU/E0FxYFvA8+3eawnOo6pngcNAkfgrEpFOIrJYRBbv3bv3nIIJL1ueiXuuZe2jv9GxVkfrKdQYYzyZorFYVUcDowEiIyPP7eL+hg25tuHPaRmWMcZkCX6WCHYAJQOel/BeS3AeEQkHCgD7fYzJGGNMPH4mgkVARREpKyLnAe2B6HjzRAP3eY9vA35Qu53XGGPSlW9VQ6p6VkS6AbOBMGCcqq4Rkb7AYlWNBsYCH4rIJuAALlkYY4xJR762EajqLGBWvNf6BDw+CbTzMwZjjDFJy76dzhljjAEsERhjTLZnicAYY7I5SwTGGJPNSWa7WlNE9gJbz3HxosC+NAwnM7B9zh5sn7OH1OxzaVUtltCETJcIUkNEFqtqZKjjSE+2z9mD7XP24Nc+W9WQMcZkc5YIjDEmm8tuiWB0qAMIAdvn7MH2OXvwZZ+zVRuBMcaY/8puJQJjjDHxWCIwxphsLksmAhFpKSLrRWSTiPRKYPr5IjLVm75ARMqEIMw0FcQ+Pykia0VkpYh8LyKlQxFnWkpunwPmu1VEVEQy/aWGweyziNzuvddrRGRyeseY1oL4bJcSkR9FZJn3+b4+FHGmFREZJyJ7RGR1ItNFRAZ7x2OliFye6o2qapb6w3V5/TtQDjgPWAFExJunKzDSe9wemBrquNNhnxsDebzHXbLDPnvz5QPmAvOByFDHnQ7vc0VgGVDIe35hqONOh30eDXTxHkcAW0Iddyr3uQFwObA6kenXA18BAlwNLEjtNrNiieAqYJOqblbV08BHQNt487QFJniPpwFNJXMPYpzsPqvqj6p63Hs6HzdiXGYWzPsM8ArwOnAyPYPzSTD7/BAwTFUPAqjqnnSOMa0Fs88K5PceFwB2pmN8aU5V5+LGZ0lMW+ADdeYDBUXkktRsMysmguLAtoDn273XEpxHVc8Ch4Ei6RKdP4LZ50BRuDOKzCzZffaKzCVVdWZ6BuajYN7nSkAlEflFROaLSMt0i84fwezzS8DdIrIdN/7Jo+kTWsik9PuerEwxeL1JOyJyNxAJNAx1LH4SkRzA20DHEIeS3sJx1UONcKW+uSJSQ1UPhTIon90JjFfVgSJSFzfqYXVVjQ11YJlFViwR7ABKBjwv4b2W4DwiEo4rTu5Pl+j8Ecw+IyLXAb2BG1X1VDrF5pfk9jkfUB2YIyJbcHWp0Zm8wTiY93k7EK2qZ1T1D2ADLjFkVsHscxTwMYCq/grkwnXOllUF9X1PiayYCBYBFUWkrIich2sMjo43TzRwn/f4NuAH9VphMqlk91lEagOjcEkgs9cbQzL7rKqHVbWoqpZR1TK4dpEbVXVxaMJNE8F8tmfgSgOISFFcVdHmdIwxrQWzz38CTQFEpCouEexN1yjTVzRwr3f10NXAYVXdlZoVZrmqIVU9KyLdgNm4Kw7GqeoaEekLLFbVaGAsrvi4Cdco0z50EadekPv8JnAB8InXLv6nqt4YsqBTKch9zlKC3OfZQHMRWQvEAD1VNdOWdoPc5x7AGBF5Atdw3DEzn9iJyBRcMi/qtXu8COQEUNWRuHaQ64FNwHHg/lRvMxMfL2OMMWkgK1YNGWOMSQFLBMYYk81ZIjDGmGzOEoExxmRzlgiMMSabs0RgMiQRiRGR5QF/ZZKY92gabG+8iPzhbWupd4dqStfxnohEeI+fizdtXmpj9NYTd1xWi8gXIlIwmflrZfbeOI3/7PJRkyGJyFFVvSCt501iHeOBL1V1mog0B95S1ZqpWF+qY0puvSIyAdigqq8mMX9HXK+r3dI6FpN1WInAZAoicoE3jsJSEVklIv/paVRELhGRuQFnzPW915uLyK/esp+ISHI/0HOBCt6yT3rrWi0ij3uv5RWRmSKywnv9Du/1OSISKSIDgNxeHJO8aUe9/x+JyA0BMY8XkdtEJExE3hSRRV4f8w8HcVh+xetsTESu8vZxmYjME5HK3p24fYE7vFju8GIfJyILvXkT6rHVZDeh7nvb/uwvoT/cXbHLvb/puLvg83vTiuLuqowr0R71/vcAenuPw3D9DRXF/bDn9V5/BuiTwPbGA7d5j9sBC4ArgFVAXtxd2WuA2sCtwJiAZQt4/+fgjXkQF1PAPHEx3gxM8B6fh+tFMjfQCXjee/18YDFQNoE4jwbs3ydAS+95fiDce3wd8Kn3uCMwNGD5/sDd3uOCuL6I8ob6/ba/0P5luS4mTJZxQlVrxT0RkZxAfxFpAMTizoQvAnYHLLMIGOfNO0NVl4tIQ9xgJb94XWuchzuTTsibIvI8rp+aKFz/NdNV9ZgXw2dAfeBrYKCIvI6rTvo5Bfv1FfCuiJwPtATmquoJrzqqpojc5s1XANdZ3B/xls8tIsu9/V8HfBsw/wQRqYjrZiFnIttvDtwoIk95z3MBpbx1mWzKEoHJLDoAxYArVPWMuB5FcwXOoKpzvURxAzBeRN4GDgLfquqdQWyjp6pOi3siIk0TmklVN4gb6+B6oJ+IfK+qfYPZCVU9KSJzgBbAHbiBVsCNNvWoqs5OZhUnVLWWiOTB9b/zCDAYNwDPj6p6s9ewPieR5QW4VVXXBxOvyR6sjcBkFgWAPV4SaAz8Z8xlceMw/6WqY4D3cMP9zQfqiUhcnX9eEakU5DZ/Bm4SkTwikhdXrfOziFwKHFfVibjO/BIaM/aMVzJJyFRcR2FxpQtwP+pd4pYRkUreNhOkbrS57kAP+acr9biuiDsGzHoEV0UWZzbwqHjFI3G90ppszhKBySwmAZEisgq4F/gtgXkaAStEZBnubPtdVd2L+2GcIiIrcdVCVYLZoKouxbUdLMS1GbynqsuAGsBCr4rmRaBfAouPBlbGNRbH8w1uYKDv1A2/CC5xrQWWihu0fBTJlNi9WFbiBmZ5A3jN2/fA5X4EIuIai3Elh5xebGu85yabs8tHjTEmm7MSgTHGZHOWCIwxJpuzRGCMMdmcJQJjjMnmLBEYY0w2Z4nAGGOyOUsExhiTzf0/HS63BkbYDPEAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.5133815625\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkCklEQVR4nO3deZxcZZ3v8c83nZBAFjDp6EASCEskCVvABkR0RBBZhKCiDAjjwCAoElAu4uCKLCrgVRmGcBE1gBvIgFeDw3IVZBCRpSMEEzAQwtYBJiGQhJA9+d0/nlN0VafTXen06erq832/XvXqOkud+p3upL71PM9ZFBGYmVlx9at1AWZmVlsOAjOzgnMQmJkVnIPAzKzgHARmZgXnIDAzKzgHgdUNSSdK+n9VrHeNpK/3RE09QdJzkj6YPf+mpJ/XuibrWxwE1i2yD6sVkpZJ+h9J10sa0p3vERG/iIgPVbHeZyPi4u587xJJIenNbD/nS/q+pIY83qsrJA2TdIWkF7Ian8mmG2tdm/VeDgLrTkdHxBBgH6AJ+FrbFST17/Gqut9e2X6+H/gn4F9rXA8AkrYA7gZ2Aw4HhgEHAIuA/bqwvb7wt7IqOAis20XEfOAOYHd461v0mZKeBp7O5h0l6TFJiyU9IGnP0usljZH0a0kLJS2SdFU2/2RJ92fPJekHkhZIWirpb5JK73e9pEvKtneapLmSXpM0XdJ2ZctC0mclPZ3VMlWSqtzPucCfgUll2+vKfu0s6Z5s3quSfiFpm038tQN8Ctge+GhEPBER6yNiQURcHBG3l+3vLmU1vfW7knSQpBZJ/ybpFeA6SU9KOqps/f5Z/ftk0+/O9nOxpJmSDupC3VZjDgLrdpLGAEcCj5bN/giwPzBR0t7ANOAzwAjgh8B0SQOzbpbfAc8DY4FRwE3tvM2HgH8E3glsDRxH+ubbtpaDge9ky7fNttt2e0cB+wJ7ZusdVuV+jgfeB8zNpru6X8pq3A6YAIwBvllNDW18ELgzIpZ14bUl/wAMB3YATgduBE4oW34Y8GpE/FXSKOC/gEuy13wRuFXSyM14f6sBB4F1p99IWgzcD/w38O2yZd+JiNciYgXpA+aHEfFQRKyLiBuAVcC7SV0Y2wHnRcSbEbEyIu5v573WAEOB8YAi4smIeLmd9U4EpkXEXyNiFfBl4ABJY8vWuTQiFkfEC8AfKfuGvxF/lfQm8CRwL3B1Nr9L+xURcyPi9xGxKiIWAt8ndTttqhFAe7+DTbEeuCCrZQXwS2CypK2y5Z8khQPAScDtEXF71vr4PdBM+hJgdcRBYN3pIxGxTUTsEBGfyz5ISl4se74DcG7WnbA4C48xpA/KMcDzEbG2ozeKiHuAq4CpwAJJ10oa1s6q25G+hZdet4zUchhVts4rZc+XA0MAJM3OBlyXSXpf2Tr7ZOv8E6mVM3hz9kvSOyTdlA0+LwV+DnRlcHcRqdWzORZGxMrSRNb99SRwdBYGk0nhAGl/P9Fmf9/bDTVYD3MQWE8pv8zti8C3stAoPbaKiBuzZdtXM1AZEVdGxLuAiaQuovPaWe0l0gcWAJIGk745z69i+7tFxJDs8ac2yyIibgb+AnxjM/fr26Tfzx4RMYz0TbuqcYo2/gAclu3jxiwHtiqb/oc2y9u7HHGpe+gY4IksHCDt08/a7O/giLi0C7VbDTkIrBZ+BHxW0v7ZoO9gSR+WNBR4mNS9cWk2f5CkA9tuQNK+2esHAG8CK0ndGm3dCJwiaZKkgaQP3Yci4rlu2pdLgdMk/cNm7NdQYBmwJOt3by/QqvEz0ofzrZLGS+onaYSkr0gqddc8BnxSUoOkw6muC+om0pjMGbS2BiC1XI6WdFi2vUHZgPPoLtZvNeIgsB4XEc3AaaSunddJg60nZ8vWAUcDuwAvAC2kLpi2hpE+eF8ndf0sAr7bznv9Afg6cCvpg3hn4Phu3Je/AfeR+v67ul8XkrqblpAGX3/dxVpWkQaM/w78HlhKCqBG4KFstc9ndSwmjZ/8portvkxq+bwH+FXZ/BdJrYSvAAtJIXQe/lypO/KNaczMis3JbWZWcA4CM7OCcxCYmRWcg8DMrODq7qJSjY2NMXbs2FqXYWZWV2bMmPFqRLR7+Y+6C4KxY8fS3Nxc6zLMzOqKpOc3tsxdQ2ZmBecgMDMrOAeBmVnB1d0YgZlZV61Zs4aWlhZWrlzZ+cp1atCgQYwePZoBAwZU/RoHgZkVRktLC0OHDmXs2LFUeSO6uhIRLFq0iJaWFnbccceqX5db15CkaUq3EZy1keWSdKXSLQQfL936zswsLytXrmTEiBF9MgQAJDFixIhNbvHkOUZwPekG2htzBDAue5wO/J8cazEzA+izIVDSlf3LLQgi4j7gtQ5WOQb4aXaDjweBbSTld2ej6dNhwgRYvTq3tzAzq0e1PGpoFJW3L2yh8vaBb5F0uqRmSc0LFy7s2rsdcwz8/e8wd27n65qZ5aShoYFJkyax++67c/TRR7N48eK3ls2ePZuDDz6YXXfdlXHjxnHxxRdTfquAO+64g6amJiZOnMjee+/Nueee2y011cXhoxFxbUQ0RUTTyJHtniFdvVWruqcoM7Mu2HLLLXnssceYNWsWw4cPZ+rUqQCsWLGCyZMnc/755zNnzhxmzpzJAw88wNVXXw3ArFmzmDJlCj//+c954oknaG5uZpdddumWmmoZBPNJN/QuGU0V95E1M+srDjjgAObPTx97v/zlLznwwAP50Ic+BMBWW23FVVddxaWXpltAX3755Xz1q19l/PjxQGpZnHHGGd1SRy0PH50OTJF0E7A/sCS7JZ6ZWf6+8AV47LHu3eakSXDFFVWtum7dOu6++25OPfVUIHULvetd76pYZ+edd2bZsmUsXbqUWbNmdVtXUFu5BYGkG4GDgEZJLcAFwACAiLgGuB04knRf1+XAKXnVUsG35jSzGlqxYgWTJk1i/vz5TJgwgUMPPbTWJeUXBBFxQifLAzgzr/c3M+tQld/cu1tpjGD58uUcdthhTJ06lbPPPpuJEydy3333Vaw7b948hgwZwrBhw9htt92YMWMGe+21V7fXVBeDxWZmfc1WW23FlVdeyfe+9z3Wrl3LiSeeyP33388f/vAHILUczj77bL70pS8BcN555/Htb3+bp556CoD169dzzTXXdEstDgIzsxrZe++92XPPPbnxxhvZcsst+e1vf8sll1zCrrvuyh577MG+++7LlClTANhzzz254oorOOGEE5gwYQK777478+bN65Y6fK0hM7MetGzZsorp22677a3ne+yxB/fee+9GX3vUUUdx1FFHdXtNxWsReLDYzKxC8YLAzMwqOAjMrFCij/cKdGX/HARmVhiDBg1i0aJFfTYMSvcjGDRo0Ca9rniDxX30H4CZdW706NG0tLTQ5YtX1oHSHco2RfGCwMwKa8CAAZt0566iKF7XkFsEZmYVihcEZmZWwUFgZlZwxQsCdw2ZmVUoXhCYmVkFB4GZWcE5CMzMCq54QeAxAjOzCsULAjMzq+AgMDMrOAeBmVnBFS8IPEZgZlaheEFgZmYVHARmZgXnIDAzKzgHgZlZwRUvCDxYbGZWwUFgZlZwxQsCMzOr4CAwMyu4XINA0uGS5kiaK+n8dpbvIOluSY9LulfS6DzrAdw1ZGbWRm5BIKkBmAocAUwETpA0sc1q/xv4aUTsCVwEfCeveszMrH15tgj2A+ZGxLyIWA3cBBzTZp2JwD3Z8z+2s9zMzHKWZxCMAl4sm27J5pWbCXwse/5RYKikEW03JOl0Sc2SmhcuXJhLsWZmRVXrweIvAu+X9CjwfmA+sK7tShFxbUQ0RUTTyJEje7pGM7M+rX+O254PjCmbHp3Ne0tEvETWIpA0BDg2IhbnWJMHi83M2sizRfAIME7SjpK2AI4HppevIKlRUqmGLwPTcqzHzMzakVsQRMRaYApwF/AkcHNEzJZ0kaTJ2WoHAXMkPQW8A/hWXvWYmVn78uwaIiJuB25vM+8bZc9vAW7JswYzM+tYrQeLe57HCMzMKhQvCMzMrIKDwMys4IoXBO4aMjOrULwgMDOzCsULArcIzMwqFC8IzMysgoPAzKzgHARmZgVXvCDwGIGZWYXiBYGZmVVwEJiZFZyDwMys4BwEZmYFV7wg8GCxmVmF4gWBmZlVcBCYmRWcg8DMrOCKFwQeIzAzq+AgMDMruOIFgZmZVXAQmJkVnIPAzKzgihcEHiMwM6tQvCAwM7MKDgIzs4JzEJiZFVzxgsBjBGZmFYoXBGZmViHXIJB0uKQ5kuZKOr+d5dtL+qOkRyU9LunIPOsxM7MN5RYEkhqAqcARwETgBEkT26z2NeDmiNgbOB64Oq96zMysfXm2CPYD5kbEvIhYDdwEHNNmnQCGZc+3Bl7KsR4zM2tHnkEwCnixbLolm1fum8BJklqA24Gz2tuQpNMlNUtqXrhw4eZV5cFiM7MKtR4sPgG4PiJGA0cCP5O0QU0RcW1ENEVE08iRI3u8SDOzvizPIJgPjCmbHp3NK3cqcDNARPwFGAQ05liTmZm1kWcQPAKMk7SjpC1Ig8HT26zzAnAIgKQJpCDYzL6fTrhryMysQv9qVpJ0IKk/f4fsNQIiInba2GsiYq2kKcBdQAMwLSJmS7oIaI6I6cC5wI8knUMaOD45IudPageBmVmFqoIA+AlwDjADWFftxiPidtIgcPm8b5Q9fwI4sNrtmZlZ96s2CJZExB25VmJmZjVRbRD8UdJ3gV8Dq0ozI+KvuVRlZmY9ptog2D/72VQ2L4CDu7ecHuAxAjOzClUFQUR8IO9CzMysNqo6fFTS1pK+Xzq7V9L3JG2dd3FmZpa/as8jmAa8ARyXPZYC1+VVVK7Wr691BWZmvUq1QbBzRFyQXUBuXkRcCGz0HIJe7eaba12BmVmvUm0QrJD03tJEdoLZinxKytmyZbWuwMysV6n2qKEzgBuycQEBrwEn51VUrnzUkJlZhWqPGnoM2EvSsGx6aZ5FmZlZz+kwCCSdFBE/l/S/2swHICK+n2Nt+XCLwMysQmctgsHZz6F5F2JmZrXRYRBExA+znxf2TDk9wC0CM7MK1Z5QdrmkYZIGSLpb0kJJJ+VdXC6am2tdgZlZr1Lt4aMfygaIjwKeA3YBzsurqFzNb3uTNDOzYqs2CEpdSB8G/jMiluRUj5mZ9bBqzyP4naS/k04iO0PSSGBlfmWZmVlPqapFEBHnA+8BmiJiDfAmcEyehZmZWc/o7DyCgyPiHkkfK5tXvsqv8yosVytWwJZb1roKM7NeobOuofcD9wBHt7MsqNcgWLAAdtih1lWYmfUKnZ1HcEH285SeKcfMzHpatecRfFvSNmXTb5N0SW5V5a2ye8vMrNCqPXz0iIhYXJqIiNeBI3OpyMzMelS1QdAgaWBpQtKWwMAO1u/dfJkJM7O3VHsewS+AuyWVbk95CnBDPiX1AAeBmdlbqr0fwWWSZgIfzGZdHBF35VdWztatq3UFZma9RrUtAoAngbUR8QdJW0kaGhFv5FVYrm67DWbOhOuu63xdM7M+rtqjhk4DbgF+mM0aBfwmp5ryd845cP31ta7CzKxXqHaw+EzgQGApQEQ8Dbw9r6J6jLuIzMyqDoJVEbG6NCGpP+nM4g5JOlzSHElzJZ3fzvIfSHosezwlaXHVlXeH1as7X8fMrI+rdozgvyV9BdhS0qHA54DbOnqBpAZgKnAo0AI8Iml6RDxRWicizilb/yxg702sf/OsXu1rDplZ4VXbIvg3YCHwN+AzwO3A1zp5zX7A3IiYl7UmbqLjK5aeANxYZT3d46GHYPx4+MY34JFHevStzcx6i05bBNk3+9kRMR740SZsexTwYtl0C7D/Rt5jB2BH0gXues7HPgZvvgkXX5wePr/AzAqo0xZBRKwD5kjaPsc6jgduyd5rA5JOl9QsqXnhwoXd965vvlk5/dJL8OCDHkQ2s0KpdozgbcBsSQ+TbkoDQERM7uA184ExZdOjs3ntOZ50ZFK7IuJa4FqApqam/L62jxrV+nzdOuhXbc+ZmVn9qjYIvt6FbT8CjJO0IykAjgc+2XYlSeNJQfOXLrxHfhoaWp//+c8wYADsu2+afu452G472GKLmpRmZtadOrtD2SDgs8AupIHin0TE2mo2HBFrJU0B7gIagGkRMVvSRUBzREzPVj0euCmiF3fQH3hg+vnv/w5PPw1XXZWm77sP3ve+2tVlZtYN1NHnr6RfAWuAPwFHAM9HxOd7qLZ2NTU1RXNz86a/sKN7ENx5Jxx+eHr+ve/BuedWv93rroMlS+DzNf21mJl1SNKMiGhqd1knQfC3iNgje94feDgi9smnzOrkEgQRrcvXrYPXXoORIzdt+xdfDF//OqxZA/035RJOZmb56ygIOhsNXVN6Um2XUF1YujQ9li1rPWR0zZo03a8fNDam+RFp/jPPwO23w3HHwbhxsGgRDGxzO4avZ8MoAwb4DmhmVlc6++q6l6Sl2XORzixemj2PiBiWa3V5GTp0w3n9+7f/Tb5/f9hpp/Q44ojW+c8+Cx//OPzudzB8+Iavk9KA8pVXpnCY3NEBVmZmtdPZzesbOlpeaNtum44mArjiCrjjDvjCFyrD4qWXUliULFiw6V1OZmY563CMoDfqljGCvPe5mq6hN96AIUPyrcPMLLM5YwTWFaXxhdJjzz03XGfo0BQYy5b1fH1mZmUcBD1h5kxYuxa+8hX4058ql5UCQYL162tTn5kVmo9z7CkNDfCtb6Xna9akAeT21impsy47M6tfxWkR7LRTrSto1b9/a7fRtGntn51caiX4zGUzy1lxgqC3OuWUdKmKiHSGclv33+/zEswsV8UJgnroahk2LNW5cmW67MW73tW6TKqPfTCzulOcIKingdiBA+Gww6C5GQ4+uHX+vvvC8uW1q8vM+qTiBEG9fpu++254MbvR24wZMHgwXHZZbWsysz6lOEFQz0aPrpw+//x0vSMzs25QnCCo1xZBSekoo5LGxtrVYmZ9ioOg3pTfT7l0iKnvsWxmm8FBUG/69YNrr62c178/vPJKbeoxs7pXnCDoS047LY0R/OhHrfO23TaNHZiZbaLiBEFfaRGUDB8On/40rF7dOu+yy1JX0Qsv1K4uM6s7DoJ6N2BA2rfDDmudt8MOreMHUro5zvz5tavRzHq14gRBPZ1Q1hV33tl6o5y2Pv/5dAiqL1VhZu0oThD01RZBufe8J+3nunXw3e+2v06plTB+PDz3XI+WZ2a9U3GCoK+3CMr16wdf/GLlzXFaWirXmTMHdtwx3SfBzAqtOEFQhBZBR0aNSgPLn/tc5fz27otgZoXiICiSAQNg6tT0uyhdvwhSV9ENN9SuLjOrKQdBUY0ena5mWnLyyb5dpllBOQiK7OGH4be/rZzX0OCji8wKxkFQdJMnp9/N3LmV830jHLPCcBBYsvPO6Xe0bFnrvH79UiA880zt6jKz3BUnCNz3XZ3Bgze818Euu7iFYNaH5RoEkg6XNEfSXEntXhFN0nGSnpA0W9IvcyvGQVC94cPTh/6aNWnMoMQtBLM+qX9eG5bUAEwFDgVagEckTY+IJ8rWGQd8GTgwIl6X9Pa86vG32S7o3z+dcBaRQqBkl13Sz/XrPbBs1gfk2SLYD5gbEfMiYjVwE3BMm3VOA6ZGxOsAEbEgt2pGjUo/hw7N7S36rFK3UNuzkPv1g0MOqU1NZtZt8gyCUUDZWUu0ZPPKvRN4p6Q/S3pQ0uHtbUjS6ZKaJTUvXLiwa9WcdVb6edJJXXu9pW6iCHjoodZ599zj8w/M6lytB4v7A+OAg4ATgB9J2qbtShFxbUQ0RUTTyJEju/ZOpS6M8j5v65r99kuB8OMft84rnX/wxhu1q8vMuiTPIJgPjCmbHp3NK9cCTI+INRHxLPAUKRi638SJ6Wf52bS2eU49dcPuomHDPG5gVmfyDIJHgHGSdpS0BXA8ML3NOr8htQaQ1EjqKpqXSzWHHAJPPQX//M+5bL6wSt1FbbvsJJg1qzY1mdkmyS0IImItMAW4C3gSuDkiZku6SNLkbLW7gEWSngD+CJwXEYva32I3GDfO31bz0ti44RnKe+zh8w/M6oCizv6TNjU1RXNzc63LsI6sWgWDBlXOq7N/Z2Z9jaQZEdHU3rJaDxZbXzRwYPrgX7mydd7h7R4QZma9gIPA8jNwIDz6aHp+112pm2jJktrWZGYbcBBYviZNqrzpzTbbwBFH1KoaM2uHg8Dy96lPVZ5wduedDgOzXsRBYD2jdPTQtGlp+s474cwza1uTmQEOAutpp5wCV16Znl99dQqIpUtrW5NZwTkIrOeddRbcckvr9NZbp0B48sna1WRWYA4Cq41jj93wQnUTJ6ZAWLy4JiWZFZWDwGqnNG4QAZdf3jr/bW9Lyw48sHa1mRWIg8B6h/PO2/Ds4wceSIFw2WW1qcmsIBwE1ruUWgi/+13rvPPPT4Ewve01C82sOzgIrHf68IdTIGy3Xeu8Y45xIJjlwEFgvdv8+SkQLr64dV4pEF5/vXZ1mfUhDgKrD1/7WgqEz3ymdd7w4SkQXnmldnWZ9QEOAqsv11yTAmHAgNZ5226bAqGr97M2KzgHgdWn1avTeQjvfGfrvLe/PQXCrrtueI6CmW2Ug8DqlwRz5qQWwuc+1zr/qafSLTQl+OlPa1efWZ1wEFjfMHVqCoR162C33Vrn/8u/+HaZZp1wEFjf0q8fzJqVPvinTKmc/+lPu8vIrB0OAuu7/uM/YNmy1umf/KS1y+i112pXl1kv4yCwvm3w4NQ6ePPNyvkjRsAnPuEjjcxwEFhRbLVVCoT162GPPdK8W25pPdLotNN8GWwrLAeBFYsEjz8OM2bALru0zv/xj1svgz1mDKxZU7sazXqYg8CKaZ994OmnUyvhL3+pXNbSAltsAe94Rzo81ayPcxCYvfvdrVc9XbEihQDAggUwfnxqJVx0kQ9BtT7LQWBWbtAgWLUqfeh/4xut8y+4IB2CKsEvflG7+sxy4CAw25gLL0yB8OijlfNPOikFwtixbiVYn+AgMOvMpEmtRxzdems69BTg+edbWwkf/zg891wtqzTrMgeBWbUk+NjH4NVX0+ODH2xdduutsOOOaZ3SY+JEuPdeWLvWLQfr1XINAkmHS5ojaa6k89tZfrKkhZIeyx6fzrMes24zYgT8/vfpA37ZsnTjnFGjKtd58kn4wAfSJbNLLYfGRnjiidrUbLYRuQWBpAZgKnAEMBE4QdLEdlb9VURMyh4/zqses9wMHpxunNPS0nr00dKl8MUvplbB8OGt6y5alC6KJ8Fee6Wjkl5+2S0G69j69ens+OXLc9l8ni2C/YC5ETEvIlYDNwHH5Ph+Zr3H0KHw3e/C7Nnpw78UEFddlYID0oltc+ak+zKXWgwf+Ug6bNWKackSuP329MXioIPSyY1SukbWkCFw4425vG3/XLaajAJeLJtuAfZvZ71jJf0j8BRwTkS82HYFSacDpwNsv/32OZRq1kPOPDM9IlJIXHghvPQSPPBAWv7b36YHpG6k115Ll9jef//Ukiid42D1KSIF/cMPpw/9q65K403PPNP++o2N6ZpYb74JBxwATU25lKXIqUkq6ePA4RHx6Wz6n4H9I2JK2TojgGURsUrSZ4B/ioiDO9puU1NTNDc351KzWU29/DJ8+ctwww0drzdsGDz2GGyzDWy9dWpNWO+xbFm6FPqzz6az1x9/HP7+9xT8nfnSl+DAA2H33VsPPugmkmZERLtJkmeLYD4wpmx6dDbvLRGxqGzyx8DlOdZj1rttuy1cf316QPr2eN118IMfpA+WkqVLYaedNnz9yJHp2+MJJ8B739sTFRfX6tXpQ/7hh1OL7uqr4ZVXqrvfxemnpxA/7jjYc8/K+2/XSJ4tgv6k7p5DSAHwCPDJiJhdts62EfFy9vyjwL9FxLs72q5bBFZoK1akD5DFi+H++6t7zfDhcOKJ6ezo0sB1N37T7FPWr4e5c9Og7P/8D/zwh/DGG/C3v6X7Yy9Y0PH1pw49NLXYliyBY49N3Tm77prOWK+xjloEuQVB9sZHAlcADcC0iPiWpIuA5oiYLuk7wGRgLfAacEZE/L2jbToIzDZi6VL4/vfTuENXHHdcalEsXZo+0EaP7huBEZG+wT//fOpSu+02+NOf0iXIFy9O3+yr1diY+vTHjoV9903nlRxySGqN9XI1C4I8OAjMNtHDD8N558F996WuiMcf3/xtfuUr6eqsQ4akbo4FC2DnndMRUZMmpfs/QOvRUu2NY0SkD+f589ORVUOHtl66Y/DgdKTM297WGkYR6dv5/Pnw4IPpm/vQoenmQnfemb6xP/sszJzZtX2aODF9cx87Nh3a++qrcNhh6RDfYcPq4sO+Iw4CM9u4VavSh+sll6QP21mzoPR/rKEB1q2rbX2bo6EhBdTb3w7vex9MmAB7750+8EeMSB/wDQ21rrJH1Gqw2MzqwcCBafB52rTq1l+5En760zRIOnFi6jOfOzfd+e2BB1ILZH52XMj48emIGUgtiAkTUnfMqlXpNXvtlR4DBqRv4udnFyAYPDgdObN8eXr9mjWwww7pkuGNjWl+Y2MaYN9rr3S8/c47+wiqLnKLwMysADpqETg+zcwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcHV3QllkhYCz3fx5Y3Aq91YTj3wPheD97kYNmefd4iIdi+YVHdBsDkkNW/szLq+yvtcDN7nYshrn901ZGZWcA4CM7OCK1oQXFvrAmrA+1wM3udiyGWfCzVGYGZmGypai8DMzNpwEJiZFVyfDAJJh0uaI2mupPPbWT5Q0q+y5Q9JGluDMrtVFfv8vyQ9IelxSXdL2qEWdXanzva5bL1jJYWkuj/UsJp9lnRc9reeLemXPV1jd6vi3/b2kv4o6dHs3/eRtaizu0iaJmmBpFkbWS5JV2a/j8cl7bPZbxoRfeoBNADPADsBWwAzgYlt1vkccE32/HjgV7Wuuwf2+QPAVtnzM4qwz9l6Q4H7gAeBplrX3QN/53HAo8Dbsum317ruHtjna4EzsucTgedqXfdm7vM/AvsAszay/EjgDkDAu4GHNvc9+2KLYD9gbkTMi4jVwE3AMW3WOQa4IXt+C3CIJPVgjd2t032OiD9GxPJs8kFgdA/X2N2q+TsDXAxcBqzsyeJyUs0+nwZMjYjXASJiQQ/X2N2q2ecAhmXPtwZe6sH6ul1E3Ae81sEqxwA/jeRBYBtJ227Oe/bFIBgFvFg23ZLNa3ediFgLLAFG9Eh1+ahmn8udSvpGUc863eesyTwmIv6rJwvLUTV/53cC75T0Z0kPSjq8x6rLRzX7/E3gJEktwO3AWT1TWs1s6v/3TvXfrHKs7kg6CWgC3l/rWvIkqR/wfeDkGpfS0/qTuocOIrX67pO0R0QsrmVROTsBuD4ivifpAOBnknaPiPW1Lqxe9MUWwXxgTNn06Gxeu+tI6k9qTi7qkeryUc0+I+mDwFeByRGxqodqy0tn+zwU2B24V9JzpL7U6XU+YFzN37kFmB4RayLiWeApUjDUq2r2+VTgZoCI+AswiHRxtr6qqv/vm6IvBsEjwDhJO0ragjQYPL3NOtOBf8mefxy4J7JRmDrV6T5L2hv4ISkE6r3fGDrZ54hYEhGNETE2IsaSxkUmR0RzbcrtFtX82/4NqTWApEZSV9G8Hqyxu1Wzzy8AhwBImkAKgoU9WmXPmg58Kjt66N3Akoh4eXM22Oe6hiJiraQpwF2kIw6mRcRsSRcBzRExHfgJqfk4lzQoc3ztKt58Ve7zd4EhwH9m4+IvRMTkmhW9marc5z6lyn2+C/iQpCeAdcB5EVG3rd0q9/lc4EeSziENHJ9cz1/sJN1ICvPGbNzjAmAAQERcQxoHORKYCywHTtns96zj35eZmXWDvtg1ZGZmm8BBYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGbtkLRO0mOSZkm6TdI23bz957Lj/JG0rDu3bbapHARm7VsREZMiYnfSuSZn1rogs7w4CMw69xeyi3pJ2lnSnZJmSPqTpPHZ/HdI+r+SZmaP92Tzf5OtO1vS6TXcB7ON6nNnFpt1J0kNpMsX/CSbdS3w2Yh4WtL+wNXAwcCVwH9HxEez1wzJ1v/XiHhN0pbAI5Jureczfa1vchCYtW9LSY+RWgJPAr+XNAR4D62X6QAYmP08GPgUQESsI13aHOBsSR/Nno8hXQDOQWC9ioPArH0rImKSpK1I17k5E7geWBwRk6rZgKSDgA8CB0TEckn3ki6IZtareIzArAPZXd3OJl3YbDnwrKRPwFv3jt0rW/Vu0i1AkdQgaWvS5c1fz0JgPOlS2Ga9joPArBMR8SjwOOkGKCcCp0qaCcym9baJnwc+IOlvwAzSvXPvBPpLehK4lHQpbLNex1cfNTMrOLcIzMwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMyu4/w+bh8TdLs2WEwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.5485117350198137\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.9703\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CIFAR100 vs. SVHN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar100.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  3.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covariance norm: 14.771052072849098\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar100 = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "cifar100_train_dataloader = DataLoader(\n",
    "    dataset=cifar100, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, cifar100_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 12.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:18<00:00, 14.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 175.51826 id variance: 3052.4336\n",
      "ood mean score: 132.59648 ood variance: 1724.3635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=cifar100_test_dataloader,\n",
    "    out_of_dist_dataloader=svhn_test_dataloader,\n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCb0lEQVR4nO3dd3gUVRfA4d8hgPSOivReRZAAKlWRIgJWFKwogoCIDfmsqKjYFUFAQBAUBBVFUZqKIgoqHaRIkd57r0nO98ed6BJDsoFsJtk97/Pk2d2Z2Zkzu5s5c++duVdUFWOMMZErk98BGGOM8ZclAmOMiXCWCIwxJsJZIjDGmAhnicAYYyKcJQJjjIlwlgjCnIgsE5HGfseRXojIUyLygU/bHikiL/mx7dQmIreLyHdn+V77TaYzlgjSkIisF5FjInJYRLZ7B4ZcodymqlZV1Rmh3EY8ETlPRF4RkY3efq4WkcdFRNJi+4nE01hENgdOU9W+qnpfiLYnItJDRJaKyBER2Swin4vIxaHY3tkSkedFZPS5rENVx6hqsyC29Z/kd7a/SRHJ6sW+2vt814vICBEpldJ1mdNZIkh7rVU1F1ADqAk86W84KScimc8w63OgCdASyA3cCXQG3g1BDCIi6e33+y7wENADKABUAL4Crk3tDSXxHYScj9seD7QBbgPyApcA83G/uRTx8/NLl1TV/tLoD1gPXB3w+nVgUsDry4DZwH5gMdA4YF4B4ENgK7AP+CpgXitgkfe+2UD1hNsELgKOAQUC5tUEdgNZvNf3Aiu89U8DSgYsq8ADwGpgXSL71gQ4DhRPML0uEAuU817PAF4B5gAHga8TxJTUZzADeBmY5e1LOeAeL+ZDwFrgfm/ZnN4yccBh7+8i4HlgtLdMKW+/7gY2ep/F0wHbyw6M8j6PFUAvYPMZvtvy3n7WSeL7HwkMBCZ58f4BlA2Y/y6wyftc5gMNAuY9jzsQjvbm3wfUAX7zPqttwHtA1oD3VAW+B/YCO4CngBbASeCU95ks9pbNCwz31rMFeAmI8uZ18D7zd4A93rwOwK/efPHm7fRi+xOohjsJOOVt7zDwTcL/AyDKi+tv7zOZT4LfkLfc1d73+Z95Sfx/JfZdd/S+65nAFKB7gnUsBm70nlcK+PxWArf4fQwJ2bHJ7wAi6S/BP0Ax7x/mXe91Ue+frCWupNbUe13Ymz8J+BTID2QBGnnTa3r/gHW9f6q7ve2cl8g2fwQ6BcTzBvC+9/w6YA1QGcgMPAPMDlhWvX+KAkD2RPbtVeDnM+z3Bv49QM/wDjTVcAfrLwL+WZP7DGZ4/8RVvRiz4M62y+IORo2Ao8Cl3vKNSXDgPsPBYRjuoH8JcAKoHLhP3mdeDFiScH0B6+0CbEjm+x/p7U8dL/4xwLiA+XcABb15jwHbgWwBcZ8Crvc+m+xALVzizOztywrgYW/53LiD+mNANu913YSfQcC2JwBDvO/kfFyijv/OOgAxwIPetrJzeiJojjuA5/O+h8pAkYB9fimJ/4PHcf8HFb33XgIUTMnvK7H1JvFdf+TtY3bgLmBWwPJVcEn1PG+ZTbgTjcz8e9JUxe/jSCj+0lvROhJ8JSKHcD+yncBz3vQ7gMmqOllV41T1e2Ae0FJEigDXAF1UdZ+qnlLVn733dQaGqOofqhqrqqNwB7PLEtn2J0B7cFUrQDtvGrgD2SuqukJVY4C+QA0RKRnw/ldUda+qHktk3YVwB57EbPPmx/tYVZeq6hHgWeAWEYlK6jMIeO9IVV2mqjHe5zBJVf9W52fgO6DBGeI4kxdU9ZiqLsadEV7iTb8F6Ot95puB/kmso2AS+x9ogqrO8T7jMbgqQgBUdbSq7vH27S3cAaliwHt/U9WvvM/mmKrOV9XfveXX4w7kjbxlWwHbVfUtVT2uqodU9Y/EAhKRC3Cf8cOqekRVd+LO8NsFLLZVVQd420r4/Z/CJZpKgHi/oWA+C3Alm2dUdaX3HS5W1T2JLBfs55uc5719PIZLfoG/8duBL1X1BO7zW6+qH3r7vBB30tI2FWJIdywRpL3rVTU37my1Ev8eIEsCbUVkf/wfUB8oAhQH9qrqvkTWVxJ4LMH7iuOqQRL6ArjcSywNcdUmvwSs592AdezFnaEVDXj/piT2a7cXa2KKePMTW88G3Jl9IZL+DBKNQUSuEZHfRWSvt3xLTk86wdge8PwoEN+Af1GC7SW1/3s48/4Hsy1EpKeIrBCRA96+5OX0fUm47xVE5FvvwoODuOQdv3xxXHVLMErivoNtAZ/7EFzJINFtB1LVH3HVUgOBnSIyVETyBLntYOMM9vNNzj/7oaqHcCXt+ITXHpecwX0mdRP8Fm8HLkyFGNIdSwQ+8c5eRwJvepM24c6U8wX85VTVV715BUQkXyKr2gS8nOB9OVR1bCLb3Ic7Y74V1+A2TlU1YD33J1hPdlWdHbiKJHbpB9w/TvHAiSJSF/fP/mPA5MBlSuDOKHcn8xn8JwYROQ+X3N4ELlDVfMBkXAJLLt5gbMNVCSUWd0LTgWIiEn02GxKRBrg2iFuA/N6+HODffYH/7s9g4C+gvKrmwdW1xy+/CShzhs0lXM8mXCmyUMDnnkdVqybxntNXqNpfVWvhqlcq4Kp8kn2ft+2yySwD7vdVR0SKJbHMESBHwOvEDtoJ4xkLtBeRy3FVaD8FxPVzgt9iLlXtGkSsGY4lAn/1A5qKyCW4RsDWItJcRKJEJJt3+WMxr5g9BRgkIvlFJIuINPTWMQzoIiJ1vStpcorItSKS+wzb/ARXN3oz/1YLAbwPPCkiVQFEJK+IBF0MVtUfcAfDL0SkqrcPl3n7NVhVVwcsfoeIVBGRHEAfYLyqxib1GZxhs1lx1Se7gBgRuQYIvKRxB1BQRPIGux8JfIb7TPKLSFGg+5kW9PZvEDDWizmrF387EXkiiG3lxtXD7wIyi0hvILmz6ty4xtnDIlIJCDxIfQsUEZGHxV3Wm9tLyuA+l1LxV115v6/vgLdEJI+IZBKRsiLSiCCISG3v95cFdzA+jittxm/rTAkJ4APgRREp7/1+q4tIwYQLeb+v74EJIlJLRDJ7+9RFRO71FlsEtPP+P6Jxv/HkTMad/fcBPlXV+Li/BSqIyJ3e+rJ4+1k5iHVmOJYIfKSqu3CNV71VdROuwfYp3MFgE+6sKv47uhN35vwXrm3hYW8d84BOuKL5PlyDb4ckNjsRd4XLdq9OPD6WCcBrwDivmmEprl0iJW7CnVFNxV0lMhp3JcqDCZb7GFca2o47C+vhxZDcZ3Aar2jfA3fA3ocr5UwMmP8X7oxvrVe8T6y6LCl9gM3AOtwZ6XjcmfOZ9ODfKpL9uCqPG4BvgtjWNNzntgpXXXacpKuiAHri9vkQ7oTg0/gZ3mfTFGiN+5xXA1d6sz/3HveIyALv+V24xLoc91mOJ/iqmDze9vd5se/BXYgA7vuv4n3+XyXy3rdx3993uKQ2HNeQm5ibcQfuT3GlpaVANO67AdfeVNaL4wVOP9FJlNce8CXuqqRPAqYfwp1UtMNdqbcd9/9xXnLrzIjk35oBY0JPRGbgruTw5e7ecyEiXYF2qhrUmbIxGYWVCIw5AxEpIiL1vKqSirhLMSf4HZcxqc3urjPmzLLirp4pjavqGYdrBzAmrFjVkDHGRDirGjLGmAiX4aqGChUqpKVKlfI7DGOMyVDmz5+/W1ULJzYvwyWCUqVKMW/ePL/DMMaYDEVENpxpnlUNGWNMhLNEYIwxEc4SgTHGRLgM10aQmFOnTrF582aOHz/udyghky1bNooVK0aWLFn8DsUYE2bCIhFs3ryZ3LlzU6pUKcSf4XFDSlXZs2cPmzdvpnTp0n6HY4wJMyGrGhI3qPROEVl6hvkiIv1FZI2ILBGRS892W8ePH6dgwYJhmQQARISCBQuGdYnHGOOfULYRjMSNj3om1+B6wSyPG2Vr8LlsLFyTQLxw3z9jjH9CVjWkqjNFpFQSi1wHfOQNjPK7iOQTkSIpGOLOGGPC2/79MHUqRzavZdehHZRqdQfUrp3qm/HzqqGinN7f+mZOHxbxHyLSWUTmici8Xbt2pUlwKRUVFUWNGjWoVq0arVu3Zv/+/f/MW7ZsGVdddRUVK1akfPnyvPjiiwT28TRlyhSio6OpUqUKNWvW5LHHHvNhD4wx6YYqDBkCpUrx41Ptqb7paW7c0Z+4eXNDsrkMcfmoqg5V1WhVjS5cONE7pH2XPXt2Fi1axNKlSylQoAADBw4E4NixY7Rp04YnnniClStXsnjxYmbPns2gQa4Ty6VLl9K9e3dGjx7N8uXLmTdvHuXKlfNzV4wxfnvwQfY/3IVON2Wlyd2QqUwZ3vnfj2Tq2i0km/MzEWzh9DFgi3nTMrzLL7+cLVvcrnzyySfUq1ePZs3cCIo5cuTgvffe49VX3TC8r7/+Ok8//TSVKlUCXMmia9ewHBbVGBOMKVOIHTSQKx7MzoiSe+h1RS+WdFtKo9JXJv/es+Tn5aMTge4iMg6oCxxIlfaBhx+GRYvOeTWnqVED+vULatHY2FimT59Ox44dAVctVKtWrdOWKVu2LIcPH+bgwYMsXbrUqoKMMQDsWfQbBdreTFT+Arzc8iWKV6hN9EXRId9uyBKBiIwFGgOFRGQz8ByQBUBV38eNPdoSN8buUeCeUMWSFo4dO0aNGjXYsmULlStXpmnTpn6HZIzJIFSVMQtG8tAX9/Fq9Sx0GvU7N5Qvn2bbD+VVQ+2Tma/AA6m+4SDP3FNbfBvB0aNHad68OQMHDqRHjx5UqVKFmTNnnrbs2rVryZUrF3ny5KFq1arMnz+fSy65xJe4jTH+2nRgE10mdWHy6slcthPqdXkZ0jAJQAZpLM5IcuTIQf/+/XnrrbeIiYnh9ttv59dff+WHH34AXMmhR48e9OrVC4DHH3+cvn37smrVKgDi4uJ4//33fYvfGJN2xv45lqqDqjJj3Qz6TYFfd7aiyl1pX1VsiSAEatasSfXq1Rk7dizZs2fn66+/5qWXXqJixYpcfPHF1K5dm+7duwNQvXp1+vXrR/v27alcuTLVqlVj7dq1Pu+BMSYt5M+en7rF6rJ058089AdEdfHnQpEMN2ZxdHS0JhyYZsWKFVSuXNmniNJOpOynMeEqJi6Gd357h5OxJ3m64dMA6M6dSOXKsHcvxMVBiHoREJH5qppoy3NYdDpnjDHp3eLti+k4sSPzt83nlqq3oKqICPLooy4J/PhjyJJAcqxqyBhjQuhEzAme/fFZoodFs+ngJj5v+znjbhrn+g9bvRrGjHGXqF8ZuvsEkhM2JYL47BquMloVnjHGWb13Na/Neo3bLr6Nt5u9TcEcBf+d2bmzexw92p/gPGFRIsiWLRt79uwJ24Nl/HgE2bJl8zsUY0wQDp88zJglYwCodn41/ur+F6OuH3V6Eti1C2bMgObNoWpVfwL1hEWJoFixYmzevJn02iFdaogfocwYk759//f3dP62Mxv2b+DSIpdSuXBlyuQv898F+/Z1j089lbYBJiIsEkGWLFls5C5jjK/2HdtHz+96MmLRCCoUrMDPHX6mcuEzXOV37Ji7+TVfPmjQIC3DTFRYJAJjjPFTbFws9UbUY9WeVTxZ/0l6N+pNtsxJVOU+/7x77N/ftyuFAlkiMMaYs7T76G4KZC9AVKYo+jbpS4m8Jbi0SBCj7sZ3O3PbbaENMEhh0VhsjDFpSVX5aPFHVBhQgQ8WfADA9ZWuDy4JrFwJv/8ODzwAUVEhjjQ4ViIwxpgU2LB/A/d/ez/T/p7GFcWvoGHJhilbQe/e7vGRR1I/uLNkicAYY4I0eslouk7qiqoy4JoBdKvdjUySgoqVdetg8mS45BIoWzZ0gaaQJQJjjAlS4RyFqVe8HkNaDaFkvpIpX0Hr1nDkCAwfnvrBnQNLBMYYcwanYk/x1m9vcSr2FM82epbm5ZrTrGyzs+vFYOdOWLYMbr4ZEoxa6DdLBMYYk4iF2xbScWJHFm5fSLtq7f7tJO5sL/eM70YiHY5JblcNGWNMgOMxx3lq+lPUHlabrYe28sUtXzD2prHn3pfZiBGQPTtcdVXqBJqKrERgjDEB1uxdw5uz3+SuS+7irWZvkT97/nNf6dq1rlrovvvOfV0hYInAGBPxDp88zIQVE7jzkjupdn41VnZfSen8qdhtzeefu8eHHkq9daYiqxoyxkS0aWumUXVQVe7+6m5W7FoBkLpJANxVQuXKQbVqqbveVGKJwBgTkfYc3cPdX91NizEtyJElB7/c88uZO4k7Fxs2uAFomjdP/XWnEqsaMsZEnPhO4tbsXcPTDZ7mmYbPJN1J3LkYNco93n13aNafCiwRGGMixq4juyiYoyBRmaJ47erXKJmvJDUurBHajY4ZA4ULQ+3aod3OObCqIWNM2FNVPlz4IRXeq8Cw+cMAuK7SdaFPAseOwapV6e4GsoSsRGCMCWvr96+n8zed+X7t9zQo0YArS6fhIPHx1UIdO6bdNs+CJQJjTNj6ePHHdJ3UFRFhUMtB3B99f8o6iTsXqv8mgjZt0mabZ8kSgTEmbF2Q6wIalmzI+63ep0TeEmm78XXr3LgDzz8PWbOm7bZTyBKBMSZsnIo9xeuzXidWY+ndqDfNyjajWdlm/gQzdap7rFPHn+2ngDUWG2PCwoJtC6g9rDbP/PQMK/esRFX9DWjYMChRAlq08DeOIFgiMMZkaMdOHeOJH56gzrA67Diygwm3TmDMjWPOvZO4c7FrFyxa5EoD6WBw+uSENBGISAsRWSkia0TkiUTmlxCRn0RkoYgsEZGWoYzHGBN+1u5by9u/vU2HGh1Y3m0511e63u+Q4L333GM67HI6MRKq4pOIRAGrgKbAZmAu0F5VlwcsMxRYqKqDRaQKMFlVSyW13ujoaJ03b15IYjbGZAwHTxzkyxVf0qFGB8CNI3xWI4aFQlwcXHQRxMTAjh3pZoB6EZmvqtGJzQtliaAOsEZV16rqSWAccF2CZRTI4z3PC2wNYTzGmDAwefVkqg2qRseJHf/pJC7dJAFwYxLv2AFvvplukkByQpkIigKbAl5v9qYFeh64Q0Q2A5OBBxNbkYh0FpF5IjJv165doYjVGJPO7T66mzsn3Mm1n1xL7vNyM+veWaHpJO5cPfggZMsG7dr5HUnQ/G4sbg+MVNViQEvgY5H/3u2hqkNVNVpVowsXLpzmQRpj/BXfSdy4pePo3bA3Czov4LJil/kd1n/t3Qvr10PRoi4ZZBChvI9gC1A84HUxb1qgjkALAFX9TUSyAYWAnSGMyxiTQew4vIPCOQsTlSmKN5u+Scl8Jal+QXW/wzqzAQPc48sv+xtHCoWyRDAXKC8ipUUkK9AOmJhgmY1AEwARqQxkA6zux5gIp6oMXzCciu9VZOj8oQC0rtg6fSeB+C4lsmWDtm39jiZFQlYiUNUYEekOTAOigBGqukxE+gDzVHUi8BgwTEQewTUcd1Df7wIxxvhp7b61dPqmEz+u+5FGJRtxdZmr/Q4pODNmuG4lXn8dMvld654yIbt8NFTs8lFjwteoRaPoNrkbURLFG03foFOtTmnXSdy5atwYfv7ZXTF0/vl+R/MfSV0+an0NGWPSjYtyX8RVpa9i8LWDKZanmN/hBO/UKZcEsmZNl0kgOZYIjDG+ORl7kld/fZU4jeP5xs/TtGxTmpZt6ndYKff22+6xZ09/4zhLGaTMZYwJN3O3zKXW0Fo8N+M51u5b638ncefiuefc44sv+hvHWbJEYIxJU0dPHaXndz25bPhl7Du2j4ntJvLRDR/520ncuZg+HU6cgHvuyXCNxPGsasgYk6bW7VvHgDkD6HRpJ167+jXyZsvrd0hn78QJ6NDBtQ306+d3NGfNEoExJuQOHD/Alyu+5J6a91D1/KqseXANxfMWT/6N6V2HDrB5sxt7IE+eZBdPrzJmOcYYk2FMWjWJqoOqct839/HX7r8AwiMJzJsH48ZB6dLpfnD65FgiMMaExK4ju7j9y9tpNbYV+bPn57eOv1GpUCW/w0o98VcITZ2aIQafSYpVDRljUl1sXCz1P6zPun3reKHxCzxR/wmyRqXvAdxTZN8+mDULLr0UKlTwO5pzZonAGJNqth/ezvk5zycqUxRvNXuLUvlKUe38an6HlfqefNINPPPuu35HkiqCrhoSkRyhDMQYk3HFaRxD5g2hwoAKDJk3BIBWFVqFZxIYOxaGDIG774b69f2OJlUkmwhE5AoRWQ785b2+REQGhTwyY0yGsGbvGpp81IQuk7pQu2htmpdr7ndIobN3L9x7L+TMCYPC5zAYTNXQO0BzvC6kVXWxiDQMaVTGmAzhw4Uf0m1yN7JGZWVY62F0rNkx494YFowePeD4cZgyBXKETyVJUG0EqropwZcbG5pwjDEZSYm8JWhetjkDWw6kaJ6EI9GGmb/+gk8/hSuugBYt/I4mVQWTCDaJyBWAikgW4CFgRWjDMsakRydiTvDKr68Qp3H0ubIPTco0oUmZJn6HlTb694fYWNdGEGaCaSzuAjyAG3h+C1AD6BbCmIwx6dAfm/+g1tBavPDzC2w8sDFjdxKXUnv2wOjRcNVVUKKE39GkumBKBBVV9fbACSJSD5gVmpCMMenJkZNHePanZ+n3ez+K5inKt+2/5doK1/odVtp67DE4dAj69PE7kpAIpkQwIMhpxpgwtOHABgbNHUSX6C4s67Ys8pLA8eNuLOJGjVz7QBg6Y4lARC4HrgAKi8ijAbPy4MYgNsaEqf3H9zN++Xjuu/Q+qhSuwpoeazLWiGGp6dNP3WPTDDhgTpCSqhrKCuTylskdMP0gcHMogzLG+Ofrv76m66Su7Dyyk/ol6lOpUKXITQKxsfDCC1CkSIYdfSwYZ0wEqvoz8LOIjFTVDWkYkzHGBzuP7KTHlB58uuxTql9QnYntJ4ZXJ3Fn46uvYN06dyfxeef5HU3IBNNYfFRE3gCqAtniJ6rqVSGLyhiTpmLjYqk3oh4bD2zkpStfole9XmSJyuJ3WP4bPtw9tm3rbxwhFkwiGAN8CrTCXUp6N7ArlEEZY9LG1kNbuTDXhURliuLdFu9SKl8pqhSu4ndY6cOcOe4O4muvhfz5/Y4mpIK5aqigqg4HTqnqz6p6L2ClAWMysDiNY/DcwVR6rxLvz3sfgJblW1oSiKfqxiAuUCCs+hQ6k2BKBKe8x20ici2wFSgQupCMMaG0as8qOn3TiZkbZnJ1mau5ptw1foeU/nz9NSxfDgMHhuUNZAkFkwheEpG8wGO4+wfyAA+HMihjTGgMXzCc7lO6ky1zNka0GUGHGh3Cu5O4s/XGG5A9O9x3n9+RpIlkE4Gqfus9PQBcCf/cWWyMyWBK5SvFNeWuYWDLgRTJXcTvcNKnxYth9my44QbIGkajqiUhqRvKooBbcH0MTVXVpSLSCngKyA7UTJsQjTFn60TMCV6c+SIAL131UmR1Enc24uKgm9eV2ttv+xtLGkqqRDAcKA7MAfqLyFYgGnhCVb9Kg9iMMedg9qbZdJzYkb92/8W9Ne5FVa0aKDnvvONKA++9B6VK+R1NmkkqEUQD1VU1TkSyAduBsqq6J21CM8acjcMnD/P09KcZMGcAxfMWZ+rtU8N71LDUsnmzu3u4bl3o2tXvaNJUUpePnlTVOABVPQ6sTWkSEJEWIrJSRNaIyBNnWOYWEVkuIstE5JOUrN8Y818bD2xkyPwhPFD7AZZ2XWpJIBiqcLPXc07//pAp6OHcw0JSJYJKIrLEey5AWe+1AKqq1ZNasdfGMBBoCmwG5orIRFVdHrBMeeBJoJ6q7hOR889hX4yJWPuO7ePz5Z/TuVZnqhSuwtqH1nJR7ov8Divj+N//4I8/4P77oU4dv6NJc0klgsrnuO46wBpVXQsgIuOA64DlAct0Agaq6j4AVd15jts0JuJMWDGBbpO7sevILhqVbETFQhUtCaTEypWuYbhJExg82O9ofJFUp3Pn2tFcUWBTwOvNQN0Ey1QAEJFZuK6tn1fVqQlXJCKdgc4AJSLg5g5jgrH98HYenPIg45ePp8aFNZh02yQqFqrod1gZT9eurkO5UaMgQhvTgxq8PsTbLw80BooBM0XkYlXdH7iQqg4FhgJER0dH0Ph4xiQuNi6WBh82YNOBTfS9qi89r+hpncSdjYED4aef4JlnoGhRv6PxTSgTwRbc5afxinnTAm0G/lDVU8A6EVmFSwxzQxiXMRnW5oObuSj3RURliqJ/i/6Uzl/auoo+W7NmQffuUK4cPPmk39H4KqimcRHJLiIpLXPOBcqLSGkRyQq0AyYmWOYrXGkAESmEqypam8LtGBP24jSOAX8MoNJ7lRg819VjX1P+GksCZ+vkSXjgAfd86lTIkcPfeHyWbCIQkdbAImCq97qGiCQ8oP+HqsYA3YFpwArgM1VdJiJ9RKSNt9g0YI+ILAd+Ah63+xSMOd1fu/+i4YcN6TG1B/VL1KdVhVZ+h5TxPf6460ri/fehbFm/o/GdqCZd5S4i83HdTs9Q1ZretD9V9eI0iO8/oqOjdd68eX5s2pg098GCD+g+uTs5suSgX4t+3Fn9Trs7+FzNnOkGor/lFhg3LmIaiEVkvqpGJzYvqG6oVfVAgh+fNdgakwbK5i9L64qtee+a97gg1wV+h5PxqbqG4WzZXENxhCSB5ASTCJaJyG1AlHcDWA9gdmjDMiYyHY85Tp+f+wDQt0lfrix9JVeWvtLnqMLIqFHwyy/wyitQqJDf0aQbwTQWP4gbr/gE8AmuO+qHQxiTMRFp1sZZ1Hi/Bq/8+gq7juwiuWpbk0Jz57p7BmrUcG0E5h/BlAgqqerTwNOhDsaYSHToxCGemv4UA+cOpGS+kky7YxrNyjbzO6zw8vnncMcdULCgex4V5XdE6UowieAtEbkQGA98qqpLQxyTMRFl88HNfLDwAx6s8yAvN3mZXFlz+R1SeNm7F9q3d+0Cs2ZB6dJ+R5TuJFs1pKpX4kYm2wUMEZE/ReSZkEdmTBjbc3TPP/cDVC5cmbU91vLuNe9aEkhtsbGuKig21t0vYEkgUUHdUKaq21W1P9AFd09B71AGZUy4UlXGLx9PlUFV6DG1Byt3rwSwYSND5dlnYdMmuOkmqF/f72jSrWBuKKssIs+LyJ+4wetn47qLMMakwLZD27jps5to+3lbiucpzrxO86yTuFB69ll3dVD9+jB2rN/RpGvBtBGMAD4Fmqvq1hDHY0xYiu8kbsuhLbx+9es8cvkjZM7kd5+PYeyDD+Cll6BFC/jyS8hiHfIlJdlfoqpenhaBGBOONh3YRNE8RYnKFMXAlgMpnb80FQpW8Dus8LZkyb+dyU2Y4BqJTZLOWDUkIp95j3+KyJKAvz8DRi4zxiQiNi6W/n/0p9LAfzuJa16uuSWBUNu3z3UfkTUrTJtmSSBISZUIHvIerYcrY1Jgxa4VdJzYkd82/8Y15a6hdcXWfocUGU6ehGuugf374euvoUwZvyPKMM5YIlDVbd7Tbqq6IfAP6JY24RmTsQydP5QaQ2qwas8qPr7hYybdNokSeW1UvZA7fNi1B/zxBzz4ILRpk/x7zD+CuXy0aSLTrkntQIwJB+ULlOeGSjew/IHl3FH9DuspNC2cPOkuD/3pJ3jnHejf3++IMpwzVg2JSFfcmX+ZBG0CuYFZoQ7MmIzg2KljPD/jeUSEV69+1TqJS2sxMa466Mcf4YUX4OGH/Y4oQ0qqjeATYArwCvBEwPRDqro3pFEZkwHM3DCT+ybex+q9q+lSqwuqaiWAtBQTA7fe6pJA167Q2+5zPVtJJQJV1fUi8kDCGSJSwJKBiVQHTxzkiR+eYPC8wZTJX4bpd03nqtJX+R1WZNm6FZo3h6VLoVcvePVVvyPK0JIrEbQC5uMGogk81VHAmuRNRNp6aCsjF43k0csepc+VfciZNaffIUWWY8fgqqtg1SrXJmDVQefsjIlAVVt5j9ZLk4l4u4/u5rNln9GtdjcqFarEuofW2YhhfoiNhdatYeVKGDYM7rvP74jCQjB9DdUTkZze8ztE5G0RsevhTERQVT5d+ilVBlbh4akPs2rPKgBLAn555BGYPh2eftqSQCoK5vLRwcBREbkEeAz4G/g4pFEZkw5sPbSV6z+9nnZftKNkvpLM7zzf7gz205AhbpzhMmVch3Im1QTT61WMqqqIXAe8p6rDRaRjqAMzxk+xcbE0/LAhWw5t4c2mb/LQZQ9ZJ3F+GjECunSBWrXc/QLnned3RGElmF/2IRF5ErgTaCAimQDrys+EpQ37N1AsTzGiMkUx6NpBlMlfhnIFyvkdVuTatcvdHxBfEpgxA3LZ4D2pLZiqoVtxA9ffq6rbcWMRvBHSqIxJY7Fxsbz929tUHliZwfNcJ3HNyjazJOCnpUuhShWXBNq3h9mzLQmESDDdUG8XkTFAbRFpBcxR1Y9CH5oxaWPpzqV0nNiROVvm0KpCK66vdL3fIZkdO6BlS3ep6IwZrkdREzLBXDV0CzAHaAvcAvwhIjeHOjBj0sL7897n0iGXsnbfWj658RMmtptIsTw2AJ+vDh50l4hu2uRGFrMkEHLBtBE8DdRW1Z0AIlIY+AEYH8rAjAml+O4gKheqTNuqbenXvB+Fcxb2Oyyzf7+7WWzhQjfKWGvrwjstBJMIMsUnAc8eghz03pj05uipo/T+qTdREsVrTV+jUalGNCplZ5zpwr59rtuIRYtg8GDoaBcnppVgEsFUEZkGxI/+fCswOXQhGRMaM9bP4L6J9/H3vr/pFt3NOolLT06dgnbtXElgzBjXOGzSTDCNxY+LyI1AfW/SUFWdENqwjEk9B44foNf3vRi6YChl85flx7t+tK6i0xNVuOce+O4713mcJYE0l9R4BOWBN4GywJ9AT1XdklaBGZNath3exug/R9Pz8p68cOUL5MiSw++QTLy4OFcFNGaMSwb/+5/fEUWkpOr6RwDfAjfheiAdkNKVi0gLEVkpImtE5IkklrtJRFREolO6DWMSs+vILgb84X6ylQpVYv1D63mj2RuWBNKTVaugTh0YORI6dXKdyBlfJFU1lFtV47+ZlSKyICUrFpEoYCBuqMvNwFwRmaiqyxMslxt4CPgjJes3JjGqytilY+kxpQcHTxykebnmVChYwa4ISm/GjYN774UsWeC99+CB/wx7YtJQUokgm4jU5N9xCLIHvlbV5BJDHWCNqq4FEJFxwHXA8gTLvQi8BjyewtiNOc2mA5voOqkrk1ZPom7RugxvM9w6iUuPRo1y1UD588PPP0O1an5HFPGSSgTbgLcDXm8PeK1AckMyFQU2BbzeDNQNXEBELgWKq+okETljIhCRzkBngBIlrAds818xcTE0HtWY7Ye3807zd3iwzoNEZYryOyyT0PDhrvvosmVh5ky46CK/IzIkPTBNSC+r8DqvexvokNyyqjoUGAoQHR2toYzLZCzr96+neJ7iZM6UmSGthlAmfxnK5LfB89KlF16A55+H6tXdmAKFCvkdkfGE8sawLUDxgNfFvGnxcgPVgBkish64DJhoDcYmGDFxMbw5+00qD6zMoLmDALi6zNWWBNKjmBjo0MElgRYtYNYsSwLpTCg7WJ8LlBeR0rgE0A64LX6mqh4A/vk1iMgM3CWq80IYkwkDS3YsoePEjszbOo/rKl7HTVVu8jskcyYnTsD118PUqXDttTBxImSyjgnSm5B9I6oaA3QHpgErgM9UdZmI9BGRNqHarglvg+YOotbQWmzYv4FPb/6UCbdO4KLcVs+cLi1eDFdc4ZLAc8/Bt99aEkinki0RiLsH/3agjKr28cYrvlBV5yT3XlWdTILuKFS19xmWbRxUxCYixXcHUe38arSr1o53mr9DoRxWvZAuxcVBz57wzjuQM6cbXeyee/yOyiQhmKqhQUAc7iqhPsAh4AugdgjjMgaAIyeP8MyPz5A5U2beaPYGDUs2pGHJhn6HZc5kxgx45hnXDnDDDTBgABQt6ndUJhnBlNPqquoDwHEAVd0HZA1pVMYA09dO5+LBF9Pvj36ciD2Bql0wlm6tXOl6Dr3yStdxXP/+8MUXlgQyiGBKBKe8u4QV/hmPIC6kUZmItv/4fnp+15PhC4dTvkB5ZnaYSYOSDfwOy5zJ8OHw0EP/dh7Xrx/kyeN3VCYFgikR9AcmAOeLyMvAr0DfkEZlItqOwzsYt3Qc/6v3PxZ3WWxJIL06fhx69HA3iF16qRtjeMQISwIZUDDdUI8RkflAE1z3Eter6oqQR2YiSvzB/6HLHqJioYqsf3i9NQanZ3FxcNttMGEC3HgjfPIJnHee31GZsxTMmMUlgKPAN8BE4Ig3zZhzpqqMXjKaKoOq0OuHXqzesxrAkkB6tmuXuyx0wgTo3du1BVgSyNCCaSOYhGsfECAbUBpYCVQNYVwmAmw8sJEu33ZhypopXF7scoa3GU75guX9Dssk5dtvoUsX2LrVDSJj4weEhWCqhi4OfO11FNctZBGZiBATF0PjkY3ZeWQn/Vv0p1vtbtZJXHr255+uLWDOHChSxN0k1qyZ31GZVJLiLiZUdYGI1E1+SWP+a+2+tZTMW5LMmTIzrPUwyhYoS6l8pfwOy5yJKrz+Ojz1FGTODK+9Bt27Qw4b4CecBHNn8aMBLzMBlwJbQxaRCUsxcTG8NfstnpvxHK83fZ0edXvQpEwTv8MySYmNdQf999+Hhg1h7FjrNjpMBVMiyB3wPAbXZvBFaMIx4WjR9kV0nNiRBdsWcEOlG2hbpa3fIZnk7NwJ7drBTz+5KqH334coq7oLV0kmAu9Gstyq2jON4jFh5r057/HItEcomL0g49uOt55CM4Lff4e2bf9tEO7VC0SSf5/JsM6YCEQks6rGiEi9tAzIhIf4TuKqX1Cd2y++nbebv02B7AX8DsskJS4O2reHzz6DXLlgyhRrEI4QSZUI5uDaAxaJyETgc+BI/ExV/TLEsZkM6PDJwzw9/WmyRGXhzWZvWidxGcWxY3DXXTB+PFx1lUsGBQv6HZVJI8F0MZEN2IPrfbQV0Np7NOY03/39HdUGVWPAnAGcij1lncRlFL//DpUquSRw333w/feWBCJMUiWC870rhpby7w1l8ew/3Pxj37F9PPrdo4xcNJKKBSsy856Z1C9R3++wTDDefhseewyyZXOJ4CZrw4lESSWCKCAXpyeAeJYIzD92HtnJ+OXjebL+k/Ru1JtsmbP5HZJJzrZtcP/98M030KSJuyqoXDm/ozI+SSoRbFPVPmkWiclQth/eztg/x/LI5Y+4TuIeWk/BHFadkO6pusFinn0WDh6EJ5+EF16ALFn8jsz4KKk2ArtezPyHqjJq0SiqDKzCk9Of/KeTOEsCGcCMGXDxxW7sgCpV4NdfoW9fSwImyURgt32a06zfv54WY1rQ4esOVClchUVdFlkncRnBzp1w991u9LBdu2DwYDeUZD27Mtw4Z6waUtW9aRmISd9i4mK4ctSV7D66m4EtB9IluguZJJiLzoyvvv4a7rgDDh92yeDddyFvXr+jMulMijudM5Flzd41lM5XmsyZMjOizQjK5C9DyXwl/Q7LJOfvv93NYXPnut5Cv/sOLr/c76hMOmWndCZRp2JP0feXvlQdVJWBcwcCcGXpKy0JpHeqrurn4otdEnjoIVi92pKASZKVCMx/LNi2gI4TO7Jo+yLaVmnLrVVv9TskkxxVN2hM796waBGULAlffQU1avgcmMkIrERgTtP/j/7UGVaH7Ye38+UtX/JZ28+4INcFfodlkvLzz65biDZtYMcOd3no0qWWBEzQrERggH87iat5YU3uuuQu3mr2Fvmz5/c7LJOUVavcUJFffQWFCsFLL0HPnjZ+sEkxSwQR7tCJQzw5/UnOizqPt5q/RYOSDWhQsoHfYZmkLF4Mzz/v7goGN3jMa6/ZqGHmrFnVUASbumYq1QZXY9DcQShqncSldydOuDuBa9VypYC2bd3VQQMGWBIw58RKBBFoz9E9PPrdo3y0+CMqF6rMrHtncXlxu6okXZs5E+65B9auhcaNXd9AFSv6HZUJE1YiiEB7ju1hwooJPNvwWRbev9CSQHq2ahXccgs0agQbN7oeQn/80ZKASVUhTQQi0kJEVorIGhF5IpH5j4rIchFZIiLTRcQuUg+RbYe28ebsN1FVKhSswIaHN9Dnyj6cl9kaFtOl2bOheXOoXNm1BXTpAuvXu26ibdhIk8pClgi88Y4HAtcAVYD2IlIlwWILgWhVrQ6MB14PVTyRSlUZsXAElQdW5tmfnmXN3jUAdkVQerV6NbRq5foBmjEDevSANWvcTWJFi/odnQlToSwR1AHWqOpaVT0JjAOuC1xAVX9S1aPey9+BYiGMJ+Ks27eOZqOb0XFiRy658BIWd1lsncSlV/FdQlSs6LqDeOIJ2L4d3nnHEoAJuVA2FhcFNgW83gzUTWL5jsCUxGaISGegM0CJEiVSK76wFhMXw1UfXcWeo3sYfO1gOtfqbJ3EpUcbNsDjj7urgLJkcYPF9OoFpUv7HZmJIOniqiERuQOIBholNl9VhwJDAaKjo+0axySs3rOaMvnLkDlTZj687kPK5i9L8bzF/Q7LJHT0qBscpl8/iIqC225zA8SUtGYyk/ZCeYq4BQg8AhXzpp1GRK4GngbaqOqJEMYT1k7FnuKlmS9RbXA13pvzHgCNSzW2JJDenDzprvypWdONF3zrrfDnnzBypCUB45tQlgjmAuVFpDQuAbQDbgtcQERqAkOAFqq6M4SxhLV5W+fRcWJHluxYQrtq7Wh/cXu/QzIJ7d4Nw4ZB//6u7r90aZgwAa6/3u/IjAldIlDVGBHpDkwDooARqrpMRPoA81R1IvAGkAv4XNwlcRtVtU2oYgpH7/7+Lo9+9ygX5rqQr9t9TZuK9vGlK8uWwfDhMGIEHDgAtWu7wWFuvBEyp4uaWWNC20agqpOByQmm9Q54fnUotx/O4juJi74omo41O/J609fJly2f32GZeBs3uqqfAQPcdf8tW7o2gdq1/Y7MmP+wU5IM5uCJg/zv+/+RLXM23mnxDvVK1KNeCRt7Nt1YuNB1CDd5MsTEuKqfwYPhwgv9jsyYM7LrCTOQyasnU3VQVYYuGErmTJmtk7j04uRJ+PhjqF8fLr0Upk93dwL//bdrB7AkYNI5KxFkALuP7ubhqQ8z5s8xVC1clfFtx1O3WFK3ZJiQU3UdwX30kfuLiYGLLnIjhHXvDoUL+x2hMUGzRJAB7Du2j29WfcNzjZ7jqQZPkTUqq98hRS5VmDTJ3fH744+QO7fr/+f6691jlix+R2hMilkiSKe2HNzCmD/H8PgVj1O+YHk2PLzBGoP9tGuXu9b/ww9hxQrIn9+NCPbIIzYWgMnwLBGkM6rKBws+oOf3PTkVe4obK99IuQLlLAn4Zf16eOYZ+OwzOHUKoqNhyBDo0AGyWsnMhAdLBOnI33v/ptM3nfhp/U80LtWYYa2HUa5AOb/DijzHj7u7f9980w0LmSmT6xH0+efdHcHGhBlLBOlETFwMTT5qwt5jexnSagj3XXqfdRKX1pYtc4PBT5rkXhcpAvfdB08/DaVK+RqaMaFkicBnK3evpGyBsmTOlJlR14+ibIGyFMtjvXGnmd27YexYGDfODQaTJYsbDaxjR9cttN39ayKAnXL65GTsSV6Y8QIXD76YgXMGAtCoVCNLAmnh5EkYPRratXMdvfXoATt3wosvwtatbkCYO++0JGAihv3SfTBnyxw6TuzI0p1Lue3i27i9+u1+hxT+Tp2CL75wwz5++inExkK+fK73zwcecDeC2RCQJkJZIkhj/X7vx2PfPUaRXEX4pv03tKrQyu+QwtfJk/DLLzBoEEyd6sYAANf3f6tWLglkskKxMZYI0kh8J3F1itah06WdeO3q18ibLa/fYYWfkyfhhx9g4ED3ePIkZMsGDRq4ht9rrnE3gRlj/mGJIMQOHD9Ar+97kT1Ldvq16McVxa/giuJX+B1WeNm6Fb7/HiZOdPX7e/dCzpxwzz1w9dXQogXkyuV3lMakW5YIQuibld/QZVIXth/eTs/Le/5TKjDnKCbG3d07erQb63fVKje9UCF34G/bFq69FrJn9zVMYzIKSwQhsOvILh6a+hBjl47l4vMv5qtbv6J2UeuH/pzs2wdTpsB338GoUf9Or1zZXft/882uwdfq/I1JMUsEIXDgxAEmr57MC41f4In6T1gncSml6g788+a5ev7p02HBAjcvXz532efll7uz/rJlfQ3VmHBgiSCVbDqwidFLRvNE/ScoV6AcGx7eYI3BwYqLc9U7c+e6qp6vv3aXdwJERUG9em50ryZNXKOvnfUbk6osEZyjOI1j6Pyh9Pq+F7EaS9uqbSlXoJwlgeQcPOjO8idMcHX9e/e66fnyuTr+okWhWTOoU8dNM8aEjCWCc7B6z2o6fdOJnzf8TJPSTRjaeihl8pfxO6z0accOd+D/8Ud3Q9emTW56/Hi+N94ItWq5On/r1dOYNGWJ4CzFxMXQ9OOm7D++n+FthnNPjXvsiqBAcXGuqueHH9xZ//z5bnrmzK6q57rroGlTV9WTP7+/sRoT4SwRpNCKXSsoX7A8mTNl5uMbPqZsgbJclPsiv8NKH44edVf1jB8PY8b8O71qVejTxx30L7nEDvzGpDOWCIJ0IuYEfX/pS99f+/JG0zd4+LKHaVCygd9h+WfvXvj5Z/jrLzdI+9y5rhvn+D58br/dHfRvv92N5WuMSbcsEQTh982/03FiR5bvWs6d1e/kzup3+h1S2lGFbdtg0SJ31+5vv7krfHbu/HeZnDnd5ZytWrnHZs2snt+YDMQSQTLemv0Wj3//OMXyFGPybZO5pvw1focUGjExsGED/Pmna8j9669/D/pHjrhlsmSB2rWhdWt3/X7dulCtGhQubD13GpOBWSI4gziNI5Nk4vLil9MluguvXv0qec7L43dYqePQIXeGv2IFrFkD06bBkiWnL5Mnj7uK5557oFIldzVP3bru7N8YE1YsESSw//h+Hpv2GDmy5GBAywEZu5O4nTth1ixXd79iBfz6q+uXf9u205erUcMd8IsWdaNzlS8PJUrYWb4xEcISQYCv/vqKbpO6sfPITnrV65UxOok7fhwWLoR162DLFvf3889w4ICbFq9IEXdmnyULdO0K5cq51xUrQo4c/sVvjPGdJQJg55GddJ/cnc+Xf06NC2vw7W3fcmmRS/0Oy1F14+ru2uUO7Js3w/Llrkpn8uT/Lp8zJ5QuDcWKwb33ujP86tUhr93pbIxJnCUC4OCJg3y/9ntevuplHr/icbJEZUnbAFRdNc5vv7nG2pUr3SWZO3acflYfL1s2qFABrr8ezjvPdb1cq5Y7y8+Vy6p0jDEpErGJYOOBjXy8+GOeavAU5QqUY+PDG8l9XohGroqNhe3bYeNGWLvWHfA3bnR3365b587wEypQwNXVX3utO5uvWtUNtF6qFJx/vg2sboxJNSE9mohIC+BdIAr4QFVfTTD/POAjoBawB7hVVdeHMqY4jeP9ee/zvx/+R5zGcWu1WylXoFzKk8CJE+4yy1273AhZu3e7RtgdO9zNVocOuQN8liyuGicxRYtCdDQ0b+6qdGrVcgf8MmVcr5vGGJMGQpYIRCQKGAg0BTYDc0VkoqoGnv52BPapajkRaQe8BtwaqphW7l5Jp2868cvGX2haqglDL+9LqS3H4O+5rnuEbdtcNc3atbBnj3t+4ADs3+8O7Nu3u6tv4rtITkzhwq4LhTx53Nl7XJzrVyd/fqhZ000rXdpGzzLGpBuhLBHUAdao6loAERkHXAcEJoLrgOe95+OB90REVFVTO5iY4cNo/lc3DkTF8OF3wt0LpyPUTf6NRYu6g3ju3O4AXq2au2s2Xz73unhxdwZfuDBccIErARhjTAYSykRQFNgU8Hoz/OfI+88yqhojIgeAgsDuwIVEpDPQGaBEiRJnFUzmwhcw+teGlM12EUWqAjdVgoIF3Zl7XJw74GfO7Kblzu3q6HPksIZXY0zYyxAtjqo6FBgKEB0dfXalhTZtqN+mTWqGZYwxYSGUY/5tAYoHvC7mTUt0GRHJDOTFNRobY4xJI6FMBHOB8iJSWkSyAu2AiQmWmQjc7T2/GfgxFO0DxhhjzixkVUNenX93YBru8tERqrpMRPoA81R1IjAc+FhE1gB7ccnCGGNMGgppG4GqTgYmJ5jWO+D5caBtKGMwxhiTtFBWDRljjMkALBEYY0yEs0RgjDERzhKBMcZEOMloV2uKyC5gw1m+vRAJ7lqOALbPkcH2OTKcyz6XVNXCic3IcIngXIjIPFWN9juOtGT7HBlsnyNDqPbZqoaMMSbCWSIwxpgIF2mJYKjfAfjA9jky2D5HhpDsc0S1ERhjjPmvSCsRGGOMScASgTHGRLiwTAQi0kJEVorIGhF5IpH554nIp978P0SklA9hpqog9vlREVkuIktEZLqIlPQjztSU3D4HLHeTiKiIZPhLDYPZZxG5xfuul4nIJ2kdY2oL4rddQkR+EpGF3u+7pR9xphYRGSEiO0Vk6Rnmi4j09z6PJSJy6TlvVFXD6g/X5fXfQBkgK7AYqJJgmW7A+97zdsCnfsedBvt8JZDDe941EvbZWy43MBP4HYj2O+40+J7LAwuB/N7r8/2OOw32eSjQ1XteBVjvd9znuM8NgUuBpWeY3xKYAghwGfDHuW4zHEsEdYA1qrpWVU8C44DrEixzHTDKez4eaCKSoQcnTnafVfUnVT3qvfwdN2JcRhbM9wzwIvAacDwtgwuRYPa5EzBQVfcBqOrONI4xtQWzzwrk8Z7nBbamYXypTlVn4sZnOZPrgI/U+R3IJyJFzmWb4ZgIigKbAl5v9qYluoyqxgAHgIJpEl1oBLPPgTrizigysmT32SsyF1fVSWkZWAgF8z1XACqIyCwR+V1EWqRZdKERzD4/D9whIptx4588mDah+Sal/+/JyhCD15vUIyJ3ANFAI79jCSURyQS8DXTwOZS0lhlXPdQYV+qbKSIXq+p+P4MKsfbASFV9S0Qux416WE1V4/wOLKMIxxLBFqB4wOti3rRElxGRzLji5J40iS40gtlnRORq4GmgjaqeSKPYQiW5fc4NVANmiMh6XF3qxAzeYBzM97wZmKiqp1R1HbAKlxgyqmD2uSPwGYCq/gZkw3XOFq6C+n9PiXBMBHOB8iJSWkSy4hqDJyZYZiJwt/f8ZuBH9VphMqhk91lEagJDcEkgo9cbQzL7rKoHVLWQqpZS1VK4dpE2qjrPn3BTRTC/7a9wpQFEpBCuqmhtGsaY2oLZ541AEwARqYxLBLvSNMq0NRG4y7t66DLggKpuO5cVhl3VkKrGiEh3YBruioMRqrpMRPoA81R1IjAcV3xcg2uUaedfxOcuyH1+A8gFfO61i29U1Ta+BX2OgtznsBLkPk8DmonIciAWeFxVM2xpN8h9fgwYJiKP4BqOO2TkEzsRGYtL5oW8do/ngCwAqvo+rh2kJbAGOArcc87bzMCflzHGmFQQjlVDxhhjUsASgTHGRDhLBMYYE+EsERhjTISzRGCMMRHOEoFJl0QkVkQWBfyVSmLZw6mwvZEiss7b1gLvDtWUruMDEaniPX8qwbzZ5xqjt574z2WpiHwjIvmSWb5GRu+N04SeXT5q0iUROayquVJ72STWMRL4VlXHi0gz4E1VrX4O6zvnmJJbr4iMAlap6stJLN8B1+tq99SOxYQPKxGYDEFEcnnjKCwQkT9F5D89jYpIERGZGXDG3MCb3kxEfvPe+7mIJHeAngmU8977qLeupSLysDctp4hMEpHF3vRbvekzRCRaRF4FsntxjPHmHfYex4nItQExjxSRm0UkSkTeEJG5Xh/z9wfxsfyG19mYiNTx9nGhiMwWkYrenbh9gFu9WG71Yh8hInO8ZRPrsdVEGr/73rY/+0vsD3dX7CLvbwLuLvg83rxCuLsq40u0h73Hx4CnvedRuP6GCuEO7Dm96f8DeieyvZHAzd7ztsAfQC3gTyAn7q7sZUBN4CZgWMB783qPM/DGPIiPKWCZ+BhvAEZ5z7PiepHMDnQGnvGmnwfMA0onEufhgP37HGjhvc4DZPaeXw184T3vALwX8P6+wB3e83y4vohy+v1925+/f2HXxYQJG8dUtUb8CxHJAvQVkYZAHO5M+AJge8B75gIjvGW/UtVFItIIN1jJLK9rjay4M+nEvCEiz+D6qemI679mgqoe8WL4EmgATAXeEpHXcNVJv6Rgv6YA74rIeUALYKaqHvOqo6qLyM3ecnlxncWtS/D+7CKyyNv/FcD3AcuPEpHyuG4Wspxh+82ANiLS03udDSjhrctEKEsEJqO4HSgM1FLVU+J6FM0WuICqzvQSxbXASBF5G9gHfK+q7YPYxuOqOj7+hYg0SWwhVV0lbqyDlsBLIjJdVfsEsxOqelxEZgDNgVtxA62AG23qQVWdlswqjqlqDRHJget/5wGgP24Anp9U9QavYX3GGd4vwE2qujKYeE1ksDYCk1HkBXZ6SeBK4D9jLosbh3mHqg4DPsAN9/c7UE9E4uv8c4pIhSC3+QtwvYjkEJGcuGqdX0TkIuCoqo7GdeaX2Jixp7ySSWI+xXUUFl+6AHdQ7xr/HhGp4G0zUepGm+sBPCb/dqUe3xVxh4BFD+GqyOJNAx4Ur3gkrldaE+EsEZiMYgwQLSJ/AncBfyWyTGNgsYgsxJ1tv6uqu3AHxrEisgRXLVQpmA2q6gJc28EcXJvBB6q6ELgYmONV0TwHvJTI24cCS+IbixP4Djcw0A/qhl8El7iWAwvEDVo+hGRK7F4sS3ADs7wOvOLte+D7fgKqxDcW40oOWbzYlnmvTYSzy0eNMSbCWYnAGGMinCUCY4yJcJYIjDEmwlkiMMaYCGeJwBhjIpwlAmOMiXCWCIwxJsL9H/g5u0PsVtO8AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.2627712738168408\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjdklEQVR4nO3de7xXVZ3/8ddb5CqgBlQKCASkIiDo8ZaaZt5DnHLG+5TlyGiRVo5l2pijZlaT+vMR/tRMc3K81xQmZoYXwgsBiQYYSmRwUEdEkZA7fOaPtY98OR4433M4+3zP9+z38/E4j7Mv67u/n8XR/dl77bXXUkRgZmbFtV2lAzAzs8pyIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwKrGpLOkPTbMsrdJOnfWyOm1iDpFUlHZsuXS7qz0jFZ++JEYC0iO1mtkrRC0v9K+qmk7i35HRHx3xFxdBnlzo2IK1vyu+tICknvZvVcLOlaSR3y+K7mkNRT0vWSFmYx/iVb713p2KztciKwlnRCRHQH9gFqgG/VLyBp+1aPquXtndXzMOAU4AsVjgcASZ2AycBewLFAT+AgYCmwfzOO1x7+VlYGJwJrcRGxGHgYGA7vXUV/SdLLwMvZtjGSZklaJulpSSPrPi+pv6RfSFoiaamkH2Xbz5I0NVuWpOskvSFpuaQ/Sar7vp9KuqrkeOdImi/pLUkTJe1asi8knSvp5SyWCZJUZj3nA08Bo0qO15x6DZb0WLbtTUn/LWmnJv6zA3wW2A34dETMjYiNEfFGRFwZEZNK6jukJKb3/q0kHS6pVtI3JL0O3C7pRUljSspvn8W/T7Z+YFbPZZKel3R4M+K2CnMisBYnqT9wPPBcyeZ/AA4AhkkaDdwG/CvQC7gZmCipc9bM8mvgb8BAoC9wTwNfczTwceCjwI7AyaQr3/qxHAF8N9u/S3bc+scbA+wHjMzKHVNmPfcADgXmZ+vNrZeyGHcF9gT6A5eXE0M9RwK/iYgVzfhsnQ8DHwAGAOOAu4HTSvYfA7wZEX+U1Bd4CLgq+8y/AT+X1Gcbvt8qwInAWtIvJS0DpgJPAleX7PtuRLwVEatIJ5ibI2JaRGyIiDuANcCBpCaMXYGLIuLdiFgdEVMb+K51QA9gD0AR8WJEvNZAuTOA2yLijxGxBvgmcJCkgSVlromIZRGxEHickiv8LfijpHeBF4EngBuz7c2qV0TMj4hHI2JNRCwBriU1OzVVL6Chf4Om2Ah8O4tlFXAXMFZSt2z/6aTkAHAmMCkiJmV3H48CM0gXAVZFnAisJf1DROwUEQMi4ovZiaTOopLlAcCFWXPCsix59CedKPsDf4uI9Vv7ooh4DPgRMAF4Q9Itkno2UHRX0lV43edWkO4c+paUeb1keSXQHUDSnOyB6wpJh5aU2ScrcwrpLmeHbamXpA9Juid7+LwcuBNozsPdpaS7nm2xJCJW161kzV8vAidkyWAsKTlAqu8/1avvIS0Qg7UyJwJrLaXD3C4CvpMljbqfbhFxd7Zvt3IeVEbEDRGxLzCM1ER0UQPFXiWdsACQtAPpynlxGcffKyK6Zz+/r7cvIuI+4Bngsm2s19Wkf58REdGTdKVd1nOKen4HHJPVcUtWAt1K1j9cb39DwxHXNQ+dCMzNkgOkOv2sXn13iIhrmhG7VZATgVXCj4FzJR2QPfTdQdKnJPUA/kBq3rgm295F0sH1DyBpv+zzHYF3gdWkZo367gY+L2mUpM6kk+60iHilhepyDXCOpA9vQ716ACuAd7J294YSWjl+Rjo5/1zSHpK2k9RL0iWS6pprZgGnS+og6VjKa4K6h/RM5jw23Q1AunM5QdIx2fG6ZA+c+zUzfqsQJwJrdRExAziH1LTzNulh61nZvg3ACcAQYCFQS2qCqa8n6cT7NqnpZynwgwa+63fAvwM/J52IBwOntmBd/gRMIbX9N7de/0FqbnqH9PD1F82MZQ3pgfGfgUeB5aQE1BuYlhW7IItjGen5yS/LOO5rpDufjwH3lmxfRLpLuARYQkpCF+HzStWRJ6YxMys2Z24zs4JzIjAzKzgnAjOzgnMiMDMruKobVKp3794xcODASodhZlZVZs6c+WZENDj8R9UlgoEDBzJjxoxKh2FmVlUk/W1L+9w0ZGZWcE4EZmYF50RgZlZwVfeMwMysudatW0dtbS2rV69uvHCV6tKlC/369aNjx45lf8aJwMwKo7a2lh49ejBw4EDKnIiuqkQES5cupba2lkGDBpX9udyahiTdpjSN4Owt7JekG5SmEHyhbuo7M7O8rF69ml69erXLJAAgiV69ejX5jifPZwQ/JU2gvSXHAUOzn3HA/88xFjMzgHabBOo0p365JYKImAK8tZUiJwL/lU3w8Sywk6T8ZjaaOhUuuwzWrs3tK8zMqlElew31ZfPpC2vZfPrA90gaJ2mGpBlLlixp3rc98wxceSWsW9e8z5uZtYAOHTowatQohg8fzgknnMCyZcve2zdnzhyOOOIIdt99d4YOHcqVV15J6VQBDz/8MDU1NQwbNozRo0dz4YUXtkhMVdF9NCJuiYiaiKjp06fBN6TNzKpC165dmTVrFrNnz+YDH/gAEyZMAGDVqlWMHTuWiy++mHnz5vH888/z9NNPc+ONNwIwe/Zsxo8fz5133sncuXOZMWMGQ4YMaZGYKpkIFpMm9K7TjzLmkTUzay8OOuggFi9Op7277rqLgw8+mKOPPhqAbt268aMf/YhrrklTQH//+9/n0ksvZY899gDSncV5553XInFUsvvoRGC8pHuAA4B3sinxzMzy95WvwKxZLXvMUaPg+uvLKrphwwYmT57M2WefDaRmoX333XezMoMHD2bFihUsX76c2bNnt1hTUH25JQJJdwOHA70l1QLfBjoCRMRNwCTgeNK8riuBz+cVi5lZW7Fq1SpGjRrF4sWL2XPPPTnqqKMqHVJ+iSAiTmtkfwBfyuv7zcy2qswr95ZW94xg5cqVHHPMMUyYMIHzzz+fYcOGMWXKlM3KLliwgO7du9OzZ0/22msvZs6cyd57793iMVXFw2Izs/amW7du3HDDDfzwhz9k/fr1nHHGGUydOpXf/e53QLpzOP/88/n6178OwEUXXcTVV1/NSy+9BMDGjRu56aabWiQWJwIzswoZPXo0I0eO5O6776Zr16786le/4qqrrmL33XdnxIgR7LfffowfPx6AkSNHcv3113Paaaex5557Mnz4cBYsWNAicXisITOzVrRixYrN1h988MH3lkeMGMETTzyxxc+OGTOGMWPGtHhMviMwMys4JwIzs4JzIjCzQikdsqE9ak79nAjMrDC6dOnC0qVL220yqJuPoEuXLk36nB8Wm1lh9OvXj9raWpo9eGUVqJuhrCmcCMysMDp27NikmbuKwk1DZmYF50RgZlZwTgRmZgXnRGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgXnRGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgXnRGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgXnRGBmVnC5JgJJx0qaJ2m+pIsb2L+bpMclPSfpBUnH5xmPmZm9X26JQFIHYAJwHDAMOE3SsHrFvgXcFxGjgVOBG/OKx8zMGpbnHcH+wPyIWBARa4F7gBPrlQmgZ7a8I/BqjvGYmVkD8kwEfYFFJeu12bZSlwNnSqoFJgFfbuhAksZJmiFpxpIlS/KI1cyssCr9sPg04KcR0Q84HviZpPfFFBG3RERNRNT06dOn1YM0M2vP8kwEi4H+Jev9sm2lzgbuA4iIZ4AuQO8cYzIzs3ryTATTgaGSBknqRHoYPLFemYXAJwEk7UlKBG77MTNrRbklgohYD4wHHgFeJPUOmiPpCkljs2IXAudIeh64GzgrIiKvmMzM7P22z/PgETGJ9BC4dNtlJctzgYPzjMHMzLau0g+LzcyswpwIzMwKzonAzKzgnAjMzArOicDMrOCcCMzMCs6JwMys4JwIzMwKLtcXyszMrJ6NG+HNN+GNN+D11+F//zctv/EGLF0KixfDpOw93N12g4ULoWPHtPyd78App7R4SE4EZmYtYcOGdFKvrYW5c2HFinRSf+45+Pvf4cUXoVOnVKYh220HffrAW2/BTjvBsmWwzz5QUwP9+qXj5jT6shOBmVlj3n0XFi1KV+dTp8IOO8D8+TB7dkoAr7wCW5srZb/94KCDUnIYMwZ69EjrH/4wfOhD6WfHHUFqtSqVciIws2KLSFfhr7wCf/1r+v3UU7BqVbp6nzcvLTdkjz1gwADo3j2d2Pv1S1fzH/kIDByYruC3a/uPYp0IzKz9W7kyneAXLIBnn03NLrW1af3Pf4Z1697/mWHD0gm9Tx844gjo3z/dCeyxRzrhd+/e2rXIjROBmVW/jRvhtdfgL39JP08+CWvXphP9nDmpSaaUBHvtBYMGwdCh6Wp+8OC0PnBguqovECcCM6sOq1enq/r589PP5Mlp+/z56aq+vkGD0hX9vvvCUUdtWh8wILXNV6g9vi1yIjCztmPVqnRSX7AAXnoptc//9repHf/1199ffuTI1BNn/HjYc890VT94cOpq2alT68dfpZwIzKx1LV+eHsr+4Q+prf6ll1Jzzssvp3b7+g45BIYMSW31AwakK/whQ6BXL1/VtxAnAjNrWRHpBD9vXrq6f/TR1MXyt7+FNWvSg9tSffqkq/hPfCKd3EeMgFGjUtt9jx6VqEHhOBGYWdNFwKuvppP9yy+nE35dE86LL76/fKdOsPfeqfmmTx844AD44AfTtoI9mG2LnAjMbMs2bkzNOC+8kF6emjcvnej/+Mf3l91993Tlf8YZqQnnwANTV8tBg2B7n2raMv91zCxdyb/2Wjrhz5gBjz+e2vLnzt28Kad/f+jcGb74xdTPfvfd4aMfTf3qq+DFKWuYE4FZ0SxZksa/efDBdLJ/6KH0u/Slqj59Ujv9v/xL6pkzcmQ68e+wQ8XCtvw4EZi1V8uXpwe1b76ZeubMnp3a8esbOTI9oP3MZ1Kb/YgRsPPOrR+vVYwTgVm1e/dd+NOf4LHHNg2b8MwzKRHU6dw5ndyPPTa9XDV6dDrh9+5dubitzXAiMKsW69enNvyXXkrNObW1MH16asOP2FRu5MjUz/6kk9I49iedlIZNcBu+bYETgVlbtHJlGu547lz4n/+BKVMaLvfpT6fmnFGjUhv+4ME+4VuTORGYVdLGjakf/hNPpH75v/51utJ/443Nyw0ZAscdl4ZOOOqo1B/fQyhYC3EiMGst77wD06aln9//Hv72t9TMU6pbtzRJySWXpL74Y8bArrtWJl4rDCcCszwsW5Z67Pz5z3DvvWko5Pr22w/OOSf1xT/sMBg+HLp0afVQzZwIzLZFRBow7dln04n/qafSen1HHJF+hgxJY9/37+8B06zNyDURSDoW+H9AB+DWiLimgTInA5cDATwfEafnGZNZs61bl3rpvPAC3HVXGj1zzZr3lzvkEDj00NR75+ij4QMfaP1YzZogt0QgqQMwATgKqAWmS5oYEXNLygwFvgkcHBFvS/pgXvGYNdnrr6feOi+8ADffnNr4S9++7d49Ne8cdhgcf3zqudOtW8XCNWuuPO8I9gfmR8QCAEn3ACcCc0vKnANMiIi3ASLijfcdxSxvEWmWq4ceSk07s2al3jwLFmwqM2JEGkxtl11Sv/zBg920Y+1GnomgL7CoZL0WOKBemY8CSHqK1Hx0eUT8pv6BJI0DxgHstttuuQRrBbJmTRpY7YEH4Prr379/l11g//3hlFPSGPkHHdSuJio3q6+sRCDpYFI7/oDsMwIiIj7SAt8/FDgc6AdMkTQiIpaVFoqIW4BbAGpqagKzpli3LjXxTJ0Kkyaltv1SH/tYGjL5c59LV/6+0reCKfeO4CfAV4GZwIYyP7MY6F+y3i/bVqoWmBYR64C/SnqJlBiml/kdZu+3YAE8/HB6OeuVVzaf2LxzZ9hrL/jylze9oGVWcOUmgnci4uEmHns6MFTSIFICOBWo3yPol8BpwO2SepOaihZg1hQLF6a++i+9BJMnp4lUSh16aBpZ8/TT06xYZraZchPB45J+APwCeK+/XEQ0ME3Re/vWSxoPPEJq/78tIuZIugKYERETs31HS5pLutO4KCKWNrMuVhSvvgo//zn84hdpaIY6O+yQum6eemrqtnnIIZ4Zy6wM5f5fUveQt6ZkWwBHbO1DETEJmFRv22UlywF8Lfsxe78VK1L7/tNPp26cDz64+f6ePeGss9LJ/4ADPOCaWTOUlQgi4hN5B2IGpElUfvMbmDAhva1bX9euacTNs89O/fc7dGj9GM3amXJ7De0IfBv4eLbpSeCKiHgnr8CsQGbNSi9s3XprGnO/jpQe7J5zThqeYa+93KPHLAflNg3dBswGTs7W/xm4HfhMHkFZOxaRZtK6/vrUq6e+z34Wxo1L/fg7dmz18MyKqNxEMDgiTipZ/w9Js3KIx9qjiDT08n/+Z3rIW99Xvwrjx8NHtvW1FDNrjnITwSpJh0TEVHjvBbNV+YVlVW/dOrj/frjjjs0nTN9//9R//+ST0+Qqbuoxq7hyE8F5wB3ZswIBbwFn5RWUVan161Ozz403wq9+tWl7795w4YVw5pnQr1/l4jOzBpXba2gWsLekntn68jyDsioSkcbhv/pqePLJTds/9SkYOza9xOVxeszatK0mAklnRsSdkr5WbzsAEXFtjrFZW7V2berPf8UVqW9/ndGjU3v/iSem/v1mVhUauyPYIfvdI+9ArI3bsAHuuy9d4Zfq1g2+8pX006dPJSIzs2201UQQETdnv/+jdcKxNqWuq+fll6eRO0tdc03q6bPDDg1+1MyqR1nv40v6vqSekjpKmixpiaQz8w7OKmTatDQW/3bbwZFHbkoCV10Fq1alBPGNbzgJmLUT5Q7McnT2gHgM8AowBLgor6CsAmpr4aijUnfOAw9MzUAAX/96GuQtAi69FLp0qWycZtbiyu0+WlfuU8D9EfGO3P+7+q1ZA1deCd/5zubb9947zd41ZEhl4jKzVlVuIvi1pD+TXiI7T1IfYHV+YVmuXn89Pdy9995N23beOc3Ze9BBFQvLzCqjrKahiLgY+BhQk80m9i5pInqrFuvWwXXXpaafXXZJSeDww+H229O+t95yEjArqMbeIzgiIh6T9JmSbaVFfpFXYNZCamvTg9277tq0rXfvNPzD4YdXLCwzazsaaxo6DHgMOKGBfYETQdv11FNphq5SF1wA3/1uGtPfzCzT2HsE385+f751wrFtNnly6vJZ6vbb0yxeZmYNKPc9gqsl7VSyvrOkq3KLyppu6tTU/l+aBBYsSN0+nQTMbCvKfY/guIhYVrcSEW8Dx+cSkZUvAi6+OCWAQw9N27p2hUWL0r5Bgyobn5lVhXK7j3aQ1Dki1gBI6gp0zi8sa9T996cx/ev07QszZ8KHPlS5mMysKpWbCP4bmCzp9mz988Ad+YRkW3XnnfDP/7xpfejQlAB6eFxAM2uecucj+J6k54G6BugrI+KR/MKyzUSkiV2uu27TtqFD4ZlnoFevysVlZu1CuXcEAC8C6yPid5K6SeoREX/PKzDLTJ+epncstXAh9O9fmXjMrN0pt9fQOcADwM3Zpr7AL3OKyQD+8pc0w1ddEth1V1i2LN0dOAmYWQsqt9fQl4CDgeUAEfEy8MG8giq8730vDfj24INpfe5cWLwYdtyxsnGZWbtUbtPQmohYWze8hKTtSW8WW0v6wx/ggAM2rftFMDNrBeUmgiclXQJ0lXQU8EXgwfzCKpi1a6FzSW/cQYNg3jzo2LFyMZlZYZTbNPQNYAnwJ+BfgUnAt/IKqlAeeGDzJDB7dnoj2EnAzFpJo3cEkjoAcyJiD+DH+YdUECtWbN73/+yz4dZbKxePmRVWo3cEEbEBmCdpt1aIpxhuv33zJLBkiZOAmVVMuU1DOwNzsonrJ9b9NPYhScdKmidpvqSLt1LuJEkhqabcwKvSypVpXKAvfCGtf/nLqTto796VjcvMCq3ch8X/3tQDZ01KE4CjgFpguqSJETG3XrkewAXAtKZ+R1X51rc2nxv41VfTTGFmZhXW2AxlXYBzgSGkB8U/iYj1ZR57f2B+RCzIjnUPaXrLufXKXQl8D7ioCXFXj4g0BeS0LM+dcAJMbPRmysys1TTWNHQHUENKAscBP2zCsfsCi0rWa7Nt75G0D9A/Ih7a2oEkjZM0Q9KMJUuWNCGECnvhBdhuu01J4OGHnQTMrM1prGloWESMAJD0E+APLfXFkrYDrgXOaqxsRNwC3AJQU1NTHS+yXXJJmhayzoYNKSmYmbUxjZ2Z1tUtNKFJqM5ioHRQnH7Ztjo9gOHAE5JeAQ4EJlb9A+MNG9ID4bokcN11qXnIScDM2qjG7gj2lrQ8WxbpzeLl2XJERM+tfHY6MFTSIFICOBU4vW5nRLwDvNddRtITwL9FxIwm16KtGDcOflzyqsXrr3uiGDNr8xqbvL5Dcw8cEesljQceAToAt0XEHElXADMiov00lte/4j/99DSBTDY2k5lZW9aU+QiaLCImkYajKN122RbKHp5nLLkqTQIvv5xGDjUzqxK5JoJ2b+NG6NBh83XfBZhZlfETzOZau3bzJPD3vzsJmFlV8h1Bc5WOGOo7ATOrYk4EzfHxj29ajup4rcHMbEvcNNRU990Hv/99Wl65srKxmJm1ACeCpli9Gk45JS3PmAFdu1Y2HjOzFuBEUK4FCzad+A87DPbdt7LxmJm1ECeCcjz9NAwevGn9iScqFoqZWUtzImjMwoVw8MFpuabGD4fNrN1xItiahQthwIC0vPfeMH16ZeMxM8uBE8GWrF69KQkAzJpVsVDMzPLkRLAlpT2C3BxkZu2YE0FDSt8S3rixcnGYmbUCJ4L6evXatDxzpoeOMLN2z0NM1PfWW+n37Nmw116VjcXMrBX4jqBU6dW/k4CZFYQTQZ2ZMzct19ZWLg4zs1bmRFCnpib9vvZa6Nu3srGYmbUiJwKABx/ctPzVr1YuDjOzCnAimDsXxo5Ny1OnVjYWM7MKcCIofShcN6aQmVmBOBHU8dvDZlZQxU4E06ZVOgIzs4ordiI48MD0+557KhuHmVkFFTsR1Dn55EpHYGZWMcVNBHffvWnZ4wmZWYEVNxGcfnr6XfpGsZlZARU3EdTZZ59KR2BmVlHFTASvvlrpCMzM2oxiJoK6sYTOPbeycZiZtQG5JgJJx0qaJ2m+pIsb2P81SXMlvSBpsqQBDR2nRS1atGn5ggty/zozs7Yut0QgqQMwATgOGAacJmlYvWLPATURMRJ4APh+XvG8N+Xkz362advuu+f2dWZm1SLPO4L9gfkRsSAi1gL3ACeWFoiIxyNiZbb6LNAvt2huvTX9vvrq9HvKFHcbNTMj30TQFyhph6E227YlZwMPN7RD0jhJMyTNWLJkSfOimT9/8/VDDmnecczM2pk28bBY0plADfCDhvZHxC0RURMRNX369GmpL22Z45iZVbk8J69fDPQvWe+XbduMpCOBS4HDImJNjvGYmVkD8rwjmA4MlTRIUifgVGBiaQFJo4GbgbER8UaOsWxu//1b7avMzNq63BJBRKwHxgOPAC8C90XEHElXSMqmBOMHQHfgfkmzJE3cwuFa1pQprfI1ZmbVIM+mISJiEjCp3rbLSpaPzPP7N3PAAZvmH+jUqdW+1sysrWsTD4tbxXZZVW++2Q+KzcxKFCcR1E1FOXx4ZeMwM2tjipcIfDdgZrYZJwIzs4JzIjAzK7jiJYLtilNlM7NyFOesWDf6qO8IzMw2U5xE4KYhM7MGORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBFS8R+IUyM7PNFOes6BfKzMwaVJxE4KYhM7MGORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBFS8R+IUyM7PNFOes6BfKzMwaVJxE4KYhM7MGORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVXK6JQNKxkuZJmi/p4gb2d5Z0b7Z/mqSBuQXjRGBm1qDcEoGkDsAE4DhgGHCapGH1ip0NvB0RQ4DrgO/lFY8TgZlZw/K8I9gfmB8RCyJiLXAPcGK9MicCd2TLDwCflHI6U3frln536JDL4c3MqtX2OR67L7CoZL0WOGBLZSJivaR3gF7Am6WFJI0DxgHstttuzYvm0Ufh/vuhT5/mfd7MrJ2qiofFEXFLRNRERE2f5p7IhwyBb36zZQMzM2sH8kwEi4H+Jev9sm0NlpG0PbAjsDTHmMzMrJ48E8F0YKikQZI6AacCE+uVmQh8Llv+R+CxiLqnumZm1hpye0aQtfmPBx4BOgC3RcQcSVcAMyJiIvAT4GeS5gNvkZKFmZm1ojwfFhMRk4BJ9bZdVrK8GvinPGMwM7Otq4qHxWZmlh8nAjOzgnMiMDMrOCcCM7OCU7X11pS0BPhbMz/em3pvLReA61wMrnMxbEudB0REg2/kVl0i2BaSZkRETaXjaE2uczG4zsWQV53dNGRmVnBOBGZmBVe0RHBLpQOoANe5GFznYsilzoV6RmBmZu9XtDsCMzOrx4nAzKzg2mUikHSspHmS5ku6uIH9nSXdm+2fJmlgBcJsUWXU+WuS5kp6QdJkSQMqEWdLaqzOJeVOkhSSqr6rYTl1lnRy9reeI+mu1o6xpZXx3/Zukh6X9Fz23/fxlYizpUi6TdIbkmZvYb8k3ZD9e7wgaZ9t/tKIaFc/pCGv/wJ8BOgEPA8Mq1fmi8BN2fKpwL2VjrsV6vwJoFu2fF4R6pyV6wFMAZ4Faioddyv8nYcCzwE7Z+sfrHTcrVDnW4DzsuVhwCuVjnsb6/xxYB9g9hb2Hw88DAg4EJi2rd/ZHu8I9gfmR8SCiFgL3AOcWK/MicAd2fIDwCclqRVjbGmN1jkiHo+Ildnqs6QZ46pZOX9ngCuB7wGrWzO4nJRT53OACRHxNkBEvNHKMba0cuocQM9seUfg1VaMr8VFxBTS/CxbciLwX5E8C+wkaZdt+c72mAj6AotK1muzbQ2WiYj1wDtAr1aJLh/l1LnU2aQrimrWaJ2zW+b+EfFQawaWo3L+zh8FPirpKUnPSjq21aLLRzl1vhw4U1Itaf6TL7dOaBXT1P/fG5XrxDTW9kg6E6gBDqt0LHmStB1wLXBWhUNpbduTmocOJ931TZE0IiKWVTKonJ0G/DQifijpINKsh8MjYmOlA6sW7fGOYDHQv2S9X7atwTKStifdTi5tlejyUU6dkXQkcCkwNiLWtFJseWmszj2A4cATkl4htaVOrPIHxuX8nWuBiRGxLiL+CrxESgzVqpw6nw3cBxARzwBdSIOztVdl/f/eFO0xEUwHhkoaJKkT6WHwxHplJgKfy5b/EXgssqcwVarROksaDdxMSgLV3m4MjdQ5It6JiN4RMTAiBpKei4yNiBmVCbdFlPPf9i9JdwNI6k1qKlrQijG2tHLqvBD4JICkPUmJYEmrRtm6JgKfzXoPHQi8ExGvbcsB213TUESslzQeeITU4+C2iJgj6QpgRkRMBH5Cun2cT3ooc2rlIt52Zdb5B0B34P7sufjCiBhbsaC3UZl1blfKrPMjwNGS5gIbgIsiomrvdsus84XAjyV9lfTg+KxqvrCTdDcpmffOnnt8G+gIEBE3kZ6DHA/MB1YCn9/m76zify8zM2sB7bFpyMzMmsCJwMys4JwIzMwKzonAzKzgnAjMzArOicCsAZI2SJolabakByXt1MLHfyXr54+kFS15bLOmciIwa9iqiBgVEcNJ75p8qdIBmeXFicCscc+QDeolabCk30iaKen3kvbItn9I0v9Iej77+Vi2/ZdZ2TmSxlWwDmZb1O7eLDZrSZI6kIYv+Em26Rbg3Ih4WdIBwI3AEcANwJMR8ensM92z8l+IiLckdQWmS/p5Nb/pa+2TE4FZw7pKmkW6E3gReFRSd+BjbBqmA6Bz9vsI4LMAEbGBNLQ5wPmSPp0t9ycNAOdEYG2KE4FZw1ZFxChJ3Ujj3HwJ+CmwLCJGlXMASYcDRwIHRcRKSU+QBkQza1P8jMBsK7JZ3c4nDWy2EvirpH+C9+aO3TsrOpk0BSiSOkjakTS8+dtZEtiDNBS2WZvjRGDWiIh4DniBNAHKGcDZkp4H5rBp2sQLgE9I+hMwkzR37m+A7SW9CFxDGgrbrM3x6KNmZgXnOwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4L7Py0nNxuKCHvaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.5826056984519462\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.9722\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}