notebooks/encoding_distances.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "from pathlib import Path\n",
    "import re\n",
    "import sys\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import scipy.io\n",
    "from tqdm import tqdm\n",
    "sns.set_style(\"whitegrid\")\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "sys.path.append(\"../src\")\n",
    "import nearest_neighbors\n",
    "import util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder_path = Path(\"../models/decoders\")\n",
    "bert_encoding_path = Path(\"../models/bert\")\n",
    "sentences_path = Path(\"../data/sentences/stimuli_384sentences.txt\")\n",
    "brains_path = Path(\"../data/brains\")\n",
    "brains_cache_path = brains_path / \"all_encodings.npz\"\n",
    "\n",
    "bert_base_model = \"uncased_L-12_H-768_A-12\"\n",
    "finetune_desc = \"finetune-250\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_models = [\"MNLI\", \"QQP\", \"SST\", \"SQuAD\", \"LM\", \"LM_scrambled\", \"LM_scrambled_para\", \"LM_contentonly\", \"LM_lmonly\", \"LM_pos\", \"LM_randommask\"]\n",
    "subjects = [\"M02\", \"M04\", \"M07\", \"M08\", \"M09\", \"M14\", \"M15\", \"P01\"]\n",
    "target_runs = [1, 2, 3, 4, 5, 6, 7, 8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = util.load_sentences(sentences_path)\n",
    "steps = list(range(5, 255, 5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model encoding evaluation\n",
    "\n",
    "First evaluate pairwise distance between sentences in model encodings (and subject brain activations)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "PCA_DIM = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "sims, sim_stats = {}, []\n",
    "SIM_METRIC = \"cosine\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8/8 [00:00<00:00, 76.39it/s]\n"
     ]
    }
   ],
   "source": [
    "# Load subject images and calculate sentence distances.\n",
    "if brains_cache_path.exists():\n",
    "    brain_encodings = np.load(brains_cache_path)\n",
    "else:\n",
    "    brain_encodings = {}\n",
    "    for subject in tqdm(subjects):\n",
    "        brain_encodings[subject] = util.load_brain_data(brains_path / subject / \"examples_384sentences.mat\", project=PCA_DIM)\n",
    "    np.savez(brains_cache_path, **brain_encodings)\n",
    "    \n",
    "for subject in tqdm(subjects):\n",
    "    sims[subject, 0] = nearest_neighbors.eval_quant(brain_encodings[subject], metric=DIST_METRIC)\n",
    "    dist_stats.append((subject, 0, steps[-1], sims[subject, 0].mean(), sims[subject, 0].std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Preparing model encodings: 100%|██████████| 4850/4850 [00:20<00:00, 188.68it/s]\n"
     ]
    }
   ],
   "source": [
    "# Load distance data for model encodings.\n",
    "for encoding_path in tqdm(list(bert_encoding_path.glob(\"encodings.%s*.npy\" % finetune_desc)),\n",
    "                          desc=\"Preparing model encodings\"):\n",
    "    model, run, step = re.findall(r\"\\.(\\w+)-run(\\d+)-(\\d+)\\.npy\", encoding_path.name)[0]\n",
    "    run, step = int(run), int(step)\n",
    "    if model not in bert_models or run not in target_runs: continue\n",
    "    if step != steps[-1]: continue\n",
    "    \n",
    "    try:\n",
    "        encoding = util.load_encodings([encoding_path], project=PCA_DIM)\n",
    "    except:\n",
    "        continue\n",
    "        \n",
    "    sims_e = nearest_neighbors.eval_quant(encoding, metric=DIST_METRIC)\n",
    "    if step == steps[-1]:\n",
    "        sims[model, run] = sims_e\n",
    "\n",
    "    dist_stats.append((model, run, step, sims_e.mean(), sims_e.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Also add distance data for base model.\n",
    "encoding = util.load_encodings([bert_encoding_path / (\"encodings.%s.npy\" % bert_base_model)], project=PCA_DIM)\n",
    "sims_e = nearest_neighbors.eval_quant(encoding, metric=DIST_METRIC)\n",
    "dists[\"_\", 0] = sims_e\n",
    "dist_stats.append((\"_\", 0, steps[-1], sims_e.mean(), sims_e.std()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>avg_sim</th>\n",
       "      <th>std_sim</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>run</th>\n",
       "      <th>step</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>M02</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002480</td>\n",
       "      <td>0.222955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M04</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002498</td>\n",
       "      <td>0.231103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M07</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002325</td>\n",
       "      <td>0.265763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M08</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001983</td>\n",
       "      <td>0.274649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M09</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002380</td>\n",
       "      <td>0.277392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M14</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.000883</td>\n",
       "      <td>0.263538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M15</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002523</td>\n",
       "      <td>0.236741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P01</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002444</td>\n",
       "      <td>0.246464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">SQuAD</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002520</td>\n",
       "      <td>0.137353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002473</td>\n",
       "      <td>0.143298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002399</td>\n",
       "      <td>0.143696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002497</td>\n",
       "      <td>0.147748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">MNLI</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002355</td>\n",
       "      <td>0.135292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002230</td>\n",
       "      <td>0.152190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002248</td>\n",
       "      <td>0.145938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002379</td>\n",
       "      <td>0.136467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">SST</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001616</td>\n",
       "      <td>0.202865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.000613</td>\n",
       "      <td>0.241237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001596</td>\n",
       "      <td>0.216510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001394</td>\n",
       "      <td>0.224317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">QQP</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002225</td>\n",
       "      <td>0.136876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002203</td>\n",
       "      <td>0.141098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002187</td>\n",
       "      <td>0.226627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002217</td>\n",
       "      <td>0.136867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002472</td>\n",
       "      <td>0.136507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM_lmonly</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002511</td>\n",
       "      <td>0.136297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM</th>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002466</td>\n",
       "      <td>0.137580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM_lmonly</th>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002515</td>\n",
       "      <td>0.135535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM_scrambled_para</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002410</td>\n",
       "      <td>0.144649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM</th>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002467</td>\n",
       "      <td>0.135827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">LM_contentonly</th>\n",
       "      <th>5</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002469</td>\n",
       "      <td>0.139237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002449</td>\n",
       "      <td>0.136209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002456</td>\n",
       "      <td>0.136701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002467</td>\n",
       "      <td>0.137395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">LM_lmonly</th>\n",
       "      <th>5</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002507</td>\n",
       "      <td>0.133890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002499</td>\n",
       "      <td>0.135559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002493</td>\n",
       "      <td>0.135958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002496</td>\n",
       "      <td>0.133181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LM_randommask</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002484</td>\n",
       "      <td>0.141169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">LM_scrambled_para</th>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002442</td>\n",
       "      <td>0.146114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002413</td>\n",
       "      <td>0.146877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002404</td>\n",
       "      <td>0.142688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">LM_randommask</th>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002486</td>\n",
       "      <td>0.138570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002473</td>\n",
       "      <td>0.139316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002464</td>\n",
       "      <td>0.140319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">LM_pos</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001707</td>\n",
       "      <td>0.188111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001239</td>\n",
       "      <td>0.203911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002106</td>\n",
       "      <td>0.183997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.001999</td>\n",
       "      <td>0.195118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">LM_scrambled_para</th>\n",
       "      <th>5</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002413</td>\n",
       "      <td>0.146132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002416</td>\n",
       "      <td>0.145771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">LM_scrambled</th>\n",
       "      <th>1</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002440</td>\n",
       "      <td>0.144654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002419</td>\n",
       "      <td>0.145035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002417</td>\n",
       "      <td>0.145325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002456</td>\n",
       "      <td>0.144026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002430</td>\n",
       "      <td>0.145504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002417</td>\n",
       "      <td>0.146322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002432</td>\n",
       "      <td>0.143572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002447</td>\n",
       "      <td>0.142398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>_</th>\n",
       "      <th>0</th>\n",
       "      <th>250</th>\n",
       "      <td>-0.002471</td>\n",
       "      <td>0.137447</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>179 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             avg_sim   std_sim\n",
       "model             run step                    \n",
       "M02               0   250  -0.002480  0.222955\n",
       "M04               0   250  -0.002498  0.231103\n",
       "M07               0   250  -0.002325  0.265763\n",
       "M08               0   250  -0.001983  0.274649\n",
       "M09               0   250  -0.002380  0.277392\n",
       "M14               0   250  -0.000883  0.263538\n",
       "M15               0   250  -0.002523  0.236741\n",
       "P01               0   250  -0.002444  0.246464\n",
       "SQuAD             1   250  -0.002520  0.137353\n",
       "                  2   250  -0.002473  0.143298\n",
       "                  3   250  -0.002399  0.143696\n",
       "                  4   250  -0.002497  0.147748\n",
       "MNLI              1   250  -0.002355  0.135292\n",
       "                  2   250  -0.002230  0.152190\n",
       "                  3   250  -0.002248  0.145938\n",
       "                  4   250  -0.002379  0.136467\n",
       "SST               1   250  -0.001616  0.202865\n",
       "                  2   250  -0.000613  0.241237\n",
       "                  3   250  -0.001596  0.216510\n",
       "                  4   250  -0.001394  0.224317\n",
       "QQP               1   250  -0.002225  0.136876\n",
       "                  2   250  -0.002203  0.141098\n",
       "                  3   250  -0.002187  0.226627\n",
       "                  4   250  -0.002217  0.136867\n",
       "LM                1   250  -0.002472  0.136507\n",
       "LM_lmonly         1   250  -0.002511  0.136297\n",
       "LM                2   250  -0.002466  0.137580\n",
       "LM_lmonly         2   250  -0.002515  0.135535\n",
       "LM_scrambled_para 1   250  -0.002410  0.144649\n",
       "LM                3   250  -0.002467  0.135827\n",
       "...                              ...       ...\n",
       "LM_contentonly    5   250  -0.002469  0.139237\n",
       "                  6   250  -0.002449  0.136209\n",
       "                  7   250  -0.002456  0.136701\n",
       "                  8   250  -0.002467  0.137395\n",
       "LM_lmonly         5   250  -0.002507  0.133890\n",
       "                  6   250  -0.002499  0.135559\n",
       "                  7   250  -0.002493  0.135958\n",
       "                  8   250  -0.002496  0.133181\n",
       "LM_randommask     1   250  -0.002484  0.141169\n",
       "LM_scrambled_para 4   250  -0.002442  0.146114\n",
       "                  6   250  -0.002413  0.146877\n",
       "                  7   250  -0.002404  0.142688\n",
       "LM_randommask     2   250  -0.002486  0.138570\n",
       "                  3   250  -0.002473  0.139316\n",
       "                  4   250  -0.002464  0.140319\n",
       "LM_pos            1   250  -0.001707  0.188111\n",
       "                  2   250  -0.001239  0.203911\n",
       "                  3   250  -0.002106  0.183997\n",
       "                  4   250  -0.001999  0.195118\n",
       "LM_scrambled_para 5   250  -0.002413  0.146132\n",
       "                  8   250  -0.002416  0.145771\n",
       "LM_scrambled      1   250  -0.002440  0.144654\n",
       "                  2   250  -0.002419  0.145035\n",
       "                  3   250  -0.002417  0.145325\n",
       "                  4   250  -0.002456  0.144026\n",
       "                  5   250  -0.002430  0.145504\n",
       "                  6   250  -0.002417  0.146322\n",
       "                  7   250  -0.002432  0.143572\n",
       "                  8   250  -0.002447  0.142398\n",
       "_                 0   250  -0.002471  0.137447\n",
       "\n",
       "[179 rows x 2 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(dist_stats, columns=[\"model\", \"run\", \"step\", \"avg_sim\", \"std_sim\"]).set_index([\"model\", \"run\", \"step\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pairwise distance: global metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/scipy/stats/stats.py:1713: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "  return np.add.reduce(sorted[indexer] * weights, axis=axis) / sumval\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2b269a48e0b8>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3wAAAJRCAYAAAAeUbV+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzs3XuclXW99//3yDCg4gkLBpPN9iwKqZWnO5XtwIhyEBG31u320M4ebbGTqbW3KXeb2pqpt5F160ZS85B2EEUYIxU17m1KZimao4WK4YHR7SEzdAaG+f3h7fqJnEaYBfTl+Xw8fMysta651sd1Yl7ruq41NR0dHR0BAACgOJus7wEAAACoDsEHAABQKMEHAABQKMEHAABQKMEHAABQKMEHAABQKMEHAABQKMEHAABQKMEHAABQqNr1PcCaeOihh9KjR4/1PQYAAMB60dramr333nu1y/1NBl+PHj0ycODA9T0GAADAetHc3Nyp5ezSCQAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUKiqB9/s2bMzfPjwNDY2ZvLkyStc5rbbbsuIESMycuTInHHGGdUeCQAAYKNQW82Vt7e3Z+LEibnqqqvSt2/fHHPMMWloaMjOO+9cWWb+/PmZPHlybrjhhmy11VZ5+eWXqzkSAADARqOqW/jmzp2bAQMGpH///qmrq8vIkSMza9asZZb5yU9+kuOPPz5bbbVVkmTbbbet5kgAAADrzP33358vf/nLuf/++9fL9Vd1C19LS0vq6+srp/v27Zu5c+cus8z8+fOTJJ/4xCeydOnSfO5zn8shhxyyyvW2tramubm5y+cFAADoSpdffnkWLFiQV155pbKRa12qavB1dHQsd15NTc0yp9vb2/PMM8/k2muvzcKFC3P88cdnxowZ2XLLLVe63h49emTgwIFdPi8AAEBXWrp0aeVrVzZMZzeAVXWXzvr6+ixcuLByuqWlJX369Flmmb59+2bo0KHp3r17+vfvnx122KGy1Q8AAIA1V9XgGzx4cObPn58FCxakra0tTU1NaWhoWGaZYcOGZc6cOUmSV155JfPnz0///v2rORYAAMBGoaq7dNbW1mbChAk55ZRT0t7ennHjxmWXXXbJpEmTMmjQoAwdOjQHH3xw7r333owYMSLdunXLV77ylWyzzTbVHAsAAGCjUNOxogPtNnDNzc2O4QMAADZ4J554Yp577rl86EMfyjXXXNNl6+1sE1X9D68DAACwfgg+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQgk+AACAQlU9+GbPnp3hw4ensbExkydPXu7yqVOn5oADDsiYMWMyZsyY/PSnP632SAAAABuF2mquvL29PRMnTsxVV12Vvn375phjjklDQ0N23nnnZZYbMWJEJkyYUM1RAAAANjpV3cI3d+7cDBgwIP37909dXV1GjhyZWbNmVfMqAQAA+H+quoWvpaUl9fX1ldN9+/bN3Llzl1vu9ttvzwMPPJAddtgh//Zv/5Z+/fqtcr2tra1pbm7u8nkBAAC6UltbW+Xr+miYqgZfR0fHcufV1NQsc/rQQw/NqFGjUldXlxtuuCFf/epXc80116xyvT169MjAgQO7dFYAAICuVldXV/nalQ3T2Xis6i6d9fX1WbhwYeV0S0tL+vTps8wy22yzTeVGOPbYY/P73/++miMBAABsNKoafIMHD878+fOzYMGCtLW1pampKQ0NDcss8+KLL1a+v+uuu7LTTjtVcyQAAICNRlV36aytrc2ECRNyyimnpL29PePGjcsuu+ySSZMmZdCgQRk6dGiuvfba3HXXXenWrVu22mqrnH/++dUcCQAAYKNR07GiA+02cM3NzY7hAwAANngnnnhinnvuuXzoQx9a7WeVvB+dbaKq/+F1AAAA1g/BBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUKiqB9/s2bMzfPjwNDY2ZvLkyStdbubMmdltt93yyCOPVHskAACAjUJVg6+9vT0TJ07MlClT0tTUlBkzZmTevHnLLffGG2/k2muvzV577VXNcQAAADYqVQ2+uXPnZsCAAenfv3/q6uoycuTIzJo1a7nlJk2alFNOOSU9evSo5jgAAAAbldpqrrylpSX19fWV03379s3cuXOXWeaxxx7LwoULc+ihh+bKK6/s1HpbW1vT3NzcpbMCAAB0tba2tsrX9dEwVQ2+jo6O5c6rqampfL906dKcf/75Of/889/Xenv06JGBAweu9XwAAADVVFdXV/nalQ3T2Xis6i6d9fX1WbhwYeV0S0tL+vTpUzn917/+NX/4wx9y4oknpqGhIQ899FBOPfVUH9wCAADQBaq6hW/w4MGZP39+FixYkL59+6apqSkXX3xx5fItttgic+bMqZw+4YQT8pWvfCWDBw+u5lgAAAAbhaoGX21tbSZMmJBTTjkl7e3tGTduXHbZZZdMmjQpgwYNytChQ6t59QAAABu1qgZfkgwZMiRDhgxZ5rwvfvGLK1z22muvrfY4AAAAG42q/+F1AAAA1g/BBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUCjBBwAAUKja97Pw448/nueeey7t7e2V8w477LAuHwoAAIC11+ng+7d/+7c88cQT2WWXXbLJJv//hkHBBwAAsGHqdPA9/PDDue2226o5CwAAAF2o08fw7b333pk3b141ZwEAAKALdXoL39ixY3PcccflAx/4QOrq6irnT58+vSqDAQAAsHY6HXxnn312vv3tb2fXXXdd5hg+AAAANkydDr5+/fpl6NCh1ZwFAACALtTp4Ntxxx1zxhln5NBDD11ml06f0gkAALBh6nTwtba2pq6uLvfee+8y5ws+AACADVOng+/888+v5hwAAAB0sdUG3xVXXJHPfOYz+cY3vpGamprlLj/nnHOqMhgAAABrZ7XBt9NOOyVJBg0aVPVhAAAA6DqrDb6GhoYkb/8dvncsXbo0ixYtSq9evao3GQAAAGul039Q74wzzsgbb7yRRYsWZcSIETn88MMzZcqUas4GAADAWuh08M2bNy+9evXKnXfemSFDhuTuu+/OtGnTVvtzs2fPzvDhw9PY2JjJkycvd/kNN9yQ0aNHZ8yYMfnkJz+ZefPmvb//AwAAAFao08G3ZMmSLF68OHfeeWeGDh2a7t27r/BDXN6tvb09EydOzJQpU9LU1JQZM2YsF3SjR4/O9OnTM23atJxyyik+DRQAAKCLdDr4jjvuuDQ0NOTNN9/Mvvvum+eee261x/DNnTs3AwYMSP/+/VNXV5eRI0dm1qxZyyzz7nW8+eabq41IAAAAOqfTf4fvxBNPzIknnlg5vd122+Waa66pnL755puX+WCXJGlpaUl9fX3ldN++fTN37tzl1n399dfnqquuyuLFi/PDH/5wtbO0tramubm5s6MDAACsF21tbZWv66NhOh1871VTU5Pa2v//x6+55prlgq+jo2OFP/dexx9/fI4//vhMnz49l112WS644IJVXnePHj0ycODANZwcAABg3airq6t87cqG6Ww8dnqXztVZUdzV19dn4cKFldMtLS3p06fPStcxcuTI3HnnnV01EgAAwEaty4JvRVvuBg8enPnz52fBggVpa2tLU1NT5e/6vWP+/PmV7++5554MGDCgq0YCAADYqK3xLp3vtaItfLW1tZkwYUJOOeWUtLe3Z9y4cdlll10yadKkDBo0KEOHDs11112X++67L7W1tdlyyy1XuzsnAAAAndNlwfeRj3xkhecPGTIkQ4YMWea8L37xi5XvzznnnK4aAQAAgHfpdPBdddVVy53Xq1evDBo0KAMHDsyECRO6dDAAAADWTqeD79FHH82jjz6aQw89NMnbx9sNHjw4N954Yw4//PB85jOfqdqQAAAAvH+dDr7XXnstU6dOzeabb54k+fznP58vfOELuf7663P00UcLPgAAgA1Mpz+l8/nnn0/37t0rp7t3757nn38+PXv2rPxtCQAAADYcnd7CN2rUqBx33HEZOnRokuSuu+7KyJEjs2jRouy0005VGxAAAIA10+ngO+200zJkyJA8+OCD6ejoyL//+79n8ODBSZKLL764agMCAACwZjodfN/85jczYsSInHTSSdWcBwAAgC7S6eDbc889c9lll2X+/PkZNmxYRowYUdnCBwAAwIan08E3duzYjB07Nq+99lpuv/32XHTRRXnhhRdy++23V3M+AAAA1lCnP6XzHX/605/y1FNP5bnnnsuOO+5YjZkAAADoAp3ewnfhhRfmjjvuSP/+/TNixIiMHz8+W265ZTVnAwAAYC10Ovi233773HjjjVmwYEHa2tryxBNPJEn23Xffqg0HAADAmut08HXr1i0nnXRSFi5cmN133z0PP/xw9t5771xzzTXVnA8AAIA11Olj+K699tr87Gc/y3bbbZdrr702N998c3r37l3N2QAAAFgLnQ6+urq69OjRI0nS1taWnXbaKU8//XTVBgMAAGDtdHqXzvr6+rz++usZNmxYPvWpT2XLLbdMnz59qjkbAAAAa6HTwff9738/SfL5z38++++/f/7yl7/k4IMPrtpgAAAArJ1OB9+77bfffl09BwAAAF3sff/hdQAAAP42CD4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCCT4AAIBCVT34Zs+eneHDh6exsTGTJ09e7vKrrroqI0aMyOjRo3PSSSflueeeq/ZIAAAAG4WqBl97e3smTpyYKVOmpKmpKTNmzMi8efOWWWbgwIG56aabMn369AwfPjwXXnhhNUcCAADYaFQ1+ObOnZsBAwakf//+qaury8iRIzNr1qxlljnggAOy6aabJkn23nvvLFy4sJojAQAAbDRqq7nylpaW1NfXV0737ds3c+fOXenyP/vZz3LIIYesdr2tra1pbm7ukhkBAACqpa2trfJ1fTRMVYOvo6NjufNqampWuOy0adPy6KOP5rrrrlvtenv06JGBAweu9XwAAACr0rGkPTW13db45+vq6ipf16Zh3jtHZ+OxqsFXX1+/zC6aLS0t6dOnz3LL/epXv8rll1+e6667rnKDAAAArG81td3y4vduX+Ofb39tUeXr2qynz+cOW6Ofq+oxfIMHD878+fOzYMGCtLW1pampKQ0NDcss89hjj2XChAm57LLLsu2221ZzHAAAgI1KVbfw1dbWZsKECTnllFPS3t6ecePGZZdddsmkSZMyaNCgDB06NN/+9rezaNGifPGLX0yS9OvXL5dffnk1xwIAANgoVDX4kmTIkCEZMmTIMue9E3dJcvXVV1d7BAAAgI1S1f/wOgAAAOuH4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAACiU4AMAAChU1YNv9uzZGT58eBobGzN58uTlLn/ggQcyduzY7LHHHpk5c2a1xwEAANhoVDX42tvbM3HixEyZMiVNTU2ZMWNG5s2bt8wy/fr1y/nnn59Ro0ZVcxQAAICNTm01Vz537twMGDAg/fv3T5KMHDkys2bNys4771xZZvvtt0+SbLKJvUsBAAC6UlWDr6WlJfX19ZXTffv2zdy5c9d6va2trWlubl7r9QAAAKzKwIED1/cIFWvSQFUNvo6OjuXOq6mpWev19ujRY4O64QEAAKrt3Q3U2fir6n6U9fX1WbhwYeV0S0tL+vTpU82rBAAA4P+pavANHjw48+fPz4IFC9LW1pampqY0NDRU8yoBAAD4f6oafLW1tZkwYUJOOeWUjBgxIkcccUR22WWXTJo0KbNmzUry9ge7HHLIIZk5c2b+1//6Xxk5cmQ1RwIAANhoVPUYviQZMmRIhgwZssx5X/ziFyvff/jDH87s2bOrPQYAAMBGx99CAAAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKJTgAwAAKFTVg2/27NkZPnx4GhsbM3ny5OUub2try5e+9KU0NjbmH//xH/Pss89WeyQAAICNQlWDr729PRMnTsyUKVPS1NSUGTNmZN68ecss89Of/jRbbrll7rjjjpx88sm56KKLqjkSAADARqOqwTd37twMGDAg/fv3T11dXUaOHJlZs2Yts8xdd92VsWPHJkmGDx+e++67Lx0dHdUcCwAAYKNQW82Vt7S0pL6+vnK6b9++mTt37nLL9OvX7+1hamuzxRZb5NVXX03v3r1Xut7W1tY0NzdXTu+8w47p3rNHF0///i1+qzXznn5qpZfvvMMO6d6z5zqcaHmL33or855+epXL7LTDgNT13GwdTbRibW8typNPP7PKZf5W5txxxwHp0WP9ztnauihPPbXyOXfYcUB6rucZk+St1kV5uoA5B+w4IJttAHMual2UZ1Y55w7ZrMf6fU1a1PpWnnlq1a9Jfztz7pjNeqzff4sWtbbmmadW/u9Q8rcx54Add8pmPerW4UQrtqi1Lc889eRKLx+w487ZrEf3dTjRii1qXZxnnpq30st32HHn9NwA5nyrdXGeXsWcO+64c3qs5zlbWxfnqVXMmCQ77bhz6tbznG2ti/PkaubceYed073nms15//335yc/+UmOPfbYHHDAAWu0jiRZ/NbizHt65XPuvMNO6d5z/T/XF7/VlnlPr/y5vvMOO6bP5w5b4/V3+/V1yV9fTbetN1ur9ayuNVamqsG3oi11NTU173uZ9+rRo0cGDhy4zHkvXXbdGkzYtT546j8tN9d7Lbzsm+tomhWrP/Wc1c6YJI9/f8w6mGbldj9tWqfmvOeKketgmpX7h880dWrOn111+DqYZuWO+dTMVc65pL1tHU6zcrW1tau9Pb/+k+HraJpVzHDsL1Y75xHTxq2jaVbu52NuWu2cI25ev69Jt43t3GvSyKmXrYNpVq7p6FNXO2dbe/s6mmblOvMcSpIjfzZ9HUyzcrceM3q1c4696b/W0TQrd/O4g1Y753FTV/1L97rw46N3Xu2c37+5ZR1Ns3Knje272jl//uP/XkfTrNgRx32gU8+h3015cR1Ms3L7nNKnU3O+8O3n1mj9V9x3Rea/Pj9/fvrPGXBg/zVaR5L0+8qHVjtny3ceXOP1d5W+X/pop27P9a17z2Ub6N0bwFalqsFXX1+fhQsXVk63tLSkT58+yy3zwgsvpL6+PkuWLMlf/vKXbL311u/rejqWtOeDp/5Tl8y8NjqWtKemttsqLl+S+lPPWYcTrXiGmtpV3+1Ll7Rl99OmraOJVj7DJrWrfsenfUlb/uEzTetoopXP0K0Tcx7zqZnraKKVz7CqOWu7rf9315LVz7G4vS1fP/YX62iaVc/RfRWztrW35edjblqHE618jrpVzrkkt41dv69Jbe1LUtdt1a9Jbe1L0nT0qetoopXPsLo567qt/PV/XenMDG3t7bn1mNHrYJpVz7CqWdval+bmcQetw4lWPkddt5Uf/dLWvjQ/PnrndTjRyudY1ZxL2jty2ti+63Cilc9R223lb+q3t3fkiOM+sA4nWvEM3VYxY5IsXdKRfU7ps8plqm3pko5sUrvqOTuWdKTfVz60Ruvv/i/dk9eT7n27r/E63pmhZhVzdixZmr5f+ugar7+rdCxZmpra6h3pttlmmy3zdV2r6jF8gwcPzvz587NgwYK0tbWlqakpDQ0NyyzT0NCQm2++OUnyi1/8IgcccMBqt/C916oia11a3RyrC611oTMzrC601oXOzLC60FoXOjPD38qcfwtWFVnr0urmWFVkrUurm2N1AbMudGaGv5U5/1b8LYTpquJlXVrdHH8rc64qstal1c2xutBaFzozw+pCa13ozAyrCq3VOfnkk7PXXnvl5JNPXuN1dGaGakbW+1HtObrq9lxTNR1V/oSUX/7ylznvvPPS3t6ecePG5dRTT82kSZMyaNCgDB06NK2trTnrrLPS3NycrbbaKpdcckn691/1puPm5ua/ic2uAAAA1dDZJqp68FWD4AMAADZmnW2iDWM7KgAAAF1O8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEX85h7AAAgAElEQVQAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABRK8AEAABSqdn0PsCZaW1vT3Ny8vscAAABYL1pbWzu1XE1HR0dHlWcBAABgPbBLJwAAQKEEHwAAQKEEHwAAQKEEHwAAQKEEHwAAQKGKD77ddtstZ511VuX0kiVLcsABB+Szn/1skqSjoyPf/OY309jYmNGjR+f3v/99kqS5uTnHHXdcRo4cmdGjR+e2227bIOd8xxtvvJGDDz44EydO3CDnvP/++zNmzJjKf4MHD86dd965wc2ZJN/+9rczcuTIHHHEEfnmN7+Zan6Q7drMeeGFF2bUqFEZNWrUen98PvnkkznuuOMyaNCg/OAHP1ju59vb23PUUUdVlt/QZmxoaMjo0aMzZsyYHH300VWb8d0GDhyYMWPGZNSoUfnCF76QN998M0kye/bsDB8+PI2NjZk8eXJl+euuuy6NjY3Zbbfd8sorr3TZHJdddlnldW7MmDF5+OGH09bWlv/4j//IsGHD0tjYmM9+9rN5/vnnO7W+JUuWZP/998/FF1+8zPknnHBChg8fntGjR+fwww/PxIkT8/rrr6/x3Ku7v6dOnZrdd989jz/+eGWZUaNG5dlnn03y9n3+3ttx6tSpVX8NXdHtfffdd+eoo47KkUcemREjRuTGG2/MZZddVnm9fOexMmbMmFxzzTVVnW/hwoU59dRTc9hhh2Xo0KGZOHFi2trakiS/+c1vcswxx+Twww/P8OHDc/3111d+7tJLL83BBx9ceUzPmjWrqnO+2z777LPceZdeeml22223PPPMM5Xzrr766uy222555JFH1tlsABuC4oNvs802yx//+Me89dZbSZJ77703ffv2rVw+e/bszJ8/P7fffnu+8Y1v5Otf/3qSpGfPnrngggvS1NSUKVOm5LzzzlurX06qNec7vvOd72S//far2nxrO+cBBxyQadOmZdq0afnhD3+YTTfdNB//+Mc3uDl/+9vf5re//W1uvfXWzJgxI4888kh+/etfb3Bz3nPPPXnsscdyyy235Cc/+UmmTJmSN954Y73NufXWW+drX/taPv3pT6/w56+55prstNNOVZuvK2b84Q9/mGnTpmXq1KlVnfMdPXv2zLRp0zJjxox07949N954Y9rb2zNx4sRMmTIlTU1NmTFjRubNm5ck+chHPpKrrroqH/rQh7psht/97ne55557cvPNN2f69Om56qqrUl9fn0suuSR//etf84tf/CJ33HFHDjvssIwfPz5Lly5d7Trvvffe7LDDDvn5z3++3JslF110UaZPn55bb701dXV1GT9+/BrPvrr7O0nq6+tz+eWXr/F1dLUV3d4f+MAHcu655+byyy/PrbfemltuuSX77bdfTj311Mpr5juPlWnTpuXEE0+s2nwdHR353Oc+l2HDhuX222/P7bffnrfeeivf/va389JLL+XMM8/Mv//7v2fmzJm54YYbctNNN+WOO+6o/PzJJ5+cadOmZdKkSTn77LM79Xippl133TVNTU2V0zNnzszOO+/8vtbRVTG5zz77LPcmQzWidEXzdpV3vyHy3uvZZ5993tf/z6WXXrrCNwfX1Ouvv155A2JN77OpU6empaVljWfozBtGJ5xwwjp7w+H93g7f/e533/cbXiW82bKq++39Pp+6+nHdlYoPviQ55JBDcs899yRJmpqaMnLkyMpls2bNylFHHZWamprsvffeef311/Piiy9mhx12yN///d8nSfr27ZvevXt36bvqXTVnkjz66KN5+eWXqxpQXTHnO37xi1/k4IMPzqabbrrBzVlTU5O2trYsXry48vUDH/jABjfnvHnzsu+++6a2tjabbbZZdt9998yePXu9zbntttvmwx/+cGpra5f7uYULF+aee+7JMcccU9X51mbG9e1jH/tYnnnmmcydOzcDBgxI//79U1dXl5EjR1a2lOyxxx7Zfvvtu/R6X3rppWyzzTapq6tLkvTu3Ttbbrllpk6dmrPPPjvdunVLkowbNy6bbbZZfvWrX+XZZ5/NqFGjKuv4wQ9+kEsvvbRyesaMGTnxxBPTr1+/PPTQQyu83rq6upx11ll5/vnnl9kC936t6v5Okn/4h3/IvHnz8tRTT63xdXSlFd3em2++edrb27P11lsnefu22XHHHdfLfPfff3969OiRcePGJUm6deuWs88+O9OmTcsVV1yRsWPHZs8996zMftZZZ63wl5uddtoptbW1efXVV9fp/O81bNiwyvNnwYIF2WKLLdK7d+8uWXdXxGRXrmd12tvbu3ydK7Ku/n/e6/XXX88NN9yQJCvdK2d1s918883L/b7ybuvqNqy2dXEflfa4LsFGEXwjRozIbbfdltbW1jzxxBPZa6+9Kpe1tLSkvr6+crq+vn65d3jmzp2bxYsX5+/+7u82uDmXLl2aCy64IF/5yleqOtvazvluTU1Ny/zCuCHNuc8++2T//ffPQQcdlIMOOigHH3xw1bdMrcmc7wTem2++mVdeeSVz5szJwoUL19ucq3LeeeflrLPOyiabVP/lZk1nTJJPf/rTOfroo/PjH/+4ihMub8mSJZk9e3Z23XXX5e7vvn37rtU7zqvz8Y9/PC+88EKGDx+er3/96/n1r3+dZ555Jv369UuvXr2WWXbQoEGVrY0r89Zbb+W+++7LoYcemlGjRi3zD/57devWLbvvvvtaxdjq7u9NNtkkp5xySv7zP/9zja+jK63o9t56663T0NCQQw89NF/+8pdz6623rrctY3/84x8rQfeOXr165UMf+lAef/zx5S5b2WPi4YcfTk1NTZfF1Zrq1atX+vXrlz/84Q+ZMWNGRowY0WXrXpuYfPbZZ3P44Ydn1qxZ+e///u9cffXV+dWvfpWxY8emubm58obAa6+9lvHjx2f06NE59thj8/jjj6e9vT1jxozJvvvum7333jv77bdfrrnmmjzzzDM5+eST8+abb2bs2LEZP358/s//+T854YQTcsYZZ2S//fbLrFmzMn78+Bx99NEZOXLkMq91++yzTy688MIcffTROfnkkzN37tyccMIJGTp06DK7577wwgv59Kc/nTfffDPf+973Vni7/PjHP864ceMyfPjwvPzyy5Xb5bLLLsvw4cNz8skn5+mnn678zOzZszN27NgceeSROemkk5Ikn/zkJzNs2LDstdde2WuvvTJ9+vQkyTnnnJMDDzww++yzTwYPHpxLLrkkSXL66afnySefzEc/+tG89dZb+etf/5rGxsbstddeGTx4cGbOnJlhw4Zl5syZOeKII/KlL30pTzzxRF588cW0tbVl5syZefTRR3PmmWdmzJgxldey3XffPQceeGAOOOCAPPDAA/ne976XQYMG5cADD8z++++fUaNG5cknn8wJJ5yQn//855k5c2YaGxvz8Y9/PIsXL85bb72Vz3/+89lzzz3zhS98obJHQvL2m2OjR4/OqFGjcuGFF76v+2Lq1KkZP358/uVf/iUNDQ257rrrctVVV+Woo47Ksccem9deey1J8pOf/CTjxo3LkUcemc9//vNZvHhxhg0blqlTp2bUqFE5/PDD8+STTy732L3nnnty3HHHVTZ07LPPPvnWt76VsWPH5qSTTqqcv3Tp0hx77LEZPXp0TjvttPz5z39OkvTp0yeXX355Ro8enc9+9rOrfX60t7fnX//1XzNq1KiMHj06V199dZJUHtdHHnlkxo4dmz/96U+ZM2dO5XE9evToJOmyx/Xw4cNX+LhOkilTpmTcuHEZPXp0vvvd71bOX9njekOzUQTf7rvvnmeffTYzZszIkCFDlrlsRe8E1dTUVL5/8cUXc9ZZZ+X888+v+i+sazLnj370oxxyyCHp169fVWd7t7W9Pf/whz/koIMO2iDnfOaZZ/Lkk0/ml7/8ZWbPnp37778/DzzwwAY350EHHZQhQ4bkE5/4RM4444zsvffela0x62POlbn77rvTu3fvDBo0qKqzvWNNZkySG264ITfffHOuuOKKXH/99VW/z5O342jMmDEZN25ctttuuxxzzDGrff50tc0337yyO0vv3r1z+umnZ86cOSu8zs4cy3r33Xdn//33z6abbprDDjssd9xxxyrfgV3b42M7c3+PGjUqDz30UBYsWLBW19UVVnR7T506Nf/xH/+Rq6++Oh/+8Idz5ZVX5uyzz14v83V0dKz0vl/ZZe929dVXZ8yYMbngggvyne98p6qP3c4aMWJEmpqacuedd6axsbHL1ru2MfmnP/0pe+21Vz796U9n6dKlue6663LYYYflH//xHyvHy1566aXZY489Mn369Jx++un56le/mubm5vz1r3/NTjvtlF//+tf56U9/mu9///s544wzcvzxx2fTTTfNjTfemE9+8pP5v//3/+aRRx7JZz7zmWy11VYZMmRIzjvvvEydOjU33XRTrr322spW2EWLFmW//fbL1KlTs/nmm+c73/lOrrzyynz/+99f5pfbRx55JBdddFF69uyZmTNnLreL3ksvvZRu3brlvPPOy5gxY7LZZpvl9ddfz5NPPpnbbrstt9xyS773ve9Vfu6VV17Jueeem+9+97u59dZbM2nSpCRvR/EWW2yRhx9+OKeffnrlObHNNtvk7/7u7zJnzpxcdNFF+cEPfpDFixfn8MMPT01NTWbNmpVNN900v/nNb/LRj340Dz30UO6+++48/fTTeeWVV/LBD34w8+fPT+/evTNhwoR069Yt9913Xw4//PAMGjQoF110UaZNm5aampr867/+azo6OnLuuefmkEMOyeOPP55/+qd/Sp8+fTJ+/Pgccsgh+chHPpIrr7wyydvxc/jhh+ecc85JTU1NfvnLX+aGG27ISy+9lBNOOCHjx4+vHIff0tKSiy66KD/84Q9zyy235JFHHql8rkFn74s//vGPueiii/Kzn/0sl1xySXr27Jlbbrkle++9d2655ZYkSWNjY2666abceuut2XHHHfPYY4+lV69eefXVV/O1r30tY8aMyemnn77MfXjHHXdk8uTJmTx5ciXSFi1alD322CM333xz9t1330oUtba25swzz8z06dOz6667Vs7/3e9+l4MOOigXX3xxBg4cuNrnR3Nzc1paWjJjxoxMnz69ciz9mWeemeOPPz633nprbrzxxnzwgx+sPA6/9KUvVT6/oKse19OmTVvh4/q//uu/8swzz+RnP/tZpk2blt///vd54IEH8uijj67wcb0h2iiCL3n7AP13Pozj3err65fZMrJw4cL06dMnydsfhPLZz342X/rSl7L33ntvkHP+7ne/y/XXX5+GhoZccMEFueWWW3LRRRdtcHO+4+c//3kaGxvTvXv3qs+4JnPecccd2WuvvbL55ptn8803z8EHH7zS3dLW55xJKsf4XHXVVUlS2QV5fcy5Mr/97W9z1113paGhIV/+8pdz//3358wzz9ygZkxSOfZr2223TWNjY+bOnVut8SrefVzWueeem7q6uuXu75aWlmWeP9XQrVu37L///vnCF76Qc889N/fcc0+ef/755Y4JfeyxxzJo0KDU1tYuswWqtbW18n1TU1Puu+++NDQ0ZNy4cXnttdcyZ86cFV5ve3t7/vCHP6z17ouru79ra2vzz//8z7niiivW6nq6yntv79tvvz3J2x9Cc/LJJ+fKK6/ML37xi/Uy2y677JJHH310mfPeeOONvPzyy9lzzz2Xu+zRRx9d5s2cd47h+9GPfpSPfexj62Tm1WloaMitt96a7bbbbrmt1mtrbWJy++23zwc+8IHU1NRkt912S5LceeedOeaYYyrPqQcffDBjxoxJkhx44IF57bXXss022+TPf/5zli5dmvvvvz/9+/fP1ltvnYULF1Zm6NGjRw4++OC0tLRk4MCB+d3vfpfhw4entrY21157bY488sgce+yxeeGFFyrHWXXv3j2HHHJIkrd3x9t3333TvXv37Lrrrnnuuecqc/+P//E/ss0226SmpiaNjY158MEHl/n/mj9/fhYtWpRPfepT+c///M+88cYbeeutt9Lc3Jxhw4Zl0003Ta9evdLQ0JAkeeihh/Kxj30s/fv3T5LKrs1vvPFGTj755CRvP66WLFmS559/Pq2trfnLX/6So48+uhIXL7/8cpK331B55+fvvffe3H777dlrr73S0NCQ9vb2tLS05NBDD02vXr3y8MMPp7GxMZtvvnleeuml5e6fp59+Ottvv326deuW4cOHZ+zYsfnNb36TOXPm5MUXX8z111+f+++/P926davcPu9sAd9zzz2zySab5KabbsoDDzyQ119/PUcffXR23333yn39yCOPZL/99kvv3r1TW1ub0aNHV95o7Ox9sf/++6dXr17p3bt3tthii8pt+u7l/vjHP+Z//s//mdGjR2f69OmVLXODBw/OOeeck5/+9KeV60qSOXPm5IorrsjkyZOz1VZbVc7fZJNNKtE2ZsyYPPjgg/nLX/6SJJXPj3jnNkqSD37wg/nv//7vXHLJJfnlL3+52udH//79s2DBgnzjG9/I7Nmz06tXr7zxxhtpaWlZ5nH9zqFAgwcPrjxmknTZ47pnz54rfFzfe++9uffee3PUUUdl7NixeeqppzJ//vz85je/WeHjekO00QTfMccck/Hjx1eebO9oaGjILbfcko6Ojjz00EPZYost0qdPn7S1teW0007LmDFjcsQRR2ywc1588cW55557ctddd+WrX/1qjjrqqKr/Qr0mc75jRcfZbEhzbrfddnnggQeyZMmSLF68OA888EDVd+lckznb29sr72A9/vjjeeKJJ9bJMZwrm3NlzjjjjMyePTt33XVX/vf//t854IADqv6GxPudcdGiRZW4WbRoUe69997ssssu1RxxpQYPHpz58+dnwYIFaWtrS1NTU1X/AXnnH613NDc3Z4cddshRRx2Vb33rW5Wtc7fcckt69OiRj370o9l2223z8ssv59VXX01bW1vlGLo33ngjDz74YOX16K677sqECRMyY8aM5a538eLFufjii9OvX7/svvvua/X/0Jn7e+zYsbnvvvuqfhz26qzo9t52222XieLHH3+8Sz+Y5/048MAD8+abb1a2DrS3t+db3/pWjj/++PzzP/9zbr755jQ3NydJXn311XznO99Zqw/eWRd69uyZM888M//yL//S5etem5h8Z7fNJNluu+3y29/+drn1rGgL+FZbbZVPfOIT+fu///v86Ec/yte+9rVssskmK1z24x//eP785z9n6tSpOfroozNnzpz86le/yo9//OPceuut2WOPPSpx2b1798oW2U022aQy3yabbLLMVvr3brV97+mOjo6cdtpp6dGjRw455JDMmjWr8jvAyrYer2xL8Iqua86c/6+9e4+puv7jOP48HO6XMSDKNBk6FWJhQ3cUtjg4umA0KCdY6rAsUsrTKHEqlIe5udXCdZG4TFtkluZm4YFakmVj66ZUrOac5lhiQDLHpSGXuJzz+4PxHf7EawQKr8dfB86Xw+fz/X7O5X0+78/7c5Tw8HA+++wzSkpKcLlc9Pf3X3J8XV0d4eHh/Pzzz/z2228EBAQwZ84c4uLi6OrqMs61yWQaMQth6Hx6eXkZ2TMDAwNs3bqV4OBg9u7dy7Jly+jr6zP+fmh9uJubG+7u7jQ2NtLW1obT6WTOnDkj9nEk13otho8hNzc344v04cdt3rwZu91OZWUlNpvN+H1paSk9PT14eXmxcuVK+vr6gMHAq7Oz86qpiVebvU9LS8Nms3H06FHOnDmDt7f3FY8PDAzE4XCwYMECY1xfia+vr3F7rMb1mjVrjC9pDx8+THp6+ojH3qwmTcA3ZcoUIzd8uISEBKZPn86DDz7Ili1byM/PBwZnon766SfKy8uNcthDb3Q3UzvHy420s6Ghgb/++mtMqoneaDuTkpIICwszSqZHRkaOyTc219vO/v5+Vq5cSXJyMna7nYKCgjEpRnK5dp4/fx6r1UpZWRklJSVYrdb/tGrolVxvG1taWlixYgWpqamkp6eTkJBw0TeeY8nd3R273U5mZibJyck8/PDDRvD5wQcfYLVaOXfuHKmpqVd9Q7wWXV1dbN68meTkZFJSUqirq8Nms5GTk4O3tzeLFy8mPj6esrIyiouLMZlMeHh4sG7dOpYtW8batWuNGbovv/yS2NjYiz6E3H///Rw5csQo679hwwZjzUp3dzfFxcX/ug+Xu97DeXp6kpGRYcwEDElNTcVqtWK1Wnn11Vf/dVuu5nLn+9133yUpKYlHH32UHTt2jElbRmIymSgqKuLQoUM89NBDLFy4EDc3N5577jluv/12CgoK2LJlC0lJScTHx5ORkTGmr+eX093dbVzHoef4cI888sgl6w9Hw2gFk2azmdTU1Esex2KxUFFRAQx+qA0KCjKeSxEREWRnZ3PixAnMZjOhoaFGSmBvby/d3d3Ex8cb25DMnj2bjo4OAgMD8fHxoa6u7oayV7777jva29txuVx89dVXzJs376L7Z8yYQUVFBS+88AJZWVk0NzfT19dHVFQUhw8fpqenhwsXLvDNN98Ag2usampqjJTrobVnAQEB7N69Gxh87fPw8ODOO+/kn3/+MYLi8vJy4/96enoaQQsMzqCeO3eO3t5efvzxRzo6Oujq6sLLy4vbbrttxGvm5+dHZ2cnADNnzqSxsdHIZnA4HEaml9lspqur66oz8Y899hinTp0yZh1///13Tp06BcDcuXOpqamhtbWVgYEBPv/8cywWy1XP//Xq7OwkNDSUvr4+Yx0kDC6vycvLY/v27ReNq6lTp1JYWMimTZs4ffq0cbzT6TT6W1lZyfz58wkICAAwZvUcDgcWiwWXy0VHRwdWq5X8/Hw8PDzo6uq6YjtbW1txuVwkJSUZ49rf358pU6ZcMq7/32iO656enhHH9X333ccnn3xijI/m5mZaWlqwWCwjjuubkcn1X24yJiIit7Tz58+TmZnJihUrePzxx8e7OTKGfvnlF3JycigsLLxkHe5HH33Exx9/zIcffnhR6tdEExkZeVGWyurVq7lw4QK+vr6XbPGSkZHBxo0biY6OHvFxTCYTISEhxuzRE088YWwvsmjRIhYvXmxUwd2zZw/Tp08nNzeXhoYGfHx8jNLxmZmZRlGc9evXU1BQwCuvvEJxcTHHjh0jKiqKt99+m6amJrKzs8nOzmb58uX09vby/PPP09zczIwZM2hra8Nms7Fw4UJiYmKora0FBtcODu/f0H2ffvop1dXVdHd3U11djZ+fnxF8tba2snbtWnx9fXF3d+fAgQPA4EyM0+nEbrfz7bffcvDgQaZNm8Ydd9zBrFmzeOaZZ6iurubNN9/E6XQSEhJCWVkZy5cvp6WlxShatW3bNlJSUsjLy+PIkSOEh4cTGxvLrl27qKqq4tixY5SWluLp6cnp06cJDQ2lpaUFp9OJl5cX/f39pKen8+yzz5KVlWVkHjzwwAPExsaybds2qqqqeOONN/D29mb//v3U1tayevVqZs+ezT333MPWrVspKipi165dxMTEEBYWhtlsNtIHLRYL7e3t2Gw20tLS2L9/P4mJiSQkJFBfX8/dd9/N2bNnefnll4mOjqayspKdO3ficrmwWq1G8b1rvRbHjx/HbrcDg7PNBw4cIDg42Lhv7969+Pv709nZidlsJjo6mq6uLlJTU6mtraW+vh6Xy0VsbCwnT55kwYIFtLe3Y7fbOXHiBBs2bKC0tJSwsDBiYmJ48sknjXTLt956i+DgYCIjI3F3d8flcmE2m8nKyqK3t5eKigr8/PxwuVykpqayZs2aKz4/Tp48SW5urhFgr1+/noSEBM6cOYPdbqetrQ0PDw9jXL/33ntGMa7RHNf19fWkpKRgs9kuuRa7d+++aFwXFBQQFhZGSUnJiOP6ZqOAT0RERGSC6e7uJiUlhfLycmM25lZxpeDgVnHo0CG+/vrriypw3qqGBz5yfdra2oz1qMO9//77BAUFjVk7br4NqURERETkhn3//ffk5eXx1FNP3XLB3kQwVHxk586d490UGWdBQUE4HI7xboZm+ERERERGS3p6urEmasjrr79+zYWkRvtxbhYj9WfmzJmX7MW5atUqli5dOpZNm3Dn+kaNxXnQuR4fCvhEREREREQmqElTpVNERERERGSyUcAnIiIiIiIyQSngExERGQWJiYlX3dz9Wo4REREZTQr4REREREREJihtyyAiIpNWQ0MDmZmZzJ8/n19//ZWIiAiWLl3Kjh07aG1tZfv27YSFhZGXl8eff/5pbIAdGRlJW1sbOTk5tLa2MnfuXIbXQHM4HOzZs4e+vj7uvfde8vPzMZvN49hTERGZrDTDJyIik9rZs2dZtWoVFRUV/PHHH1RWVrJv3z42btxIaWkphYWFREVFUVlZyUsvvcSmTZsAKCoqYt68eRw8eJDExESampoAqKur44svvmDfvn04HA7c3NyorKwczy6KiMgkphk+ERGZ1O666y5jD6hZs2YRFxeHyWQiIiKCxsZGmpqaKCwsBCAuLo729nY6OjqoqanhnXfeAWDRokUEBgYC8MMPP3D8+HHS0tIA6OnpISQkZBx6JiIiooBPREQmOU9PT+O2m5ub8bPJZGJgYGDEVEyTyXTZx3O5XCxZsoScnJzRb6yIiMh1UkqniIjIFVgsFioqKgA4evQoQUFB+Pv7Y7FYjFTN6upq/v77b2BwFrCqqoqWlhYA2tvbaWxsHJ/Gi4jIpKcZPhERkSuw2Wzk5uaSkpKCj48Pr732GgDr1q0jJyeHJUuWYLFYmDp1KjCYFvriiy/y9NNP43Q68fDwwG63M23atPHshoiITFIm1/CyYiIiIiIiIjJhKKVTRERERERkglLAJyIiIiIiMkEp4BMREREREZmgFPCJiIiIiIhMUAr4REREREREJigFfCIiIiIiIhOUAuwKrJQAAAAYSURBVD4REREREZEJSgGfiIiIiIjIBPU/J3xnPLK0NeAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, ax = plt.subplots(figsize=(15, 10))\n",
    "final_dists = df.xs(steps[-1], level=\"step\")\n",
    "order = final_dists.avg_sim.argsort()\n",
    "sns.barplot(data=final_dists.reset_index(), x=\"model\", y=\"avg_sim\", ax=ax)#, order=order.index[order])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pairwise distance: local evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argsmallest_n(a, n):\n",
    "    ret = np.argpartition(a, n)[:n]\n",
    "    b = np.take(a, ret)\n",
    "    return np.take(ret, np.argsort(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "tu = np.triu_indices(np.ceil(np.sqrt(2 * len(next(iter(sims.values()))))), 1)\n",
    "def nearest_neighbor_sentences(measure, n=10, reverse=True):\n",
    "    closest = argsmallest_n(measure if not reverse else -measure, n)\n",
    "    pairs = np.column_stack((np.take(tu[0], closest),\n",
    "                             np.take(tu[1], closest))) + 1\n",
    "    \n",
    "    ret = []\n",
    "    for (s1_id, s2_id), sim_id in zip(pairs, closest):\n",
    "        ret.append((measure[sim_id], sentences[s1_id], sentences[s2_id]))\n",
    "        \n",
    "    return ret"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8226685864745145,\n",
       "  'A luthier is a carpenter that makes and repairs musical instruments.',\n",
       "  'Depending on the size and format, scissors can cut paper, hair or fabric.'),\n",
       " (0.7899059227095679,\n",
       "  'A desert is a natural environment where almost no rain falls.',\n",
       "  'The blades are usually metallic, with plastic or metal handles.'),\n",
       " (0.7801688728758386,\n",
       "  'A desert is a natural environment where almost no rain falls.',\n",
       "  \"Theft is the taking of person's property without their consent.\"),\n",
       " (0.7776972812048104,\n",
       "  'Tables are also used as surfaces to work or write on.',\n",
       "  'A television is an appliance with a screen that displays moving images.'),\n",
       " (0.7743732223497682,\n",
       "  'Blenders have a glass or plastic container with a rotating blade in the bottom.',\n",
       "  'Televisions are mostly used for watching programs or films.'),\n",
       " (0.772858920696943,\n",
       "  'Blenders have a glass or plastic container with a rotating blade in the bottom.',\n",
       "  'Whales breathe through blowholes on their heads when surfacing.'),\n",
       " (0.769909802069989,\n",
       "  'Fingers are used for grasping, writing, grooming and other activities.',\n",
       "  'The nail protects the tip of the finger and can be grown or decorated.'),\n",
       " (0.7662517989955857,\n",
       "  'Forks are usually made of metal, or plastic, if disposable.',\n",
       "  'The blades are usually metallic, with plastic or metal handles.'),\n",
       " (0.763585016399199,\n",
       "  'Blenders have a glass or plastic container with a rotating blade in the bottom.',\n",
       "  'Mosquitoes have mouthparts for piercing the skin of plants and animals.'),\n",
       " (0.7586508575211898,\n",
       "  'A desert is a natural environment where almost no rain falls.',\n",
       "  'Fingers are used for grasping, writing, grooming and other activities.')]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"P01\", 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.899349724015977,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.8865050057285732,\n",
       "  'A tropical cyclone can move inland and seriously damage property.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.8566101144339264,\n",
       "  'Tropical cyclone can also cause high waves and coastal flooding.',\n",
       "  'A tropical cyclone can move inland and seriously damage property.'),\n",
       " (0.816843840009133,\n",
       "  'A butterfly is a flying insect with four large wings.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.'),\n",
       " (0.8137871264511599,\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.',\n",
       "  'There are many varieties of tomato, with different sizes and tastes.'),\n",
       " (0.7994702553669911,\n",
       "  'The term is used to indicate sex distinctions relative to girls.',\n",
       "  'Manhood is associated with masculinity and virility.'),\n",
       " (0.7965665589793874,\n",
       "  'Blenders are also used to make cold alcoholic drinks with crushed ice.',\n",
       "  'A boy is a young male child who has not undergone puberty.'),\n",
       " (0.7912244764799875,\n",
       "  'A bicycle is propelled by the pedalling of a cyclist on the saddle.',\n",
       "  'The wheels have rubber tires with an inner tube full of air.'),\n",
       " (0.7860142260481275,\n",
       "  'Arson is the criminal act of burning a building or wildland.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.'),\n",
       " (0.7857786318565708,\n",
       "  'Arson is the criminal act of burning a building or wildland.',\n",
       "  'A butterfly is a flying insect with four large wings.')]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"LM_scrambled\", 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8985496517058501,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.8906737350762995,\n",
       "  'A tropical cyclone can move inland and seriously damage property.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.8338768753469367,\n",
       "  'A butterfly is a flying insect with four large wings.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.'),\n",
       " (0.8258726344638743,\n",
       "  'Tropical cyclone can also cause high waves and coastal flooding.',\n",
       "  'A tropical cyclone can move inland and seriously damage property.'),\n",
       " (0.8180066855747953,\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.',\n",
       "  'There are many varieties of tomato, with different sizes and tastes.'),\n",
       " (0.8107069883736137,\n",
       "  'Blenders are also used to make cold alcoholic drinks with crushed ice.',\n",
       "  'A boy is a young male child who has not undergone puberty.'),\n",
       " (0.8081750422225392,\n",
       "  'A bicycle is propelled by the pedalling of a cyclist on the saddle.',\n",
       "  'The wheels have rubber tires with an inner tube full of air.'),\n",
       " (0.805372423392914,\n",
       "  'Arson is the criminal act of burning a building or wildland.',\n",
       "  'A butterfly is a flying insect with four large wings.'),\n",
       " (0.7958991024533594,\n",
       "  'Cyclones can produce very powerful winds and torrential rain.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.7910858454106751,\n",
       "  'Arson is the criminal act of burning a building or wildland.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.')]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"LM_scrambled_para\", 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9563107068686492,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.8883446792268924,\n",
       "  'A tropical cyclone can move inland and seriously damage property.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.8665087316821483,\n",
       "  'Tropical cyclone can also cause high waves and coastal flooding.',\n",
       "  'A tropical cyclone can move inland and seriously damage property.'),\n",
       " (0.8571678938749929,\n",
       "  'At puberty the body of a girl develops into that of a woman.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.8340738159799185,\n",
       "  'Blenders are also used to make cold alcoholic drinks with crushed ice.',\n",
       "  'A boy is a young male child who has not undergone puberty.'),\n",
       " (0.8194505734708437,\n",
       "  'At puberty the body of a girl develops into that of a woman.',\n",
       "  'A boy becomes a man after passing through puberty.'),\n",
       " (0.8142834161172635,\n",
       "  'A bicycle is propelled by the pedalling of a cyclist on the saddle.',\n",
       "  'The wheels have rubber tires with an inner tube full of air.'),\n",
       " (0.8000733842924581,\n",
       "  'Carpenters build and repair houses, furniture, boats and other objects.',\n",
       "  'Cats are small furry animals with four legs and a tail.'),\n",
       " (0.7996929969870776,\n",
       "  'Tropical cyclone can also cause high waves and coastal flooding.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.7992816472444773,\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.',\n",
       "  'There are many varieties of tomato, with different sizes and tastes.')]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"LM_pos\", 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9584460221891828,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.8250132035394377,\n",
       "  'A tropical cyclone can move inland and seriously damage property.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.7893542109259628,\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.',\n",
       "  'There are many varieties of tomato, with different sizes and tastes.'),\n",
       " (0.778891940977054,\n",
       "  \"Most earthquakes are caused by the release of energy in the Earth's crust.\",\n",
       "  'In a flood a river will flow out of its channel or break dams and levees.'),\n",
       " (0.7771691556723739,\n",
       "  'The term is used to indicate sex distinctions relative to girls.',\n",
       "  'At puberty the body of a girl develops into that of a woman.'),\n",
       " (0.7732678585659691,\n",
       "  'Tropical cyclone can also cause high waves and coastal flooding.',\n",
       "  'A tropical cyclone can move inland and seriously damage property.'),\n",
       " (0.7568334423792358,\n",
       "  'Extreme drought will eventually turn areas into desert and cause famine.',\n",
       "  'Large numbers of people have to migrate away from dust bowls.'),\n",
       " (0.7507686416611901,\n",
       "  'A clarinet is a long black tube with a flare at the bottom.',\n",
       "  'A cod is a fish that lives in the colder areas of the Atlantic.'),\n",
       " (0.7459571754890446,\n",
       "  'Bees spread pollen between flowers as they collect nectar to make honey.',\n",
       "  'The wings of butterflies can be very colorful and patterned.'),\n",
       " (0.7437025525413182,\n",
       "  'Arson is the criminal act of burning a building or wildland.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.')]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"MNLI\", 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.8896671574631109,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.8230850764025711,\n",
       "  'A tropical cyclone can move inland and seriously damage property.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.8124404103040963,\n",
       "  'Tropical cyclone can also cause high waves and coastal flooding.',\n",
       "  'A tropical cyclone can move inland and seriously damage property.'),\n",
       " (0.7610745344410159,\n",
       "  'Arson is the criminal act of burning a building or wildland.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.'),\n",
       " (0.7544694437857943,\n",
       "  'The term is used to indicate sex distinctions relative to girls.',\n",
       "  'Manhood is associated with masculinity and virility.'),\n",
       " (0.7514461480703941,\n",
       "  'Freshly squeezed orange juice often contains pulp as well.',\n",
       "  'Salmon can be eaten raw in sushi or cooked by grilling, poaching or in other ways.'),\n",
       " (0.7500651350597597,\n",
       "  'Blenders are also used to make cold alcoholic drinks with crushed ice.',\n",
       "  'A boy is a young male child who has not undergone puberty.'),\n",
       " (0.7398599275158704,\n",
       "  'A butterfly is a flying insect with four large wings.',\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.'),\n",
       " (0.737757479931513,\n",
       "  'Lettuce is considered fairly easy to grow and a suitable crop for beginners.',\n",
       "  'There are many varieties of tomato, with different sizes and tastes.'),\n",
       " (0.7338685571793564,\n",
       "  'Bees spread pollen between flowers as they collect nectar to make honey.',\n",
       "  'The wings of butterflies can be very colorful and patterned.')]"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"QQP\", 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.924750836737331,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.7772689279784516,\n",
       "  'The term is used to indicate sex distinctions relative to girls.',\n",
       "  'At puberty the body of a girl develops into that of a woman.'),\n",
       " (0.737254352308359,\n",
       "  'Blenders have a glass or plastic container with a rotating blade in the bottom.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.'),\n",
       " (0.7298741713490393,\n",
       "  'The raspberry is a fruit that grows in forest clearings or fields.',\n",
       "  'Ravens have large brains and are capable of problem solving.'),\n",
       " (0.6889709420024499,\n",
       "  'A bed is made of a mattress and a box spring, plus sheets, pillows and covers.',\n",
       "  'Chairs consist of a seat raised above floor level, supported by legs.'),\n",
       " (0.6852407618168672,\n",
       "  'The term is used to indicate sex distinctions relative to girls.',\n",
       "  'Manhood is associated with masculinity and virility.'),\n",
       " (0.6594409122303901,\n",
       "  'Broccoli is a plant in the cabbage family.',\n",
       "  'A woman can become pregnant and bear children.'),\n",
       " (0.6505233880196236,\n",
       "  'People often put carpets on a floor, for comfort and to dampen noise.',\n",
       "  \"Women's shoes often have high heels and come in many styles and colors.\"),\n",
       " (0.6483002403475655,\n",
       "  'A desk may have drawers to store supplies and papers.',\n",
       "  'A mug is thick and has a ring base for thermal insulation.'),\n",
       " (0.6410860712362328,\n",
       "  'Freshly squeezed orange juice often contains pulp as well.',\n",
       "  'Salmon can be eaten raw in sushi or cooked by grilling, poaching or in other ways.')]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"SQuAD\", 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.9467139047844539,\n",
       "  'A boy becomes a man after passing through puberty.',\n",
       "  'A woman is stereotypically seen as a caregiver.'),\n",
       " (0.9111868771571411,\n",
       "  'Freshly squeezed orange juice often contains pulp as well.',\n",
       "  'Salmon can be eaten raw in sushi or cooked by grilling, poaching or in other ways.'),\n",
       " (0.90420951004844,\n",
       "  'Metamorphosis is the transformation of a caterpillar into a butterfly.',\n",
       "  'A carpenter is an artist or manual laborer who works with wood.'),\n",
       " (0.8965680346806192,\n",
       "  'Freshly squeezed orange juice often contains pulp as well.',\n",
       "  'A tomato softens and changes color from green to red as it gets more ripe.'),\n",
       " (0.8869241941558758,\n",
       "  'Salmon can be eaten raw in sushi or cooked by grilling, poaching or in other ways.',\n",
       "  'A tomato softens and changes color from green to red as it gets more ripe.'),\n",
       " (0.8845068057702591,\n",
       "  'A tropical cyclone can move inland and seriously damage property.',\n",
       "  'The violin is an instrument with four strings that is played with a bow.'),\n",
       " (0.8805769779979179,\n",
       "  'Extreme drought will eventually turn areas into desert and cause famine.',\n",
       "  'Large numbers of people have to migrate away from dust bowls.'),\n",
       " (0.863993148536969,\n",
       "  'Parrots have brightly-colored feathers and a sharp, curved beak.',\n",
       "  'Ravens feed on carrion, insects, berries or small animals.'),\n",
       " (0.838629052183327,\n",
       "  'A swamp is covered with shallow water, mud and vegetation.',\n",
       "  'Swamps are nesting and breeding grounds for birds and reptiles.'),\n",
       " (0.8194364484414364,\n",
       "  'Large numbers of people have to migrate away from dust bowls.',\n",
       "  'A floor is a horizontal surface inside a building, used for walking on.')]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nearest_neighbor_sentences(sims[\"SST\", 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LM_pos</th>\n",
       "      <th>LM_scrambled_para</th>\n",
       "      <th>pos_vs_scrambled_para</th>\n",
       "      <th>abs_pos_vs_scrambled_para</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>72408</th>\n",
       "      <td>-0.140894</td>\n",
       "      <td>0.572202</td>\n",
       "      <td>-0.713096</td>\n",
       "      <td>0.713096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16198</th>\n",
       "      <td>-0.286826</td>\n",
       "      <td>0.422098</td>\n",
       "      <td>-0.708924</td>\n",
       "      <td>0.708924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70249</th>\n",
       "      <td>0.564239</td>\n",
       "      <td>-0.143931</td>\n",
       "      <td>0.708170</td>\n",
       "      <td>0.708170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33303</th>\n",
       "      <td>0.547607</td>\n",
       "      <td>-0.149764</td>\n",
       "      <td>0.697370</td>\n",
       "      <td>0.697370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13566</th>\n",
       "      <td>-0.364057</td>\n",
       "      <td>0.329469</td>\n",
       "      <td>-0.693527</td>\n",
       "      <td>0.693527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57199</th>\n",
       "      <td>0.622043</td>\n",
       "      <td>-0.061629</td>\n",
       "      <td>0.683672</td>\n",
       "      <td>0.683672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40593</th>\n",
       "      <td>0.670382</td>\n",
       "      <td>0.015423</td>\n",
       "      <td>0.654959</td>\n",
       "      <td>0.654959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54183</th>\n",
       "      <td>0.658134</td>\n",
       "      <td>0.004803</td>\n",
       "      <td>0.653331</td>\n",
       "      <td>0.653331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48513</th>\n",
       "      <td>0.519580</td>\n",
       "      <td>-0.129519</td>\n",
       "      <td>0.649098</td>\n",
       "      <td>0.649098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47682</th>\n",
       "      <td>0.485529</td>\n",
       "      <td>-0.157376</td>\n",
       "      <td>0.642905</td>\n",
       "      <td>0.642905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40412</th>\n",
       "      <td>0.501553</td>\n",
       "      <td>-0.140815</td>\n",
       "      <td>0.642367</td>\n",
       "      <td>0.642367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27777</th>\n",
       "      <td>0.476793</td>\n",
       "      <td>-0.162398</td>\n",
       "      <td>0.639191</td>\n",
       "      <td>0.639191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73494</th>\n",
       "      <td>-0.452476</td>\n",
       "      <td>0.184447</td>\n",
       "      <td>-0.636923</td>\n",
       "      <td>0.636923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61898</th>\n",
       "      <td>-0.264694</td>\n",
       "      <td>0.372036</td>\n",
       "      <td>-0.636730</td>\n",
       "      <td>0.636730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9072</th>\n",
       "      <td>0.485283</td>\n",
       "      <td>-0.145225</td>\n",
       "      <td>0.630508</td>\n",
       "      <td>0.630508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64337</th>\n",
       "      <td>0.417079</td>\n",
       "      <td>-0.207095</td>\n",
       "      <td>0.624174</td>\n",
       "      <td>0.624174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57203</th>\n",
       "      <td>0.508357</td>\n",
       "      <td>-0.114640</td>\n",
       "      <td>0.622998</td>\n",
       "      <td>0.622998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53986</th>\n",
       "      <td>0.598320</td>\n",
       "      <td>-0.023868</td>\n",
       "      <td>0.622187</td>\n",
       "      <td>0.622187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52579</th>\n",
       "      <td>0.602596</td>\n",
       "      <td>-0.019371</td>\n",
       "      <td>0.621967</td>\n",
       "      <td>0.621967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9256</th>\n",
       "      <td>0.424548</td>\n",
       "      <td>-0.196365</td>\n",
       "      <td>0.620913</td>\n",
       "      <td>0.620913</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         LM_pos  LM_scrambled_para  pos_vs_scrambled_para  \\\n",
       "72408 -0.140894           0.572202              -0.713096   \n",
       "16198 -0.286826           0.422098              -0.708924   \n",
       "70249  0.564239          -0.143931               0.708170   \n",
       "33303  0.547607          -0.149764               0.697370   \n",
       "13566 -0.364057           0.329469              -0.693527   \n",
       "57199  0.622043          -0.061629               0.683672   \n",
       "40593  0.670382           0.015423               0.654959   \n",
       "54183  0.658134           0.004803               0.653331   \n",
       "48513  0.519580          -0.129519               0.649098   \n",
       "47682  0.485529          -0.157376               0.642905   \n",
       "40412  0.501553          -0.140815               0.642367   \n",
       "27777  0.476793          -0.162398               0.639191   \n",
       "73494 -0.452476           0.184447              -0.636923   \n",
       "61898 -0.264694           0.372036              -0.636730   \n",
       "9072   0.485283          -0.145225               0.630508   \n",
       "64337  0.417079          -0.207095               0.624174   \n",
       "57203  0.508357          -0.114640               0.622998   \n",
       "53986  0.598320          -0.023868               0.622187   \n",
       "52579  0.602596          -0.019371               0.621967   \n",
       "9256   0.424548          -0.196365               0.620913   \n",
       "\n",
       "       abs_pos_vs_scrambled_para  \n",
       "72408                   0.713096  \n",
       "16198                   0.708924  \n",
       "70249                   0.708170  \n",
       "33303                   0.697370  \n",
       "13566                   0.693527  \n",
       "57199                   0.683672  \n",
       "40593                   0.654959  \n",
       "54183                   0.653331  \n",
       "48513                   0.649098  \n",
       "47682                   0.642905  \n",
       "40412                   0.642367  \n",
       "27777                   0.639191  \n",
       "73494                   0.636923  \n",
       "61898                   0.636730  \n",
       "9072                    0.630508  \n",
       "64337                   0.624174  \n",
       "57203                   0.622998  \n",
       "53986                   0.622187  \n",
       "52579                   0.621967  \n",
       "9256                    0.620913  "
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sims_df = pd.DataFrame({\"LM_pos\": sims[\"LM_pos\", 2], \"LM_scrambled_para\": sims[\"LM_scrambled_para\", 2]})\n",
    "sims_df[\"pos_vs_scrambled_para\"] = sims_df.LM_pos - sims_df.LM_scrambled_para\n",
    "sims_df[\"abs_pos_vs_scrambled_para\"] = np.abs(sims_df.pos_vs_scrambled_para)\n",
    "sims_df = sims_df.sort_values(\"abs_pos_vs_scrambled_para\", ascending=False)\n",
    "sims_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def calc_row_idx(k, n):\n",
    "    return int(math.ceil((1/2.) * (- (-8*k + 4 *n**2 -4*n - 7)**0.5 + 2*n -1) - 1))\n",
    "\n",
    "def elem_in_i_rows(i, n):\n",
    "    return i * (n - 1 - i) + (i*(i + 1))/2\n",
    "\n",
    "def calc_col_idx(k, i, n):\n",
    "    return int(n - elem_in_i_rows(i + 1, n) + k)\n",
    "\n",
    "def condensed_to_square(k, n):\n",
    "    i = calc_row_idx(k, n)\n",
    "    j = calc_col_idx(k, i, n)\n",
    "    return i, j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('A toaster is a kitchen appliance for toasting bread using heating elements.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  -0.14089391236409643,\n",
       "  0.5722016569939911),\n",
       " ('A blender is a kitchen appliance used to mix ingredients or puree food.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  -0.2868264685588837,\n",
       "  0.42209791730213),\n",
       " ('A bicycle is propelled by the pedalling of a cyclist on the saddle.',\n",
       "  'It can be hard to climb stairs or ride a bike in a dress.',\n",
       "  -0.3640573252121033,\n",
       "  0.32946928655184615),\n",
       " ('The blubber in a whale serves as an energy reservoir and as insulation.',\n",
       "  'When open, a window will also let air and sound through.',\n",
       "  -0.4524760292024097,\n",
       "  0.18444652166096054),\n",
       " ('A mug is thick and has a ring base for thermal insulation.',\n",
       "  'The blubber in a whale serves as an energy reservoir and as insulation.',\n",
       "  -0.2646940609595365,\n",
       "  0.3720358919379858),\n",
       " ('Milk is a white liquid produced by female mammals to feed their young.',\n",
       "  'Newborn animals must be fed milk before they can eat solid food.',\n",
       "  -0.03691471817113001,\n",
       "  0.5787996214047455),\n",
       " ('Liquid produced from soybeans and coconuts is sometimes called milk.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  -0.37740366380260837,\n",
       "  0.23677486253292046),\n",
       " ('Dairy products such as cheese and butter are made from milk.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  -0.3038691584888098,\n",
       "  0.3060492401525363),\n",
       " ('Coffee is made from coffee beans, the seeds of the coffee plant.',\n",
       "  'Liquid produced from soybeans and coconuts is sometimes called milk.',\n",
       "  -0.06063349626895165,\n",
       "  0.5486547280614967),\n",
       " ('A bicycle is a vehicle with two wheels and pedals on a frame.',\n",
       "  'It can be hard to climb stairs or ride a bike in a dress.',\n",
       "  -0.36535118149103263,\n",
       "  0.23986153660812604),\n",
       " ('A blacksmith hammers and bends hot iron on an anvil to change its shape.',\n",
       "  'Carpenters use tools such as hammers, saws, pliers and planes.',\n",
       "  -0.1169258670761335,\n",
       "  0.4705984224877031),\n",
       " ('Their appearance and abilities makes them popular pets.',\n",
       "  \"Women's shoes often have high heels and come in many styles and colors.\",\n",
       "  -0.38654987829229426,\n",
       "  0.19953187138846662),\n",
       " ('The driver steers the car on roads, other passengers just sit.',\n",
       "  'A bicycle is propelled by the pedalling of a cyclist on the saddle.',\n",
       "  -0.29223515988873805,\n",
       "  0.2926468460181757),\n",
       " ('Air pockets in snow insulate the space, and body heat warms igloos.',\n",
       "  'The blubber in a whale serves as an energy reservoir and as insulation.',\n",
       "  -0.43170853932496933,\n",
       "  0.14856945059098325),\n",
       " ('A blacksmith hammers and bends hot iron on an anvil to change its shape.',\n",
       "  'A hammer is most often used to drive nails into wood.',\n",
       "  0.004872607506434812,\n",
       "  0.5796166479571047),\n",
       " ('A house has a roof and walls to keep out cold and rain, and windows to let light in.',\n",
       "  'The blubber in a whale serves as an energy reservoir and as insulation.',\n",
       "  -0.5142441367886774,\n",
       "  0.056891388363558226),\n",
       " ('Carpenters use tools such as hammers, saws, pliers and planes.',\n",
       "  'The blades are usually metallic, with plastic or metal handles.',\n",
       "  -0.3206690106628862,\n",
       "  0.24242593877462326),\n",
       " ('The driver steers the car on roads, other passengers just sit.',\n",
       "  'A bicycle is a vehicle with two wheels and pedals on a frame.',\n",
       "  -0.34668408758078484,\n",
       "  0.21509151890748346),\n",
       " ('A blacksmith hammers and bends hot iron on an anvil to change its shape.',\n",
       "  'Shears are large scissors used for cutting shrubs or pieces of metal.',\n",
       "  -0.2409614443603274,\n",
       "  0.31166533979699995),\n",
       " ('Walls delineate a building, support the roof, and divide the space into rooms.',\n",
       "  'The blubber in a whale serves as an energy reservoir and as insulation.',\n",
       "  -0.5428015608934338,\n",
       "  0.0075865166014820185)]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(sentences[condensed_to_square(idx, 384)[0]], sentences[condensed_to_square(idx, 384)[1]], row.LM_pos, row.LM_scrambled_para)\n",
    " for idx, row in sims_df.sort_values(\"pos_vs_scrambled_para\").head(20).iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  'Walls delineate a building, support the roof, and divide the space into rooms.',\n",
       "  0.7122544322365467,\n",
       "  0.10018211754362016),\n",
       " ('A house has a roof and walls to keep out cold and rain, and windows to let light in.',\n",
       "  'A tomato softens and changes color from green to red as it gets more ripe.',\n",
       "  0.5647060125841246,\n",
       "  -0.04882222355636934),\n",
       " ('A girl is a female human who has not yet reached maturity.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.5324532885221487,\n",
       "  -0.08213889525989004),\n",
       " (\"Women's shoes often have high heels and come in many styles and colors.\",\n",
       "  'When open, a window will also let air and sound through.',\n",
       "  0.5550248125961711,\n",
       "  -0.06116087318815322),\n",
       " ('The tomato is a shiny and smooth red fruit, with a savory taste.',\n",
       "  'Walls delineate a building, support the roof, and divide the space into rooms.',\n",
       "  0.4529855567531772,\n",
       "  -0.16549162031063624),\n",
       " ('A banana is a long fruit that grows in bunchs, with a soft edible inside.',\n",
       "  'Walls delineate a building, support the roof, and divide the space into rooms.',\n",
       "  0.42454786663767763,\n",
       "  -0.1963653933604128),\n",
       " ('Some patients go there for specialist diagnosis or therapy, leaving the same day.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.6025956791479921,\n",
       "  -0.01937083006580287),\n",
       " ('Igloos are built in places where snow covers the land most of the time.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.5983197102933082,\n",
       "  -0.023867760726500098),\n",
       " ('People can go to public libraries and borrow books to take home for several weeks.',\n",
       "  'A tomato softens and changes color from green to red as it gets more ripe.',\n",
       "  0.5083574731037136,\n",
       "  -0.11464027739844451),\n",
       " ('Peaches are sweet and delicate, and must be harvested after ripening.',\n",
       "  'Walls delineate a building, support the roof, and divide the space into rooms.',\n",
       "  0.41707858735836123,\n",
       "  -0.2070951251403692),\n",
       " ('A banana is a long fruit that grows in bunchs, with a soft edible inside.',\n",
       "  'A house has a roof and walls to keep out cold and rain, and windows to let light in.',\n",
       "  0.4852832640660447,\n",
       "  -0.145225199228167),\n",
       " ('Coffee is made from coffee beans, the seeds of the coffee plant.',\n",
       "  'When open, a window will also let air and sound through.',\n",
       "  0.47679310263669206,\n",
       "  -0.16239819722668725),\n",
       " ('Farms usually have a house for farmers, a silo and a barn.',\n",
       "  'A girl is a female human who has not yet reached maturity.',\n",
       "  0.5015525800195918,\n",
       "  -0.1408149081540233),\n",
       " ('A girl is a female human who has not yet reached maturity.',\n",
       "  'A house has a roof and walls to keep out cold and rain, and windows to let light in.',\n",
       "  0.4855292445069105,\n",
       "  -0.157375797543013),\n",
       " ('Girls sometimes grow long hair and wear dresses and skirts.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.5195797128548599,\n",
       "  -0.12951877017609958),\n",
       " ('Igloos are often built in the shape of a dome, with an entrance tunnel.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.6581339661691775,\n",
       "  0.004802640361383448),\n",
       " ('Farms usually have a house for farmers, a silo and a barn.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.6703815037724932,\n",
       "  0.015422715590864722),\n",
       " ('People can go to public libraries and borrow books to take home for several weeks.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.622043270638285,\n",
       "  -0.06162886122682654),\n",
       " ('It can be hard to climb stairs or ride a bike in a dress.',\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.5476067562920437,\n",
       "  -0.1497636238544513),\n",
       " (\"Women's shoes often have high heels and come in many styles and colors.\",\n",
       "  'Pop-up toasters toast the bread placed in a slot and eject it once done.',\n",
       "  0.5642388998702866,\n",
       "  -0.14393147501424397)]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[(sentences[condensed_to_square(idx, 384)[0]], sentences[condensed_to_square(idx, 384)[1]], row.LM_pos, row.LM_scrambled_para)\n",
    " for idx, row in sims_df.sort_values(\"pos_vs_scrambled_para\").tail(20).iterrows()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-69-59b96c191ddb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mbrain_nearest_neighbors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msubject\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msubjects\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mbrain_nearest_neighbors\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msubject\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnearest_neighbor_sentences\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdists\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msubject\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-59-c9641aa4305c>\u001b[0m in \u001b[0;36mnearest_neighbor_sentences\u001b[0;34m(measure, n, reverse)\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0ms1_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms2_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdist_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpairs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m         \u001b[0mret\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeasure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdist_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentences\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms1_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msentences\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0ms2_id\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mret\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "brain_nearest_neighbors = {}\n",
    "for subject in subjects:\n",
    "    brain_nearest_neighbors[subject] = nearest_neighbor_sentences(dists[subject, 0], n=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check Jaccard measure for nearest-neighbor predictions between subjects.\n",
    "neighbor_jaccards = []\n",
    "for s1, s2 in itertools.combinations(SUBJECTS, 2):\n",
    "    s1_neighbors = set((sent1, sent2) for _, sent1, sent2 in brain_nearest_neighbors[s1])\n",
    "    s2_neighbors = set((sent1, sent2) for _, sent1, sent2 in brain_nearest_neighbors[s2])\n",
    "    neighbor_jaccards.append((s1, s2, len(s1_neighbors & s2_neighbors) / len(s1_neighbors | s2_neighbors)))\n",
    "    \n",
    "neighbor_jaccards = pd.DataFrame(neighbor_jaccards, columns=[\"s1\", \"s2\", \"jaccard\"]).set_index([\"s1\", \"s2\"])\n",
    "neighbor_jaccards.sort_values(\"jaccard\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pairwise distance: collapse analysis\n",
    "\n",
    "Under the collapse theory, we should find that some pairs of sentences which are well separated in brain representations are not well separated in model activations (case I) or that pairs well separated in model activations are well separated in brain representations (case II). We can measure this by computing, for each sentence pair $(s_1, s_2)$,\n",
    "\n",
    "$$q(s_1, s_2) = \\frac{dist(m(s_1), m(s_2))}{dist(b(s_1),b(s_2))}$$\n",
    "\n",
    "for a model representation $m$ and brain representation $b$.\n",
    "\n",
    "If $q$ is large, then the model distinguishes $s_1, s_2$ along the major axis in a way not captured by the brain. If $q$ is small, then the brain distinguishes $s_1, s_2$ along the major axis in a way not captured by the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_measures = {}\n",
    "\n",
    "for subject in subjects:\n",
    "    subject_dists = dists[subject, 0].copy()\n",
    "    subject_dists -= subject_dists.min()\n",
    "    subject_dists /= subject_dists.max() - subject_dists.min()\n",
    "    \n",
    "    for model, run in zip(bert_models, target_runs):\n",
    "        model_dists = dists[model, run].copy()\n",
    "        model_dists -= model_dists.min()\n",
    "        model_dists /= model_dists.max() - model_dists.min()\n",
    "        \n",
    "        q_measures[model, run, subject] = pd.Series(model_dists / (subject_dists + 1e-5))\n",
    "        \n",
    "q_measures = pd.DataFrame(pd.concat(q_measures, names=[\"model\", \"run\", \"subject\", \"pair\"]))\n",
    "q_measures.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_means = q_measures.reset_index().groupby([\"model\", \"pair\"])[0].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_means.hist(by=\"model\", bins=30, figsize=(15,10), sharex=True, range=(1, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_means.groupby(\"model\").mean().sort_values(0).plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rank change analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoding_preds = {}\n",
    "for encoding in bert_models:\n",
    "    for run in target_runs:\n",
    "        for subject in subjects:\n",
    "            try:\n",
    "                encoding_preds[encoding, steps[-1], subject] = \\\n",
    "                    pd.read_csv(decoder_path / (\"encodings.%s.%s.%s-run%i-250-%s.pred.csv\" % (finetune_desc, bert_base_model, encoding, run, subject)),\n",
    "                                index_col=[0, 1])\n",
    "            except:\n",
    "                continue\n",
    "    \n",
    "encoding_preds = pd.concat(encoding_preds, names=[\"model\", \"step\", \"subject\"])\n",
    "encoding_preds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_changes = encoding_preds.reset_index().set_index([\"step\", \"idx\"]).groupby([\"model\", \"subject\"]) \\\n",
    "    .apply(lambda xs: xs.loc[steps[-1]] - xs.loc[steps[0]]).rename(columns={\"rank\": \"rank_change\"})\n",
    "rank_changes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average across subjects.\n",
    "avg_rank_changes = rank_changes.mean(level=[\"model\", \"idx\"])\n",
    "avg_rank_changes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.barplot(data=avg_rank_changes.mean(level=\"model\").reset_index(), x=\"model\", y=\"rank_change\")\n",
    "plt.title(\"Average sentence rank change\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 20\n",
    "topn = {}\n",
    "bottomn = {}\n",
    "\n",
    "for (model, subject), rank_changes_m in rank_changes.reset_index().groupby([\"model\", \"subject\"]):\n",
    "    print(\"\\n\\n========\\n %s // %s\" % (model, subject))\n",
    "    \n",
    "    rank_changes_m = rank_changes_m.set_index(\"idx\")\n",
    "    top_sentences = rank_changes_m.index[rank_changes_m.rank_change.argsort()[::-1]]\n",
    "    topn[model, subject] = top_sentences[:n]\n",
    "    bottomn[model, subject] = top_sentences[::-1][:n]\n",
    "    \n",
    "    for sent_id in topn[model, subject][:5]:\n",
    "        print(rank_changes_m.loc[sent_id].rank_change, sentences[sent_id])\n",
    "    print()\n",
    "    for sent_id in bottomn[model, subject][:5]:\n",
    "        print(rank_changes_m.loc[sent_id].rank_change, sentences[sent_id])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute pairwise subject Jaccard cofficients by comparing maximum-rank-change sets for each model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "jaccards = []\n",
    "for model in ENCODINGS:\n",
    "    for s1, s2 in itertools.combinations(SUBJECTS, 2):\n",
    "        topn_s1, topn_s2 = set(topn[model, s1]), set(topn[model, s2])\n",
    "        bottomn_s1, bottomn_s2 = set(bottomn[model, s1]), set(bottomn[model, s2])\n",
    "        \n",
    "        jaccard_positive = len(topn_s1 & topn_s2) / len(topn_s1 | topn_s2)\n",
    "        jaccard_negative = len(bottomn_s1 & bottomn_s2) / len(bottomn_s1 | bottomn_s2)\n",
    "        jaccards.append((model, s1, s2, jaccard_positive, jaccard_negative))\n",
    "        \n",
    "jaccards = pd.DataFrame(jaccards, columns=[\"model\", \"s1\", \"s2\", \"jaccard_positive\", \"jaccard_negative\"]).set_index([\"model\", \"s1\", \"s2\"]).sort_index()\n",
    "jaccards.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "jaccards.sort_values(\"jaccard_positive\", ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Per-subject rank changes\n",
    "\n",
    "Let's go deeper and see how some of the sentences with most highly unstable rank are behaving within-subject."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "std_rank_changes = rank_changes.std(level=[\"model\", \"idx\"]).rename(columns={\"rank_change\": \"rank_change_std\"})\n",
    "std_rank_changes = std_rank_changes.sort_values(\"rank_change_std\", ascending=False)\n",
    "std_rank_changes.loc[\"QQP\"][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, std in std_rank_changes.loc[\"QQP\"][:10].iterrows():\n",
    "    print(sentences[idx], std.rank_change_std)\n",
    "    print(rank_changes.loc[\"QQP\", :, idx].sort_values(\"rank_change\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f, ax = plt.subplots(figsize=(20, 8))\n",
    "sns.barplot(data=rank_changes.loc[\"MNLI\", :, std_rank_changes.loc[\"MNLI\"][:15].index].reset_index(),\n",
    "            x=\"idx\", y=\"rank_change\", hue=\"subject\", ax=ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pair separation analysis\n",
    "\n",
    "For each model, find sentence pairs whose distance changes maximally between the base model and the end of training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for model in ENCODINGS:\n",
    "    dist_start = dists[\"LM\"]\n",
    "    dist_end = dists[model]\n",
    "    dist_changes = dist_end - dist_start\n",
    "    \n",
    "    print(model)\n",
    "    for score, sent1, sent2 in nearest_neighbor_sentences(dist_changes):\n",
    "        print(score, sent1, \"//\", sent2)\n",
    "    for score, sent1, sent2 in nearest_neighbor_sentences(dist_changes, reverse=True):\n",
    "        print(score, sent1, \"//\", sent2)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}