tables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = [\n",
    "    (\"cnp\", \"CNP\", \"loglik\"),\n",
    "    (\"acnp\", \"ACNP\", \"loglik\"),\n",
    "    (\"convcnp\", \"ConvCNP\", \"loglik\"),\n",
    "    (\"gnp\", \"GNP\", \"loglik\"),\n",
    "    (\"agnp\", \"AGNP\", \"loglik\"),\n",
    "    (\"convgnp\", \"ConvGNP\", \"loglik\"),\n",
    "    (\"fullconvgnp\", \"FullConvGNP\", \"loglik\"),\n",
    "    (\"np\", \"NP\", \"loglik\"),\n",
    "    (\"anp\", \"ANP\", \"loglik\"),\n",
    "    (\"convnp\", \"ConvNP\", \"loglik\"),\n",
    "    (\"np\", \"NP\", \"elbo\"),\n",
    "    (\"anp\", \"ANP\", \"elbo\"),\n",
    "    (\"convnp\", \"ConvNP\", \"elbo\"),\n",
    "]\n",
    "\n",
    "datasets = [\n",
    "    \"sawtooth\",\n",
    "    # \"mixture\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_line(string, interpolation, extrapolation, out_of_range, phantom=True):\n",
    "    \n",
    "    p = \"\\hphantom{-}\" if phantom else \"\"\n",
    "    \n",
    "    if interpolation is not None:\n",
    "        p1 = p if interpolation[0] > 0 else \"\"\n",
    "        inter = f\"${p1}{interpolation[0]:8.2f} {{\\small \\pm {interpolation[1]:6.2f}}}$\" if interpolation[0] > -20. else \" \" * 13 + \"F\" + \" \" * 14\n",
    "    else:\n",
    "        inter = f\"{'Missing':<19}\"\n",
    "        \n",
    "    if interpolation is not None:\n",
    "        p2 = p if extrapolation[0] > 0 else \"\"\n",
    "        extra = f\"${p2}{extrapolation[0]:8.2f} {{\\small \\pm {extrapolation[1]:6.2f}}}$\" if extrapolation[0] > -20. else \" \" * 13 + \"F\" + \" \" * 14\n",
    "    else:\n",
    "        extra = f\"{'Missing':<19}\"\n",
    "        \n",
    "    if out_of_range is not None:\n",
    "        p3 = p if out_of_range[0] > 0 else \"\"\n",
    "        oor = f\"${p3}{out_of_range[0]:8.2f} {{\\small \\pm {out_of_range[1]:6.2f}}}$\" if out_of_range[0] > -20. else \" \" * 13 + \"F\" + \" \" * 14\n",
    "    else:\n",
    "        oor = f\"{'Missing':<19}\"\n",
    "    \n",
    "    return f\"{string:<20} & {inter}  & {extra}  & {oor} \\\\\\\\\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNP                  & $   -0.30 {\\small \\pm   0.00}$  & $   -0.41 {\\small \\pm   0.00}$  & $   -0.35 {\\small \\pm   0.00}$ \\\\\n",
      "CNP (AR)             & $   -0.29 {\\small \\pm   0.00}$  & $   -1.19 {\\small \\pm   0.02}$  & $   -0.89 {\\small \\pm   0.02}$ \\\\\n",
      "ACNP                 & $   -0.08 {\\small \\pm   0.01}$  & $   -0.41 {\\small \\pm   0.01}$  & $   -0.42 {\\small \\pm   0.01}$ \\\\\n",
      "ACNP (AR)            & $\\hphantom{-}    1.01 {\\small \\pm   0.01}$  & $   -0.47 {\\small \\pm   0.01}$  & $   -0.45 {\\small \\pm   0.01}$ \\\\\n",
      "ConvCNP              & $\\hphantom{-}    0.87 {\\small \\pm   0.03}$  & $\\hphantom{-}    0.88 {\\small \\pm   0.02}$  & $   -0.28 {\\small \\pm   0.00}$ \\\\\n",
      "ConvCNP (AR)         & $\\hphantom{-}    2.23 {\\small \\pm   0.01}$  & $\\hphantom{-}    2.24 {\\small \\pm   0.01}$  & $\\hphantom{-}    1.94 {\\small \\pm   0.01}$ \\\\\n",
      "GNP                  & $   -0.03 {\\small \\pm   0.00}$  &              F                &              F               \\\\\n",
      "AGNP                 & $\\hphantom{-}    0.47 {\\small \\pm   0.01}$  & $   -0.38 {\\small \\pm   0.01}$  & $   -0.70 {\\small \\pm   0.08}$ \\\\\n",
      "ConvGNP              & Missing              & Missing              & Missing             \\\\\n",
      "FullConvGNP          & $\\hphantom{-}    1.06 {\\small \\pm   0.03}$  & $\\hphantom{-}    1.05 {\\small \\pm   0.04}$  & $\\hphantom{-}    0.11 {\\small \\pm   0.00}$ \\\\\n",
      "NP                   & $   -0.32 {\\small \\pm   0.00}$  & $   -0.32 {\\small \\pm   0.00}$  & $   -0.32 {\\small \\pm   0.00}$ \\\\\n",
      "ANP                  & $   -0.21 {\\small \\pm   0.00}$  & $   -0.33 {\\small \\pm   0.00}$  & $   -0.33 {\\small \\pm   0.00}$ \\\\\n",
      "ConvNP               & $\\hphantom{-}    1.88 {\\small \\pm   0.03}$  & $\\hphantom{-}    1.89 {\\small \\pm   0.03}$  & $   -0.11 {\\small \\pm   0.01}$ \\\\\n",
      "NP (EL., ML.)        & $   -0.08 {\\small \\pm   0.01}$  &              F                & $   -1.37 {\\small \\pm   0.00}$ \\\\\n",
      "NP (ELBO)            & $   -0.08 {\\small \\pm   0.01}$  &              F                & $   -1.37 {\\small \\pm   0.00}$ \\\\\n",
      "ANP (EL., ML.)       & $   -0.04 {\\small \\pm   0.01}$  & $   -1.09 {\\small \\pm   0.02}$  & $   -0.35 {\\small \\pm   0.00}$ \\\\\n",
      "ANP (ELBO)           & $   -0.05 {\\small \\pm   0.01}$  & $   -1.14 {\\small \\pm   0.03}$  & $   -0.35 {\\small \\pm   0.00}$ \\\\\n",
      "ConvNP (EL., ML.)    & $   -2.98 {\\small \\pm   0.10}$  & $   -2.98 {\\small \\pm   0.10}$  & $   -6.74 {\\small \\pm   0.04}$ \\\\\n",
      "ConvNP (ELBO)        & $\\hphantom{-}    2.01 {\\small \\pm   0.01}$  & $\\hphantom{-}    2.01 {\\small \\pm   0.01}$  & $\\hphantom{-}    1.47 {\\small \\pm   0.01}$ \\\\\n"
     ]
    }
   ],
   "source": [
    "rows = []\n",
    "inters = []\n",
    "\n",
    "datasets = [\"sawtooth\"]\n",
    "\n",
    "for (model, string, objective), dataset in product(models, datasets):\n",
    "    \n",
    "    root = f\"_experiments/{dataset}/x1_y2/{model}\"\n",
    "    \n",
    "    if \"conv\" in model:\n",
    "        root = root + \"/unet\"\n",
    "        \n",
    "    root = root + f\"/{objective}/log_evaluate_out.txt\"\n",
    "    \n",
    "    _file = open(root, \"r\").read()\n",
    "    \n",
    "    try:\n",
    "        \n",
    "        if objective == \"loglik\":\n",
    "\n",
    "            file = _file.split(\"| Loglik:\")[1].strip().split(\"\\n\")[:6]\n",
    "\n",
    "            process = lambda x: float(x.strip())\n",
    "\n",
    "            interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "            row = format_line(string, interpolation, extrapolation, out_of_range)\n",
    "            inters.append(interpolation[0])\n",
    "            rows.append(row)\n",
    "            print(row)\n",
    "\n",
    "            if \"cnp\" in model and \"| AR:\" in _file:\n",
    "\n",
    "                file = _file.split(\"| AR:\")[1].strip().split(\"\\n\")[:6]\n",
    "\n",
    "                process = lambda x: float(x.strip())\n",
    "\n",
    "                interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "                extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "                out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "                row = format_line(string + \" (AR)\", interpolation, extrapolation, out_of_range)\n",
    "                inters.append(interpolation[0])\n",
    "                rows.append(row)\n",
    "                print(row)\n",
    "\n",
    "        else:\n",
    "\n",
    "            file = _file.split(\"| Loglik:\")[1].strip().split(\"\\n\")[:6]\n",
    "\n",
    "            process = lambda x: float(x.strip())\n",
    "\n",
    "            interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "            row = format_line(string + ' (EL., ML.)', interpolation, extrapolation, out_of_range)\n",
    "            inters.append(interpolation[0])\n",
    "            rows.append(row)\n",
    "            print(row)\n",
    "\n",
    "            file = _file.split(\"| ELBO:\")[1].strip().split(\"\\n\")[:6]\n",
    "\n",
    "            process = lambda x: float(x.strip())\n",
    "\n",
    "            interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "            row = format_line(string + ' (ELBO)', interpolation, extrapolation, out_of_range)\n",
    "            inters.append(interpolation[0])\n",
    "            rows.append(row)\n",
    "            print(row)\n",
    "            \n",
    "            \n",
    "    except:\n",
    "        row = format_line(string, None, None, None)\n",
    "        inters.append(-100000)\n",
    "        print(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CNP                  & $   -0.17 {\\small \\pm   0.01}$  & $   -0.18 {\\small \\pm   0.01}$  & $   -0.18 {\\small \\pm   0.01}$ \\\\\n",
    "CNP (AR)             & $   -0.17 {\\small \\pm   0.01}$  & $   -0.18 {\\small \\pm   0.01}$  & $   -0.18 {\\small \\pm   0.01}$ \\\\\n",
    "ACNP                 & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$ \\\\\n",
    "ACNP (AR)            & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$ \\\\\n",
    "ConvCNP              & $\\hphantom{-}    2.40 {\\small \\pm   0.06}$  & $\\hphantom{-}    2.39 {\\small \\pm   0.05}$  & $\\hphantom{-}    0.01 {\\small \\pm   0.01}$ \\\\\n",
    "ConvCNP (AR)         & $\\hphantom{-}    3.62 {\\small \\pm   0.01}$  & $\\hphantom{-}    3.62 {\\small \\pm   0.01}$  & $\\hphantom{-}    3.36 {\\small \\pm   0.02}$ \\\\\n",
    "GNP                  & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$ \\\\\n",
    "AGNP                 & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$ \\\\\n",
    "ConvGNP              & $\\hphantom{-}    2.87 {\\small \\pm   0.05}$  & $\\hphantom{-}    2.87 {\\small \\pm   0.05}$  & $   -0.01 {\\small \\pm   0.01}$ \\\\\n",
    "FullConvGNP          & $\\hphantom{-}    2.88 {\\small \\pm   0.07}$  & $\\hphantom{-}    2.90 {\\small \\pm   0.07}$  & $\\hphantom{-}    0.30 {\\small \\pm   0.01}$ \\\\\n",
    "NP                   & $\\hphantom{-}    0.07 {\\small \\pm   0.00}$  & $   -0.05 {\\small \\pm   0.00}$  & $   -0.05 {\\small \\pm   0.00}$ \\\\\n",
    "ANP                  & $\\hphantom{-}    0.19 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$  & $   -0.18 {\\small \\pm   0.00}$ \\\\\n",
    "ConvNP               & $\\hphantom{-}    3.06 {\\small \\pm   0.04}$  & $\\hphantom{-}    3.06 {\\small \\pm   0.04}$  & $\\hphantom{-}    0.64 {\\small \\pm   0.01}$ \\\\\n",
    "NP (EL., ML.)        & $\\hphantom{-}    0.10 {\\small \\pm   0.01}$  &              F                &              F               \\\\\n",
    "NP (ELBO)            & $\\hphantom{-}    0.09 {\\small \\pm   0.01}$  &              F                &              F               \\\\\n",
    "ANP (EL., ML.)       & $\\hphantom{-}    0.50 {\\small \\pm   0.02}$  & $   -0.24 {\\small \\pm   0.00}$  & $   -6.28 {\\small \\pm   0.46}$ \\\\\n",
    "ANP (ELBO)           & $\\hphantom{-}    0.50 {\\small \\pm   0.02}$  & $   -6.28 {\\small \\pm   0.04}$  & $  -10.81 {\\small \\pm   0.48}$ \\\\\n",
    "ConvNP (EL., ML.)    &              F                &              F                &              F               \\\\\n",
    "ConvNP (ELBO)        & $\\hphantom{-}    3.51 {\\small \\pm   0.02}$  & $\\hphantom{-}    3.52 {\\small \\pm   0.02}$  & $\\hphantom{-}    2.68 {\\small \\pm   0.02}$ \\\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CNP                  & $   -0.30 {\\small \\pm   0.00}$  & $   -0.41 {\\small \\pm   0.00}$  & $   -0.35 {\\small \\pm   0.00}$ \\\\\n",
    "CNP (AR)             & $   -0.29 {\\small \\pm   0.00}$  & $   -1.19 {\\small \\pm   0.02}$  & $   -0.89 {\\small \\pm   0.02}$ \\\\\n",
    "ACNP                 & $   -0.08 {\\small \\pm   0.01}$  & $   -0.41 {\\small \\pm   0.01}$  & $   -0.42 {\\small \\pm   0.01}$ \\\\\n",
    "ACNP (AR)            & $\\hphantom{-}    1.01 {\\small \\pm   0.01}$  & $   -0.47 {\\small \\pm   0.01}$  & $   -0.45 {\\small \\pm   0.01}$ \\\\\n",
    "ConvCNP              & $\\hphantom{-}    0.87 {\\small \\pm   0.03}$  & $\\hphantom{-}    0.88 {\\small \\pm   0.02}$  & $   -0.28 {\\small \\pm   0.00}$ \\\\\n",
    "ConvCNP (AR)         & $\\hphantom{-}    2.23 {\\small \\pm   0.01}$  & $\\hphantom{-}    2.24 {\\small \\pm   0.01}$  & $\\hphantom{-}    1.94 {\\small \\pm   0.01}$ \\\\\n",
    "GNP                  & $   -0.03 {\\small \\pm   0.00}$  &              F                &              F               \\\\\n",
    "AGNP                 & $\\hphantom{-}    0.47 {\\small \\pm   0.01}$  & $   -0.38 {\\small \\pm   0.01}$  & $   -0.70 {\\small \\pm   0.08}$ \\\\\n",
    "ConvGNP              & Missing              & Missing              & Missing             \\\\\n",
    "FullConvGNP          & $\\hphantom{-}    1.06 {\\small \\pm   0.03}$  & $\\hphantom{-}    1.05 {\\small \\pm   0.04}$  & $\\hphantom{-}    0.11 {\\small \\pm   0.00}$ \\\\\n",
    "NP                   & $   -0.32 {\\small \\pm   0.00}$  & $   -0.32 {\\small \\pm   0.00}$  & $   -0.32 {\\small \\pm   0.00}$ \\\\\n",
    "ANP                  & $   -0.21 {\\small \\pm   0.00}$  & $   -0.33 {\\small \\pm   0.00}$  & $   -0.33 {\\small \\pm   0.00}$ \\\\\n",
    "ConvNP               & $\\hphantom{-}    1.88 {\\small \\pm   0.03}$  & $\\hphantom{-}    1.89 {\\small \\pm   0.03}$  & $   -0.11 {\\small \\pm   0.01}$ \\\\\n",
    "NP (EL., ML.)        & $   -0.08 {\\small \\pm   0.01}$  &              F                & $   -1.37 {\\small \\pm   0.00}$ \\\\\n",
    "NP (ELBO)            & $   -0.08 {\\small \\pm   0.01}$  &              F                & $   -1.37 {\\small \\pm   0.00}$ \\\\\n",
    "ANP (EL., ML.)       & $   -0.04 {\\small \\pm   0.01}$  & $   -1.09 {\\small \\pm   0.02}$  & $   -0.35 {\\small \\pm   0.00}$ \\\\\n",
    "ANP (ELBO)           & $   -0.05 {\\small \\pm   0.01}$  & $   -1.14 {\\small \\pm   0.03}$  & $   -0.35 {\\small \\pm   0.00}$ \\\\\n",
    "ConvNP (EL., ML.)    & $   -2.98 {\\small \\pm   0.10}$  & $   -2.98 {\\small \\pm   0.10}$  & $   -6.74 {\\small \\pm   0.04}$ \\\\\n",
    "ConvNP (ELBO)        & $\\hphantom{-}    2.01 {\\small \\pm   0.01}$  & $\\hphantom{-}    2.01 {\\small \\pm   0.01}$  & $\\hphantom{-}    1.47 {\\small \\pm   0.01}$ \\\\"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index 18 is out of bounds for axis 0 with size 18",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_11341/2184575270.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margsort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: index 18 is out of bounds for axis 0 with size 18"
     ]
    }
   ],
   "source": [
    "for r in np.array(rows)[np.argsort(-np.array(inters))]:\n",
    "    print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19, 19)"
      ]
     },
     "execution_count": 355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(rows), len(inters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNP                  &    -1.03 {\\small \\pm   0.02}  &    -1.10 {\\small \\pm   0.02}  &    -1.17 {\\small \\pm   0.02} \\\\\n",
      "ACNP                 &    -0.46 {\\small \\pm   0.02}  &    -1.24 {\\small \\pm   0.02}  &    -1.22 {\\small \\pm   0.01} \\\\\n",
      "ConvCNP              &     0.07 {\\small \\pm   0.04}  &     0.06 {\\small \\pm   0.04}  &    -1.21 {\\small \\pm   0.01} \\\\\n",
      "ConvCNP (AR)         &     0.56 {\\small \\pm   0.04}  &     0.56 {\\small \\pm   0.04}  &     0.34 {\\small \\pm   0.04} \\\\\n",
      "GNP                  &    -0.69 {\\small \\pm   0.01}  &    -2.58 {\\small \\pm   0.07}  &    -1.74 {\\small \\pm   0.04} \\\\\n",
      "AGNP                 &    -0.34 {\\small \\pm   0.02}  &    -0.94 {\\small \\pm   0.02}  &    -1.29 {\\small \\pm   0.03} \\\\\n",
      "ConvGNP              &     0.16 {\\small \\pm   0.06}  &     0.16 {\\small \\pm   0.06}  &    -0.91 {\\small \\pm   0.02} \\\\\n",
      "FullConvGNP          &     0.30 {\\small \\pm   0.03}  &     0.30 {\\small \\pm   0.03}  &    -0.46 {\\small \\pm   0.01} \\\\\n",
      "NP                   &    -0.63 {\\small \\pm   0.01}  &    -1.43 {\\small \\pm   0.01}  &    -1.25 {\\small \\pm   0.02} \\\\\n",
      "ANP                  &    -0.25 {\\small \\pm   0.02}  &    -1.04 {\\small \\pm   0.02}  &    -1.04 {\\small \\pm   0.02} \\\\\n",
      "ConvNP               &     0.38 {\\small \\pm   0.04}  &     0.39 {\\small \\pm   0.04}  &    -0.85 {\\small \\pm   0.02} \\\\\n",
      "NP (ELBO)            &    -0.59 {\\small \\pm   0.01}  &    -2.72 {\\small \\pm   0.03}  &    -1.88 {\\small \\pm   0.01} \\\\\n",
      "ANP (ELBO)           &    -0.40 {\\small \\pm   0.02}  &    -2.81 {\\small \\pm   0.07}  &    -3.27 {\\small \\pm   0.04} \\\\\n",
      "ConvNP (ELBO)        &     0.45 {\\small \\pm   0.04}  &     0.45 {\\small \\pm   0.04}  &    -0.10 {\\small \\pm   0.04} \\\\\n"
     ]
    }
   ],
   "source": [
    "rows = []\n",
    "inters = []\n",
    "datasets = [\"mixture\"]\n",
    "\n",
    "for (model, string, objective), dataset in product(models, datasets):\n",
    "    \n",
    "    root = f\"_experiments/{dataset}/x1_y1/{model}\"\n",
    "    \n",
    "    if \"conv\" in model:\n",
    "        root = root + \"/unet\"\n",
    "        \n",
    "    root = root + f\"/{objective}/log_evaluate_out.txt\"\n",
    "    \n",
    "    _file = open(root, \"r\").read()\n",
    "\n",
    "\n",
    "    try:\n",
    "        if objective == \"loglik\":\n",
    "\n",
    "            file = _file.split(\"| Loglik:\")[1].strip().split(\"\\n\")[:12]\n",
    "            file = file[:2] + file[4:6] + file[8:10]\n",
    "\n",
    "            process = lambda x: float(x.strip())\n",
    "\n",
    "            interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "            row = format_line(string, interpolation, extrapolation, out_of_range)\n",
    "            print(row)\n",
    "\n",
    "            rows.append(row)\n",
    "            inters.append(interpolation[0])\n",
    "\n",
    "            if \"cnp\" in model and \"| AR:\" in _file:\n",
    "\n",
    "                file = _file.split(\"| AR:\")[1].strip().split(\"\\n\")[:12]\n",
    "                file = file[:2] + file[4:6] + file[8:10]\n",
    "\n",
    "                process = lambda x: float(x.strip())\n",
    "\n",
    "                interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "                extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "                out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "                row = format_line(string + \" (AR)\", interpolation, extrapolation, out_of_range)\n",
    "                print(row)\n",
    "\n",
    "                rows.append(row)\n",
    "                inters.append(interpolation[0])\n",
    "\n",
    "        else:\n",
    "\n",
    "            file = _file.split(\"| Loglik:\")[1].strip().split(\"\\n\")[:12]\n",
    "            file = file[:2] + file[4:6] + file[8:10]\n",
    "\n",
    "            process = lambda x: float(x.strip())\n",
    "\n",
    "            interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "            row = format_line(string + ' (ELBO, ML)', interpolation, extrapolation, out_of_range)\n",
    "\n",
    "            file = _file.split(\"| ELBO:\")[1].strip().split(\"\\n\")[:12]\n",
    "            file = file[:2] + file[4:6] + file[8:10]\n",
    "\n",
    "            process = lambda x: float(x.strip())\n",
    "\n",
    "            interpolation = list(map(process, file[1].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            extrapolation = list(map(process, file[3].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "            out_of_range = list(map(process, file[5].split(\"|\")[1].strip().split(\"Loglik (V):\")[1].split(\"(\")[0].split(\"+-\")))\n",
    "\n",
    "            row = format_line(string + ' (ELBO)', interpolation, extrapolation, out_of_range)\n",
    "            print(row)\n",
    "\n",
    "            rows.append(row)\n",
    "            inters.append(interpolation[0])\n",
    "\n",
    "    except:\n",
    "        row = format_line(string, None, None, None)\n",
    "        print(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ConvCNP (AR)         &     0.56 {\\small \\pm   0.04}  &     0.56 {\\small \\pm   0.04}  &     0.34 {\\small \\pm   0.04} \\\\\n",
      "ConvNP (ELBO)        &     0.45 {\\small \\pm   0.04}  &     0.45 {\\small \\pm   0.04}  &    -0.10 {\\small \\pm   0.04} \\\\\n",
      "ConvNP               &     0.38 {\\small \\pm   0.04}  &     0.39 {\\small \\pm   0.04}  &    -0.85 {\\small \\pm   0.02} \\\\\n",
      "FullConvGNP          &     0.30 {\\small \\pm   0.03}  &     0.30 {\\small \\pm   0.03}  &    -0.46 {\\small \\pm   0.01} \\\\\n",
      "ConvGNP              &     0.16 {\\small \\pm   0.06}  &     0.16 {\\small \\pm   0.06}  &    -0.91 {\\small \\pm   0.02} \\\\\n",
      "ConvCNP              &     0.07 {\\small \\pm   0.04}  &     0.06 {\\small \\pm   0.04}  &    -1.21 {\\small \\pm   0.01} \\\\\n",
      "ANP                  &    -0.25 {\\small \\pm   0.02}  &    -1.04 {\\small \\pm   0.02}  &    -1.04 {\\small \\pm   0.02} \\\\\n",
      "AGNP                 &    -0.34 {\\small \\pm   0.02}  &    -0.94 {\\small \\pm   0.02}  &    -1.29 {\\small \\pm   0.03} \\\\\n",
      "ANP (ELBO)           &    -0.40 {\\small \\pm   0.02}  &    -2.81 {\\small \\pm   0.07}  &    -3.27 {\\small \\pm   0.04} \\\\\n",
      "ACNP                 &    -0.46 {\\small \\pm   0.02}  &    -1.24 {\\small \\pm   0.02}  &    -1.22 {\\small \\pm   0.01} \\\\\n",
      "NP (ELBO)            &    -0.59 {\\small \\pm   0.01}  &    -2.72 {\\small \\pm   0.03}  &    -1.88 {\\small \\pm   0.01} \\\\\n",
      "NP                   &    -0.63 {\\small \\pm   0.01}  &    -1.43 {\\small \\pm   0.01}  &    -1.25 {\\small \\pm   0.02} \\\\\n",
      "GNP                  &    -0.69 {\\small \\pm   0.01}  &    -2.58 {\\small \\pm   0.07}  &    -1.74 {\\small \\pm   0.04} \\\\\n",
      "CNP                  &    -1.03 {\\small \\pm   0.02}  &    -1.10 {\\small \\pm   0.02}  &    -1.17 {\\small \\pm   0.02} \\\\\n"
     ]
    }
   ],
   "source": [
    "for r in np.array(rows)[np.argsort(-np.array(inters))]:\n",
    "    print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv-np",
   "language": "python",
   "name": "venv-np"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}