CP_NA_Synthetic_Plots.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import generation as gen\n",
    "import prediction as prediction\n",
    "import utils\n",
    "import files\n",
    "import os\n",
    "import numpy as np\n",
    "from tqdm.autonotebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "from mpl_toolkits.axes_grid1.inset_locator import zoomed_inset_axes, mark_inset\n",
    "from matplotlib.backends.backend_pgf import FigureCanvasPgf\n",
    "mpl.backend_bases.register_backend('pdf', FigureCanvasPgf)\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.lines as mlines\n",
    "import pandas as pd\n",
    "from matplotlib import cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "size=19\n",
    "mpl.rcParams.update({\n",
    "    \"pgf.texsystem\": \"pdflatex\",\n",
    "    'font.family': 'serif',\n",
    "    'font.serif': 'Times',\n",
    "    'text.usetex': True,\n",
    "    'pgf.rcfonts': False,\n",
    "    'font.size': size,\n",
    "    'axes.labelsize':size,\n",
    "    'axes.titlesize':size,\n",
    "    'figure.titlesize':size,\n",
    "    'xtick.labelsize':size,\n",
    "    'ytick.labelsize':size,\n",
    "    'legend.fontsize':size,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.1\n",
    "d = 10\n",
    "phi = 0.8\n",
    "regression = 'Linear'\n",
    "n_rep = 100\n",
    "beta = np.array([1, 2, -1, 3, -0.5, -1, 0.3, 1.7, 0.4, -0.3])\n",
    "\n",
    "train_size = 500\n",
    "cal_size = 250\n",
    "params_test = {'iid':{'test_size': 2000}, \n",
    "               'fixed_nb_sample_pattern':{'nb_sample_pattern': 100}, \n",
    "               'fixed_nb_sample_pattern_size':{'nb_sample_pattern': 100}}\n",
    "params_test = gen.process_test(params_test, d=d)\n",
    "\n",
    "params_reg = {'regression':regression, 'beta': beta, 'phi': phi}\n",
    "\n",
    "params_noise = {'noise':'Gaussian'}\n",
    "\n",
    "prob_missing = 0.2\n",
    "var_missing = np.full(d, 1)\n",
    "params_missing = {'prob_missing':prob_missing, 'var_missing':var_missing, 'mechanism': 'MCAR'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "methods = ['QR_TrainCal','CQR','CQR_MDA']\n",
    "\n",
    "basemodel = 'NNet'\n",
    "mask = 'Yes'\n",
    "protection = 'No'\n",
    "imputation = 'iterative_ridge'\n",
    "\n",
    "name_pipeline_to_plot = []\n",
    "for method in methods: \n",
    "    if method == 'CQR_MDA':\n",
    "        name_temp = files.get_name_method(method, basemodel, mask=mask, protection=protection, exact=True)\n",
    "        if not name_temp in name_pipeline_to_plot:\n",
    "            name_pipeline_to_plot.append(name_temp)\n",
    "            \n",
    "    name_temp = files.get_name_method(method, basemodel, mask=mask, protection=protection, exact=False)\n",
    "    if not name_temp in name_pipeline_to_plot:\n",
    "        name_pipeline_to_plot.append(name_temp)\n",
    "    \n",
    "current_pipeline = method+'_'+basemodel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_cov = dict.fromkeys(name_pipeline_to_plot)\n",
    "dict_len = dict.fromkeys(name_pipeline_to_plot)\n",
    "\n",
    "for pipeline in name_pipeline_to_plot:\n",
    "    dict_cov[pipeline] = {}\n",
    "    dict_len[pipeline] = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys_pattern = np.arange(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "impute_inf = True\n",
    "replace_inf = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "key = -1\n",
    "\n",
    "nb_boxplot = len(keys_pattern)+1\n",
    "\n",
    "name_method = []\n",
    "\n",
    "for pipeline in tqdm(name_pipeline_to_plot):\n",
    "\n",
    "    name_method = np.append(name_method, '_'.join([imputation, pipeline]))\n",
    "\n",
    "    data, results = utils.get_data_results(pipeline, train_size, cal_size, params_test, n_rep, d=d, imputation=imputation,\n",
    "                                           params_reg=params_reg, params_noise=params_noise, params_missing=params_missing,\n",
    "                                           parent_results='results', parent_data='data', extension='xz')\n",
    "\n",
    "    contains, lengths = utils.compute_PI_metrics(data, results, 'iid')\n",
    "    \n",
    "    if replace_inf:\n",
    "        max_y_train = np.max(data['Y']['Train'], axis=1)\n",
    "        max_y_cal = np.max(data['Y']['Cal'], axis=1)\n",
    "        min_y_train = np.min(data['Y']['Train'], axis=1)\n",
    "        min_y_cal = np.min(data['Y']['Cal'], axis=1)\n",
    "        max_length_traincal = np.maximum(max_y_train, max_y_cal)-np.minimum(min_y_train, min_y_cal)\n",
    "        for k in range(n_rep):\n",
    "            idx_inf = np.where(np.isinf(lengths[k,:]))[0]\n",
    "            if len(idx_inf)>0:\n",
    "                lengths[k,:][idx_inf] = max_length_traincal[k]\n",
    "    \n",
    "    metrics = utils.compute_metrics_cond(n_rep, data, results, 'fixed_nb_sample_pattern_size', cond='Pattern_Size',\n",
    "                                         replace_inf=replace_inf)\n",
    "    \n",
    "    dict_cov[pipeline][key] = np.mean(contains, axis=1)\n",
    "    dict_len[pipeline][key] = np.mean(lengths, axis=1)\n",
    "\n",
    "    #key += 1\n",
    "\n",
    "    for key_pattern in keys_pattern:\n",
    "\n",
    "        dict_cov[pipeline][key_pattern] = metrics[key_pattern]['avg_cov']\n",
    "        dict_len[pipeline][key_pattern] = metrics[key_pattern]['avg_len']\n",
    "\n",
    "        #key += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'phi' in params_reg:\n",
    "    phi = params_reg['phi']\n",
    "else:\n",
    "    phi = 0.8\n",
    "cov = np.full((d,d),phi)+(1-phi)*np.eye(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len_oracle_marginal = []\n",
    "M_test = data['M']['Test']['iid']\n",
    "test_size = M_test.shape[1]\n",
    "for i in range(n_rep):\n",
    "    \n",
    "    M_test_i = M_test[i,:,:]\n",
    "    patterns = np.unique(M_test_i, axis=0)\n",
    "    oracles_len_per_pattern = list(map(functools.partial(prediction.oracle_len_pattern, beta=beta, cov=cov, alpha=0.1), patterns))\n",
    "\n",
    "    len_oracle = np.empty(test_size)\n",
    "    \n",
    "    for idp, pattern in enumerate(patterns):\n",
    "        pattern_id = utils.pattern_to_id(pattern.astype(int))\n",
    "        M_test_id = list(map(utils.pattern_to_id, M_test_i.astype(int)))\n",
    "        len_oracle[np.where(np.array(M_test_id) == pattern_id)] = oracles_len_per_pattern[idp]\n",
    "    len_oracle_marginal = np.append(len_oracle_marginal, np.mean(len_oracle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len_oracle = {}\n",
    "patterns_by_size = dict.fromkeys(np.arange(0,d))\n",
    "for k in range(d):\n",
    "    patterns_by_size[k] = []\n",
    "patterns_id = np.arange(0, 2**d-1)\n",
    "for pattern_id in patterns_id:\n",
    "    vec_pattern = utils.bin_to_vec(bin(pattern_id), d)\n",
    "    size_pattern = utils.pattern_to_size(vec_pattern)\n",
    "    patterns_by_size[size_pattern] = np.append(patterns_by_size[size_pattern], pattern_id)\n",
    "for k in range(d):\n",
    "    list_len = []\n",
    "    for pattern_id in patterns_by_size[k]:\n",
    "        vec_pattern = utils.bin_to_vec(bin(np.int(pattern_id)), d)\n",
    "        list_len = np.append(list_len, prediction.oracle_len_pattern(vec_pattern, beta, cov))\n",
    "    len_oracle[k] = np.mean(list_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_methods = {'QR_TrainCal_NNet_Mask': 'QR',\n",
    "                'QR_NNet_Mask': 'QR',\n",
    "                'CQR_NNet_Mask': 'CQR', \n",
    "                'CQR_MDA_Exact_NNet_Mask': 'CQR-MDA-Exact',\n",
    "                'CQR_MDA_Nested_NNet_Mask': 'CQR-MDA-Nested'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ((ax1, ax2, ax3, ax4), (ax5, ax6, ax7, ax8)) = plt.subplots(2, 4, figsize=(15,6), sharey='row')\n",
    "\n",
    "name_ticks = list(map(utils.name_tick, name_method))\n",
    "colors_palette = sns.color_palette(\"husl\", nb_boxplot)\n",
    "colors = colors_palette \n",
    "\n",
    "ax = [ax1, ax2, ax3, ax4]\n",
    "nb_subplots = len(ax)\n",
    "for axi in ax:\n",
    "    axi.axhline(1-alpha, color='black', ls='--')\n",
    "\n",
    "axl = [ax5, ax6, ax7, ax8]\n",
    "    \n",
    "for idp, pipeline in enumerate(name_pipeline_to_plot):\n",
    "    \n",
    "    ax[idp].set_title(dict_methods[pipeline])\n",
    "    \n",
    "    box = ax[idp].violinplot(dict_cov[pipeline].values(), showmeans=True, showextrema=False)#, quantiles=[[0.25, 0.75]]*nb_boxes)#, patch_artist=True)\n",
    "    for pc,color in zip(box['bodies'], colors):\n",
    "        pc.set_facecolor(color)\n",
    "        pc.set_edgecolor('black')\n",
    "        pc.set_alpha(1)\n",
    "    box['cmeans'].set_color('black')\n",
    "    \n",
    "    box = axl[idp].violinplot(dict_len[pipeline].values(), showmeans=True, showextrema=False)#, quantiles=[[0.25, 0.75]]*nb_boxes)#, patch_artist=True)\n",
    "    for pc,color in zip(box['bodies'], colors):\n",
    "        pc.set_facecolor(color)\n",
    "        pc.set_edgecolor('black')\n",
    "        pc.set_alpha(1)\n",
    "    box['cmeans'].set_color('black')\n",
    "    \n",
    "idx = np.arange(d+1)\n",
    "idy = np.append([np.mean(len_oracle_marginal)], np.array(list(len_oracle.values())))\n",
    "\n",
    "for axi in axl:\n",
    "    axi.scatter(idx+1, idy, color=colors, zorder=2, marker='*', s=100, edgecolor='black')\n",
    "\n",
    "for axi in ax:\n",
    "    axi.set_xticks([])\n",
    "    \n",
    "name_ticks_missing = []\n",
    "for k in range(d):\n",
    "    name_ticks_missing = np.append(name_ticks_missing, str(k)+r' \\texttt{NA}')\n",
    "name_ticks = np.append(['Marg.'], name_ticks_missing)\n",
    "\n",
    "for axi in axl:\n",
    "    ticks = np.arange(0,d+1)\n",
    "    axi.set_xticks(ticks+1)\n",
    "    axi.set_xticklabels(name_ticks, rotation=70)\n",
    "\n",
    "ax1.set_ylabel('Average coverage')\n",
    "ax5.set_ylabel('Average length')\n",
    "\n",
    "ax5.legend(handles = [mlines.Line2D([], [], marker=\"*\", linestyle='None', markersize=15, markeredgecolor='black', markerfacecolor='White')],\n",
    "           labels=['Oracle length'], loc='upper left', handletextpad=10**(-60))\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "name_plot = 'plots/synthetic/Linear_d_'+str(d)+'_NA_'+str(prob_missing)+'_imputation_'+str(imputation)+'_basemodel_'+basemodel\n",
    "if mask == 'Yes':\n",
    "    name_plot = name_plot + '_mask' \n",
    "name_plot = name_plot + '_train_'+str(train_size) + '_cal_'+str(cal_size) +'_rep_'+str(n_rep)\n",
    "if mask == 'No':\n",
    "    name_plot = name_plot+'_nomask'\n",
    "if impute_inf:\n",
    "    name_plot = name_plot+'_replaceinf'\n",
    "plt.savefig(name_plot+'.pdf',bbox_inches='tight', dpi=300)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}