BinaryVAE_text-BASELINES.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package reuters to /home/fmena/nltk_data...\n",
      "[nltk_data]   Package reuters is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/fmena/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import keras,gc,nltk\n",
    "from keras.layers import *\n",
    "from keras.models import Sequential,Model\n",
    "from keras import backend as K\n",
    "from astropy.table import Table\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.special import expit\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "from scipy.io import loadmat\n",
    "nltk.download('reuters')\n",
    "nltk.download('wordnet')\n",
    "%matplotlib inline\n",
    "\n",
    "from base_networks import *\n",
    "\n",
    "np.random.seed(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data\n",
    "\n",
    "### 20 Newsgroup\n",
    "---\n",
    "\n",
    "Utilizado en trabajos previos de *Hashing* de texto (http://people.csail.mit.edu/jrennie/20Newsgroups), también disponible en **sklearn**. El dataset contiene textos de usuarios asociados a temáticas de noticias etiquetados como pertenenciente a uno de 20 grupos de noticias, el detalle de los conjuntos se detalla a continuación:\n",
    "\n",
    "|Tipo set|Datos|Label|\n",
    "|---|---|---|\n",
    "|Entrenamiento|11.314|20|\n",
    "|Pruebas|7.532|20|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Datos de entrenamiento:  (11314,)\n",
      "Datos de prueba:  (7532,)\n"
     ]
    }
   ],
   "source": [
    "dat_n = \"20News\"\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "newsgroups_t = fetch_20newsgroups(subset='train')\n",
    "newsgroups_test = fetch_20newsgroups(subset='test')\n",
    "labels = newsgroups_t.target_names\n",
    "\n",
    "texts_t = newsgroups_t.data\n",
    "y_t = newsgroups_t.target\n",
    "labels_t = [labels[valor] for valor in y_t]\n",
    "\n",
    "texts_test = newsgroups_test.data\n",
    "y_test = newsgroups_test.target\n",
    "labels_test = [labels[valor] for valor in y_test]\n",
    "\n",
    "print(\"Datos de entrenamiento: \",y_t.shape)\n",
    "print(\"Datos de prueba: \",y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Es el dataset con los documentos más extensos en información o textos, lo que podría ser un indicador que es el más difícil ya que se deberá comprimir toda la información presente en estos textos.\n",
    "\n",
    "### Reuters21578\n",
    "---\n",
    "Similar a 20NewsGroup es un datataset de textos de noticias del periodico de Reuters en 1987, citado también en el estado del arte (https://www.nltk.org/book/ch02.html) y disponible en la librería **nltk**. El detalle de los conjuntos se muestra a continuación:\n",
    "\n",
    "|Tipo set|Datos|Label|\n",
    "|---|---|---|\n",
    "|Entrenamiento|7.769|90|\n",
    "|Pruebas|3.019|90|\n",
    "\n",
    "Los documentos pueden pertenecer a **múltiples tópicos** dentro de 90 disponibles en el dataset manualmente etiquetadas, ésto es porque un texto asociado a una noticia puede hablar de varios tópicos a la vez. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10788 documents\n",
      "7769 total training documents\n",
      "3019 total test documents\n"
     ]
    }
   ],
   "source": [
    "dat_n = \"Reuters\"\n",
    "\n",
    "from nltk.corpus import reuters\n",
    "documents_stat = reuters.fileids()\n",
    "print(str(len(documents_stat)) + \" documents\")\n",
    "\n",
    "train_docs_stat = list(filter(lambda doc: doc.startswith(\"train\"), documents_stat))\n",
    "print(str(len(train_docs_stat)) + \" total training documents\")\n",
    "test_docs_stat = list(filter(lambda doc: doc.startswith(\"test\"), documents_stat))\n",
    "print(str(len(test_docs_stat)) + \" total test documents\")\n",
    "\n",
    "texts_t = [reuters.raw(archivo) for archivo in train_docs_stat]\n",
    "labels_t = [reuters.categories(archivo) for archivo in train_docs_stat]\n",
    "\n",
    "texts_test = [reuters.raw(archivo) for archivo in test_docs_stat]\n",
    "labels_test = [reuters.categories(archivo) for archivo in test_docs_stat]\n",
    "\n",
    "labels = reuters.categories()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### top 20 labels (as VDSH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "category with most data (earn) has = 3964, the top-K category (nat-gas) has = 105\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['earn',\n",
       " 'acq',\n",
       " 'money-fx',\n",
       " 'grain',\n",
       " 'crude',\n",
       " 'trade',\n",
       " 'interest',\n",
       " 'ship',\n",
       " 'wheat',\n",
       " 'corn',\n",
       " 'dlr',\n",
       " 'money-supply',\n",
       " 'oilseed',\n",
       " 'sugar',\n",
       " 'coffee',\n",
       " 'gnp',\n",
       " 'veg-oil',\n",
       " 'gold',\n",
       " 'soybean',\n",
       " 'nat-gas']"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import get_topK_labels,set_newlabel_list,enmask_data\n",
    "new_labels = get_topK_labels(labels_t+labels_test, labels, K=20)\n",
    "labels = new_labels\n",
    "new_labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reuters Corpus Volume 1\n",
    "---\n",
    "Corpus de Reuters extendido, con los 103 tópicos originales. Disponible en trabajos previos (https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multilabel.html) y a través de **sklearn** (https://scikit-learn.org/0.17/datasets/rcv1.html). La representación son las *features* ya extraídas.\n",
    "\n",
    "|Tipo set|Datos|Label|\n",
    "|---|---|---|\n",
    "|Entrenamiento|23.149|103|\n",
    "|Pruebas|781.265|103|\n",
    "\n",
    "Representación de **sklearn** viene pre-procesada con la transformación logarítmica de TF-IDF."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-e85c480aa03f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mfetch_rcv1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mrcv1_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfetch_rcv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mrcv1_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfetch_rcv1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mX_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrcv1_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/sklearn/datasets/_rcv1.py\u001b[0m in \u001b[0;36mfetch_rcv1\u001b[0;34m(data_home, subset, download_if_missing, random_state, shuffle, return_X_y)\u001b[0m\n\u001b[1;32m    191\u001b[0m             \u001b[0mremove\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    192\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m         \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_id\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_refresh_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0msamples_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_id_path\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m9\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    194\u001b[0m         \u001b[0;31m# TODO: Revert to the following two lines in v0.23\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m         \u001b[0;31m# X = joblib.load(samples_path)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/sklearn/datasets/_base.py\u001b[0m in \u001b[0;36m_refresh_cache\u001b[0;34m(files, compress)\u001b[0m\n\u001b[1;32m    926\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    927\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 928\u001b[0;31m                 \u001b[0mjoblib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcompress\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcompress\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    929\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mIOError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    930\u001b[0m             message = (\"This dataset will stop being loadable in scikit-learn \"\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(value, filename, compress, protocol, cache_size)\u001b[0m\n\u001b[1;32m    500\u001b[0m         with _write_fileobject(filename, compress=(compress_method,\n\u001b[1;32m    501\u001b[0m                                                    compress_level)) as f:\n\u001b[0;32m--> 502\u001b[0;31m             \u001b[0mNumpyPickler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    503\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mis_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    504\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36mdump\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    406\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart_framing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 408\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    409\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSTOP\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    410\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend_framing\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    293\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    518\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m         \u001b[0;31m# Save the reduce() output and finally memoize the object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 520\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_reduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mrv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    521\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    522\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpersistent_id\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_reduce\u001b[0;34m(self, func, args, state, listitems, dictitems, obj)\u001b[0m\n\u001b[1;32m    625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    626\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 627\u001b[0;31m             \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    628\u001b[0m             \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBUILD\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    629\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    293\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj, save_persistent_id)\u001b[0m\n\u001b[1;32m    473\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    474\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mf\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 475\u001b[0;31m             \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Call unbound method with explicit self\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    476\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36msave_dict\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    813\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmemoize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 814\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_setitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    815\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    816\u001b[0m     \u001b[0mdispatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msave_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/pickle.py\u001b[0m in \u001b[0;36m_batch_setitems\u001b[0;34m(self, items)\u001b[0m\n\u001b[1;32m    838\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtmp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    839\u001b[0m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 840\u001b[0;31m                     \u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    841\u001b[0m                 \u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSETITEMS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36msave\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    290\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    291\u001b[0m             \u001b[0;31m# And then array bytes are written right after the wrapper.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 292\u001b[0;31m             \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    293\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/joblib/numpy_pickle.py\u001b[0m in \u001b[0;36mwrite_array\u001b[0;34m(self, array, pickler)\u001b[0m\n\u001b[1;32m    102\u001b[0m                                            \u001b[0mbuffersize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbuffersize\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    103\u001b[0m                                            order=self.order):\n\u001b[0;32m--> 104\u001b[0;31m                 \u001b[0mpickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfile_handle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtostring\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'C'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munpickler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/.local/lib/python3.5/site-packages/joblib/compressor.py\u001b[0m in \u001b[0;36mwrite\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    506\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtobytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    507\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 508\u001b[0;31m             \u001b[0mcompressed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compressor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompress\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    509\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcompressed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    510\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pos\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "dat_n = \"RCV1\"\n",
    "\n",
    "from sklearn.datasets import fetch_rcv1\n",
    "rcv1_train = fetch_rcv1(subset='train')\n",
    "rcv1_test = fetch_rcv1(subset='test')\n",
    "X_t = rcv1_train.data\n",
    "y_t = rcv1_train.target\n",
    "X_test = rcv1_test.data\n",
    "y_test = rcv1_test.target\n",
    "labels = rcv1_train.target_names\n",
    "\n",
    "del rcv1_train,rcv1_test\n",
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_t = np.concatenate((X_t,X_test),axis=0)\n",
    "labels_t = np.concatenate((X_t,X_test),axis=0) #labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "labels_t = np.asarray(labels_t)\n",
    "X_t,X_test,labels_t,labels_test  = train_test_split(X_t,labels_t,random_state=20,test_size=0.1)\n",
    "\n",
    "X_train,X_val,labels_train,labels_val  = train_test_split(X_t,labels_t,random_state=20,test_size=0.1)\n",
    "\n",
    "del X_t, labels_t\n",
    "gc.collect()\n",
    "\n",
    "print(\"Vocabulario size = \",X_train.shape[1])\n",
    "print(\"Cantidad de datos Entrenamiento: \",len(labels_train))\n",
    "print(\"Cantidad de datos Validación: \",len(labels_val))\n",
    "print(\"Cantidad de datos Pruebas: \",len(labels_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TMC\n",
    "---\n",
    "textos relacionados con reportes aeros de tráfico provistos por la NASA y usados en la competencia de text mining SIAM. Cada texto tiene 22 etiquetas no exlucenyetes.\n",
    "\n",
    "\n",
    "|Tipo set|Datos|Label|\n",
    "|---|---|---|\n",
    "|Entrenamiento|21,519|22|\n",
    "|Validación|3,498|22|\n",
    "|Pruebas|3,498|22|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dat_n = \"TMC\"\n",
    "\n",
    "labels = ['a','b','c','d','e','f','e','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "El dataset anterior también es extenso en sus documentos pero no tan extremo como 20 Newsgroup.\n",
    "\n",
    "### SearchSnipet\n",
    "---\n",
    "Dataset de Google search *snippets*-- pequeñas porciones de texto que le dan a usuarios una idea de lo que hay en el sitio web en el buscador de Google. Pertenecientes a 8 clases únicas (dominio). Disponibles a través de http://jwebpro.sourceforge.net/data-web-snippets.tar.gz.\n",
    "\n",
    "\n",
    "|Tipo set|Datos|Label|\n",
    "|---|---|---|\n",
    "|Entrenamiento|10.060|8|\n",
    "|Pruebas|2.280|8|"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Datos de entrenamiento:  10060\n",
      "Datos de pruebas:  2280\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['computers',\n",
       " 'education-science',\n",
       " 'politics-society',\n",
       " 'culture-arts-entertainment',\n",
       " 'business',\n",
       " 'health',\n",
       " 'sports',\n",
       " 'engineering']"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dat_n = \"Snippets\"\n",
    "\n",
    "def read_file(archivo,symb=' '):\n",
    "    with open(archivo,'r') as f:\n",
    "        lineas = f.readlines()\n",
    "        tokens_f = [linea.strip().split(symb) for linea in lineas]\n",
    "        labels = [tokens[-1] for tokens in tokens_f]\n",
    "        tokens = [' '.join(tokens[:-1]) for tokens in tokens_f]\n",
    "    return labels,tokens\n",
    "labels_t,texts_t = read_file(\"Data/data-web-snippets/train.txt\")\n",
    "labels_test,texts_test = read_file(\"Data/data-web-snippets/test.txt\")\n",
    "print(\"Datos de entrenamiento: \",len(texts_t))\n",
    "print(\"Datos de pruebas: \",len(texts_test))\n",
    "\n",
    "labels = list(set(labels_t))\n",
    "labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cantidad de datos Entrenamiento:  9054\n",
      "Cantidad de datos Validación:  1006\n",
      "Cantidad de datos Pruebas:  2280\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "labels_t = np.asarray(labels_t)\n",
    "labels_test = np.asarray(labels_test)\n",
    "texts_train,texts_val,labels_train,labels_val  = train_test_split(texts_t,labels_t,random_state=20,test_size=0.1)\n",
    "\n",
    "print(\"Cantidad de datos Entrenamiento: \",len(texts_train))\n",
    "print(\"Cantidad de datos Validación: \",len(texts_val))\n",
    "print(\"Cantidad de datos Pruebas: \",len(texts_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-process\n",
    "---\n",
    "Para obtener la representación de los datos, en primer lugar se normalizan con experimentar dos tipos de pre-procesamiento:\n",
    "\n",
    "1. El primero es el siguiente (Basado en Deep Semantic Hashing)\n",
    "    * Pasar letras a minúsculas\n",
    "    * Eliminar extra espacios (saltos de línea por ejemplo)\n",
    "    * Remover stop words\n",
    "    * Borrar todo lo que no sea letras (eliminar números y puntuaciones)\n",
    "    * Conservar las top $k$ palabras/*tokens* más frecuentes\n",
    "2. El segundo añade lo siguiente (Basado en Semantic Hashing)\n",
    "    * Se realiza un *stemming* (Snowball) -- Lemmatization\n",
    "    * Remover palabras de menos de 3 largo\n",
    "\n",
    "\n",
    "Para la representación vectorial se utiliza lo siguiente:\n",
    "* TF-IDF: \n",
    "$$ w_f(d) \\cdot \\left(1 + log\\left( \\frac{1+n_d}{1+df_w} \\right) \\right)$$\n",
    "* **TF (*term frequency*)**: $$ w_f(d) $$\n",
    "* Binary: $$I(w_f(d) \\neq 0)$$\n",
    "\n",
    "La representación *term frequency* es la que se toma como base, puesto que resulta natural al momento de reconstruir el dato y poder utilizar la función de pérdida *cross entropy*. Ya que para la representación *tf-idf* requeriría una función de pérdida adecuada para variable continua, que podría ser *mse*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fmena/.local/lib/python3.5/site-packages/sklearn/feature_extraction/text.py:507: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'\n",
      "  warnings.warn(\"The parameter 'token_pattern' will not be used\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.56 s, sys: 2.7 ms, total: 1.57 s\n",
      "Wall time: 1.56 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(9054, 10000)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "#analyzer = TfidfVectorizer(ngram_range=(1, 3)).build_analyzer()\n",
    "tokenizer = TfidfVectorizer().build_tokenizer()\n",
    "stemmer = SnowballStemmer(\"english\") \n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "\"\"\"Extract features from raw input\"\"\"\n",
    "def preProcess(s): #String processor\n",
    "    return s.lower().strip().strip('-').strip('_')\n",
    "def number_normalize(doc):\n",
    "    results = []\n",
    "    for token in tokenizer(doc):\n",
    "        token_pro = preProcess(token)\n",
    "        if len(token_pro) != 0 and not token_pro[0].isdigit():\n",
    "            results.append(token_pro)\n",
    "    return results\n",
    "def stemmed_words(doc):\n",
    "    results = []\n",
    "    for token in tokenizer(doc):\n",
    "        pre_pro = preProcess(token)\n",
    "        #token_pro = stemmer.stem(pre_pro) #aumenta x10 el tiempo de procesamiento\n",
    "        token_pro = lemmatizer.lemmatize(pre_pro) #so can explain/interpretae -- aumenta x5 el tiempo de proce\n",
    "        if len(token_pro) > 2 and not token_pro[0].isdigit(): #elimina palabra largo menor a 2\n",
    "            results.append(token_pro)\n",
    "    return results\n",
    "\n",
    "def get_transform_representation(mode, analizer,min_count,max_feat):\n",
    "    smooth_idf_b = False\n",
    "    use_idf_b = False\n",
    "    binary_b = False\n",
    "\n",
    "    if mode == 'binary':\n",
    "        binary_b = True\n",
    "    elif mode == 'tf':     \n",
    "        pass #default is tf\n",
    "    elif mode == 'tf-idf':\n",
    "        use_idf_b = True\n",
    "        smooth_idf_b = True #inventa 1 conteo imaginario (como priors)--laplace smoothing\n",
    "    return TfidfVectorizer(stop_words='english',tokenizer=analizer,min_df=min_count, max_df=0.8, max_features=max_feat\n",
    "                                ,binary=binary_b, use_idf=use_idf_b, smooth_idf=smooth_idf_b,norm=None\n",
    "                                  ,ngram_range=(1, 3)) \n",
    "\n",
    "min_count = 1 #default = 1\n",
    "max_feat = 10000 #Best: 10000 -- Hinton (2000)\n",
    "\n",
    "\n",
    "vectorizer = get_transform_representation(\"tf\", stemmed_words,min_count,max_feat)\n",
    "\n",
    "%time vectorizer.fit(texts_train)\n",
    "vectors_train = vectorizer.transform(texts_train)\n",
    "vectors_val = vectorizer.transform(texts_val)\n",
    "vectors_test = vectorizer.transform(texts_test)\n",
    "\n",
    "token2idx = vectorizer.vocabulary_\n",
    "idx2token = {idx:token for token,idx in token2idx.items()}\n",
    "\n",
    "#vectorizer2 = get_transform_representation(\"tf-idf\", stemmed_words,min_count,max_feat)\n",
    "\n",
    "#%time vectorizer2.fit(texts_train)\n",
    "#vectors_train2 = vectorizer2.transform(texts_train)\n",
    "#vectors_val2 = vectorizer2.transform(texts_val)\n",
    "#vectors_test2 = vectorizer2.transform(texts_test)\n",
    "vectors_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "En la celda anterior se calculan dos representaciones deseables de los datos, *term frequency* y *tf-idf*. A continuación se muestra cómo quedó el vocabulario."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., ..., 0., 0., 0.])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#todense --get representation\n",
    "X_train = np.asarray(vectors_train.todense())\n",
    "X_val = np.asarray(vectors_val.todense())\n",
    "X_test = np.asarray(vectors_test.todense())\n",
    "\n",
    "#X_train2 = np.asarray(vectors_train2.todense())\n",
    "#X_val2 = np.asarray(vectors_val2.todense())\n",
    "#X_test2 = np.asarray(vectors_test2.todense())\n",
    "\n",
    "del vectors_train,vectors_val,vectors_test#,vectors_train2,vectors_val2,vectors_test2\n",
    "gc.collect()\n",
    "\n",
    "X_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFL1JREFUeJzt3X2QXXV9x/H3p7sE5UEezAqYBxNmIpoZRWFBbG2Nzwk6\nzTjDjMQH1DGTYSqOfZgpME51OrZTqX2wViRGTKm1JVphNKVBsNiKlkHZKEICBJZEyQY0y4M4Bmyy\nybd/3LN4WXf3nnvvuQ+/3/28ZnZyzzm/e+/3d+7ms+f8zu/eq4jAzMzy8lu9LsDMzKrncDczy5DD\n3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDI03KsnXrhwYSxbtqxXT29mlqTt27c/\nGhEjjdr1LNyXLVvG2NhYr57ezCxJkn5Spp2HZczMMuRwNzPLkMPdzCxDDcNd0mZJ+yXtaNDuHElT\nki6orjwzM2tFmSP3a4DV8zWQNARcAdxcQU1mZtamhuEeEbcCjzdo9iHgOmB/FUWZmVl72h5zl7QI\neDtwVYm2GySNSRqbnJxs96nNzGwOVVxQ/RRwaUQcadQwIjZFxGhEjI6MNJyDbwPsF786xNfv3Nfr\nMvrG1OEjfGVsL0eO+GsxG9mx70l++NATHX+eb+x4hAcnf9nx52lVFW9iGgW2SAJYCJwvaSoivlbB\nY9uAuvSrd3Hjjp9yxqnH85JTn9frcnru6u/u4RM33kdE8I5zlva6nL72tn/8LgA//sRbO/o8F3/p\nB5w+cizf+pNVHX2eVrUd7hGxfPq2pGuAGxzs1q6Hn/wVAE8fPNzjSvrD4wcOAvDzpw71uBKrt3vy\nQK9LmFPDcJd0LbAKWChpAvgYcBRARGzsaHVmZtaShuEeEevKPlhEvK+taszMrBJ+h6qZWYYc7mZm\nGXK4myXEEyGtLIe7WQLU6wIsOQ53swT4iN2a5XA3S4iP4K0sh7uZWYYc7mZmGXK4myXEY+9WlsPd\n+prDrMZj7dYsh7v1JYeZWXsc7mZmGXK4m5llyOFulgBfe7BmOdzNEuJrEVaWw90sAdOh7iN4K8vh\nbmaWIYe7mVmGHO5mZhlqGO6SNkvaL2nHHNvfJekuSXdLuk3SmdWXaWZmzShz5H4NsHqe7XuA10bE\ny4CPA5sqqMsGnC8cmrVnuFGDiLhV0rJ5tt9Wt3g7sLj9ssxqPPXv2cJ/9aykqsfcPwDcWPFj2gBz\nlpm1puGRe1mSXkct3F8zT5sNwAaApUuXVvXUliEfsc9O3jFWUiVH7pJeDlwNrI2Ix+ZqFxGbImI0\nIkZHRkaqeGozM5tF2+EuaSlwPfCeiLi//ZLMzKxdDYdlJF0LrAIWSpoAPgYcBRARG4GPAs8HPqva\nOeNURIx2qmAzM2uszGyZdQ22rwfWV1aRmZm1ze9QNUtBcSHVUyGtLIe7mVmGHO5mKfARuzXJ4W6W\nEM9zt7Ic7mZmGXK4m5llyOFulgLPlrEmOdytrznMzFrjcLe+5AuHZu1xuJuZZcjhbmaWIYe7mVmG\nHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuFtf8jtTzdrjcLe+5neq1gjvCGuOw90sAeFv67Am\nNQx3SZsl7Ze0Y47tkvRpSeOS7pJ0VvVl2qDy8IxZa8ocuV8DrJ5n+xpgRfGzAbiq/bJs0Hk4xqw9\nDcM9Im4FHp+nyVrgi1FzO3CipNOqKtDMzJpXxZj7ImBv3fJEse43SNogaUzS2OTkZAVPbWZms+nq\nBdWI2BQRoxExOjIy0s2nNkuaZ8tYs6oI933AkrrlxcU6M6uYZ81YWVWE+1bgomLWzHnAkxHxSAWP\na2ZmLRpu1EDStcAqYKGkCeBjwFEAEbER2AacD4wDTwHv71SxZmZWTsNwj4h1DbYH8MHKKjKz3+Dh\nGGuW36FqlhBfWLWyHO5mZhlyuJslYPqI3cMzVpbD3cwsQw53M7MMOdytz3kYwqwVDnfrS54TYtYe\nh7uZWYYc7mYJ8Tx3K8vhbpYQT4W0shzuZmYZcribmWXI4W5mliGHu/Uljyybtcfhbn3Os0MA5N1g\nTXK4myUkfEpjJTnczRLgULdmOdzNEuLhGSvL4W59zoesZq0oFe6SVkvaJWlc0mWzbD9B0n9I+pGk\nnZL8JdnWFh+gmrWnYbhLGgKuBNYAK4F1klbOaPZB4J6IOBNYBfytpAUV12o2sDwcY80qc+R+LjAe\nEbsj4iCwBVg7o00Ax0sScBzwODBVaaVm5gurVlqZcF8E7K1bnijW1fsM8FLgYeBu4MMRcaSSCs3M\nrGlVXVB9C3An8ELgFcBnJD1vZiNJGySNSRqbnJys6KnNzGymMuG+D1hSt7y4WFfv/cD1UTMO7AFe\nMvOBImJTRIxGxOjIyEirNZsNHA/HWLPKhPsdwApJy4uLpBcCW2e0eQh4A4CkU4AzgN1VFmpmZuUN\nN2oQEVOSLgFuAoaAzRGxU9LFxfaNwMeBayTdTW0W26UR8WgH6zYbKNOzZTxrxspqGO4AEbEN2DZj\n3ca62w8Db662NDMza5XfoWqWEI+9W1kOdzOzDDnczcwy5HC3vuZhCLPWONytL8nTQp7Fe8Oa5XA3\nS4BPYKxZDnczsyZFAuOFDnczsww53M3MMjTQ4f7k04f4q233cuhwWp9O/PTBw/zlf97Drw4d7nUp\nHTN92nvt9/c2aDn7ff/+m/cz8cRTVZdlloyBDvcrvnEfn7t1N1vvfLjXpTTl89/Zzee/s4fN/7un\n16V03HU/mGj6Pg9O/pJ/uOUBLv7S9g5UZJaGgQ73g1O1I/bDCVwcqTd9pjF1OK26u+VIsVv+71Ba\nZ2Tz8VRIa9ZAh7uZWStSOB50uJslIIEssT7jcDczy5DD3cwsQw53SPacN4Vxv17y7rFBNtDhnuoM\nhFTrbkY7HxyW4/7JsU/WWQMd7mZmuXK4m5k1KYUhv1LhLmm1pF2SxiVdNkebVZLulLRT0rerLdPM\nzJox3KiBpCHgSuBNwARwh6StEXFPXZsTgc8CqyPiIUkv6FTBZmbWWJkj93OB8YjYHREHgS3A2hlt\n3glcHxEPAUTE/mrL7KxI4iTrN6Vad7ek8JnbZp1SJtwXAfUfzTdRrKv3YuAkSf8jabuki2Z7IEkb\nJI1JGpucnGyt4gol+01uyRbeHd49ZtVdUB0GzgbeCrwF+DNJL57ZKCI2RcRoRIyOjIxU9NStS/bA\nLtnCu8O7x6zEmDuwD1hSt7y4WFdvAngsIg4AByTdCpwJ3F9JlR2mRGcRp1p3t/hLtq1TUhjyK3Pk\nfgewQtJySQuAC4GtM9p8HXiNpGFJxwCvAu6ttlQzMyur4ZF7RExJugS4CRgCNkfETkkXF9s3RsS9\nkr4B3AUcAa6OiB2dLNzMzOZWZliGiNgGbJuxbuOM5U8Cn6yutO5JddZJqnV3SwqnzmadMtDvUE12\nSDbZwrsjx92TY5+sswY63K1/OcvM2uNwN0uAR5j6Swovh8PdzCxDDnczsww53En3lDfVusuoomsZ\n7x6zhgY63FN9h2eaVXdTfnvIs2WsWQMd7mZmuXK4m5llaKDDPdV3eKZZdTflu4f8rtv+kMLLMNDh\nPi3V8cxU6+6WnHZPCmFi/cXhTrr/cVKtu1u8e2yQDXS4e7ZMrvLbQ9Nnaf6MeitroMPdzCxXDncz\nsww53K0vefBhdp4t0x9SmGnncDczy5DDnXRnVaRad9d4B9kAc7ibmWWoVLhLWi1pl6RxSZfN0+4c\nSVOSLqiuxM5JdVZZqnV3i/ePWYlwlzQEXAmsAVYC6yStnKPdFcDNVRdpZjWe525llTlyPxcYj4jd\nEXEQ2AKsnaXdh4DrgP0V1mdmdTxbpj+k8DKUCfdFwN665Yli3TMkLQLeDlxVXWlmZtaqqi6ofgq4\nNCKOzNdI0gZJY5LGJicnK3rq1qXw13c2qdbdLd4/ZjBcos0+YEnd8uJiXb1RYEsxHrgQOF/SVER8\nrb5RRGwCNgGMjo76v6CZWYeUCfc7gBWSllML9QuBd9Y3iIjl07clXQPcMDPY+1Gq16ZSrbtbctw/\nqX7InfVOw3CPiClJlwA3AUPA5ojYKeniYvvGDtdoZmZNKnPkTkRsA7bNWDdrqEfE+9ovy8zqpfBZ\nJtZf/A5V60s5Dq2YdZPD3fqSZ7yYtcfhTsJBkmzh3eG9Y4NsoMM91VN/z5yYX457x6+5NWugw93M\nLFcOdzOzDDnczRLiyyz9IYXXweFuZpYhhzvpvkEkzaq7xx+Pa4PM4W6WkFRneFn3OdxJd5pZmlV3\nj7+1yAaZw93MLEMOdzOzDDncrS95RGV2vkbcH1KYhOFwJ40XajZpVt09ni1jg2zAwz3Nw0Mf1c4v\nxwupGXbJOmzAw90sDT4JsWY53M3MMuRwN0uIh2esLIe7WUI8PNMfUngdSoW7pNWSdkkal3TZLNvf\nJekuSXdLuk3SmdWX2jkpvFCzSbXubvHusUHWMNwlDQFXAmuAlcA6SStnNNsDvDYiXgZ8HNhUdaFm\nZlZemSP3c4HxiNgdEQeBLcDa+gYRcVtEPFEs3g4srrbMzkh1/DLRsrsmx/2T6u+q9U6ZcF8E7K1b\nnijWzeUDwI2zbZC0QdKYpLHJycnyVXZIqsMaiZbdNd4/ZhVfUJX0Omrhfuls2yNiU0SMRsToyMhI\nlU/dllSPilKtu1u8e2yQDZdosw9YUre8uFj3LJJeDlwNrImIx6opz8zg12eZPivpDym8DmWO3O8A\nVkhaLmkBcCGwtb6BpKXA9cB7IuL+6svsrGSHZxKtu1u8e2yQNTxyj4gpSZcANwFDwOaI2Cnp4mL7\nRuCjwPOBzxaf6zEVEaOdK9tsMHmoycoqMyxDRGwDts1Yt7Hu9npgfbWldV6qY9aJlt01Oe6fVH9X\nrXf8DlUzsww53M3MMuRwNzPLkMOddGdVpPoNUt2S42yiDLuUpBS+5cvhbmaWoYEO91QnIHjmxPy8\nf8wGPNzNUuO/W1aWw93MLEMDHe79f0lkdglcy+kp7x+zAQ93s9T471Z/SOF1cLhbX5JHl83aMtDh\nnmp8eDbI/HLcPxl2yTpsoMPdzCxXDnezBKQwxmv9xeFuZpYhh7uZWZNSmG7rcIc0XqlZJFp21/iD\n1WyQDXS4pzqrQqkW3iU5TqPMr0fWaQMd7mZmuSoV7pJWS9olaVzSZbNsl6RPF9vvknRW9aWamVlZ\nDcNd0hBwJbAGWAmsk7RyRrM1wIriZwNwVcV1mplZE8ocuZ8LjEfE7og4CGwB1s5osxb4YtTcDpwo\n6bSKazUzs5LU6OuiJF0ArI6I9cXye4BXRcQldW1uAD4REd8tlm8BLo2Isbked3R0NMbG5tw8p2/f\nP8lf3HBP0/ebzQP7f/nM7RUvOK6Sx+yGVOtuRjt9fPrQYSaeeLql+/arQXjNqzK9rzq5nw5HsHvy\nQMvP845zlrD+d09v6bklbY+I0Ubthlt69BZJ2kBt2IalS5e29BjHHT3MilOqedFOPeE5fOeBR3nj\nS09hwXA68xGWnnwMt9y3n1VnjHDMgqFel9MRJx5zFHf8+AmAll7viSee5uWLT2DxSc+turSeWL7w\nWG6+52e8eeUpDA+l87vaCw///GkOHDxcWU7MZffkAV5y6vGcPnJs0/ddeNzRHajo2cqE+z5gSd3y\n4mJds22IiE3AJqgduTdVaeHsF53E2S86u5W7mpkNjDJj7ncAKyQtl7QAuBDYOqPNVuCiYtbMecCT\nEfFIxbWamVlJDY/cI2JK0iXATcAQsDkidkq6uNi+EdgGnA+MA08B7+9cyWZm1kipMfeI2EYtwOvX\nbay7HcAHqy3NzMxa5XeompllyOFuZpYhh7uZWYYc7mZmGXK4m5llqOHHD3TsiaVJ4Cct3n0h8GiF\n5aTAfR4M7vNgaKfPL4qIkUaNehbu7ZA0VuazFXLiPg8G93kwdKPPHpYxM8uQw93MLEOphvumXhfQ\nA+7zYHCfB0PH+5zkmLuZmc0v1SN3MzObR3Lh3ujLulMhaYmk/5Z0j6Sdkj5crD9Z0jclPVD8e1Ld\nfS4v+r1L0lvq1p8t6e5i26cl9fW3OUgakvTD4hu8su+zpBMlfVXSfZLulfTqAejzHxW/1zskXSvp\nObn1WdJmSfsl7ahbV1kfJR0t6cvF+u9JWtZUgRGRzA+1jxx+EDgdWAD8CFjZ67pa7MtpwFnF7eOB\n+6l9AflfA5cV6y8Drihuryz6ezSwvNgPQ8W27wPnAQJuBNb0un8N+v7HwL8BNxTLWfcZ+GdgfXF7\nAXBizn0GFgF7gOcWy18B3pdbn4HfA84CdtStq6yPwB8AG4vbFwJfbqq+Xu+gJnfmq4Gb6pYvBy7v\ndV0V9e3rwJuAXcBpxbrTgF2z9ZXa5+u/umhzX936dcDnet2fefq5GLgFeH1duGfbZ+CEIug0Y33O\nfV4E7AVOpvax4jcAb86xz8CyGeFeWR+n2xS3h6m96Ulla0ttWGb6l2baRLEuacXp1iuB7wGnxK+/\nxeqnwCnF7bn6vqi4PXN9v/oU8KfAkbp1Ofd5OTAJ/FMxFHW1pGPJuM8RsQ/4G+Ah4BFq38x2Mxn3\nuU6VfXzmPhExBTwJPL9sIamFe3YkHQdcB/xhRPyiflvU/mRnM51J0tuA/RGxfa42ufWZ2hHXWcBV\nEfFK4AC10/Vn5NbnYpx5LbU/bC8EjpX07vo2ufV5Nr3uY2rhXuqLuFMh6Shqwf6vEXF9sfpnkk4r\ntp8G7C/Wz9X3fcXtmev70e8Avy/px8AW4PWSvkTefZ4AJiLie8XyV6mFfc59fiOwJyImI+IQcD3w\n2+Td52lV9vGZ+0gapjbE91jZQlIL9zJf1p2E4or4F4B7I+Lv6jZtBd5b3H4vtbH46fUXFlfQlwMr\ngO8Xp4C/kHRe8ZgX1d2nr0TE5RGxOCKWUXvtvhUR7ybvPv8U2CvpjGLVG4B7yLjP1IZjzpN0TFHr\nG4B7ybvP06rsY/1jXUDt/0v5M4FeX5Bo4QLG+dRmljwIfKTX9bTRj9dQO2W7C7iz+Dmf2pjaLcAD\nwH8BJ9fd5yNFv3dRN2sAGAV2FNs+QxMXXXrY/1X8+oJq1n0GXgGMFa/114CTBqDPfw7cV9T7L9Rm\niWTVZ+BaatcUDlE7Q/tAlX0EngP8OzBObUbN6c3U53eompllKLVhGTMzK8HhbmaWIYe7mVmGHO5m\nZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhn6f1O74uP1E5ahAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ffb2797ab38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "##representacion soft para TF ---mucho mejor!\n",
    "X_train_input = np.log(X_train+1) \n",
    "X_val_input = np.log(X_val+1) \n",
    "X_test_input = np.log(X_test+1) \n",
    "plt.plot(X_train_input[0])\n",
    "plt.show()\n",
    "\n",
    "#soft para tf-idf\n",
    "#X_train_input2 = np.log(X_train2+1) \n",
    "#X_val_input2 = np.log(X_val2+1) \n",
    "#X_test_input2 = np.log(X_test2+1) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Architecture\n",
    "---\n",
    "La arquitectura para afrontar el problema es una red clásica *Feed Forward* basado en el *baseline* de *Variational Deep Semantic Hashing* (VDSH), la cual es una arquitectura de *autoencoder* no simétrico con dos capas escondidas en el *encoder* y sin capas escondidas en el *decoder*.\n",
    "\n",
    "\n",
    "> Input(|V|) -> Relu(500) -> Relu (500) -> Laten variable(32)-> Sampling -> Softmax(|V|)\n",
    "\n",
    "\n",
    "Tamién se experimenta con una arquitectura de autoencoder simétrico, lo cual asimila de mejor manera lo que es una RBM (red bidireccional) como se realiza en el trabajo de *Semantic Hashing*.\n",
    "\n",
    "La primera experimentación se realiza con 32 *bits* en la representación latente, ya que los trabajos previos han mostrado que esta cantidad de *bits* parece ser lo suficiente antes de empezar a realizar *overfitting*."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def REC_loss(x_true, x_pred):\n",
    "    x_pred = K.clip(x_pred, K.epsilon(), 1)\n",
    "    return - K.sum(x_true*K.log(x_pred), axis=-1) #keras.losses.categorical_crossentropy(x_true, x_pred)\n",
    "\n",
    "def info_BKL_loss(logits_b):\n",
    "    ep = K.epsilon()\n",
    "    p_b_x = keras.activations.sigmoid(logits_b) #B_j = Q(b_j|x) probability of b_j\n",
    "    p_b = K.mean(p_b_x, axis=0, keepdims=True) #marginalize over minibatch\n",
    "    Nb = K.int_shape(p_b)[1]\n",
    "    def infoKL(y_true, y_pred):\n",
    "        return Nb*np.log(2) + K.sum( p_b*K.log(p_b + ep) + (1-p_b)* K.log(1-p_b +ep),axis=1) #sum\n",
    "    return infoKL\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "\n",
    "def binary_AE(data_dim,Nb,units,layers_e,layers_d,opt='adam',BN=True, summ=True,beta=0,lamb=0.):\n",
    "    pre_encoder = define_pre_encoder(data_dim, layers=layers_e,units=units,BN=BN)\n",
    "    if summ:\n",
    "        print(\"pre-encoder network:\")\n",
    "        pre_encoder.summary()\n",
    "    generator = define_generator(Nb,data_dim,layers=layers_d,units=units,BN=BN)\n",
    "    if summ:\n",
    "        print(\"generator network:\")\n",
    "        generator.summary()\n",
    "\n",
    "    x = Input(shape=(data_dim,))\n",
    "    hidden = pre_encoder(x)\n",
    "    dist_b = Dense(Nb, activation='sigmoid')(hidden) #p(b) #otra forma de modelarlo\n",
    "    encoder = Model(x, dist_b)\n",
    "\n",
    "\n",
    "    output = generator(dist_b)\n",
    "        \n",
    "    Recon_loss = REC_loss\n",
    "    #kl_loss = BKL_loss(logits_b)\n",
    "    #info_kl_loss = info_BKL_loss(logits_b) #over minibatch\n",
    "    #definirlas en base a probabilidad\n",
    "    def BAE_loss(y_true, y_pred): \n",
    "        return Recon_loss(y_true, y_pred) #+ beta*kl_loss(y_true, y_pred) + lamb*info_kl_loss(y_true,y_pred)\n",
    "\n",
    "    binary_vae = Model(x, output)\n",
    "    binary_vae.compile(optimizer=opt, loss=BAE_loss )#, metrics = [Recon_loss,kl_loss,info_kl_loss])\n",
    "    return binary_vae, encoder,generator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train details\n",
    "---\n",
    "\n",
    "* 30* epochs* \n",
    "* *batch size* de 100\n",
    "* optimizador Adam\n",
    "* Inicializador de Glorot (para los pesos)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from utils import  compare_hist_train, add_hist_plot\n",
    "from utils import find_lambda\n",
    "\n",
    "\n",
    "batch_size = 100\n",
    "epochs = 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cantidad de datos Entrenamiento:  11016\n",
      "Cantidad de datos Validación:  3667\n",
      "Cantidad de datos Pruebas:  3668\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(11016, 10000)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from utils import Load_Dataset\n",
    "\n",
    "#https://github.com/unsuthee/VariationalDeepSemanticHashing\n",
    "## load data used for VDSH - for sake comparison\n",
    "\n",
    "\n",
    "filename = 'Data/ng20.tfidf.mat'\n",
    "#filename = 'Data/reuters.tfidf.mat'\n",
    "#filename = 'Data/tmc.tfidf.mat'\n",
    "data = Load_Dataset(filename)\n",
    "\n",
    "X_train_input = data[\"train\"]\n",
    "X_train = X_train_input.copy() \n",
    "X_val_input = data[\"cv\"]\n",
    "X_val = X_val_input.copy()\n",
    "X_test_input = data[\"test\"]\n",
    "X_test = X_test_input.copy()\n",
    "\n",
    "\n",
    "if \"ng20\" in filename:\n",
    "    labels_train = np.asarray([labels[value.argmax(axis=-1)] for value in data[\"gnd_train\"]])\n",
    "    labels_val = np.asarray([labels[value.argmax(axis=-1)] for value in data[\"gnd_cv\"]])\n",
    "    labels_test = np.asarray([labels[value.argmax(axis=-1)] for value in data[\"gnd_test\"]])\n",
    "elif \"reuters\" in filename or \"tmc\" in filename:\n",
    "    labels = np.asarray(labels)\n",
    "    labels_train = np.asarray([labels[value.astype(bool)] for value in data[\"gnd_train\"]])\n",
    "    labels_val = np.asarray([labels[value.astype(bool)] for value in data[\"gnd_cv\"]])\n",
    "    labels_test = np.asarray([labels[value.astype(bool)] for value in data[\"gnd_test\"]])\n",
    "\n",
    "print(\"Cantidad de datos Entrenamiento: \",len(labels_train))\n",
    "print(\"Cantidad de datos Validación: \",len(labels_val))\n",
    "print(\"Cantidad de datos Pruebas: \",len(labels_test))\n",
    "X_train_input.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMPORTANTE EJECUTAR, nueva modificación!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output target normalizado en dataset  20News\n"
     ]
    }
   ],
   "source": [
    "#outputs as probabolities -- normalized over datasets..\n",
    "X_train = X_train/X_train.sum(axis=-1,keepdims=True) \n",
    "X_val = X_val/X_val.sum(axis=-1,keepdims=True)\n",
    "X_test = X_test/X_test.sum(axis=-1,keepdims=True)\n",
    "print(\"Output target normalizado en dataset \",dat_n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train[np.isnan(X_train)] = 0\n",
    "X_val[np.isnan(X_val)] = 0\n",
    "X_test[np.isnan(X_test)] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***************************************\n",
      "*********** SUMMARY RESULTS ***********\n",
      "***************************************\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lambda</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.000000e-07</td>\n",
       "      <td>0.5327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000e-06</td>\n",
       "      <td>0.5320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.000000e-05</td>\n",
       "      <td>0.5396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.000000e-04</td>\n",
       "      <td>0.5399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.000000e-03</td>\n",
       "      <td>0.5389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.000000e-02</td>\n",
       "      <td>0.5307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.000000e-01</td>\n",
       "      <td>0.5264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.5423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>0.5237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.000000e+02</td>\n",
       "      <td>0.3985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.000000e+03</td>\n",
       "      <td>0.1991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.000000e+04</td>\n",
       "      <td>0.0764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.000000e+05</td>\n",
       "      <td>0.0639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.000000e+06</td>\n",
       "      <td>0.0646</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          lambda   score\n",
       "0   1.000000e-07  0.5327\n",
       "1   1.000000e-06  0.5320\n",
       "2   1.000000e-05  0.5396\n",
       "3   1.000000e-04  0.5399\n",
       "4   1.000000e-03  0.5389\n",
       "5   1.000000e-02  0.5307\n",
       "6   1.000000e-01  0.5264\n",
       "7   1.000000e+00  0.5423\n",
       "8   1.000000e+01  0.5237\n",
       "9   1.000000e+02  0.3985\n",
       "10  1.000000e+03  0.1991\n",
       "11  1.000000e+04  0.0764\n",
       "12  1.000000e+05  0.0639\n",
       "13  1.000000e+06  0.0646"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best value is 1 with lambda 1.000000\n",
      "Worst value is 0 with lambda 100000.000000\n",
      "***************************************\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def create_model_B(lambda_V):\n",
    "    return binary_VAE(X_train_input.shape[1],Nb=32,units=500,layers_e=2,layers_d=0\n",
    "                                                                  ,beta=0, lamb=lambda_V, summ=False)\n",
    "\n",
    "lambda_B = find_lambda(create_model_B, X_train_input, X_train, X_val_input,labels_train,labels_val,\n",
    "                   binary=True, BS=batch_size)\n",
    "#20news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lambda_V_found = {\"20news\":1e-0, \"reuters\":1e-1, \"tmc\":1e-2, \"snippets\":1e-0} #values for sum\n",
    "#tmc puede ser 1e1\n",
    "#reuters puede ser  1e-2\n",
    "#snippets puede ser 1e0\n",
    "\n",
    "lambda_B = lambda_V_found[dat_n.lower()]\n",
    "lambda_B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre-encoder network:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 10000)             0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 500)               5000500   \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 500)               2000      \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 500)               2000      \n",
      "=================================================================\n",
      "Total params: 5,255,000\n",
      "Trainable params: 5,253,000\n",
      "Non-trainable params: 2,000\n",
      "_________________________________________________________________\n",
      "generator network:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 10000)             330000    \n",
      "=================================================================\n",
      "Total params: 330,000\n",
      "Trainable params: 330,000\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 11016 samples, validate on 3667 samples\n",
      "Epoch 1/30\n",
      "11016/11016 [==============================] - 3s - loss: 8.6707 - val_loss: 8.3899\n",
      "Epoch 2/30\n",
      "11016/11016 [==============================] - 1s - loss: 8.3703 - val_loss: 8.3605\n",
      "Epoch 3/30\n",
      "11016/11016 [==============================] - 1s - loss: 8.2950 - val_loss: 8.2971\n",
      "Epoch 4/30\n",
      "11016/11016 [==============================] - 1s - loss: 8.2183 - val_loss: 8.2064\n",
      "Epoch 5/30\n",
      "11016/11016 [==============================] - 1s - loss: 8.1494 - val_loss: 8.1380\n",
      "Epoch 6/30\n",
      "11016/11016 [==============================] - 1s - loss: 8.0898 - val_loss: 8.0882\n",
      "Epoch 7/30\n",
      "11016/11016 [==============================] - 1s - loss: 8.0367 - val_loss: 8.0490\n",
      "Epoch 8/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.9874 - val_loss: 8.0134\n",
      "Epoch 9/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.9407 - val_loss: 7.9803\n",
      "Epoch 10/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.8966 - val_loss: 7.9503\n",
      "Epoch 11/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.8544 - val_loss: 7.9219\n",
      "Epoch 12/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.8139 - val_loss: 7.8962\n",
      "Epoch 13/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.7748 - val_loss: 7.8709\n",
      "Epoch 14/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.7373 - val_loss: 7.8480\n",
      "Epoch 15/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.7013 - val_loss: 7.8260\n",
      "Epoch 16/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.6659 - val_loss: 7.8045\n",
      "Epoch 17/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.6314 - val_loss: 7.7844\n",
      "Epoch 18/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.5982 - val_loss: 7.7653\n",
      "Epoch 19/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.5658 - val_loss: 7.7470\n",
      "Epoch 20/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.5350 - val_loss: 7.7300\n",
      "Epoch 21/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.5047 - val_loss: 7.7135\n",
      "Epoch 22/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.4746 - val_loss: 7.6977\n",
      "Epoch 23/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.4460 - val_loss: 7.6818\n",
      "Epoch 24/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.4174 - val_loss: 7.6668\n",
      "Epoch 25/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.3900 - val_loss: 7.6510\n",
      "Epoch 26/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.3637 - val_loss: 7.6411\n",
      "Epoch 27/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.3377 - val_loss: 7.6258\n",
      "Epoch 28/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.3129 - val_loss: 7.6133\n",
      "Epoch 29/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.2876 - val_loss: 7.6012\n",
      "Epoch 30/30\n",
      "11016/11016 [==============================] - 1s - loss: 7.2634 - val_loss: 7.5895\n"
     ]
    }
   ],
   "source": [
    "binary_ae,encoder_Bae,generator_Bae= binary_AE(X_train.shape[1],Nb=32,units=500,layers_e=2,layers_d=0)\n",
    "\n",
    "hist2 = binary_ae.fit(X_train_input, X_train, epochs=epochs, batch_size=batch_size\n",
    "                           ,validation_data=(X_val_input,X_val) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'REC_loss'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-27-60201a7c2e9c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0madd_hist_plot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'g'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_n\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"B-VAE\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Epochs'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Value'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"upper right\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfancybox\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/Desktop/DiscreteVAE/utils.py\u001b[0m in \u001b[0;36madd_hist_plot\u001b[0;34m(hist, c, model_n)\u001b[0m\n\u001b[1;32m     37\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0madd_hist_plot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'b'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_n\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m     \u001b[0mhistory_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 39\u001b[0;31m     \u001b[0mrec_loss_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'REC_loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     40\u001b[0m     \u001b[0mkl_loss_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'KL'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0mrec_val_loss_values\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_REC_loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'REC_loss'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5f26677780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15,6))\n",
    "add_hist_plot(hist2, c='g', model_n = \"B-VAE\")\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Value')\n",
    "plt.legend(loc=\"upper right\", fancybox= True)\n",
    "plt.title(\"VAE loss \"+dat_n)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Another intrinsic measure: *Classification*\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#function to define and train model\n",
    "from sklearn.metrics import jaccard_score\n",
    "from utils import define_fit, visualize_probas, visualize_mean, calculate_hash, visualize_probas_byB\n",
    "\n",
    "results = []\n",
    "results_S = []\n",
    "results_B = []\n",
    "results_O_B = [] #original testing on tresholded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 1., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#codify input data (binarize -- or aprox)\n",
    "X_train_Bcode = encoder_Bae.predict(X_train_input)\n",
    "X_val_Bcode = encoder_Bae.predict(X_test_input)\n",
    "\n",
    "##codify labels\n",
    "labels_aux = np.asarray(labels)\n",
    "def codify_labels(inputs):\n",
    "    inputs = np.asarray(inputs)\n",
    "    matrix_labels = np.zeros((inputs.shape[0],labels_aux.shape[0]))\n",
    "    for i,aux_labels in enumerate(inputs):\n",
    "        if type(aux_labels) == list or type(aux_labels) == np.ndarray :\n",
    "            for aux_label in aux_labels:\n",
    "                idx = np.where(aux_label==labels_aux)[0]\n",
    "                matrix_labels[i,idx] = 1 #various-multiple\n",
    "        else:\n",
    "            idx = np.where(aux_labels==labels_aux)[0]\n",
    "            matrix_labels[i,idx] = 1 #only one\n",
    "    return matrix_labels\n",
    "\n",
    "C_train = codify_labels(labels_train)\n",
    "#C_val = codify_labels(labels_val)\n",
    "C_val = codify_labels(labels_test)\n",
    "C_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20News\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAFhCAYAAADQncj9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4XXd95/H3V/tmW7YsJ16zN4lpSUJMWEohlCkkdEnb\np88USssyZdJMCS0z0wGe6QylpZ2nLWWGdgKkgWaAblD2lIZCS1na0gBOCNkTTOLE+y7JsrXrN3+c\nI/talizZvvpdSX6/nkeP7j3nd+/53nOvzv3onN/5nUgpIUmSlEtdrQuQJEnnFsOHJEnKyvAhSZKy\nMnxIkqSsDB+SJCkrw4ckScrK8KFFIyJuj4j/Wes65puI+HBE/O4ZPvadEfEXp5j/cERcP7ltRGyI\niP6IqD+jouexiPiViHjvabTfGBGbIyLK+1sj4t+dov3WiLhwFs97XkQ8GhHNFdPeHBF/MNvapFox\nfGjBKDfKA+WX2qGI+LuIWD8xP6V0S0rpXWXb6yNie+2qPTeklJ6VUvrqFNOfSSl1pJTGACLiqxHx\nxuwFVllENAH/A3j3aTzsXcAfpSoPqpRS2gN8Bbi5YvIHgddExKpqLkuqNsOHFpqfTCl1AKuBPcD/\nrXE9NRURDbWu4RxzE/BYSmnHbBpHxGrgpcBn56ievwR+ZeJOSmkQ+ALw2jlanlQVhg8tSOVG9pPA\nxolpE4cXIqKdYgO8ptxL0h8RayY/R9n+/RHxhbLNv0bE+RHx3nLPymMRcU1F+zUR8amI2BcRT0XE\nr1XMuy4i/i0ieiJiV0TcVv6XPDE/RcQtEfG9ss37JnbDT1HXOyPikxHx8Yg4HBH3RcRVFfO3RsTb\nIuIB4EhENETEleXehZ7yUMhPTXralRHxD+XzfS0iLqh4vj+OiG0R0RcR90bEj0x6bMsMtZx0CCEi\nLixfc0NE/B7wI8Bt5Xq+rXz975n0mLsi4j9Ps05SRPxquf4OR8S7IuKSiPhGWfffTFrfPxER95fr\n4xsR8eyKeW+PiO+Xz/NIRPxMxbzXR8S/RMQflZ+BpyLixopSbgS+NsXrvDkidpbv/W9UtP8x4L7y\n81rpueWyD0XE/4uIlmle99si4psTITMi/lP5/k60/yZwceX7CXwV+PGpnk+aLwwfWpAiog34eeCe\nyfNSSkcoviR2lrv+O1JKO6d5qn9PsRt9JTAE/BtwX3n/k8D/LpdXB/wt8F1gLfAy4C0R8YryecaA\n/1w+7gXl/F+dtKyfAJ4LPLtc7iuY3k3AJ4AVwF8Bn42Ixor5r6b4gukEoqztS8Aq4M3AX0bE5RXt\nX0Ox+38lcD/Ff8wTvg1cXbGsT0z6MpypllNKKf0m8M/AreV7cSvwEeDV5XolIlYC/658/um8ArgW\neD7wVuAO4BeB9cAPluuEMjDeSbFHoAv4U+CuON434vsUYWgZ8NvAX0Sxh2LC84DHKdbVHwJ/VhEU\nf6icN9lLgcuAlwNvqwhk07V/Tfl6LgF+gOIzOJV3U3wu/0dEXAb8L+AXJ8JMSmkU2AJcVfGYRyfd\nl+Ydw4cWms9GRA/QS/Ff5ekce5/KZ1JK95Yb888Agymlj5Z9FT4OTOz5eC7QnVL6nZTScErpSYrj\n668CKJ/jnpTSaEppK8UX3ksmLev3U0o9KaVnKI7VX32Kuu5NKX0ypTRCEYBaKL50J/xJSmlbSmmg\nnN5RPv9wSumfgM9TfhmX/i6l9PWU0hDwm8ALouwvk1L6i5TSgbL29wDNQGVwmamW05ZS+hbFe/iy\nctKrgK+W/Rim84cppb6U0sPAQ8CXUkpPppR6KfZ0TbxXNwN/mlL6ZkppLKX0EYov8OeXy/5ESmln\nSmk8pfRx4HvAdRXLeTql9MHyM/ARikN855XzOoHDU9T22ymlIymlB4H/x/F1P13728r37yDwe5z4\nXh2TUhqnOITya8Bd5Tr4zqRmh8vlVN5fNtXzSfOF4UMLzU+nlDopvgBvBb4WEeefxfNVftkNTHG/\no7x9AcVhnJ6JH+C/U34pRcQPRMTnI2J3RPRR/Ie6ctKydlfcPlrx3FPZNnGj/ALaDqyZan45fVvZ\nbsLTFHtopnq+fuDgxPNFxG9EcdZEb/m6lk2qfaZaztRHKPZcUP7+8xnan8579V8nvVfrOf56X1tx\nSKaHYq9J5es99j6llI6WNyee+xCwZIraKt+Ppzm+fk63/UnKMPsV4ELgfVM0WQL0TLrfO93zSfOB\n4UMLUvkf7acpDne8aKomVV7kNuCplFJnxc+SlNIry/kfAB4DLkspLaUIJlP26ZilY2fxlIcm1gGV\nh44qX99OYP3EIYzSBqCyU2Tl83VQHELZWfbveCvFYaDlZbDrnVT7TLXMxlTvx18AN5V9SK6kep0y\ntwG/N+m9aksp/XXZN+KDFMG1q3y9DzH79+oBisMkk62vuL2B4+vndNufJCJ+nOJQ3peZtKev7Aty\nKcXhwAlXTrovzTuGDy1IUbgJWE5xjHuyPUBXRFRr9/O3gMNlB8DWiKiPiB+MiOeW85cAfUB/RFwB\n/KezXN61EfGz5ZfLWygOG5zUv6X0TYo9KW+NiMYoxt34SeBjFW1eGREvKjtlvgu4J6W0rax7FNgH\nNETEO4ClZ1HLdPYAF1dOSCltp+hv8ufAp8pDSNXwQeCWiHhe+Tlpj4gfj4glQDtFENoHEBFvoNjz\nMVt3c/LhNID/GRFtEfEs4A0Uh+wA/gF4zhQdSt8UEesiYgXFYbCPM4WyL8yHgDcCrwN+MiJeWdHk\nOmBrSunpimkvoTgMJc1bhg8tNH8bEf0UX/S/B7yu7ANwgpTSY8BfA0+Wu9fP6jBBefz/Jyj6aTwF\n7Kf4UpgIN78B/ALF8fYPMs2XyWn4HEWH2kPALwE/W/a5mKq2YYqwcWNZ1/uB15brYMJfAb9Fcbjl\nWo4f7vgi8PfAExS7/wc58ZDAadVyCn8M/Fx5dsefVEz/CEWnzJkOucxaSmkz8B+B2yhq3gK8vpz3\nCPAeio7Fe8pl/+tpPP3fAldM8Xn6WrmcL1OM6fGlcnl7gH+i6LRb6a8oOgg/SdEBdrpB4O4APpdS\nujuldAD4ZeBDEdFVzn8NcPtE4zLkvJJivUrzVlR53BtJZyki3glcmlL6xZnaLnQR8WKKwy8XVHsQ\nrrkSETcDG1NKb4liJNKngMbyzJOp2m+kCAPXzeY1RsRW4Pqyr8ep2q2iCD3XTJz9EhFvBtanlN46\n6xck1YADFEmqifJ03V8HPrRQggdASumO02z/CMXZUtWuYy9F/47Kaef0oHtaODzsIim7iLiS4gyN\n1cCsr5NyjngvJ569Ii06HnaRJElZuedDkiRlZfiQJElZGT4kSVJWhg9JkpSV4UOSJGVl+JAkSVkZ\nPiRJUlaGD0mSlJXhQ5IkZWX4kCRJWRk+JElSVoYPSZKUleFDkiRlZfiQJElZGT4kSVJWhg9JkpSV\n4UOSJGVl+JAkSVnNGD4i4s6I2BsRD00zPyLiTyJiS0Q8EBHPqX6ZkiRpsZjNno8PAzecYv6NwGXl\nz83AB86+LEmStFjNGD5SSl8HDp6iyU3AR1PhHqAzIlZXq0BJkrS4VKPPx1pgW8X97eU0SZKkkzTk\nXFhE3ExxaIb29vZrr7jiipyLlzSNe++9d39KqbvWdcyG2xFpfjqd7Ug1wscOYH3F/XXltJOklO4A\n7gDYtGlT2rx5cxUWL+lsRcTTta5httyOSPPT6WxHqnHY5S7gteVZL88HelNKu6rwvJIkaRGacc9H\nRPw1cD2wMiK2A78FNAKklG4H7gZeCWwBjgJvmKtiJUnSwjdj+EgpvXqG+Ql4U9UqkiRJi5ojnEqS\npKwMH5IkKSvDhyRJysrwIUmSsjJ8SJKkrAwfkiQpK8OHJEnKyvAhSZKyMnxIkqSsDB+SJCkrw4ck\nScrK8CFJkrIyfEiSpKwMH5IkKSvDhyRJysrwIUmSsjJ8SJKkrAwfkiQpK8OHJEnKyvAhSZKyMnxI\nkqSsDB+SJCkrw4ckScrK8CFJkrIyfEiSpKwMH5IkKSvDhyRJysrwIUmSsjJ8SJKkrAwfkiQpK8OH\nJEnKyvAhSZKyMnxIkqSsDB+SJCkrw4ckScrK8CFJkrIyfEiSpKwMH5IkKSvDhyRJysrwIUmSsjJ8\nSJKkrAwfkiQpK8OHJEnKyvAhSZKyMnxIkqSsDB+SJCmrWYWPiLghIh6PiC0R8fYp5i+LiL+NiO9G\nxMMR8YbqlypJkhaDGcNHRNQD7wNuBDYCr46IjZOavQl4JKV0FXA98J6IaKpyrZIkaRGYzZ6P64At\nKaUnU0rDwMeAmya1ScCSiAigAzgIjFa1UkmStCjMJnysBbZV3N9eTqt0G3AlsBN4EPj1lNL45CeK\niJsjYnNEbN63b98ZlizpXOZ2RFr4qtXh9BXA/cAa4GrgtohYOrlRSumOlNKmlNKm7u7uKi1a0rnE\n7Yi08M0mfOwA1lfcX1dOq/QG4NOpsAV4CriiOiVKkqTFZDbh49vAZRFxUdmJ9FXAXZPaPAO8DCAi\nzgMuB56sZqGSJGlxaJipQUppNCJuBb4I1AN3ppQejohbyvm3A+8CPhwRDwIBvC2ltH8O65YkSQvU\njOEDIKV0N3D3pGm3V9zeCby8uqVJkqTFyBFOJUlSVoYPSZKUleFDkiRlZfiQJElZGT4kSVJWhg9J\nkpSV4UOSJGVl+JAkSVkZPiRJUlaGD0mSlJXhQ5IkZWX4kCRJWRk+JElSVoYPSZKUleFDkiRlZfiQ\nJElZGT4kSVJWhg9JkpSV4UOSJGVl+JAkSVkZPiRJUlaGD0mSlJXhQ5IkZWX4kCRJWRk+JElSVoYP\nSZKUleFDkiRlZfiQJElZGT4kSVJWhg9JkpSV4UOSJGVl+JAkSVkZPiRJUlaGD0mSlJXhQ5IkZWX4\nkLRgjY0nUkq1LkPSaTJ8SFqwHt3Vx33PHKp1GZJOk+FD0oIVEdz2T1vc+yEtMIYPSQvWeUub+crj\n+3jrJx9gYHis1uVImiXDh6QFa2VHM7/2o5fyiXu386Pv+Sof+cZWDg+O1LosSTMwfEha0P7Lyy/n\nb37lBaxe1sJv3fUwm373H7nlz+/lE5u3sbt3sNblSZpCQ60LkKSzdd1FK/j0r/4w92/r4TP3befu\nh3bz9w/vBuCCrjaes2E512zoZOPqpVyxeikdzW76pFryL1DSonH1+k6uXt/Jb/3ks3hs92H+dct+\nNj99kH/Zsp/PfGfHsXYbVrRx2aoOLl3VwSWrOriku50NK9pZ2dFERNTwFUjnBsOHpEWnri7YuGYp\nG9cs5T9yMSkldvUO8uiuPh7Z2cdjuw+zZW8///y9/QyPjR97XHtTPRd0tbNhRRvrlreyvvy9bnkb\nqztbWNrSWMNXJS0ehg9Ji15EsKazlTWdrbzsyvOOTR8dG2fboQG27j/C0weOsPXAUbYeOMKWff18\n9Ym9DI6Mn/A8S5obWN3ZwprOVlYva2X1shbOX9rC+ctaWL2shfOWtbCkucG9J9IMZhU+IuIG4I+B\neuBDKaXfn6LN9cB7gUZgf0rpJVWsU5KqrqG+jotWtnPRyvaT5qWU2N8/zPZDR9l+aIBdvQPs7Blk\nZ88AO3sHeHB7LweODJ/0uLames5b2sJ5S5s5b2kRTlZV3D9vSQurljbT0lif4yVK89KM4SMi6oH3\nAT8GbAe+HRF3pZQeqWjTCbwfuCGl9ExErJqrgiUph4ige0kz3UuauWbD8inbDI2OsbdviF29g+zq\nHWB37yB7Dw+xu2+QvX2D3PfMIfb0DTE8On7SY5e1Nh4LJKuWVISTY79b6F7STGO9JyVq8ZnNno/r\ngC0ppScBIuJjwE3AIxVtfgH4dErpGYCU0t5qFypJ801zQz3rV7SxfkXbtG1SSvQOjLCnb4g9fYPs\n6SsDSu8gew8PsqdviO/v3c/ew0OMjp84UmtEMZbJ+WUYWb2sOMRzfnl7dWdx6Me9KFpoZhM+1gLb\nKu5vB543qc0PAI0R8VVgCfDHKaWPVqVCSVrAIoLOtiY625q4/Pwl07YbH08cODJchpNBdvcWe1D2\n9A6y5/Ag2w8dZfPTB+k5evIgal3tTUVflGVFv5Z1y1tZ21l0lF23vJXOtkb7oWheqVaH0wbgWuBl\nQCvwbxFxT0rpicpGEXEzcDPAhg0bZv3kF779706atvX3f7zqj5E0/1VuR1aev5a/+uYzNa5o7pxf\n9hmpNDw6zuHBEXoGRuid+Dk6Qs/AMPdv6+GrT+w76TBPc0MdK9qbWN7WRFd7Eys6muhqb6aro4ll\nrY3ULYJg8gvPm/13impvNuFjB7C+4v66clql7cCBlNIR4EhEfB24CjghfKSU7gDuANi0aZNXgqqC\nqUIWGLTOhsF1fqvcjlx85bPPue1IU0MdXR3NdHU0Tzk/pcTAyBg9R0foOTrMwaMjHDwyzKEjw+w7\nPMQTew6fcHinoS5Y0d7Eyo6if8vE7+6OZlqbPJyjuTGb8PFt4LKIuIgidLyKoo9Hpc8Bt0VEA9BE\ncVjm/1SzUEnSzCKCtqYG2poaWNPZetL88ZToGxjhwJFhDvYPs//IEPsPD7Hv8BCP7z7MWMUVgjua\nG451ul1VBpLuJc0sa/Uwjs7OjOEjpTQaEbcCX6Q41fbOlNLDEXFLOf/2lNKjEfH3wAPAOMXpuA/N\nZeFzwf94JS12dRV9UC7pPnHe2Hji0NFiD8mxn/4hHtjec8KYJ031daxcUu4t6WhmZbnHZGV7E812\nftUszKrPR0rpbuDuSdNun3T/3cC7q1eaJCmn+rooQkRHM1euPj49pUT/0OixMDIRTLYdPMqD23up\nPPbV0dxAV0fRt2R5e9nHpK243eEAbCo5wqnOWe7pkmYnIljS0siSlkYu7u44Yd7I2DgH+ofZ3z/E\ngSPDHOgfYn//MFv29tM3OHpC24a6YENX27EzcdZWjBa7trOV85e10NTguCbnAsOHJOmMNdbXFWOP\nLGs5ad7I2DiHjgwXHV4HRug5MsyS1ga2Hxrg4Z27OTjFCLErO5qOjWUy8fu8E37sc7IYnHPhw/92\nNRf8XEkna6yvY1U5vPyEylNiB4bH2NU7wK7eQXb0DLCrZ5DdfYPs7h1g+6EB7n36EIemGNekuaGO\nVUubWbWkhVVlZ9hV5YiwxzrHLmmmq72Z+jpDynx0zoUPnVsMBdL81dpUz8XdHScdyqk0OFIMYb/n\ncDE67O7e46PE7u0rTh3+ly37OTzpEA9AXUBXx/Ew0t3RzKqlE2fttBy7vWppM21Nfh3m5NqWTsN8\nDjPzuTbpTLU01rOhq40NXdMPYQ/FXpSiQ+wg+w4PsbfijJ2J24/tOsz+/pOHsQdob6ov9p6UYaS4\n5k75u7y/elmLIaVKXIuSpGxyjUjbUFfH6mVFZ9ZK4ylxdHiM/sFRDg+OcHholMODo/QPjtA3OMre\nw0N8f18/fYMjjIydHFJaGutY1trIstZGlrY00tnWxPK2RpaXZ/QsaWlYFCPGVpqL0WMNH7Nwqv8o\n/W9TkhaOugg6mhvoaG6YspPshJQSQ6PjHB4cpW9whL6BIpz0DhS3ewdG2NkzSP/QyWf0dHU0HTtl\nefWyFtYtb2O519c5geFDkqRJIoKWxnpaGuvpXjL1UPZQnNHTc3SEQ0eHOXS0GDV2X39xBeNHd/Ux\ncYSnvameZ61dxgsu7uK8pdOHnnOF4WOOuLdEkha/xvq6Y2fZTDY6Ps6e3iG29xzl0V19fOupgzy8\ns4+333DFOX8WjuFjgah1YDnTC9h5ReLq8kKC0vw2Oj5O79ER9vcfH55+x6HidGKAtZ3u9QDDhxYJ\nA4ukaqvsaDk+nugZGKk4g+b4WTW7y3FKdvYMsK9/iIpr87GivYkrzl/Cv3/uel54yUqee+Fy+35g\n+FAV+MV/aq4faf4ZHh3n6PAoR4fHyp/RY7/7h8Y4MjTK5x/YycEjw+zvL/pzjE1xim5rYz2rlxXD\nxF9/eTdrOltZ09nKxSvbubi7gxXtTTV4dfOf4UOqEQ+hSGdn4oyUgeExjo6MMTA8xsDE7+HRY9OO\nVkyfCBlTjfUxoaWxjvamBi5c2c76FW1cvb6TFe3FGSzHBykrRlVtb6p3T8YZMHxIkmompcTw6DgD\nIyeGhMGRiiAx+XbF7+kjRHHaa1tTPa1N9bQ2NrCivYm1na20NdWXPw20NtXT1lzcbi/bNtQVF7eb\ni/EtVFjQ4cPd2WfG/7glVdvYeJpyr8OxUHHs9uhJIeIUOyGoi2KU09bGiRBRz4r2pmP32yqmtzY1\nHJ/eVE9jvVfIna8WdPjQqS22cLbYXo80X42Np2OHJ44Mj3J0qPh9ZOjEvhGV/SUGR8ZP+ZwtjXXF\nnoYyHHS2NR0PDY1FWGipCA4T05sa6jyssQgZPiQpk1ruxh8cGTt26ufEGRv7+4c40D/MgSND7O8f\n5kD/EAeODNMzxZVkJ3Q0N9BZDid+3tKWYljxtuPDjHe2NdE58XtiGPLWxnN+XAudyPAhSQtYSone\ngRF29Ayw49BAeUn64tL0e/oG2dNXjLY51VVfATrbGulqb6Kro5nLz19CV3szK9qb6OpoYnlbE13t\nxTVLVrQXoaK5oT7zK9RiZPiQpHlucGSMZw4eZev+Izxz8CjbDh4tfh8qxpY4Ojx2Qvv6ujh2RdZL\nuzv44Uu6jl2xdWI0zu4lRciwX4RqwfAxj9gRVDq3HRka5Yk9h/nenn4e33OY7+3t56n9/Ww/NHDC\nwFVLmhvY0NXGJd3tvPiybtZ0trC2HF9i9bIWujqaPcyhec3woZqw86jOdYcHR3hwRy8PbO/loR29\nPLKzj6cOHDkWMloa67iku4Or1y/nZ69Zx8Xd7VzY1c4FXW0sa/UKqVrYIqVTnSU9dzYtWZI2X3vt\nrNre8+SBk6Y9/+KuaacvpsfM9HzVfsx0zmQ50z3Ox+R/H2YSX/vavSmlTbNqPI9c1daRvnTFD9W6\njFlIjIwlmhrq6B8a5fDgKAPDx/tgNDcUY020NzUcG4OiubEe44UWktPZjrjnQ5Kqrhg4a3B0nMGR\nYsCsiaG56+uCjpZGutrb6GhuoL25gcZ6Y4bOLbXb87FpU9q8efOs2k63i/5MLlu/0B4z0/NV+zHT\nOZPlTPc4H5P/fZhJRCzIPR8XX/ns9Lsf/nyty2BkbJzthwZ4+sARth44wtMHjjI0Wox70dnWyIVd\nxSGTN730Ei7p7qDO/hhahE5nO+KeD53AvhjSzPoGR3jmQHHGydMHjrCzZ5Cx8h+5VUuauWpdJxeu\nbOfCrjY6245fWOyy85bUqmRpXjF8SNIpjIyNs7NngG2HBthWnubaM1AMwtVQF6ztbOWHL+3igq52\nNqxoo73Zzao0E/9KpHnIPVCzs6K9qaqjhqaU2HrgKN955hD3b+vh/m09PLqrj5GxYq/G2s5WXnTZ\nSq7ZsJznbOjkWWuW0dTgOBnS6TJ8SDpnHR0e5f5nerj36UPc98whvrOt59jQ4u1N9Tx7XSdv/JGL\nuXp9J9es72TV0pYaVywtDoYPSeeM3oERvvXUQb755AG+tfUgD+/sO3YWymWrOnj5xvN4zoblXLNh\nOZeu6nCgLmmOGD4kLVrj44kHd/Ty5cf28vUn9vHA9h7GEzQ11HHN+k5+9fpLuPaCImwsa22sdbnS\nOcPwIWnReWhHL5++bwd3P7iL3X2D1AVctb6TW196KS+8dCVXr++kpdELpEm1YviQtCiMjyf+7sFd\nfPCfn+SB7b001dfx4h/o5r+94nJeesUqVrQ3zfwkkrIwfEha8B7a0cvbPvUAD+/s4+Ludn7npmdx\n01VrWdbmoRRpPjJ8SFrQPnf/Dv7bJx6gs62R9/781fzUVWscQVSa5wwfkhasI0Oj/Je/+S6bLljO\nn/7StSeMJipp/nJ0HEkL1s7eQdZ0tvBnr3+uwUNaQAwfkhaswZExbnnJJXQ4pLm0oBg+JC1oL76s\nu9YlSDpNhg9JC9bSlkbWdrbWugxJp8nwIWnBuqCrzTNbpAXI8CFJkrIyfEiSpKwMH5IkKSvDhyRJ\nysrwIUmSsppV+IiIGyLi8YjYEhFvP0W750bEaET8XPVKlCRJi8mM4SMi6oH3ATcCG4FXR8TGadr9\nAfClahcpSZIWj9ns+bgO2JJSejKlNAx8DLhpinZvBj4F7K1ifZIkaZGZTfhYC2yruL+9nHZMRKwF\nfgb4QPVKkyRJi1G1Opy+F3hbSmn8VI0i4uaI2BwRm/ft21elRUs6l7gdkRa+2YSPHcD6ivvrymmV\nNgEfi4itwM8B74+In578RCmlO1JKm1JKm7q7vRiUpNPndkRa+GZzHepvA5dFxEUUoeNVwC9UNkgp\nXTRxOyI+DHw+pfTZKtYpSZIWiRnDR0ppNCJuBb4I1AN3ppQejohbyvm3z3GNkiRpEZnNng9SSncD\nd0+aNmXoSCm9/uzLkiRJi5UjnEqSpKwMH5IkKSvDhyRJysrwIUmSsjJ8SJKkrAwfkiQpK8OHJEnK\nyvAhSZKyMnxIkqSsDB+SJCkrw4ckScrK8CFJkrIyfEiSpKwMH5IkKSvDhyRJysrwIUmSsjJ8SJKk\nrAwfkiQpK8OHJEnKyvAhSZKyMnxIkqSsDB+SJCkrw4ckScrK8CFJkrIyfEiSpKwMH5IkKSvDhyRJ\nysrwIUmSsjJ8SJKkrAwfkiQpK8OHJEnKyvAhSZKyMnxIkqSsDB+SJCkrw4ckScrK8CFJkrIyfEiS\npKwMH5IkKSvDhyRJysrwIUmSsjJ8SJKkrAwfkiQpK8OHJEnKyvAhSZKyMnxIkqSsDB+SJCmrWYWP\niLghIh6PiC0R8fYp5r8mIh6IiAcj4hsRcVX1S5UkSYvBjOEjIuqB9wE3AhuBV0fExknNngJeklL6\nIeBdwB3VLlSSJC0Os9nzcR2wJaX0ZEppGPgYcFNlg5TSN1JKh8q79wDrqlumJElaLGYTPtYC2yru\nby+nTeeXgS9MNSMibo6IzRGxed++fbOvUpJKbkekha+qHU4j4qUU4eNtU81PKd2RUtqUUtrU3d1d\nzUVLOke4HZEWvoZZtNkBrK+4v66cdoKIeDbwIeDGlNKB6pQnSZIWm9ns+fg2cFlEXBQRTcCrgLsq\nG0TEBuCco/WYAAAI90lEQVTTwC+llJ6ofpmSJGmxmHHPR0ppNCJuBb4I1AN3ppQejohbyvm3A+8A\nuoD3RwTAaEpp09yVLUmSFqrZHHYhpXQ3cPekabdX3H4j8MbqliZJkhYjRziVJElZGT4kSVJWhg9J\nkpSV4UOSJGVl+JAkSVkZPiRJUlaGD0mSlJXhQ5IkZWX4kCRJWRk+JElSVoYPSZKUleFDkiRlZfiQ\nJElZGT4kSVJWhg9JkpSV4UOSJGVl+JAkSVkZPiRJUlaGD0mSlJXhQ5IkZWX4kCRJWRk+JElSVoYP\nSZKUleFDkiRlZfiQJElZGT4kSVJWhg9JkpSV4UOSJGVl+JAkSVkZPiRJUlaGD0mSlJXhQ5IkZWX4\nkCRJWRk+JElSVoYPSZKUleFDkiRlZfiQJElZGT4kSVJWhg9JkpSV4UOSJGVl+JAkSVkZPiRJUlaG\nD0mSlJXhQ5IkZWX4kCRJWc0qfETEDRHxeERsiYi3TzE/IuJPyvkPRMRzql+qJElaDGYMHxFRD7wP\nuBHYCLw6IjZOanYjcFn5czPwgSrXKUmSFonZ7Pm4DtiSUnoypTQMfAy4aVKbm4CPpsI9QGdErK5y\nrZIkaRGYTfhYC2yruL+9nHa6bSRJkoiU0qkbRPwccENK6Y3l/V8CnpdSurWizeeB308p/Ut5/8vA\n21JKmyc9180Uh2UALgceP816VwL7T/Mx1VbrGmq9fGuYH8uvdg0XpJS6q/Rcc2rSduQHgYdqWM6E\nxfZ5WMg1wPyo41ysYdbbkYZZtNkBrK+4v66cdrptSCndAdwxm8KmEhGbU0qbzvTx1VDrGmq9fGuY\nH8ufLzXUQuV2ZL6sg/lQhzXMrzqs4dRmc9jl28BlEXFRRDQBrwLumtTmLuC15Vkvzwd6U0q7qlyr\nJElaBGbc85FSGo2IW4EvAvXAnSmlhyPilnL+7cDdwCuBLcBR4A1zV7IkSVrIZnPYhZTS3RQBo3La\n7RW3E/Cm6pY2pTM+ZFNFta6h1ssHa5gPy4f5UUOtzZd1MB/qsIbj5kMd1nAKM3Y4lSRJqiaHV5ck\nSVktmPAx0xDvGZa/NSIejIj7I2LzzI+oyjLvjIi9EfFQxbQVEfEPEfG98vfyGtTwzojYUa6L+yPi\nlXO4/PUR8ZWIeCQiHo6IXy+nZ1sPp6ghy3qIiJaI+FZEfLdc/m+X07N+Fmqp1pd4mO4zMKnN9RHR\nW/F5eEc1a6hYzim3RRnWxeUVr/H+iOiLiLdMajMn6+JstonV+g6ZpoZ3R8Rj5fr+TER0TvPYqnyP\nnM12uVrr4ayllOb9D0VH1+8DFwNNwHeBjZlr2AqszLzMFwPPAR6qmPaHwNvL228H/qAGNbwT+I1M\n62A18Jzy9hLgCYph/rOth1PUkGU9AAF0lLcbgW8Cz8/9WajVz2z+/ik6vH+hXFfPB76Z4zMwqc31\nwOczrI9Tbovmel1M8d7sphjfYc7XxZluE6v5HTJNDS8HGsrbfzDd32K1vkfOdLtczfVwtj8LZc/H\nbIZ4X3RSSl8HDk6afBPwkfL2R4CfrkEN2aSUdqWU7itvHwYepRg9N9t6OEUNWaRCf3m3sfxJZP4s\n1FDNL/FQ68/Aacp5uYuXAd9PKT09R89/grPYJlbtO2SqGlJKX0opjZZ376EY62rOnMV2ed58ly6U\n8DEfhm9PwD9GxL1RjLBYK+el42Oo7AbOq1Edby53Md6Za3d/RFwIXEPxn39N1sOkGiDTeoiI+oi4\nH9gL/ENKqWbroAbm1SUepvgMVHph+Xn4QkQ8ay6Wz8zbopzby1cBfz3NvBzrAmb3d5BznfwHij1P\nU5nr75GZtkfz4bsUWDjhYz54UUrpaoor+L4pIl5c64JSsR+tFqcrfYBit93VwC7gPXO9wIjoAD4F\nvCWl1Fc5L9d6mKKGbOshpTRWfv7WAddFxA9Oml+rz8I55VSfQ+A+YENK6dnA/wU+O0dlzIttURSD\nTv4U8IkpZudaFyeo9d9BRPwmMAr85TRN5vK9y75dPhsLJXzMavj2uZRS2lH+3gt8hmL3VS3smdiF\nWv7em7uAlNKe8stwHPggc7wuIqKRYoP/lymlT5eTs66HqWrIvR7KZfYAXwFuYB58FjKp2iUezsY0\nn8NjUkp9E4fHUjE2UmNErKxmDeVzz7QtyrW9vBG4L6W0Z4oas6yL0mz+DnJ8Pl4P/ATwmjIEnWQu\nv0dmuT2q+XfphIUSPmYzxPuciYj2iFgycZuic1GtLmZ1F/C68vbrgM/lLmDS8eOfYQ7XRUQE8GfA\noyml/10xK9t6mK6GXOshIrones9HRCvwY8BjzIPPQiY1v8TDKT6HlW3OL9sREddRbF8PVKuG8nln\nsy3KdbmLVzPNIZcc66LCbP4O5vQ7JCJuAN4K/FRK6eg0beb0e2SW26OafpeeoBa9XM/kh6IH9xMU\nPXV/M/OyL6boFfxd4OFcy6f4w94FjFAcm/tloAv4MvA94B+BFTWo4c+BB4EHKD64q+dw+S+i2I36\nAHB/+fPKnOvhFDVkWQ/As4HvlMt5CHhHOT3rZ6GWP1P9/QO3ALeUtwN4Xzn/QWBTps9AZQ23ltuH\n71J0OnzhHKyHKbdFOddFuYx2ijCxrGLanK+L09kmAmuAu0/1GapiDVso+lJMfDZun1zDdO9dFWuY\ncns0V+vhbH8c4VSSJGW1UA67SJKkRcLwIUmSsjJ8SJKkrAwfkiQpK8OHJEnKyvChWYuIsfJqid+N\niPsi4oXl9DUR8cny9tXTXU1RkiTAU201exHRn1LqKG+/AvjvKaWXTGrzeopxBW6tQYmSpAXAPR86\nU0uBQ1BcaCsiHipHzPsd4OfLPSQ/X9MKJUnzUkOtC9CC0lpeWbUFWA38aOXMlNJwRLwD93xIkk7B\n8KHTMZCKKzISES8APjr56qqSJM3Ewy46IymlfwNWAt21rkWStLAYPnRGIuIKoJ6Tr1R5GFiSvyJJ\n0kJh+NDpaC07kt4PfBx4XUppbFKbrwAb7XAqSZqOp9pKkqSs3PMhSZKyMnxIkqSsDB+SJCkrw4ck\nScrK8CFJkrIyfEiSpKwMH5IkKSvDhyRJyur/A885A2+hQFDQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5f26f4b668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(dat_n)\n",
    "visualize_probas_byB(X_train_Bcode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "aux = [dat_n]\n",
    "multi_label = \"Reuters\" in aux or \"TMC\" in aux\n",
    "\n",
    "model1_O = define_fit(multi_label,X_train_Bcode,C_train)\n",
    "\n",
    "if not multi_label:\n",
    "    aux.append(model1_O.evaluate(X_train_Bcode,C_train,verbose=0)[1])\n",
    "    aux.append(model1_O.evaluate(X_val_Bcode,C_val,verbose=0)[1])\n",
    "else:\n",
    "    aux.append(jaccard_score(C_train, (model1_O.predict(X_train_Bcode)>=0.5)*1, average='micro'))\n",
    "    aux.append(jaccard_score(C_val, (model1_O.predict(X_val_Bcode)>=0.5)*1, average='micro'))\n",
    "    \n",
    "results.append(aux)\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Accuracy on dataset:  20News\n",
      "Binary VAE (train-val): 0.803740 - 0.703381\n"
     ]
    }
   ],
   "source": [
    "for valores in results:\n",
    "    print(\"\\nAccuracy on dataset: \",valores[0])\n",
    "    print(\"Binary VAE (train-val): %f - %f\"%(valores[1],valores[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ahora es mejor para reconstruir ya que el modelo VAE al concentrarse mas en la KL (peso mas alto) olvida mejorar la reconstrucción y los patrones de los datos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, ..., 0, 1, 0],\n",
       "       [1, 1, 1, ..., 0, 0, 1],\n",
       "       [1, 0, 0, ..., 1, 0, 1],\n",
       "       ...,\n",
       "       [1, 0, 1, ..., 0, 0, 1],\n",
       "       [1, 0, 1, ..., 0, 1, 0],\n",
       "       [0, 1, 0, ..., 0, 0, 0]], dtype=int32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## binary // treshold it\n",
    "X_train_Bcode_B = calculate_hash(X_train_Bcode, from_probas=True, from_logits=False)\n",
    "X_val_Bcode_B = calculate_hash(X_val_Bcode, from_probas=True, from_logits=False)\n",
    "\n",
    "X_val_Bcode_B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "### trained models over encoder representation testing on tresholded representation (see how decrease)\n",
    "\n",
    "aux = [dat_n]\n",
    "multi_label = \"Reuters\" in aux or \"TMC\" in aux\n",
    "if not multi_label:\n",
    "    aux.append(model1_O.evaluate(X_train_Bcode_B,C_train,verbose=0)[1])\n",
    "    aux.append(model1_O.evaluate(X_val_Bcode_B,C_val,verbose=0)[1])\n",
    "else:\n",
    "    aux.append(jaccard_score(C_train, (model1_O.predict(X_train_Bcode_B)>=0.5)*1, average='micro'))\n",
    "    aux.append(jaccard_score(C_val, (model1_O.predict(X_val_Bcode_B)>=0.5)*1, average='micro'))\n",
    "    \n",
    "results_O_B.append(aux)\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification on Binary (thresholded) representation\n",
      "\n",
      "Accuracy on dataset:  20News\n",
      "Binary VAE (train-val): 0.800472 - 0.683479\n"
     ]
    }
   ],
   "source": [
    "print(\"Classification on Binary (thresholded) representation\")\n",
    "for valores in results_O_B:\n",
    "    print(\"\\nAccuracy on dataset: \",valores[0])\n",
    "    print(\"Binary VAE (train-val): %f - %f\"%(valores[1],valores[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "aux = [dat_n]\n",
    "multi_label = \"Reuters\" in aux or \"TMC\" in aux\n",
    "\n",
    "model1 = define_fit(multi_label,X_train_Bcode_B,C_train)\n",
    "\n",
    "if not multi_label:\n",
    "    aux.append(model1.evaluate(X_train_Bcode_B,C_train,verbose=0)[1])\n",
    "    aux.append(model1.evaluate(X_val_Bcode_B,C_val,verbose=0)[1])\n",
    "else:\n",
    "    aux.append(jaccard_score(C_train, (model1.predict(X_train_Bcode_B)>=0.5)*1, average='micro'))\n",
    "    aux.append(jaccard_score(C_val, (model1.predict(X_val_Bcode_B)>=0.5)*1, average='micro'))\n",
    "    \n",
    "results_B.append(aux)\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification on Binary representation\n",
      "\n",
      "Accuracy on dataset:  20News\n",
      "Binary VAE (train-val): 0.805374 - 0.686205\n"
     ]
    }
   ],
   "source": [
    "print(\"Classification on Binary representation\")\n",
    "for valores in results_B:\n",
    "    print(\"\\nAccuracy on dataset: \",valores[0])\n",
    "    print(\"Binary VAE (train-val): %f - %f\"%(valores[1],valores[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "---\n",
    "#### Proceso de evaluación (*content-based retrieval*)\n",
    "> *Query*: **Documento**\n",
    "\n",
    "1. Calcular código hashing/binario de cada dato/documento\n",
    "    * Para *VAE* tradicional se utiliza la mediana de cada componente como *treshold*\n",
    "    * Para *VAE* binario se utiliza el *treshold* de 0.5 en la probabilidad\n",
    "2. Recuperar documentos basado en *match* perfecto, *top K* o *ball search* de distancia *hamming* de un documento consulta.\n",
    "    * En el mismo conjunto/*set*\n",
    "    * Con *query* recupero sobre *database*\n",
    "3. Medir *precision* y *recall* en base a algún criterio de relevancia.\n",
    "    * Documento relevante a otro o no -> Si comparten al menos una clase.\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from utils import MedianHashing, get_similar, measure_metrics, calculate_hash "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experimentando variando el #Bits\n",
    "---\n",
    "A continación se realizan las experimentaciones correspondientes para encontrar la mejor arquitectura y mejor configuración del modelo propuesto en base a las métricas *precision* y *recall* del conjunto de validación en el **top 100** de elementos recuperados."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_hashing(encoder,train,val,labels_train, labels_val, traditional=True,tipo=\"topK\"):\n",
    "    encode_train = encoder.predict(train)\n",
    "    encode_val = encoder.predict(val)\n",
    "    \n",
    "    train_hash = calculate_hash(encode_train, from_probas=~traditional )\n",
    "    val_hash = calculate_hash(encode_val, from_probas=~traditional)\n",
    "\n",
    "    val_similares_train =  get_similar(val_hash, train_hash, tipo=tipo,K=100) \n",
    "    return measure_metrics(labels,val_similares_train,labels_query=labels_val,labels_source=labels_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "binary_dat = {\"20news\":{\"p\":[],\"r\":[]},\n",
    "              \"snippets\":{\"p\":[],\"r\":[]},\n",
    "              \"reuters\":{\"p\":[],\"r\":[]},\n",
    "             \"tmc\":{\"p\":[],\"r\":[]}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20news\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'binary_dat' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-b20bada3f9ab>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mbinary_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mbinary_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"r\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'binary_dat' is not defined"
     ]
    }
   ],
   "source": [
    "Nbits = np.asarray([4,8,16,32,64])\n",
    "dataset = dat_n.lower()\n",
    "print(dataset)\n",
    "\n",
    "binary_dat[dataset][\"p\"] = []\n",
    "binary_dat[dataset][\"r\"] = []\n",
    "\n",
    "for Nbit in Nbits:    \n",
    "    binary_ae,encoder_Bae,generator_Bae = binary_AE(X_train.shape[1],Nb=Nbit,units=500,layers_e=2,layers_d=0,lamb=lambda_B)\n",
    "    binary_ae.fit(X_train_input, X_train, epochs=epochs, batch_size=batch_size,verbose=0 )\n",
    "    p_b,r_b = evaluate_hashing(encoder_Bae,X_train_input,X_val_input,labels_train,labels_val,traditional=False,tipo=\"topK\")\n",
    "    binary_dat[dataset][\"p\"].append(p_b) \n",
    "    binary_dat[dataset][\"r\"].append(r_b) \n",
    "    keras.backend.clear_session()\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resultados de Precision en validación\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'binary_dat' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-dec34c322fe6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#Table()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"N bits\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNbits\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"20News\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbinary_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"20news\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Reuters\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbinary_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"reuters\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Snippets\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbinary_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"snippets\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'binary_dat' is not defined"
     ]
    }
   ],
   "source": [
    "print(\"Resultados de Precision en validación\")\n",
    "t = pd.DataFrame() #Table()\n",
    "t[\"N bits\"] = Nbits\n",
    "t[\"20News\"] = binary_dat[\"20news\"][\"p\"]\n",
    "t[\"Reuters\"] = binary_dat[\"reuters\"][\"p\"]\n",
    "t[\"Snippets\"] = binary_dat[\"snippets\"][\"p\"]\n",
    "t[\"TMC\"] = binary_dat[\"tmc\"][\"p\"]\n",
    "print(\"\\n*** VAE Binary***\")\n",
    "print(t.T)\n",
    "\n",
    "print(\"\\nResultados de Recall en validación\")\n",
    "t = pd.DataFrame() #Table()\n",
    "t[\"N bits\"] = Nbits\n",
    "t[\"20News\"] = binary_dat[\"20news\"][\"r\"]\n",
    "t[\"Reuters\"] = binary_dat[\"reuters\"][\"r\"]\n",
    "t[\"Snippets\"] = binary_dat[\"snippets\"][\"r\"]\n",
    "t[\"TMC\"] = binary_dat[\"tmc\"][\"r\"]\n",
    "print(\"\\n*** VAE Binary***\")\n",
    "print(t.T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Best models facing test set*\n",
    "Luego de la selección de la mejor configuración de los modelos se evalua de diferentes formas sobre el conjunto de pruebas de los distintos *datasets*, los cambios realizados son:\n",
    "* Se entrenan sobre todos los datos disponibles (*train+val*)\n",
    "* Se aumentó el número de *epochs* a 50.\n",
    "\n",
    "El mejor modelo:\n",
    "* Simétrica para *Binary VAE* y Base para *Traditional VAE*\n",
    "* 32 Bits en la codificación (número de variables latentes)\n",
    "* Representación sobre las top 10 mil *tokens* más frecuentes y *Term Frequency*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_hashing(encoder,train,test,labels_trainn,labels_testt,traditional=True,tipo=\"topK\"):\n",
    "    \"\"\"\n",
    "        Evaluate Hashing correclty: Query and retrieve on a different set\n",
    "    \"\"\"\n",
    "    encode_train = encoder.predict(train)\n",
    "    encode_test = encoder.predict(test)\n",
    "    train_hash = calculate_hash(encode_train, from_probas=~traditional, from_logits=False )\n",
    "    test_hash = calculate_hash(encode_test, from_probas=~traditional, from_logits=False)\n",
    "\n",
    "    test_similares_train =  get_similar(test_hash,train_hash,tipo=\"topK\",K=100)\n",
    "    return measure_metrics(labels,test_similares_train,labels_query=labels_testt,labels_source=labels_trainn)\n",
    "\n",
    "#to save results\n",
    "results_dat = {\"20news\":{\"p\":[],\"r\":[]},\n",
    "               \"snippets\":{\"p\":[],\"r\":[]},\n",
    "               \"reuters\":{\"p\":[],\"r\":[]},\n",
    "              \"tmc\":{\"p\":[],\"r\":[]},\n",
    "              } #for topk\n",
    "BITS_S = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pre-encoder network:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_9 (InputLayer)         (None, 10000)             0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 500)               5000500   \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 500)               2000      \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 500)               2000      \n",
      "=================================================================\n",
      "Total params: 5,255,000\n",
      "Trainable params: 5,253,000\n",
      "Non-trainable params: 2,000\n",
      "_________________________________________________________________\n",
      "generator network:\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 10000)             330000    \n",
      "=================================================================\n",
      "Total params: 330,000\n",
      "Trainable params: 330,000\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1011"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_total_input = np.concatenate((X_train_input,X_val_input),axis=0)\n",
    "X_total = np.concatenate((X_train,X_val),axis=0)\n",
    "labels_total = np.concatenate((labels_train,labels_val),axis=0)\n",
    "\n",
    "binary_ae,encoder_Bae,generator_Bae = binary_AE(X_train.shape[1],Nb=BITS_S,units=500,layers_e=2,layers_d=0)\n",
    "binary_ae.fit(X_total_input, X_total, epochs=epochs, batch_size=batch_size,verbose=0)\n",
    "\n",
    "#save model\n",
    "#encoder_Bvae.save(\"saved_models/\"+dat_n+\"_BVAE_\"+str(BITS_S)+\"b_E_text_L?avg.h5\")\n",
    "#generator_Bvae.save_weights(\"saved_models/\"+dat_n+\"_BVAE_\"+str(BITS_S)+\"b_D_text_L?avg_w.h5\")\n",
    "\n",
    "###keras.backend.clear_session()\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "source": [
    "##load model\n",
    "encoder_Bvae = keras.models.load_model(\"saved_models/\"+dat_n+\"_BVAE_\"+str(BITS_S)+\"b_E_text_L?avg.h5\")\n",
    "#generator_Bvae = keras.models.load_model(\"saved_models/\"+name_dat+\"_BVAE_\"+str(BITS_S)+\"b_D_defaultCNN.h5\")\n",
    "encoder_Bvae.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Realizado sobre dataset  20news con precision y recall para B-VAE 0.5524482006543083 0.0741014772507479\n"
     ]
    }
   ],
   "source": [
    "dataset = dat_n.lower()\n",
    "\n",
    "p_b,r_b = evaluate_hashing(encoder_Bae,X_total_input,X_test_input,labels_total,labels_test,traditional=False,tipo=\"topK\")\n",
    "results_dat[dataset][\"p\"] = [p_b]\n",
    "results_dat[dataset][\"r\"] = [r_b]\n",
    "print(\"Realizado sobre dataset \",dataset,\"con precision y recall para B-VAE\",p_b,r_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*************Top 100 most similar retrieve*************\n",
      "\n",
      "Resultados de Precision en pruebas\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Length of values does not match length of index",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-49-08a386411639>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Modelo\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m\"Binario\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"20News\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"20news\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Reuters\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"reuters\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"TMC\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"tmc\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Snippets\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresults_dat\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"snippets\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"p\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__setitem__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   2517\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2518\u001b[0m             \u001b[0;31m# set column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2519\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2520\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2521\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_setitem_slice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_set_item\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   2583\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2584\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_valid_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2585\u001b[0;31m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sanitize_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2586\u001b[0m         \u001b[0mNDFrame\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_item\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2587\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_sanitize_column\u001b[0;34m(self, key, value, broadcast)\u001b[0m\n\u001b[1;32m   2758\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2759\u001b[0m             \u001b[0;31m# turn me into an ndarray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2760\u001b[0;31m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_sanitize_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2761\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2762\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/fmena/anaconda3/envs/tesis/lib/python3.5/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m_sanitize_index\u001b[0;34m(data, index, copy)\u001b[0m\n\u001b[1;32m   3119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3120\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3121\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Length of values does not match length of '\u001b[0m \u001b[0;34m'index'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3123\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mPeriodIndex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Length of values does not match length of index"
     ]
    }
   ],
   "source": [
    "print(\"*************Top 100 most similar retrieve*************\")\n",
    "\n",
    "print(\"\\nResultados de Precision en pruebas\")\n",
    "t = pd.DataFrame() #Table()\n",
    "t[\"Modelo\"] = [\"Binario\"]\n",
    "t[\"20News\"] = results_dat[\"20news\"][\"p\"]\n",
    "t[\"Reuters\"] = results_dat[\"reuters\"][\"p\"]\n",
    "t[\"TMC\"] = results_dat[\"tmc\"][\"p\"]\n",
    "t[\"Snippets\"] = results_dat[\"snippets\"][\"p\"]\n",
    "print(t)\n",
    "\n",
    "print(\"\\n\\nResultados de Recall en pruebas\")\n",
    "t = pd.DataFrame()\n",
    "t[\"Modelo\"] = [\"Binario\"]\n",
    "t[\"20News\"] = results_dat[\"20news\"][\"r\"]\n",
    "t[\"Reuters\"] = results_dat[\"reuters\"][\"r\"]\n",
    "t[\"TMC\"] = results_dat[\"tmc\"][\"r\"]\n",
    "t[\"Snippets\"] = results_dat[\"snippets\"][\"r\"]\n",
    "print(t)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tesis]",
   "language": "python",
   "name": "conda-env-tesis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}