process_data.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3",
   "language": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import codecs\n",
    "import json\n",
    "import configparser\n",
    "import os\n",
    "import sys\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import defaultdict\n",
    "from dotenv import load_dotenv\n",
    "from num2words import num2words\n",
    "from sklearn import linear_model\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "import dask.array as da\n",
    "import dask.dataframe as dd\n",
    "import h5py\n",
    "\n",
    "import spacy\n",
    "import lemminflect\n",
    "sp = spacy.load(\"en\")\n",
    "sp.Defaults.stop_words.add(\"left\")\n",
    "sp.Defaults.stop_words.add(\"right\")\n",
    "sp.Defaults.stop_words.add(\"near\")\n",
    "sp.Defaults.stop_words.add(\"far\")\n",
    "sp.Defaults.stop_words.add(\"middle\")\n",
    "sp.Defaults.stop_words.add(\"center\")\n",
    "sp.Defaults.stop_words.add(\"furthest\")\n",
    "sp.Defaults.stop_words.add(\"first\")\n",
    "sp.Defaults.stop_words.add(\"second\")\n",
    "sp.Defaults.stop_words.add(\"third\")\n",
    "sp.Defaults.stop_words.add(\"fourth\")\n",
    "sp.Defaults.stop_words.add(\"fifth\")\n",
    "sp.Defaults.stop_words.add(\"front\")\n",
    "sp.Defaults.stop_words.add(\"back\")\n",
    "sp.Defaults.stop_words.add(\"leftmost\")\n",
    "sp.Defaults.stop_words.add(\"rightmost\")\n",
    "sp.Defaults.stop_words.add(\"upper\")\n",
    "sp.Defaults.stop_words.add(\"lower\")\n",
    "sp.Defaults.stop_words.add(\"nearest\")\n",
    "sp.Defaults.stop_words.add(\"closest\")\n",
    "sp.Defaults.stop_words.add(\"uppermost\")\n",
    "sp.Defaults.stop_words.add(\"lowermost\")\n",
    "sp.Defaults.stop_words.add(\"highest\")\n",
    "sp.Defaults.stop_words.add(\"lowest\")\n",
    "sp.Defaults.stop_words.add(\"closer\")\n",
    "sp.Defaults.stop_words.add(\"nearer\")\n",
    "sp.Defaults.stop_words.add(\"farther\")\n",
    "sp.Defaults.stop_words.add(\"further\")\n",
    "sp.Defaults.stop_words.add(\"farthest\")\n",
    "\n",
    "\n",
    "import random\n",
    "random.seed(42)\n",
    "\n",
    "from time import localtime, strftime\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "\n",
    "from bet_utils.semparse import prob_at_least"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_FEATS = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load up config file (needs path; adapt env var if necessary); local imports\n",
    "load_dotenv()\n",
    "\n",
    "# load config file, set up paths, make project-specific imports\n",
    "config_path = os.getenv('VISCONF')\n",
    "if not config_path:\n",
    "    # try default location, if not in environment\n",
    "    default_path_to_config = '../Config/default.cfg'\n",
    "    if os.path.isfile(default_path_to_config):\n",
    "        config_path = default_path_to_config\n",
    "\n",
    "assert config_path is not None, 'You need to specify the path to the config file via environment variable VISCONF.'        \n",
    "\n",
    "config = configparser.ConfigParser()\n",
    "with codecs.open(config_path, 'r', encoding='utf-8') as f:\n",
    "    config.read_file(f)\n",
    "\n",
    "corpora_base = config.get('DEFAULT', 'corpora_base')\n",
    "preproc_path = config.get('DSGV-PATHS', 'preproc_path')\n",
    "dsgv_home = config.get('DSGV-PATHS', 'dsgv_home')\n",
    "\n",
    "sys.path.insert(0,dsgv_home + \"/Utils\")\n",
    "from utils import icorpus_code, plot_labelled_bb, get_image_filename, query_by_id\n",
    "from utils import plot_img_cropped, plot_img_ax, invert_dict, get_a_by_b, get_image_part\n",
    "sys.path.insert(0,dsgv_home + \"/WACs/WAC_Utils\")\n",
    "from wac_utils import create_word2den, is_relational, filter_refdf_by_filelist, filter_relational_expr\n",
    "from wac_utils import filter_X_by_filelist, make_mask_matrix, make_X_id_index, train_this_word\n",
    "from wac_utils import get_X_for_word\n",
    "\n",
    "from data_utils import load_dfs\n",
    "\n",
    "from apply_utils import apply_wac_set_matrix, logreg\n",
    "\n",
    "#sys.path.append(dsgv_home + '/Preproc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load up preprocessed DataFrames. Slow!\n",
    "# These DataFrames are the result of pre-processing the original corpus data,\n",
    "# as per dsg-vision/Preprocessing/preproc.py\n",
    "\n",
    "df_names = ['refcoco_refdf', 'refcocoplus_refdf']\n",
    "df = load_dfs(preproc_path, df_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(preproc_path + '/refcoco_splits.json', 'r') as f:\n",
    "    rc_splits = json.load(f)"
   ]
  },
  {
   "source": [
    "## RefCoco"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mscoco_bbdf_pattern = '/Volumes/BigData_SSD/Data/Computed/ExtractOut/vgg/mscoco_bbdf_vgg19-fc2/mscoco_bbdf_vgg19-fc2_%d.hdf5'\n",
    "# model_path_prefix = '../TrainWACs/ModelsOut/01_refcoco_vgg'\n",
    "mscoco_bbdf_pattern = '../../data/Models/ForBToma/mscoco_bbdf_rsn50-max/mscoco_bbdf_rsn50-max_%d.hdf5'\n",
    "model_path_prefix = '../../data/Models/ForBToma/01_refcoco_rsn'"
   ]
  },
  {
   "source": [
    "### Load Image Features"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "das = []\n",
    "fhs = []\n",
    "for n in range(1,8):\n",
    "    f = h5py.File(mscoco_bbdf_pattern % (n), 'r')\n",
    "    fhs.append(f)\n",
    "    das.append(da.from_array(f['img_feats'], chunks=(1000, 4106)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = da.concatenate(das)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(602408, 2058)"
      ]
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rc_all_test = rc_splits['testA'] + rc_splits['testB'] # rc_splits['val']  # rc_splits['testA'] + rc_splits['testB']\n",
    "X_ts = filter_X_by_filelist(X, rc_all_test)\n",
    "refdf_test = filter_refdf_by_filelist(df['refcoco_refdf'], rc_all_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is small enough to be fully in memory\n",
    "X_ts = X_ts[:].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Meaning that I can already close the file handles\n",
    "for fh in fhs:\n",
    "    fh.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2den_ts = create_word2den(refdf_test)\n",
    "X_idx_ts = make_X_id_index(X_ts)\n",
    "mask_matrix_ts = make_mask_matrix(X_ts, X_idx_ts, word2den_ts, word2den_ts.keys())"
   ]
  },
  {
   "source": [
    "### Load WACs"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exp2indseq(w2i, exp):\n",
    "    return [word2ind[w] for w in exp.split() if w in word2ind] #the list of indices for each \"word\" in exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def imageid2rows(idx, ic, ii):\n",
    "    '''return all regions that belong to an image, as indices into X (via idx)'''\n",
    "    # or should this be a separate dictionary?\n",
    "    return [v for k,v in idx.items() if k[0] == ic and k[1] == ii]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "with h5py.File(model_path_prefix + '.hdf5', 'r') as f:\n",
    "    wacs = f['wac_weights'][:]   # slice, to actually read into memory (as ndarray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open(model_path_prefix + '.json', 'r') as f:\n",
    "    modelpars, wordlist = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   i_corpus  image_id  region_id r_corpus  rex_id                      refexp  \\\n",
       "0         1    581563    1345868  refcoco      71  lower left corner darkness   \n",
       "1         1    581563    1345868  refcoco      72            bpttom left dark   \n",
       "2         1    581563    1345868  refcoco      73   black van in front of cab   \n",
       "3         1    581563     344837  refcoco      74                        taxi   \n",
       "4         1    581563     344837  refcoco      75   the taxi cab bottom right   \n",
       "\n",
       "                                              tagged  \n",
       "0  [[lower, JJR], [left, VBD], [corner, JJR], [da...  \n",
       "1            [[bpttom, NN], [left, VBD], [dark, NN]]  \n",
       "2  [[black, JJ], [van, NN], [in, IN], [front, NN]...  \n",
       "3                                       [[taxi, NN]]  \n",
       "4  [[the, DT], [taxi, NN], [cab, NN], [bottom, NN...  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>i_corpus</th>\n      <th>image_id</th>\n      <th>region_id</th>\n      <th>r_corpus</th>\n      <th>rex_id</th>\n      <th>refexp</th>\n      <th>tagged</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>581563</td>\n      <td>1345868</td>\n      <td>refcoco</td>\n      <td>71</td>\n      <td>lower left corner darkness</td>\n      <td>[[lower, JJR], [left, VBD], [corner, JJR], [da...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>581563</td>\n      <td>1345868</td>\n      <td>refcoco</td>\n      <td>72</td>\n      <td>bpttom left dark</td>\n      <td>[[bpttom, NN], [left, VBD], [dark, NN]]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>581563</td>\n      <td>1345868</td>\n      <td>refcoco</td>\n      <td>73</td>\n      <td>black van in front of cab</td>\n      <td>[[black, JJ], [van, NN], [in, IN], [front, NN]...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>581563</td>\n      <td>344837</td>\n      <td>refcoco</td>\n      <td>74</td>\n      <td>taxi</td>\n      <td>[[taxi, NN]]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>581563</td>\n      <td>344837</td>\n      <td>refcoco</td>\n      <td>75</td>\n      <td>the taxi cab bottom right</td>\n      <td>[[the, DT], [taxi, NN], [cab, NN], [bottom, NN...</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "refdf_test.head()"
   ]
  },
  {
   "source": [
    "### Apply all WACs to all regions"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_applied = apply_wac_set_matrix(X_ts[:, ID_FEATS:], wacs.T, net=logreg) #applies all wac and returns a matrix of results, rows are entities and columns are words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2ind = {w[0]:n for n,w in enumerate(wordlist)} #creates dictionary from word to its index in the wordlist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dependency parse each refexp to make generation a bit easier hopefully\n",
    "refdf_test[\"DepParse\"] = refdf_test.apply(lambda row: sp(row.refexp), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this function will go through and create some sort of representation of every referring expression (and hopefully also every word?) per picture\n",
    "ref_dict = {}\n",
    "for i in refdf_test[\"image_id\"].unique():\n",
    "    ref_dict[i] = {refdf_test[\"refexp\"][j] : j for j in refdf_test[refdf_test[\"image_id\"] == i].index}\n",
    "ref_vocab = defaultdict(set)\n",
    "for k, v in ref_dict.items():\n",
    "    for exp in v:\n",
    "        for word in refdf_test.loc[v[exp]][\"DepParse\"]:\n",
    "            if word.text in word2ind and not word.is_stop:\n",
    "                ref_vocab[k].add(word.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "singularity_exceptions = {\"sprinkles\", \"stripes\", \"jeans\", \"pants\", \"shorts\", \"glasses\", \"spots\", \"clothes\"}\n",
    "\n",
    "def get_good_nps(image_id):\n",
    "    nps = set()\n",
    "    for k, v in ref_dict[image_id].items():\n",
    "        for chunk in refdf_test.loc[v][\"DepParse\"].noun_chunks:\n",
    "            if all([token.text in word2ind for token in chunk]) and not chunk.root.is_stop:\n",
    "                nps.add(\" \".join([token._.inflect(\"NN\") if (token == chunk.root and token.text not in singularity_exceptions) else token.text for token in chunk if not token.is_stop]))\n",
    "    return nps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_ids = list(refdf_test.image_id.unique())\n",
    "ref_dict_per_id = [ref_dict[x] for x in image_ids]\n",
    "ref_voc_per_id = [ref_vocab[x] for x in image_ids]\n",
    "good_nps_per_id = [get_good_nps(x) for x in image_ids]\n",
    "im_ref_df = pd.DataFrame({\"image_id\": image_ids, \"ref_exps\": ref_dict_per_id, \"vocab\": ref_voc_per_id, \"NPs\": good_nps_per_id}).set_index(\"image_id\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                                   ref_exps  \\\n",
       "image_id                                                      \n",
       "581563    {'lower left corner darkness': 0, 'bpttom left...   \n",
       "581518    {'top donut': 8, 'top of donuts': 7, 'donut wc...   \n",
       "581346    {'left zebra butt': 13, 'zebra on the left': 1...   \n",
       "581282    {'person bottom left': 18, 'left black shirt':...   \n",
       "580668    {'man on right': 20, 'right man': 21, 'man sit...   \n",
       "\n",
       "                                                      vocab  \\\n",
       "image_id                                                      \n",
       "581563                           {dark, van, corner, black}   \n",
       "581518                                   {sprinkles, donut}   \n",
       "581346                                        {zebra, butt}   \n",
       "581282                               {person, black, shirt}   \n",
       "580668    {yellow, white, sitting, blue, man, bench, car...   \n",
       "\n",
       "                                                     NPs  \n",
       "image_id                                                  \n",
       "581563                                       {black van}  \n",
       "581518                                {sprinkles, donut}  \n",
       "581346                               {zebra, zebra butt}  \n",
       "581282                                     {black shirt}  \n",
       "580668    {blue car, man, white car, bench, car, person}  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ref_exps</th>\n      <th>vocab</th>\n      <th>NPs</th>\n    </tr>\n    <tr>\n      <th>image_id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>581563</th>\n      <td>{'lower left corner darkness': 0, 'bpttom left...</td>\n      <td>{dark, van, corner, black}</td>\n      <td>{black van}</td>\n    </tr>\n    <tr>\n      <th>581518</th>\n      <td>{'top donut': 8, 'top of donuts': 7, 'donut wc...</td>\n      <td>{sprinkles, donut}</td>\n      <td>{sprinkles, donut}</td>\n    </tr>\n    <tr>\n      <th>581346</th>\n      <td>{'left zebra butt': 13, 'zebra on the left': 1...</td>\n      <td>{zebra, butt}</td>\n      <td>{zebra, zebra butt}</td>\n    </tr>\n    <tr>\n      <th>581282</th>\n      <td>{'person bottom left': 18, 'left black shirt':...</td>\n      <td>{person, black, shirt}</td>\n      <td>{black shirt}</td>\n    </tr>\n    <tr>\n      <th>580668</th>\n      <td>{'man on right': 20, 'right man': 21, 'man sit...</td>\n      <td>{yellow, white, sitting, blue, man, bench, car...</td>\n      <td>{blue car, man, white car, bench, car, person}</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 25
    }
   ],
   "source": [
    "im_ref_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_nps = set().union(*good_nps_per_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_true_exp(rfdf, idx: \"\"\"image id\"\"\", n: \"\"\"maximum number of statements per image\"\"\"):\n",
    "    #this function will generate true sentences containing refexp for a given image\n",
    "    truths = []\n",
    "    eligible_NPs = list(rfdf.loc[idx][\"NPs\"])\n",
    "    random.shuffle(eligible_NPs)\n",
    "    while len(truths) < n:\n",
    "        if not eligible_NPs:\n",
    "            break\n",
    "        np = eligible_NPs.pop()\n",
    "        if set(np.split()).intersection(singularity_exceptions):\n",
    "            article = \"\"\n",
    "            verb = \"are\"\n",
    "        elif np.startswith(('a','e','i','o','u')):\n",
    "            article = \"an \"\n",
    "            verb = \"is\"\n",
    "        else:\n",
    "            article = \"a \"\n",
    "            verb = \"is\"\n",
    "        truths.append(f\"There {verb} {article}{np}.\")\n",
    "    return truths\n",
    "\n",
    "\n",
    "def generate_false_exp(rfdf, every_np, idx: \"\"\"image id\"\"\", n: \"\"\"number of statements per image\"\"\"):\n",
    "    #this function will generate false sentences (hopefully) for a given image\n",
    "    lies = []\n",
    "    eligible_NPs = list(every_np)\n",
    "    random.shuffle(eligible_NPs)\n",
    "    for np in eligible_NPs:\n",
    "        if len(lies) >= n:\n",
    "            break\n",
    "        elif any([word in rfdf.loc[idx][\"vocab\"] for word in np.split()]):\n",
    "            continue\n",
    "        else:\n",
    "            if np.startswith(('a','e','i','o','u')):\n",
    "                article = \"an\"\n",
    "            else:\n",
    "                article = \"a\"\n",
    "            lies.append(f\"There is {article} {np}.\")\n",
    "    return lies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = [generate_true_exp(im_ref_df, x, 2) for x in im_ref_df.index.to_series()]\n",
    "neg = [generate_false_exp(im_ref_df, all_nps, x, 2) for x in im_ref_df.index.to_series()]\n",
    "im_ref_df[\"Positives\"] = pos\n",
    "im_ref_df[\"Negatives\"] = neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "                                                   ref_exps  \\\n",
       "image_id                                                      \n",
       "581563    {'lower left corner darkness': 0, 'bpttom left...   \n",
       "581518    {'top donut': 8, 'top of donuts': 7, 'donut wc...   \n",
       "581346    {'left zebra butt': 13, 'zebra on the left': 1...   \n",
       "581282    {'person bottom left': 18, 'left black shirt':...   \n",
       "580668    {'man on right': 20, 'right man': 21, 'man sit...   \n",
       "\n",
       "                                                      vocab  \\\n",
       "image_id                                                      \n",
       "581563                           {dark, van, corner, black}   \n",
       "581518                                   {sprinkles, donut}   \n",
       "581346                                        {zebra, butt}   \n",
       "581282                               {person, black, shirt}   \n",
       "580668    {yellow, white, sitting, blue, man, bench, car...   \n",
       "\n",
       "                                                     NPs  \\\n",
       "image_id                                                   \n",
       "581563                                       {black van}   \n",
       "581518                                {sprinkles, donut}   \n",
       "581346                               {zebra, zebra butt}   \n",
       "581282                                     {black shirt}   \n",
       "580668    {blue car, man, white car, bench, car, person}   \n",
       "\n",
       "                                            Positives  \\\n",
       "image_id                                                \n",
       "581563                        [There is a black van.]   \n",
       "581518      [There are sprinkles., There is a donut.]   \n",
       "581346    [There is a zebra., There is a zebra butt.]   \n",
       "581282                      [There is a black shirt.]   \n",
       "580668    [There is a person., There is a white car.]   \n",
       "\n",
       "                                                  Negatives  \n",
       "image_id                                                     \n",
       "581563     [There is an area., There is a blue tshirt guy.]  \n",
       "581518    [There is an orange bike., There is a little p...  \n",
       "581346            [There is a coffee cup., There is a pot.]  \n",
       "581282             [There is a pic lady., There is a tree.]  \n",
       "580668          [There is a blk., There is a corner donut.]  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ref_exps</th>\n      <th>vocab</th>\n      <th>NPs</th>\n      <th>Positives</th>\n      <th>Negatives</th>\n    </tr>\n    <tr>\n      <th>image_id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>581563</th>\n      <td>{'lower left corner darkness': 0, 'bpttom left...</td>\n      <td>{dark, van, corner, black}</td>\n      <td>{black van}</td>\n      <td>[There is a black van.]</td>\n      <td>[There is an area., There is a blue tshirt guy.]</td>\n    </tr>\n    <tr>\n      <th>581518</th>\n      <td>{'top donut': 8, 'top of donuts': 7, 'donut wc...</td>\n      <td>{sprinkles, donut}</td>\n      <td>{sprinkles, donut}</td>\n      <td>[There are sprinkles., There is a donut.]</td>\n      <td>[There is an orange bike., There is a little p...</td>\n    </tr>\n    <tr>\n      <th>581346</th>\n      <td>{'left zebra butt': 13, 'zebra on the left': 1...</td>\n      <td>{zebra, butt}</td>\n      <td>{zebra, zebra butt}</td>\n      <td>[There is a zebra., There is a zebra butt.]</td>\n      <td>[There is a coffee cup., There is a pot.]</td>\n    </tr>\n    <tr>\n      <th>581282</th>\n      <td>{'person bottom left': 18, 'left black shirt':...</td>\n      <td>{person, black, shirt}</td>\n      <td>{black shirt}</td>\n      <td>[There is a black shirt.]</td>\n      <td>[There is a pic lady., There is a tree.]</td>\n    </tr>\n    <tr>\n      <th>580668</th>\n      <td>{'man on right': 20, 'right man': 21, 'man sit...</td>\n      <td>{yellow, white, sitting, blue, man, bench, car...</td>\n      <td>{blue car, man, white car, bench, car, person}</td>\n      <td>[There is a person., There is a white car.]</td>\n      <td>[There is a blk., There is a corner donut.]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 29
    }
   ],
   "source": [
    "im_ref_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbs_in_image = refdf_test.groupby(\"image_id\").apply(lambda x: x[\"region_id\"].unique())"
   ]
  },
  {
   "source": [
    "numbers = {num2words(n): n for n in range(0, 20)}\n",
    "\n",
    "def get_np_and_number(sentence: str):\n",
    "    n = 1\n",
    "    sent_parsed = sp(sentence)\n",
    "    try:\n",
    "        head = [word for word in sent_parsed if word.dep_ == \"attr\"][0]\n",
    "    except IndexError:\n",
    "        return None\n",
    "    dependents = [word for word in sent_parsed if word.head == head]\n",
    "    determiner = [word for word in dependents if word.dep_ == \"det\"]\n",
    "    if determiner:\n",
    "        det = determiner[0]\n",
    "        if det in numbers:\n",
    "            n = numbers[det]\n",
    "        refexp = \" \".join([x.text for x in sent_parsed if x == head or x in dependents and x != det])\n",
    "    else:\n",
    "        refexp = \" \".join([x.text for x in sent_parsed if x == head or x in dependents])\n",
    "    return refexp, n"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 31,
   "outputs": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_sent(w2i, all_applied, all_regs, sent, mode:\"\"\"str\"\"\" = \"fuzzy\", threshold:float = None, fuzzy_result:float=None):\n",
    "    parsed = get_np_and_number(sent) #<- this turns a \"There are...\" sentence into the relevant refexp, plus the number of entities quantified\n",
    "    if parsed:\n",
    "        refexp, n = parsed\n",
    "    else:\n",
    "        return \"PARSE_FAIL\"\n",
    "    this_exp_seq = exp2indseq(w2i, refexp) #<- this gets the row numbers for each word in the refexp\n",
    "    if not this_exp_seq:\n",
    "        return \"PARSE_FAIL\"\n",
    "    classifications = [all_applied[all_regs][:, i] for i in this_exp_seq] #<- this is the part that actually runs the classifier\n",
    "    #classical combination\n",
    "    if mode == \"classical\":\n",
    "        if not threshold:\n",
    "            raise ValueError(\"Classical parsing requires a numerical threshold\")\n",
    "        bool_classes = [[ent >= threshold for ent in bb] for bb in classifications]\n",
    "        boolean_output_vals = bool_classes[0]\n",
    "        for classif in bool_classes[1:]:\n",
    "            for i, val in enumerate(classif):\n",
    "                boolean_output_vals[i] = boolean_output_vals[i] and val\n",
    "        return sum(boolean_output_vals) >= n\n",
    "    #fuzzy combination\n",
    "    elif mode in {\"fuzzy\",\"mixed\"}:\n",
    "        if mode == \"mixed\" and fuzzy_result is not None:\n",
    "            return fuzzy_result >= threshold\n",
    "        if len(classifications) > 1:\n",
    "            all_regs_applied = np.multiply.reduce(classifications, axis=0)  #<- multiplies(?) the results\n",
    "        else:\n",
    "            try:\n",
    "                all_regs_applied = classifications[0]\n",
    "            except IndexError:\n",
    "                print(sent, this_exp_seq, classifications, this_exp_seq, refexp, n)\n",
    "                raise IndexError(\"wtf\")\n",
    "        likelihood = prob_at_least(all_regs_applied.tolist(), n)\n",
    "        if mode == \"mixed\":\n",
    "            return likelihood >= threshold\n",
    "        else:\n",
    "            return likelihood"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "metadata": {},
     "execution_count": 33
    }
   ],
   "source": [
    "eval_sent(word2ind, all_applied, imageid2rows(X_idx_ts, 1, 581563), \"There is a black van.\", mode = \"classical\", threshold = 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "eval_sent(word2ind, all_applied, imageid2rows(X_idx_ts, 1, 581563), \"There is a black van.\", mode = \"classical\", threshold = 0.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   image_id                     sentence  gold_label  \\\n",
       "0    581563        There is a black van.        True   \n",
       "1    581563            There is an area.       False   \n",
       "2    581563  There is a blue tshirt guy.       False   \n",
       "3    581518         There are sprinkles.        True   \n",
       "4    581518            There is a donut.        True   \n",
       "\n",
       "                                            all_regs  \n",
       "0  [2335, 2728, 7050, 8093, 10940, 13866, 14299, ...  \n",
       "1  [2335, 2728, 7050, 8093, 10940, 13866, 14299, ...  \n",
       "2  [2335, 2728, 7050, 8093, 10940, 13866, 14299, ...  \n",
       "3             [6082, 6104, 6106, 6111, 11057, 12906]  \n",
       "4             [6082, 6104, 6106, 6111, 11057, 12906]  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>image_id</th>\n      <th>sentence</th>\n      <th>gold_label</th>\n      <th>all_regs</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>581563</td>\n      <td>There is a black van.</td>\n      <td>True</td>\n      <td>[2335, 2728, 7050, 8093, 10940, 13866, 14299, ...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>581563</td>\n      <td>There is an area.</td>\n      <td>False</td>\n      <td>[2335, 2728, 7050, 8093, 10940, 13866, 14299, ...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>581563</td>\n      <td>There is a blue tshirt guy.</td>\n      <td>False</td>\n      <td>[2335, 2728, 7050, 8093, 10940, 13866, 14299, ...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>581518</td>\n      <td>There are sprinkles.</td>\n      <td>True</td>\n      <td>[6082, 6104, 6106, 6111, 11057, 12906]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>581518</td>\n      <td>There is a donut.</td>\n      <td>True</td>\n      <td>[6082, 6104, 6106, 6111, 11057, 12906]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "new_one = []\n",
    "for image_id, row in im_ref_df.iterrows():\n",
    "    for sent in row[\"Positives\"]:\n",
    "        new_one.append([image_id, sent, True])\n",
    "    for sent in row[\"Negatives\"]:\n",
    "        new_one.append([image_id, sent, False])\n",
    "sent_df = pd.DataFrame(new_one, columns=[\"image_id\", \"sentence\", \"gold_label\"])\n",
    "sent_df[\"all_regs\"] = sent_df.apply(lambda row: imageid2rows(X_idx_ts, 1, row[\"image_id\"]), axis=1)\n",
    "sent_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[2335, 2728, 7050, 8093, 10940, 13866, 14299, 14300, 14301]"
      ]
     },
     "metadata": {},
     "execution_count": 36
    }
   ],
   "source": [
    "sent_df.loc[0][\"all_regs\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_eval_corpus(im_df, w2i, all_applied, thresholds):\n",
    "    classical_threshes = []\n",
    "    mixed_threshes = []\n",
    "    thresh_key = {}\n",
    "    for t in thresholds:\n",
    "        col_name = \"classical@{}\".format(t)\n",
    "        im_df[col_name] = im_df.apply(lambda row: eval_sent(w2i, all_applied, row[\"all_regs\"], row[\"sentence\"], mode = \"classical\", threshold=t), axis=1)\n",
    "    im_df[\"fuzzy_result\"] = im_df.apply(lambda row: eval_sent(w2i, all_applied, row[\"all_regs\"], row[\"sentence\"], mode = \"fuzzy\"), axis=1)\n",
    "    for t in thresholds:\n",
    "        col_name = \"mixed@{}\".format(t)\n",
    "        im_df[col_name] = im_df.apply(lambda row: eval_sent(w2i, all_applied, row[\"all_regs\"], row[\"sentence\"], mode = \"mixed\", threshold=t, fuzzy_result = row[\"fuzzy_result\"]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_eval_corpus(sent_df, word2ind, all_applied, [0.0001,0.00025,0.0005,0.00075,0.001,0.0025,0.005,0.0075,0.01,0.025,0.05,0.075])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "   image_id                     sentence  gold_label  \\\n",
       "0    581563        There is a black van.        True   \n",
       "1    581563            There is an area.       False   \n",
       "2    581563  There is a blue tshirt guy.       False   \n",
       "3    581518         There are sprinkles.        True   \n",
       "4    581518            There is a donut.        True   \n",
       "\n",
       "                                            all_regs classical@0.0001  \\\n",
       "0  [2335, 2728, 7050, 8093, 10940, 13866, 14299, ...             True   \n",
       "1  [2335, 2728, 7050, 8093, 10940, 13866, 14299, ...            False   \n",
       "2  [2335, 2728, 7050, 8093, 10940, 13866, 14299, ...            False   \n",
       "3             [6082, 6104, 6106, 6111, 11057, 12906]             True   \n",
       "4             [6082, 6104, 6106, 6111, 11057, 12906]             True   \n",
       "\n",
       "  classical@0.00025 classical@0.0005 classical@0.00075 classical@0.001  \\\n",
       "0              True             True              True            True   \n",
       "1             False            False             False           False   \n",
       "2             False            False             False           False   \n",
       "3              True             True              True            True   \n",
       "4              True             True              True            True   \n",
       "\n",
       "  classical@0.0025  ... mixed@0.0005 mixed@0.00075 mixed@0.001 mixed@0.0025  \\\n",
       "0             True  ...         True          True        True         True   \n",
       "1            False  ...        False         False       False        False   \n",
       "2            False  ...        False         False       False        False   \n",
       "3            False  ...         True          True        True        False   \n",
       "4             True  ...         True          True        True         True   \n",
       "\n",
       "  mixed@0.005 mixed@0.0075 mixed@0.01 mixed@0.025 mixed@0.05 mixed@0.075  \n",
       "0        True         True       True        True       True        True  \n",
       "1       False        False      False       False      False       False  \n",
       "2       False        False      False       False      False       False  \n",
       "3       False        False      False       False      False       False  \n",
       "4        True         True       True        True       True        True  \n",
       "\n",
       "[5 rows x 29 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>image_id</th>\n      <th>sentence</th>\n      <th>gold_label</th>\n      <th>all_regs</th>\n      <th>classical@0.0001</th>\n      <th>classical@0.00025</th>\n      <th>classical@0.0005</th>\n      <th>classical@0.00075</th>\n      <th>classical@0.001</th>\n      <th>classical@0.0025</th>\n      <th>...</th>\n      <th>mixed@0.0005</th>\n      <th>mixed@0.00075</th>\n      <th>mixed@0.001</th>\n      <th>mixed@0.0025</th>\n      <th>mixed@0.005</th>\n      <th>mixed@0.0075</th>\n      <th>mixed@0.01</th>\n      <th>mixed@0.025</th>\n      <th>mixed@0.05</th>\n      <th>mixed@0.075</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>581563</td>\n      <td>There is a black van.</td>\n      <td>True</td>\n      <td>[2335, 2728, 7050, 8093, 10940, 13866, 14299, ...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>581563</td>\n      <td>There is an area.</td>\n      <td>False</td>\n      <td>[2335, 2728, 7050, 8093, 10940, 13866, 14299, ...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>581563</td>\n      <td>There is a blue tshirt guy.</td>\n      <td>False</td>\n      <td>[2335, 2728, 7050, 8093, 10940, 13866, 14299, ...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>...</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>581518</td>\n      <td>There are sprinkles.</td>\n      <td>True</td>\n      <td>[6082, 6104, 6106, 6111, 11057, 12906]</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>581518</td>\n      <td>There is a donut.</td>\n      <td>True</td>\n      <td>[6082, 6104, 6106, 6111, 11057, 12906]</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>...</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n      <td>True</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 29 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "source": [
    "sent_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_df.to_pickle(\"./data/processed_data.pkl\")"
   ]
  }
 ]
}