vqa/2-nli-eval/second_run/20220525 NLI Save (10000 images; num_answers = 2; num_choices = 2; not_redundant = True; repeated_comparisons = False, group_count = num_choices) vqa lxmert-test-mnli-EMERGENCY.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stores NLI For LXMERT 10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModelForSeq2SeqLM\n",
    "import torch\n",
    "from pysat.formula import IDPool, WCNFPlus\n",
    "from pysat.examples.rc2 import RC2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "\n",
    "# custom modules\n",
    "import sys\n",
    "sys.path.append('./nlic')\n",
    "import qa_converter\n",
    "import nli\n",
    "import solver\n",
    "\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: roberta-large-mnli\n",
      "Save path: /data/nli-consistency/vqa/lxmert-test-3pred-40token-1seed_predictions_nli-mnli.json\n",
      "Data path: /data/nli-consistency/vqa/lxmert-test-3pred-40token-1seed_predictions_nli.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at roberta-large-mnli were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']\n",
      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image #: 1 image number 2342914\n",
      "image #: 2 image number 2342916\n",
      "image #: 3 image number 2342919\n",
      "image #: 4 image number 2342921\n",
      "image #: 5 image number 2342923\n",
      "image #: 6 image number 2342924\n",
      "image #: 7 image number 2342925\n",
      "image #: 8 image number 2342926\n",
      "image #: 9 image number 2342927\n",
      "image #: 10 image number 2342933\n",
      "image #: 11 image number 2342934\n",
      "image #: 12 image number 2342937\n",
      "image #: 13 image number 2342938\n",
      "image #: 14 image number 2342939\n",
      "image #: 15 image number 2342940\n",
      "image #: 16 image number 2342941\n",
      "image #: 17 image number 2342943\n",
      "image #: 18 image number 2342944\n",
      "image #: 19 image number 2342952\n",
      "image #: 20 image number 2342953\n",
      "image #: 21 image number 2342954\n",
      "image #: 22 image number 2342956\n",
      "image #: 23 image number 2342959\n",
      "image #: 24 image number 2342960\n",
      "image #: 25 image number 2342961\n",
      "image #: 26 image number 2342963\n",
      "image #: 27 image number 2342965\n",
      "image #: 28 image number 2342966\n",
      "image #: 29 image number 2342967\n",
      "image #: 30 image number 2342971\n",
      "image #: 31 image number 2342972\n",
      "image #: 32 image number 2342976\n",
      "image #: 33 image number 2342977\n",
      "image #: 34 image number 2342978\n",
      "image #: 35 image number 2342980\n",
      "image #: 36 image number 2342981\n",
      "image #: 37 image number 2342982\n",
      "image #: 38 image number 2342983\n",
      "image #: 39 image number 2342986\n",
      "image #: 40 image number 2342987\n",
      "image #: 41 image number 2342989\n",
      "image #: 42 image number 2342990\n",
      "image #: 43 image number 2342991\n",
      "image #: 44 image number 2342992\n",
      "image #: 45 image number 2342993\n",
      "image #: 46 image number 2342995\n",
      "image #: 47 image number 2342996\n",
      "image #: 48 image number 2342999\n",
      "image #: 49 image number 2343000\n",
      "image #: 50 image number 2343002\n",
      "image #: 51 image number 2343004\n",
      "image #: 52 image number 2343006\n",
      "image #: 53 image number 2343008\n",
      "image #: 54 image number 2343009\n",
      "image #: 55 image number 2343010\n",
      "image #: 56 image number 2343011\n",
      "image #: 57 image number 2343012\n",
      "image #: 58 image number 2343014\n",
      "image #: 59 image number 2343015\n",
      "image #: 60 image number 2343016\n",
      "image #: 61 image number 2343017\n",
      "image #: 62 image number 2343018\n",
      "image #: 63 image number 2343019\n",
      "image #: 64 image number 2343020\n",
      "image #: 65 image number 2343021\n",
      "image #: 66 image number 2343022\n",
      "image #: 67 image number 2343024\n",
      "image #: 68 image number 2343025\n",
      "image #: 69 image number 2343027\n",
      "image #: 70 image number 2343028\n",
      "image #: 71 image number 2343029\n",
      "image #: 72 image number 2343030\n",
      "image #: 73 image number 2343031\n",
      "image #: 74 image number 2343032\n",
      "image #: 75 image number 2343033\n",
      "image #: 76 image number 2343034\n",
      "image #: 77 image number 2343036\n",
      "image #: 78 image number 2343037\n",
      "image #: 79 image number 2343038\n",
      "image #: 80 image number 2343039\n",
      "image #: 81 image number 2343040\n",
      "image #: 82 image number 2343043\n",
      "image #: 83 image number 2343044\n",
      "image #: 84 image number 2343045\n",
      "image #: 85 image number 2343046\n",
      "image #: 86 image number 2343047\n",
      "image #: 87 image number 2343048\n",
      "image #: 88 image number 2343049\n",
      "image #: 89 image number 2343050\n",
      "image #: 90 image number 2343051\n",
      "image #: 91 image number 2343052\n",
      "image #: 92 image number 2343053\n",
      "image #: 93 image number 2343054\n",
      "image #: 94 image number 2343056\n",
      "image #: 95 image number 2343057\n",
      "image #: 96 image number 2343058\n",
      "image #: 97 image number 2343059\n",
      "image #: 98 image number 2343060\n",
      "image #: 99 image number 2343064\n",
      "image #: 100 image number 2343067\n",
      "image #: 101 image number 2343071\n",
      "image #: 102 image number 2343072\n",
      "image #: 103 image number 2343074\n",
      "image #: 104 image number 2343076\n",
      "image #: 105 image number 2343077\n",
      "image #: 106 image number 2343078\n",
      "image #: 107 image number 2343079\n",
      "image #: 108 image number 2343081\n",
      "image #: 109 image number 2343082\n",
      "image #: 110 image number 2343083\n",
      "image #: 111 image number 2343084\n",
      "image #: 112 image number 2343085\n",
      "image #: 113 image number 2343086\n",
      "image #: 114 image number 2343087\n",
      "image #: 115 image number 2343088\n",
      "image #: 116 image number 2343092\n",
      "image #: 117 image number 2343093\n",
      "image #: 118 image number 2343094\n",
      "image #: 119 image number 2343095\n",
      "image #: 120 image number 2343097\n",
      "image #: 121 image number 2343100\n",
      "image #: 122 image number 2343101\n",
      "image #: 123 image number 2343103\n",
      "image #: 124 image number 2343104\n",
      "image #: 125 image number 2343106\n",
      "image #: 126 image number 2343107\n",
      "image #: 127 image number 2343109\n",
      "image #: 128 image number 2343111\n",
      "image #: 129 image number 2343112\n",
      "image #: 130 image number 2343114\n",
      "image #: 131 image number 2343115\n",
      "image #: 132 image number 2343116\n",
      "image #: 133 image number 2343118\n",
      "image #: 134 image number 2343121\n",
      "image #: 135 image number 2343125\n",
      "image #: 136 image number 2343126\n",
      "image #: 137 image number 2343127\n",
      "image #: 138 image number 2343128\n",
      "image #: 139 image number 2343131\n",
      "image #: 140 image number 2343132\n",
      "image #: 141 image number 2343133\n",
      "image #: 142 image number 2343134\n",
      "image #: 143 image number 2343135\n",
      "image #: 144 image number 2343136\n",
      "image #: 145 image number 2341974\n",
      "image #: 146 image number 2341975\n",
      "image #: 147 image number 2341976\n",
      "image #: 148 image number 2341977\n",
      "image #: 149 image number 2341979\n",
      "image #: 150 image number 2341983\n",
      "image #: 151 image number 2341984\n",
      "image #: 152 image number 2341985\n",
      "image #: 153 image number 2341986\n",
      "image #: 154 image number 2341987\n",
      "image #: 155 image number 2341988\n",
      "image #: 156 image number 2341989\n",
      "image #: 157 image number 2341992\n",
      "image #: 158 image number 2341994\n",
      "image #: 159 image number 2341995\n",
      "image #: 160 image number 2341996\n",
      "image #: 161 image number 2341997\n",
      "image #: 162 image number 2341999\n",
      "image #: 163 image number 2342001\n",
      "image #: 164 image number 2342002\n",
      "image #: 165 image number 2342004\n",
      "image #: 166 image number 2342005\n",
      "image #: 167 image number 2342006\n",
      "image #: 168 image number 2342007\n",
      "image #: 169 image number 2342008\n",
      "image #: 170 image number 2342009\n",
      "image #: 171 image number 2342010\n",
      "image #: 172 image number 2342011\n",
      "image #: 173 image number 2342013\n",
      "image #: 174 image number 2342016\n",
      "image #: 175 image number 2342017\n",
      "image #: 176 image number 2342020\n",
      "image #: 177 image number 2342021\n",
      "image #: 178 image number 2342022\n",
      "image #: 179 image number 2342023\n",
      "image #: 180 image number 2342024\n",
      "image #: 181 image number 2342025\n",
      "image #: 182 image number 2342026\n",
      "image #: 183 image number 2342027\n",
      "image #: 184 image number 2342028\n",
      "image #: 185 image number 2342029\n",
      "image #: 186 image number 2342030\n",
      "image #: 187 image number 2342032\n",
      "image #: 188 image number 2342033\n",
      "image #: 189 image number 2342034\n",
      "image #: 190 image number 2342036\n",
      "image #: 191 image number 2342037\n",
      "image #: 192 image number 2342038\n",
      "image #: 193 image number 2342039\n",
      "image #: 194 image number 2342040\n",
      "image #: 195 image number 2342044\n",
      "image #: 196 image number 2342045\n",
      "image #: 197 image number 2342046\n",
      "image #: 198 image number 2342047\n",
      "image #: 199 image number 2342048\n",
      "image #: 200 image number 2342051\n",
      "image #: 201 image number 2342052\n",
      "image #: 202 image number 2342054\n",
      "image #: 203 image number 2342057\n",
      "image #: 204 image number 2342064\n",
      "image #: 205 image number 2342065\n",
      "image #: 206 image number 2342066\n",
      "image #: 207 image number 2342072\n",
      "image #: 208 image number 2342075\n",
      "image #: 209 image number 2342076\n",
      "image #: 210 image number 2342080\n",
      "image #: 211 image number 2342081\n",
      "image #: 212 image number 2342084\n",
      "image #: 213 image number 2342086\n",
      "image #: 214 image number 2342087\n",
      "image #: 215 image number 2342088\n",
      "image #: 216 image number 2342090\n",
      "image #: 217 image number 2342092\n",
      "image #: 218 image number 2342093\n",
      "image #: 219 image number 2342094\n",
      "image #: 220 image number 2342096\n",
      "image #: 221 image number 2342097\n",
      "image #: 222 image number 2342098\n",
      "image #: 223 image number 2342099\n",
      "image #: 224 image number 2342100\n",
      "image #: 225 image number 2342101\n",
      "image #: 226 image number 2342102\n",
      "image #: 227 image number 2342103\n",
      "image #: 228 image number 2342104\n",
      "image #: 229 image number 2342105\n",
      "image #: 230 image number 2342108\n",
      "image #: 231 image number 2342109\n",
      "image #: 232 image number 2342110\n",
      "image #: 233 image number 2342111\n",
      "image #: 234 image number 2342112\n",
      "image #: 235 image number 2342113\n",
      "image #: 236 image number 2342114\n",
      "image #: 237 image number 2342115\n",
      "image #: 238 image number 2342116\n",
      "image #: 239 image number 2342117\n",
      "image #: 240 image number 2342120\n",
      "image #: 241 image number 2342122\n",
      "image #: 242 image number 2342123\n",
      "image #: 243 image number 2342130\n",
      "image #: 244 image number 2342132\n",
      "image #: 245 image number 2342133\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image #: 246 image number 2342134\n",
      "image #: 247 image number 2342137\n",
      "image #: 248 image number 2342138\n",
      "image #: 249 image number 2342139\n",
      "image #: 250 image number 2342141\n",
      "image #: 251 image number 2342142\n",
      "image #: 252 image number 2342143\n",
      "image #: 253 image number 2342145\n",
      "image #: 254 image number 2342147\n",
      "image #: 255 image number 2342148\n",
      "image #: 256 image number 2342150\n",
      "image #: 257 image number 2342151\n",
      "image #: 258 image number 2342152\n",
      "image #: 259 image number 2342154\n",
      "image #: 260 image number 2342155\n",
      "image #: 261 image number 2342157\n",
      "image #: 262 image number 2342158\n",
      "image #: 263 image number 2342159\n",
      "image #: 264 image number 2342160\n",
      "image #: 265 image number 2342161\n",
      "image #: 266 image number 2342162\n",
      "image #: 267 image number 2342163\n",
      "image #: 268 image number 2342164\n",
      "image #: 269 image number 2342165\n",
      "image #: 270 image number 2342166\n",
      "image #: 271 image number 2342167\n",
      "image #: 272 image number 2342169\n",
      "image #: 273 image number 2342170\n",
      "image #: 274 image number 2342171\n",
      "image #: 275 image number 2342172\n",
      "image #: 276 image number 2342174\n",
      "image #: 277 image number 2342175\n",
      "image #: 278 image number 2342176\n",
      "image #: 279 image number 2342180\n",
      "image #: 280 image number 2342181\n",
      "image #: 281 image number 2342183\n",
      "image #: 282 image number 2342185\n",
      "image #: 283 image number 2342187\n",
      "image #: 284 image number 2342188\n",
      "image #: 285 image number 2342190\n",
      "image #: 286 image number 2342191\n",
      "image #: 287 image number 2342194\n",
      "image #: 288 image number 2342196\n",
      "image #: 289 image number 2342198\n",
      "image #: 290 image number 2342201\n",
      "image #: 291 image number 2342203\n",
      "image #: 292 image number 2342206\n",
      "image #: 293 image number 2342207\n",
      "image #: 294 image number 2342208\n",
      "image #: 295 image number 2342209\n",
      "image #: 296 image number 2342210\n",
      "image #: 297 image number 2342212\n",
      "image #: 298 image number 2342213\n",
      "image #: 299 image number 2342214\n",
      "image #: 300 image number 2342216\n",
      "image #: 301 image number 2342217\n",
      "image #: 302 image number 2342218\n",
      "image #: 303 image number 2342221\n",
      "image #: 304 image number 2342222\n",
      "image #: 305 image number 2342223\n",
      "image #: 306 image number 2342224\n",
      "image #: 307 image number 2342225\n",
      "image #: 308 image number 2342226\n",
      "image #: 309 image number 2342227\n",
      "image #: 310 image number 2342229\n",
      "image #: 311 image number 2342231\n",
      "image #: 312 image number 2342233\n",
      "image #: 313 image number 2342235\n",
      "image #: 314 image number 2342236\n",
      "image #: 315 image number 2342237\n",
      "image #: 316 image number 2342238\n",
      "image #: 317 image number 2342239\n",
      "image #: 318 image number 2342240\n",
      "image #: 319 image number 2342243\n",
      "image #: 320 image number 2342247\n",
      "image #: 321 image number 2342248\n",
      "image #: 322 image number 2342251\n",
      "image #: 323 image number 2342252\n",
      "image #: 324 image number 2342253\n",
      "image #: 325 image number 2342255\n",
      "image #: 326 image number 2342256\n",
      "image #: 327 image number 2342257\n",
      "image #: 328 image number 2342260\n",
      "image #: 329 image number 2342261\n",
      "image #: 330 image number 2342264\n",
      "image #: 331 image number 2342265\n",
      "image #: 332 image number 2342266\n",
      "image #: 333 image number 2342268\n",
      "image #: 334 image number 2342269\n",
      "image #: 335 image number 2342270\n",
      "image #: 336 image number 2342271\n",
      "image #: 337 image number 2342274\n",
      "image #: 338 image number 2342275\n",
      "image #: 339 image number 2342276\n",
      "image #: 340 image number 2342277\n",
      "image #: 341 image number 2342278\n",
      "image #: 342 image number 2342280\n",
      "image #: 343 image number 2342281\n",
      "image #: 344 image number 2342282\n",
      "image #: 345 image number 2342283\n",
      "image #: 346 image number 2342284\n",
      "image #: 347 image number 2342285\n",
      "image #: 348 image number 2342286\n",
      "image #: 349 image number 2342291\n",
      "image #: 350 image number 2342292\n",
      "image #: 351 image number 2342293\n",
      "image #: 352 image number 2342294\n",
      "image #: 353 image number 2342295\n",
      "image #: 354 image number 2342296\n",
      "image #: 355 image number 2342299\n",
      "image #: 356 image number 2342304\n",
      "image #: 357 image number 2342305\n",
      "image #: 358 image number 2342307\n",
      "image #: 359 image number 2342308\n",
      "image #: 360 image number 2342311\n",
      "image #: 361 image number 2342312\n",
      "image #: 362 image number 2342313\n",
      "image #: 363 image number 2342314\n",
      "image #: 364 image number 2342316\n",
      "image #: 365 image number 2342317\n",
      "image #: 366 image number 2342319\n",
      "image #: 367 image number 2342320\n",
      "image #: 368 image number 2342321\n",
      "image #: 369 image number 2342323\n",
      "image #: 370 image number 2342324\n",
      "image #: 371 image number 2342325\n",
      "image #: 372 image number 2342327\n",
      "image #: 373 image number 2342329\n",
      "image #: 374 image number 2342330\n",
      "image #: 375 image number 2342331\n",
      "image #: 376 image number 2342332\n",
      "image #: 377 image number 2342333\n",
      "image #: 378 image number 2342336\n",
      "image #: 379 image number 2342338\n",
      "image #: 380 image number 2342340\n",
      "image #: 381 image number 2342342\n",
      "image #: 382 image number 2342343\n",
      "image #: 383 image number 2342345\n",
      "image #: 384 image number 2342346\n",
      "image #: 385 image number 2342347\n",
      "image #: 386 image number 2342350\n",
      "image #: 387 image number 2342351\n",
      "image #: 388 image number 2342352\n",
      "image #: 389 image number 2342353\n",
      "image #: 390 image number 2342355\n",
      "image #: 391 image number 2342356\n",
      "image #: 392 image number 2342357\n",
      "image #: 393 image number 2342359\n",
      "image #: 394 image number 2342362\n",
      "image #: 395 image number 2342364\n",
      "image #: 396 image number 2342365\n",
      "image #: 397 image number 2342368\n",
      "image #: 398 image number 2342370\n",
      "image #: 399 image number 2342371\n",
      "image #: 400 image number 2342374\n",
      "image #: 401 image number 2342375\n",
      "image #: 402 image number 2342377\n",
      "image #: 403 image number 2342378\n",
      "image #: 404 image number 2342380\n",
      "image #: 405 image number 2342382\n",
      "image #: 406 image number 2342384\n",
      "image #: 407 image number 2342385\n",
      "image #: 408 image number 2342387\n",
      "image #: 409 image number 2342388\n",
      "image #: 410 image number 2342391\n",
      "image #: 411 image number 2342393\n",
      "image #: 412 image number 2342394\n",
      "image #: 413 image number 2342395\n",
      "image #: 414 image number 2342396\n",
      "image #: 415 image number 2342400\n",
      "image #: 416 image number 2342401\n",
      "image #: 417 image number 2342405\n",
      "image #: 418 image number 2342410\n",
      "image #: 419 image number 2342414\n",
      "image #: 420 image number 2342415\n",
      "image #: 421 image number 2342418\n",
      "image #: 422 image number 2342419\n",
      "image #: 423 image number 2342420\n",
      "image #: 424 image number 2342421\n",
      "image #: 425 image number 2342423\n",
      "image #: 426 image number 2342425\n",
      "image #: 427 image number 2342426\n",
      "image #: 428 image number 2342427\n",
      "image #: 429 image number 2342429\n",
      "image #: 430 image number 2342430\n",
      "image #: 431 image number 2342431\n",
      "image #: 432 image number 2342432\n",
      "image #: 433 image number 2342433\n",
      "image #: 434 image number 2342434\n",
      "image #: 435 image number 2342435\n",
      "image #: 436 image number 2342437\n",
      "image #: 437 image number 2342439\n",
      "image #: 438 image number 2342440\n",
      "image #: 439 image number 2342442\n",
      "image #: 440 image number 2342444\n",
      "image #: 441 image number 2342445\n",
      "image #: 442 image number 2342447\n",
      "image #: 443 image number 2342448\n",
      "image #: 444 image number 2342451\n",
      "image #: 445 image number 2342452\n",
      "image #: 446 image number 2342453\n",
      "image #: 447 image number 2342454\n",
      "image #: 448 image number 2342455\n",
      "image #: 449 image number 2342456\n",
      "image #: 450 image number 2342457\n",
      "image #: 451 image number 2342459\n",
      "image #: 452 image number 2342461\n",
      "image #: 453 image number 2342462\n",
      "image #: 454 image number 2342463\n",
      "image #: 455 image number 2342465\n",
      "image #: 456 image number 2342467\n",
      "image #: 457 image number 2342468\n",
      "image #: 458 image number 2342470\n",
      "image #: 459 image number 2342472\n",
      "image #: 460 image number 2342474\n",
      "image #: 461 image number 2342476\n",
      "image #: 462 image number 2342479\n",
      "image #: 463 image number 2342481\n",
      "image #: 464 image number 2342482\n",
      "image #: 465 image number 2342483\n",
      "image #: 466 image number 2342484\n",
      "image #: 467 image number 2342485\n",
      "image #: 468 image number 2342486\n",
      "image #: 469 image number 2342487\n",
      "image #: 470 image number 2342488\n",
      "image #: 471 image number 2342489\n",
      "image #: 472 image number 2342490\n",
      "image #: 473 image number 2342493\n",
      "image #: 474 image number 2342494\n",
      "image #: 475 image number 2342496\n",
      "image #: 476 image number 2342497\n",
      "image #: 477 image number 2342499\n",
      "image #: 478 image number 2342500\n",
      "image #: 479 image number 2342501\n",
      "image #: 480 image number 2342502\n",
      "image #: 481 image number 2342503\n",
      "image #: 482 image number 2342504\n",
      "image #: 483 image number 2342505\n",
      "image #: 484 image number 2342506\n",
      "image #: 485 image number 2342507\n",
      "image #: 486 image number 2342510\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "image #: 487 image number 2342511\n",
      "image #: 488 image number 2342512\n",
      "image #: 489 image number 2342514\n",
      "image #: 490 image number 2342515\n",
      "image #: 491 image number 2342519\n",
      "image #: 492 image number 2342520\n",
      "image #: 493 image number 2342521\n",
      "image #: 494 image number 2342523\n",
      "image #: 495 image number 2342525\n",
      "image #: 496 image number 2342527\n",
      "image #: 497 image number 2342528\n",
      "image #: 498 image number 2342530\n",
      "image #: 499 image number 2342531\n",
      "image #: 500 image number 2342532\n",
      "image #: 501 image number 2342533\n",
      "image #: 502 image number 2342534\n",
      "image #: 503 image number 2342535\n",
      "image #: 504 image number 2342536\n",
      "image #: 505 image number 2342537\n",
      "image #: 506 image number 2342538\n",
      "image #: 507 image number 2342540\n",
      "image #: 508 image number 2342541\n",
      "image #: 509 image number 2342544\n",
      "image #: 510 image number 2342545\n",
      "image #: 511 image number 2342548\n",
      "image #: 512 image number 2342549\n",
      "image #: 513 image number 2342550\n",
      "image #: 514 image number 2342551\n",
      "image #: 515 image number 2342553\n",
      "image #: 516 image number 2342554\n",
      "image #: 517 image number 2342555\n",
      "image #: 518 image number 2342557\n",
      "image #: 519 image number 2342558\n",
      "image #: 520 image number 2342560\n",
      "image #: 521 image number 2342562\n",
      "image #: 522 image number 2342563\n",
      "image #: 523 image number 2342565\n",
      "image #: 524 image number 2342569\n",
      "image #: 525 image number 2342573\n",
      "image #: 526 image number 2342575\n",
      "image #: 527 image number 2342576\n",
      "image #: 528 image number 2342577\n",
      "image #: 529 image number 2342578\n",
      "image #: 530 image number 2342579\n",
      "image #: 531 image number 2342580\n",
      "image #: 532 image number 2342581\n",
      "image #: 533 image number 2342582\n",
      "image #: 534 image number 2342584\n",
      "image #: 535 image number 2342586\n",
      "image #: 536 image number 2342587\n",
      "image #: 537 image number 2342589\n",
      "image #: 538 image number 2342591\n",
      "image #: 539 image number 2342595\n",
      "image #: 540 image number 2342596\n",
      "image #: 541 image number 2342598\n",
      "image #: 542 image number 2342599\n",
      "image #: 543 image number 2342600\n",
      "image #: 544 image number 2342601\n",
      "image #: 545 image number 2342602\n",
      "image #: 546 image number 2342603\n",
      "image #: 547 image number 2342605\n",
      "image #: 548 image number 2342606\n",
      "image #: 549 image number 2342612\n",
      "image #: 550 image number 2342614\n",
      "image #: 551 image number 2342615\n",
      "image #: 552 image number 2342618\n",
      "image #: 553 image number 2342619\n",
      "image #: 554 image number 2342621\n",
      "image #: 555 image number 2342622\n",
      "image #: 556 image number 2342624\n",
      "image #: 557 image number 2342626\n",
      "image #: 558 image number 2342632\n",
      "image #: 559 image number 2342633\n",
      "image #: 560 image number 2342640\n",
      "image #: 561 image number 2342641\n",
      "image #: 562 image number 2342643\n",
      "image #: 563 image number 2342644\n",
      "image #: 564 image number 2342645\n",
      "image #: 565 image number 2342647\n",
      "image #: 566 image number 2342648\n",
      "image #: 567 image number 2342649\n",
      "image #: 568 image number 2342651\n",
      "image #: 569 image number 2342652\n",
      "image #: 570 image number 2342653\n",
      "image #: 571 image number 2342654\n",
      "image #: 572 image number 2342656\n",
      "image #: 573 image number 2342657\n",
      "image #: 574 image number 2342658\n",
      "image #: 575 image number 2342659\n",
      "image #: 576 image number 2342662\n",
      "image #: 577 image number 2342665\n",
      "image #: 578 image number 2342666\n",
      "image #: 579 image number 2342667\n",
      "image #: 580 image number 2342668\n",
      "image #: 581 image number 2342669\n",
      "image #: 582 image number 2342671\n",
      "image #: 583 image number 2342672\n",
      "image #: 584 image number 2342673\n",
      "image #: 585 image number 2342675\n",
      "image #: 586 image number 2342679\n",
      "image #: 587 image number 2342680\n",
      "image #: 588 image number 2342683\n",
      "image #: 589 image number 2342684\n",
      "image #: 590 image number 2342685\n",
      "image #: 591 image number 2342687\n",
      "image #: 592 image number 2342688\n",
      "image #: 593 image number 2342690\n",
      "image #: 594 image number 2342693\n",
      "image #: 595 image number 2342694\n",
      "image #: 596 image number 2342697\n",
      "image #: 597 image number 2342699\n",
      "image #: 598 image number 2342700\n",
      "image #: 599 image number 2342701\n",
      "image #: 600 image number 2342703\n",
      "image #: 601 image number 2342705\n",
      "image #: 602 image number 2342708\n",
      "image #: 603 image number 2342709\n",
      "image #: 604 image number 2342710\n",
      "image #: 605 image number 2342712\n",
      "image #: 606 image number 2342713\n",
      "image #: 607 image number 2342715\n",
      "image #: 608 image number 2342717\n",
      "image #: 609 image number 2342719\n",
      "image #: 610 image number 2342721\n",
      "image #: 611 image number 2342722\n",
      "image #: 612 image number 2342724\n",
      "image #: 613 image number 2342725\n",
      "image #: 614 image number 2342726\n",
      "image #: 615 image number 2342727\n",
      "image #: 616 image number 2342728\n",
      "image #: 617 image number 2342729\n",
      "image #: 618 image number 2342730\n",
      "image #: 619 image number 2342731\n",
      "image #: 620 image number 2342732\n",
      "image #: 621 image number 2342735\n",
      "image #: 622 image number 2342736\n",
      "image #: 623 image number 2342738\n",
      "image #: 624 image number 2342740\n",
      "image #: 625 image number 2342741\n",
      "image #: 626 image number 2342743\n",
      "image #: 627 image number 2342744\n",
      "image #: 628 image number 2342747\n",
      "image #: 629 image number 2342749\n",
      "image #: 630 image number 2342750\n",
      "image #: 631 image number 2342751\n",
      "image #: 632 image number 2342753\n",
      "image #: 633 image number 2342755\n",
      "image #: 634 image number 2342756\n",
      "image #: 635 image number 2342757\n",
      "image #: 636 image number 2342758\n",
      "image #: 637 image number 2342759\n",
      "image #: 638 image number 2342760\n",
      "image #: 639 image number 2342761\n",
      "image #: 640 image number 2342762\n",
      "image #: 641 image number 2342764\n",
      "image #: 642 image number 2342765\n",
      "image #: 643 image number 2342767\n",
      "image #: 644 image number 2342768\n",
      "image #: 645 image number 2342769\n",
      "image #: 646 image number 2342770\n",
      "image #: 647 image number 2342771\n",
      "image #: 648 image number 2342772\n",
      "image #: 649 image number 2342773\n",
      "image #: 650 image number 2342776\n",
      "image #: 651 image number 2342779\n",
      "image #: 652 image number 2342781\n",
      "image #: 653 image number 2342782\n",
      "image #: 654 image number 2342787\n",
      "image #: 655 image number 2342788\n",
      "image #: 656 image number 2342789\n",
      "image #: 657 image number 2342790\n",
      "image #: 658 image number 2342791\n",
      "image #: 659 image number 2342793\n",
      "image #: 660 image number 2342796\n",
      "image #: 661 image number 2342798\n",
      "image #: 662 image number 2342800\n",
      "image #: 663 image number 2342801\n",
      "image #: 664 image number 2342802\n",
      "image #: 665 image number 2342803\n",
      "image #: 666 image number 2342806\n",
      "image #: 667 image number 2342807\n",
      "image #: 668 image number 2342811\n",
      "image #: 669 image number 2342812\n",
      "image #: 670 image number 2342813\n",
      "image #: 671 image number 2342814\n",
      "image #: 672 image number 2342815\n",
      "image #: 673 image number 2342817\n",
      "image #: 674 image number 2342818\n",
      "image #: 675 image number 2342823\n",
      "image #: 676 image number 2342824\n",
      "image #: 677 image number 2342829\n",
      "image #: 678 image number 2342830\n",
      "image #: 679 image number 2342833\n",
      "image #: 680 image number 2342834\n",
      "image #: 681 image number 2342835\n",
      "image #: 682 image number 2342836\n",
      "image #: 683 image number 2342837\n",
      "image #: 684 image number 2342839\n",
      "image #: 685 image number 2342846\n",
      "image #: 686 image number 2342847\n",
      "image #: 687 image number 2342848\n",
      "image #: 688 image number 2342849\n",
      "image #: 689 image number 2342851\n",
      "image #: 690 image number 2342852\n",
      "image #: 691 image number 2342855\n",
      "image #: 692 image number 2342856\n",
      "image #: 693 image number 2342857\n",
      "image #: 694 image number 2342858\n",
      "image #: 695 image number 2342859\n",
      "image #: 696 image number 2342861\n",
      "image #: 697 image number 2342862\n",
      "image #: 698 image number 2342864\n",
      "image #: 699 image number 2342865\n",
      "image #: 700 image number 2342866\n",
      "image #: 701 image number 2342868\n",
      "image #: 702 image number 2342872\n",
      "image #: 703 image number 2342874\n",
      "image #: 704 image number 2342876\n",
      "image #: 705 image number 2342879\n",
      "image #: 706 image number 2342880\n",
      "image #: 707 image number 2342881\n",
      "image #: 708 image number 2342882\n",
      "image #: 709 image number 2342886\n",
      "image #: 710 image number 2342888\n",
      "image #: 711 image number 2342889\n",
      "image #: 712 image number 2342891\n",
      "image #: 713 image number 2342892\n",
      "image #: 714 image number 2342893\n",
      "image #: 715 image number 2342894\n",
      "image #: 716 image number 2342895\n",
      "image #: 717 image number 2342896\n",
      "image #: 718 image number 2342898\n",
      "image #: 719 image number 2342899\n",
      "image #: 720 image number 2342901\n",
      "image #: 721 image number 2342902\n",
      "image #: 722 image number 2342903\n",
      "image #: 723 image number 2342904\n",
      "image #: 724 image number 2342906\n",
      "image #: 725 image number 2342910\n",
      "725\n"
     ]
    }
   ],
   "source": [
    "### INSTRUCTION FOR USERS : INDICATE APPROPRIATE PATH\n",
    "data_path = '/data/nli-consistency/vqa/lxmert-test-3pred-40token-1seed_predictions_nli.json'\n",
    "\n",
    "models = [\"roberta-large-mnli\"]\n",
    "### INSTRUCTION FOR USERS : INDICATE APPROPRIATE PATH\n",
    "save_paths = ['/data/nli-consistency/vqa/lxmert-test-3pred-40token-1seed_predictions_nli-mnli.json']\n",
    "\n",
    "for m_name, save_path in zip(models, save_paths):\n",
    "    print('Model:', m_name)\n",
    "    print('Save path:', save_path)\n",
    "    print('Data path:', data_path)\n",
    "    \n",
    "    with open(data_path, 'r') as f:\n",
    "        data = json.load(f)\n",
    "    \n",
    "    nlier = nli.NLIInferencer(model_hf_name=m_name, confidence_threshold=0.0,\n",
    "                         dedup_constraints=False)\n",
    "\n",
    "    # Normalize\n",
    "\n",
    "    num_choices = 2\n",
    "    not_redundant = True # I.e. self comparisons -- it doesn't add much value since false will still yield truth value\n",
    "    repeated_comparisons = False # I.e. identical comparisons; as in what if there are multiple answer confidence levels? No because regardless based on single statement\n",
    "    group_count = num_choices\n",
    "\n",
    "    raw_correct = 0\n",
    "    new_correct = 0\n",
    "    good_change = 0\n",
    "    bad_change = 0\n",
    "    count = 0\n",
    "    questions_done = 0\n",
    "\n",
    "    for key in data.keys():\n",
    "        print('image #:', questions_done + 1, 'image number', key)\n",
    "        img_data = data[key]\n",
    "        \n",
    "        for group in img_data.keys():\n",
    "            group_list = img_data[group]['orig']\n",
    "            nli_stuff = img_data[group]['nli']\n",
    "            \n",
    "            statement_groups = nli_stuff['statement_groups']\n",
    "            converted_flat = nli_stuff['converted_flat']\n",
    "            \n",
    "            compared = nlier(converted_flat, group_count = group_count, not_redundant=not_redundant, fp_batch_size=64)\n",
    "            if not repeated_comparisons:\n",
    "                compared = list(set(compared))\n",
    "            \n",
    "            data[key][group]['nli']['compared'] = compared\n",
    "        questions_done += 1        \n",
    "\n",
    "    print(questions_done)\n",
    "    \n",
    "    with open(save_path, 'w') as f:\n",
    "        json.dump(data, f)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4af316a89dd3e3197ef817522435dd20aadd9960b7f591262313bbb961fac057"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}