Draft_Integrated_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import json\n",
    "import keras\n",
    "from keras import layers, models\n",
    "from keras.layers.core import Activation\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Flatten\n",
    "from keras.layers import Input, LSTM, Embedding, Dense, Add, Concatenate, Reshape, Dropout\n",
    "\n",
    "\n",
    "import re\n",
    "from collections import defaultdict\n",
    "import importlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.04802181, 0.90796024, 1.1933948 , ..., 1.867591  , 0.4990241 ,\n",
       "        1.9825053 ],\n",
       "       [0.787118  , 0.24431148, 0.06212842, ..., 1.5899594 , 1.5548197 ,\n",
       "        2.0951674 ],\n",
       "       [1.1351358 , 0.643366  , 1.2675045 , ..., 0.44118398, 3.6098127 ,\n",
       "        0.21706353],\n",
       "       [2.510353  , 2.1102235 , 0.42487723, ..., 0.2930224 , 0.46952274,\n",
       "        0.3113997 ],\n",
       "       [1.4251724 , 0.42584297, 0.9633813 , ..., 1.9000303 , 1.1129313 ,\n",
       "        0.12448684]], dtype=float32)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_features = np.load('Files/img_vectors_sample.npy')\n",
    "image_features[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1913, 512)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>question</th>\n",
       "      <th>question_id</th>\n",
       "      <th>vec</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>458752</td>\n",
       "      <td>What is this photo taken looking through?</td>\n",
       "      <td>458752000</td>\n",
       "      <td>(((tf.Tensor(-0.1294253, shape=(), dtype=float...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>458752</td>\n",
       "      <td>What position is this man playing?</td>\n",
       "      <td>458752001</td>\n",
       "      <td>(((tf.Tensor(-0.11989767, shape=(), dtype=floa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>458752</td>\n",
       "      <td>What color is the players shirt?</td>\n",
       "      <td>458752002</td>\n",
       "      <td>(((tf.Tensor(-0.085942656, shape=(), dtype=flo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>458752</td>\n",
       "      <td>Is this man a professional baseball player?</td>\n",
       "      <td>458752003</td>\n",
       "      <td>(((tf.Tensor(-0.11925976, shape=(), dtype=floa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>262146</td>\n",
       "      <td>What color is the snow?</td>\n",
       "      <td>262146000</td>\n",
       "      <td>(((tf.Tensor(-0.04078256, shape=(), dtype=floa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id                                     question  question_id  \\\n",
       "0    458752    What is this photo taken looking through?    458752000   \n",
       "1    458752           What position is this man playing?    458752001   \n",
       "2    458752             What color is the players shirt?    458752002   \n",
       "3    458752  Is this man a professional baseball player?    458752003   \n",
       "4    262146                      What color is the snow?    262146000   \n",
       "\n",
       "                                                 vec  \n",
       "0  (((tf.Tensor(-0.1294253, shape=(), dtype=float...  \n",
       "1  (((tf.Tensor(-0.11989767, shape=(), dtype=floa...  \n",
       "2  (((tf.Tensor(-0.085942656, shape=(), dtype=flo...  \n",
       "3  (((tf.Tensor(-0.11925976, shape=(), dtype=floa...  \n",
       "4  (((tf.Tensor(-0.04078256, shape=(), dtype=floa...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questionsdf = pickle.load(open(\"Files/bert_small.pkl\", \"rb\"))\n",
    "questionsdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 10, 768), dtype=float32, numpy=\n",
       "array([[[-0.1294253 ,  0.09955358,  0.0038021 , ..., -0.34734687,\n",
       "          0.16523981,  0.5111018 ],\n",
       "        [ 0.25323898, -0.28621358,  0.04786473, ...,  0.32535607,\n",
       "          0.22321571, -0.29962608],\n",
       "        [ 0.0961668 , -0.5219595 ,  0.7254022 , ..., -0.3481344 ,\n",
       "          0.28285167,  0.5112381 ],\n",
       "        ...,\n",
       "        [ 0.9590938 , -0.0078491 ,  0.4185476 , ...,  0.02356663,\n",
       "          0.11205805, -0.59412855],\n",
       "        [-0.05230472, -0.22541082, -0.6083301 , ..., -0.06796081,\n",
       "          0.21848793, -0.18448268],\n",
       "        [ 0.71103114,  0.07298414, -0.27923203, ...,  0.22893938,\n",
       "         -0.55665475, -0.21895313]]], dtype=float32)>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question_features = questionsdf['vec'].values\n",
    "question_features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9935,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([41, 34, 32, ..., 37, 26, 59])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lens = []\n",
    "for quest in questionsdf['question']:\n",
    "    lens.append(len(quest))\n",
    "question_length = np.array(lens)\n",
    "question_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9935,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question_length.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[8, 24, 29, 33, 35]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_ids = []\n",
    "for x in questionsdf['image_id']:\n",
    "    if x not in image_ids:\n",
    "        image_ids.append(x)\n",
    "image_ids = np.array(image_ids) - 1\n",
    "sorted(image_ids)[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1913,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_ids.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "file=r\"v2_Annotations_Train_mscoco/v2_mscoco_train2014_annotations.json\"\n",
    "with open(file,'r') as myfile:\n",
    "    data=myfile.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question_type': 'what is this',\n",
       " 'multiple_choice_answer': 'net',\n",
       " 'answers': [{'answer': 'net', 'answer_confidence': 'maybe', 'answer_id': 1},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 2},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 3},\n",
       "  {'answer': 'netting', 'answer_confidence': 'yes', 'answer_id': 4},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 5},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 6},\n",
       "  {'answer': 'mesh', 'answer_confidence': 'maybe', 'answer_id': 7},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 8},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 9},\n",
       "  {'answer': 'net', 'answer_confidence': 'yes', 'answer_id': 10}],\n",
       " 'image_id': 458752,\n",
       " 'answer_type': 'other',\n",
       " 'question_id': 458752000}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=json.loads(data)['annotations']\n",
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_vocab_answers(annotations, n_answers):\n",
    "    \"\"\"Make dictionary for top n answers and save them into text file.\"\"\"\n",
    "    answers = defaultdict(lambda: 0)\n",
    "    for annotation in annotations:\n",
    "            for answer in annotation['answers']:\n",
    "                word = answer['answer']\n",
    "                if re.search(r\"[^\\w\\s]\", word):\n",
    "                    continue\n",
    "                answers[word] += 1\n",
    "                \n",
    "    answers = sorted(answers, key=answers.get, reverse=True)\n",
    "    assert('<unk>' not in answers)\n",
    "    top_answers = ['<unk>'] + answers[:n_answers-1] # '-1' is due to '<unk>'\n",
    "    \n",
    "    with open('vocab_answers.txt', 'w') as f:\n",
    "        f.writelines([w+'\\n' for w in top_answers])\n",
    "\n",
    "    print('Make vocabulary for answers')\n",
    "    print('The number of total words of answers: %d' % len(answers))\n",
    "    print('Keep top %d answers into vocab' % n_answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make vocabulary for answers\n",
      "The number of total words of answers: 135203\n",
      "Keep top 500 answers into vocab\n"
     ]
    }
   ],
   "source": [
    "make_vocab_answers(data,500) ## this saves a text file in the same folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "question_featuresSENTENCE_SPLIT_REGEX = re.compile(r'(\\W+)')\n",
    "\n",
    "\n",
    "def tokenize(sentence):\n",
    "    tokens = SENTENCE_SPLIT_REGEX.split(sentence.lower())\n",
    "    tokens = [t.strip() for t in tokens if len(t.strip()) > 0]\n",
    "    return tokens\n",
    "def load_str_list(fname):\n",
    "    with open(fname) as f:\n",
    "        lines = f.readlines()\n",
    "    lines = [l.strip() for l in lines]\n",
    "    return lines\n",
    "\n",
    "\n",
    "class VocabDict:\n",
    "\n",
    "    def __init__(self, vocab_file):\n",
    "        self.word_list = load_str_list(vocab_file)\n",
    "        self.word2idx_dict = {w:n_w for n_w, w in enumerate(self.word_list)}\n",
    "        self.vocab_size = len(self.word_list)\n",
    "        self.unk2idx = self.word2idx_dict['<unk>'] if '<unk>' in self.word2idx_dict else None\n",
    "    def idx2word(self, n_w):\n",
    "\n",
    "        return self.word_list[n_w]\n",
    "\n",
    "    def word2idx(self, w):\n",
    "        if w in self.word2idx_dict:\n",
    "            return self.word2idx_dict[w]\n",
    "        elif self.unk2idx is not None:\n",
    "            return self.unk2idx\n",
    "        else:\n",
    "            raise ValueError('word %s not in dictionary (while dictionary does not contain <unk>)' % w)\n",
    "\n",
    "    def tokenize_and_index(self, sentence):\n",
    "        inds = [self.word2idx(w) for w in tokenize(sentence)]\n",
    "\n",
    "        return inds\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<unk>': 0,\n",
       " 'no': 1,\n",
       " 'yes': 2,\n",
       " '2': 3,\n",
       " '1': 4,\n",
       " 'white': 5,\n",
       " '3': 6,\n",
       " 'red': 7,\n",
       " 'black': 8,\n",
       " 'blue': 9,\n",
       " '0': 10,\n",
       " '4': 11,\n",
       " 'green': 12,\n",
       " 'brown': 13,\n",
       " 'yellow': 14,\n",
       " '5': 15,\n",
       " 'gray': 16,\n",
       " '6': 17,\n",
       " 'baseball': 18,\n",
       " 'nothing': 19,\n",
       " 'frisbee': 20,\n",
       " 'tennis': 21,\n",
       " 'right': 22,\n",
       " 'left': 23,\n",
       " 'orange': 24,\n",
       " 'wood': 25,\n",
       " 'bathroom': 26,\n",
       " 'pizza': 27,\n",
       " 'none': 28,\n",
       " 'pink': 29,\n",
       " 'kitchen': 30,\n",
       " '7': 31,\n",
       " '8': 32,\n",
       " 'cat': 33,\n",
       " 'dog': 34,\n",
       " 'skiing': 35,\n",
       " 'grass': 36,\n",
       " 'water': 37,\n",
       " 'man': 38,\n",
       " 'skateboarding': 39,\n",
       " 'silver': 40,\n",
       " '10': 41,\n",
       " 'kite': 42,\n",
       " 'horse': 43,\n",
       " 'black and white': 44,\n",
       " 'skateboard': 45,\n",
       " 'surfing': 46,\n",
       " 'snow': 47,\n",
       " 'giraffe': 48,\n",
       " 'tan': 49,\n",
       " '9': 50,\n",
       " 'wii': 51,\n",
       " 'surfboard': 52,\n",
       " 'living room': 53,\n",
       " 'phone': 54,\n",
       " 'cake': 55,\n",
       " 'elephant': 56,\n",
       " 'broccoli': 57,\n",
       " 'apple': 58,\n",
       " 'purple': 59,\n",
       " 'stop': 60,\n",
       " '12': 61,\n",
       " 'table': 62,\n",
       " 'sunny': 63,\n",
       " 'eating': 64,\n",
       " 'woman': 65,\n",
       " 'banana': 66,\n",
       " 'soccer': 67,\n",
       " 'food': 68,\n",
       " 'unknown': 69,\n",
       " 'hat': 70,\n",
       " 'sheep': 71,\n",
       " 'train': 72,\n",
       " 'snowboarding': 73,\n",
       " 'standing': 74,\n",
       " 'male': 75,\n",
       " 'winter': 76,\n",
       " 'beach': 77,\n",
       " 'bear': 78,\n",
       " 'cow': 79,\n",
       " 'umbrella': 80,\n",
       " 'bus': 81,\n",
       " 'laptop': 82,\n",
       " 'motorcycle': 83,\n",
       " 'wine': 84,\n",
       " 'female': 85,\n",
       " 'maybe': 86,\n",
       " 'clear': 87,\n",
       " 'outside': 88,\n",
       " 'trees': 89,\n",
       " 'zebra': 90,\n",
       " 'flowers': 91,\n",
       " 'walking': 92,\n",
       " 'camera': 93,\n",
       " 'sitting': 94,\n",
       " 'metal': 95,\n",
       " 'brick': 96,\n",
       " '20': 97,\n",
       " 'tile': 98,\n",
       " 'many': 99,\n",
       " 'bench': 100,\n",
       " 'plane': 101,\n",
       " 'bed': 102,\n",
       " 'bananas': 103,\n",
       " 'bird': 104,\n",
       " 'night': 105,\n",
       " 'bedroom': 106,\n",
       " '11': 107,\n",
       " '15': 108,\n",
       " 'car': 109,\n",
       " 'summer': 110,\n",
       " 'down': 111,\n",
       " 'kites': 112,\n",
       " 'hot dog': 113,\n",
       " 'cell phone': 114,\n",
       " 'beige': 115,\n",
       " 'cloudy': 116,\n",
       " 'tree': 117,\n",
       " 'fork': 118,\n",
       " 'sandwich': 119,\n",
       " 'sand': 120,\n",
       " 'helmet': 121,\n",
       " 'blue and white': 122,\n",
       " 'donut': 123,\n",
       " 'up': 124,\n",
       " 'chair': 125,\n",
       " 'ground': 126,\n",
       " 'horses': 127,\n",
       " 'red and white': 128,\n",
       " 'wall': 129,\n",
       " 'plate': 130,\n",
       " 'beer': 131,\n",
       " 'boat': 132,\n",
       " 'glass': 133,\n",
       " 'bat': 134,\n",
       " 'day': 135,\n",
       " 'skis': 136,\n",
       " 'fence': 137,\n",
       " 'bike': 138,\n",
       " 'ball': 139,\n",
       " 'birthday': 140,\n",
       " '13': 141,\n",
       " 'cows': 142,\n",
       " 'people': 143,\n",
       " 'girl': 144,\n",
       " 'cheese': 145,\n",
       " 'overcast': 146,\n",
       " 'airplane': 147,\n",
       " 'truck': 148,\n",
       " 'square': 149,\n",
       " 'street': 150,\n",
       " 'glasses': 151,\n",
       " 'plastic': 152,\n",
       " 'boy': 153,\n",
       " 'dirt': 154,\n",
       " 'coffee': 155,\n",
       " 'couch': 156,\n",
       " 'snowboard': 157,\n",
       " 'knife': 158,\n",
       " 'nike': 159,\n",
       " 'english': 160,\n",
       " 'blonde': 161,\n",
       " 'tv': 162,\n",
       " 'ocean': 163,\n",
       " 'not sure': 164,\n",
       " 'floor': 165,\n",
       " '14': 166,\n",
       " 'round': 167,\n",
       " 'suitcase': 168,\n",
       " 'concrete': 169,\n",
       " 'tennis racket': 170,\n",
       " 'open': 171,\n",
       " 'fruit': 172,\n",
       " 'tie': 173,\n",
       " 'fall': 174,\n",
       " 'lot': 175,\n",
       " 'afternoon': 176,\n",
       " 'person': 177,\n",
       " 'airport': 178,\n",
       " 'christmas': 179,\n",
       " 'stripes': 180,\n",
       " 'computer': 181,\n",
       " 'teddy bear': 182,\n",
       " 'toilet': 183,\n",
       " 'no one': 184,\n",
       " 'zoo': 185,\n",
       " 'toothbrush': 186,\n",
       " 'chicken': 187,\n",
       " 'old': 188,\n",
       " 'paper': 189,\n",
       " 'usa': 190,\n",
       " 'carrots': 191,\n",
       " 'scissors': 192,\n",
       " '30': 193,\n",
       " '25': 194,\n",
       " 'flying kite': 195,\n",
       " '16': 196,\n",
       " 'spoon': 197,\n",
       " 'chocolate': 198,\n",
       " 'clock': 199,\n",
       " 'cold': 200,\n",
       " 'elephants': 201,\n",
       " 'daytime': 202,\n",
       " 'carpet': 203,\n",
       " 'building': 204,\n",
       " 'stone': 205,\n",
       " 'city': 206,\n",
       " 'breakfast': 207,\n",
       " 'playing': 208,\n",
       " 'closed': 209,\n",
       " 'apples': 210,\n",
       " 'remote': 211,\n",
       " 'bicycle': 212,\n",
       " 'window': 213,\n",
       " 'sun': 214,\n",
       " 'lettuce': 215,\n",
       " 'light': 216,\n",
       " 'donuts': 217,\n",
       " 'rectangle': 218,\n",
       " '50': 219,\n",
       " 'sidewalk': 220,\n",
       " 'spring': 221,\n",
       " 'clouds': 222,\n",
       " 'dell': 223,\n",
       " 'sleeping': 224,\n",
       " 'chinese': 225,\n",
       " 'leaves': 226,\n",
       " 'birds': 227,\n",
       " 'park': 228,\n",
       " 'home': 229,\n",
       " 'africa': 230,\n",
       " 'restaurant': 231,\n",
       " 'field': 232,\n",
       " 'not possible': 233,\n",
       " 'mirror': 234,\n",
       " 'desk': 235,\n",
       " 'circle': 236,\n",
       " 'oranges': 237,\n",
       " 'sky': 238,\n",
       " 'morning': 239,\n",
       " 'wedding': 240,\n",
       " 'both': 241,\n",
       " 'wii remote': 242,\n",
       " 'white and black': 243,\n",
       " 'dinner': 244,\n",
       " 'road': 245,\n",
       " 'mountains': 246,\n",
       " 'cooking': 247,\n",
       " 'fish': 248,\n",
       " 'evening': 249,\n",
       " 'house': 250,\n",
       " 'brown and white': 251,\n",
       " 'office': 252,\n",
       " 'on table': 253,\n",
       " 'inside': 254,\n",
       " 'palm': 255,\n",
       " 'neither': 256,\n",
       " 'plaid': 257,\n",
       " 'happy': 258,\n",
       " 'sunglasses': 259,\n",
       " 'playing wii': 260,\n",
       " 'nowhere': 261,\n",
       " 'small': 262,\n",
       " 'luggage': 263,\n",
       " 'carrot': 264,\n",
       " 'gold': 265,\n",
       " 'vegetables': 266,\n",
       " 'mountain': 267,\n",
       " '100': 268,\n",
       " 'bread': 269,\n",
       " 'tomato': 270,\n",
       " 'wetsuit': 271,\n",
       " 'giraffes': 272,\n",
       " 'sign': 273,\n",
       " 'noon': 274,\n",
       " 'racket': 275,\n",
       " 'short': 276,\n",
       " 'milk': 277,\n",
       " 'hair': 278,\n",
       " 'asian': 279,\n",
       " 'picture': 280,\n",
       " 'microwave': 281,\n",
       " 'counter': 282,\n",
       " 'fire hydrant': 283,\n",
       " 'shirt': 284,\n",
       " '18': 285,\n",
       " 'american': 286,\n",
       " 'flower': 287,\n",
       " 'ski poles': 288,\n",
       " 'child': 289,\n",
       " 'bag': 290,\n",
       " 'zebras': 291,\n",
       " 'striped': 292,\n",
       " 'watch': 293,\n",
       " 'backpack': 294,\n",
       " 'bowl': 295,\n",
       " 'vase': 296,\n",
       " 'london': 297,\n",
       " 'pepperoni': 298,\n",
       " 'towel': 299,\n",
       " 'salad': 300,\n",
       " 'jeans': 301,\n",
       " 'several': 302,\n",
       " 'sink': 303,\n",
       " 'grazing': 304,\n",
       " 'on': 305,\n",
       " 'baby': 306,\n",
       " '17': 307,\n",
       " 'wii controller': 308,\n",
       " 'front': 309,\n",
       " 'catcher': 310,\n",
       " 'lunch': 311,\n",
       " 'cement': 312,\n",
       " 'refrigerator': 313,\n",
       " 'white and blue': 314,\n",
       " 'oak': 315,\n",
       " 'color': 316,\n",
       " 'graffiti': 317,\n",
       " 'hay': 318,\n",
       " 'forward': 319,\n",
       " 'solid': 320,\n",
       " 'real': 321,\n",
       " 'book': 322,\n",
       " 'books': 323,\n",
       " 'adidas': 324,\n",
       " 'rain': 325,\n",
       " 'police': 326,\n",
       " 'china': 327,\n",
       " 'electric': 328,\n",
       " 'rocks': 329,\n",
       " 'spinach': 330,\n",
       " 'keyboard': 331,\n",
       " 'middle': 332,\n",
       " 'blanket': 333,\n",
       " 'very': 334,\n",
       " 'shorts': 335,\n",
       " 'outdoors': 336,\n",
       " 'smiling': 337,\n",
       " 'hand': 338,\n",
       " '40': 339,\n",
       " 'england': 340,\n",
       " 'shoes': 341,\n",
       " 'pole': 342,\n",
       " 'talking': 343,\n",
       " 'soup': 344,\n",
       " 'drinking': 345,\n",
       " 'unsure': 346,\n",
       " 'flying': 347,\n",
       " '24': 348,\n",
       " 'lights': 349,\n",
       " 'box': 350,\n",
       " 'glove': 351,\n",
       " 'gas': 352,\n",
       " 'pine': 353,\n",
       " 'clothes': 354,\n",
       " 'floral': 355,\n",
       " 'rock': 356,\n",
       " 'sandals': 357,\n",
       " 'young': 358,\n",
       " 'ketchup': 359,\n",
       " 'motorcycles': 360,\n",
       " 'playing tennis': 361,\n",
       " 'resting': 362,\n",
       " 'posing': 363,\n",
       " 'mouse': 364,\n",
       " 'air': 365,\n",
       " 'church': 366,\n",
       " 'background': 367,\n",
       " 'lake': 368,\n",
       " 'windows': 369,\n",
       " 'new york': 370,\n",
       " 'dessert': 371,\n",
       " 'roses': 372,\n",
       " 'collar': 373,\n",
       " 'bricks': 374,\n",
       " 'leather': 375,\n",
       " 'brushing teeth': 376,\n",
       " 'controller': 377,\n",
       " 'ski': 378,\n",
       " 'cars': 379,\n",
       " 'skating': 380,\n",
       " 'oven': 381,\n",
       " 'passenger': 382,\n",
       " 'warm': 383,\n",
       " 'hot dogs': 384,\n",
       " 'straight': 385,\n",
       " 'green and white': 386,\n",
       " 'photographer': 387,\n",
       " 'baseball bat': 388,\n",
       " 'rainy': 389,\n",
       " 'red and black': 390,\n",
       " 'america': 391,\n",
       " 'pitcher': 392,\n",
       " 'off': 393,\n",
       " 'umpire': 394,\n",
       " 'kite flying': 395,\n",
       " 'umbrellas': 396,\n",
       " 'river': 397,\n",
       " 'jacket': 398,\n",
       " 'white and red': 399,\n",
       " 'rose': 400,\n",
       " 'cream': 401,\n",
       " '21': 402,\n",
       " 'clean': 403,\n",
       " 'triangle': 404,\n",
       " 'reading': 405,\n",
       " 'living': 406,\n",
       " 'lamp': 407,\n",
       " 'adult': 408,\n",
       " 'tomatoes': 409,\n",
       " 'wire': 410,\n",
       " 'sneakers': 411,\n",
       " 'taking off': 412,\n",
       " 'meat': 413,\n",
       " 'long': 414,\n",
       " 'sunset': 415,\n",
       " 'bridge': 416,\n",
       " 'canada': 417,\n",
       " 'talking on phone': 418,\n",
       " 'wilson': 419,\n",
       " 'stove': 420,\n",
       " 'heart': 421,\n",
       " 'batter': 422,\n",
       " 'lemon': 423,\n",
       " 'hotel': 424,\n",
       " 'running': 425,\n",
       " 'cup': 426,\n",
       " 'surfboards': 427,\n",
       " 'doughnut': 428,\n",
       " 'steel': 429,\n",
       " 'hot': 430,\n",
       " 'large': 431,\n",
       " 'seagull': 432,\n",
       " 'soda': 433,\n",
       " '23': 434,\n",
       " 'united states': 435,\n",
       " 'safety': 436,\n",
       " 'top': 437,\n",
       " '22': 438,\n",
       " 'shadow': 439,\n",
       " 'delta': 440,\n",
       " 'ceramic': 441,\n",
       " 'indoors': 442,\n",
       " 'purse': 443,\n",
       " 'on wall': 444,\n",
       " 'rug': 445,\n",
       " 'jumping': 446,\n",
       " 'bears': 447,\n",
       " 'in air': 448,\n",
       " 'fridge': 449,\n",
       " 'back': 450,\n",
       " 'on ground': 451,\n",
       " 'landing': 452,\n",
       " 'parking': 453,\n",
       " 'gloves': 454,\n",
       " 'nobody': 455,\n",
       " 'checkered': 456,\n",
       " 'poles': 457,\n",
       " 'boots': 458,\n",
       " 'stop sign': 459,\n",
       " 'riding': 460,\n",
       " 'train station': 461,\n",
       " 'calm': 462,\n",
       " 'coca cola': 463,\n",
       " 'plant': 464,\n",
       " 'scarf': 465,\n",
       " 'rope': 466,\n",
       " 'rainbow': 467,\n",
       " 'goggles': 468,\n",
       " 'beef': 469,\n",
       " 'wild': 470,\n",
       " 'television': 471,\n",
       " 'east': 472,\n",
       " 'cap': 473,\n",
       " 'ford': 474,\n",
       " 'paint': 475,\n",
       " 'north': 476,\n",
       " 'peppers': 477,\n",
       " 'tea': 478,\n",
       " '19': 479,\n",
       " 'human': 480,\n",
       " 'tennis ball': 481,\n",
       " 'pillow': 482,\n",
       " 'oval': 483,\n",
       " 'van': 484,\n",
       " 'blue and yellow': 485,\n",
       " 'on plate': 486,\n",
       " 'good': 487,\n",
       " 'red and yellow': 488,\n",
       " 'branch': 489,\n",
       " 'basket': 490,\n",
       " 'no idea': 491,\n",
       " 'plain': 492,\n",
       " 'watching': 493,\n",
       " 'playing frisbee': 494,\n",
       " 'flag': 495,\n",
       " 'bottle': 496,\n",
       " 'door': 497,\n",
       " 'hydrant': 498,\n",
       " 'marble': 499}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans_vocab = VocabDict('vocab_answers.txt')\n",
    "ans_vocab = ans_vocab.word2idx_dict\n",
    "ans_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  12,\n",
       "        13,  14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  24,  25,\n",
       "        26,  27,  28,  29,  30,  31,  32,  33,  34,  35,  36,  37,  38,\n",
       "        39,  40,  41,  42,  43,  44,  45,  46,  47,  48,  49,  50,  51,\n",
       "        52,  53,  54,  55,  56,  57,  58,  59,  60,  61,  62,  63,  64,\n",
       "        65,  66,  67,  68,  69,  70,  71,  72,  73,  74,  75,  76,  77,\n",
       "        78,  79,  80,  81,  82,  83,  84,  85,  86,  87,  88,  89,  90,\n",
       "        91,  92,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102, 103,\n",
       "       104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,\n",
       "       117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129,\n",
       "       130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142,\n",
       "       143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155,\n",
       "       156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168,\n",
       "       169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181,\n",
       "       182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194,\n",
       "       195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207,\n",
       "       208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220,\n",
       "       221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233,\n",
       "       234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246,\n",
       "       247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257, 258, 259,\n",
       "       260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272,\n",
       "       273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285,\n",
       "       286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298,\n",
       "       299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311,\n",
       "       312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324,\n",
       "       325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337,\n",
       "       338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350,\n",
       "       351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363,\n",
       "       364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376,\n",
       "       377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389,\n",
       "       390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402,\n",
       "       403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415,\n",
       "       416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428,\n",
       "       429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439, 440, 441,\n",
       "       442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454,\n",
       "       455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467,\n",
       "       468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480,\n",
       "       481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493,\n",
       "       494, 495, 496, 497, 498, 499])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers = np.array(list(ans_vocab.values()))\n",
    "answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_image_features = np.load(\"Files/test_img_vectors.npy\")\n",
    "# test_image_features[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_questions = pd.read_csv('Files/questions_test.csv')\n",
    "# test_questions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_questions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lens = []\n",
    "# for quest in test_questions['question']:\n",
    "#     lens.append(len(quest))\n",
    "# test_question_length = np.array(lens)\n",
    "# test_question_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def VQA_MODEL():\n",
    "    image_feature_size          = 4096\n",
    "    word_feature_size           = 300\n",
    "    number_of_LSTM              = 3\n",
    "    number_of_hidden_units_LSTM = 512\n",
    "    max_length_questions        = 30\n",
    "    number_of_dense_layers      = 3\n",
    "    number_of_hidden_units      = 1024\n",
    "    activation_function         = 'tanh'\n",
    "    dropout_pct                 = 0.5\n",
    "\n",
    "    # Image model\n",
    "    model_image = Sequential()\n",
    "    model_image.add(Reshape((image_feature_size,), input_shape=(image_feature_size,)))\n",
    "\n",
    "   # Language Model\n",
    "    model_language = Sequential()\n",
    "#     model_language.add(LSTM(number_of_hidden_units_LSTM, return_sequences=True,input_shape=(max_length_questions, word_feature_size)))\n",
    "#     model_language.add(LSTM(number_of_hidden_units_LSTM, return_sequences=True))  \n",
    "#     model_language.add(LSTM(number_of_hidden_units_LSTM, return_sequences=False))\n",
    "    model_language.add(Reshape((word_feature_size,), input_shape=(word_feature_size,)))\n",
    "\n",
    "\n",
    "    # combined model\n",
    "    x = Concatenate()([model_language.output, model_image.output])\n",
    "\n",
    "    for _ in range(number_of_dense_layers):\n",
    "        x = Dense(number_of_hidden_units, kernel_initializer='uniform', activation= activation_function)(x)\n",
    "        x = Dropout(dropout_pct)(x)\n",
    "        \n",
    "    x = Dense(50, activation='softmax')(x)\n",
    "    \n",
    "    model = Model(inputs = [model_language.input, model_image.input], outputs=x)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = 'simple_mlp'\n",
    "num_hidden_units_mlp = 1024\n",
    "num_hidden_units_lstm = 512\n",
    "num_hidden_layers_mlp = 3\n",
    "num_hidden_layers_lstm = 1\n",
    "dropout = 0.5\n",
    "activation_1 = 'tanh'\n",
    "activation_2 = 'relu'\n",
    "seed = 1337\n",
    "optimizer = 'rmsprop'\n",
    "nb_epoch = 300\n",
    "nb_iter = 200000\n",
    "model_save_interval = 19\n",
    "batch_size = 128\n",
    "word_vector = 'glove'\n",
    "word_emb_dim = 300\n",
    "vocabulary_size = 12603\n",
    "max_ques_length = 26\n",
    "data_type = 'TRAIN'\n",
    "img_vec_dim = 2048\n",
    "img_features = 'resnet'\n",
    "img_normalize = 0\n",
    "nb_classes = 500\n",
    "class_activation = 'softmax'\n",
    "loss = 'categorical_crossentropy'\n",
    "save_folder = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_common(lst):\n",
    "    return max(set(lst), key=lst.count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_data():\n",
    "\n",
    "    train_data = {}\n",
    "    print('loading image feature...')\n",
    "    img_feature = np.load('Files/img_vectors_sample.npy')\n",
    "    \n",
    "    train_data['question'] = question_features \n",
    "    train_data['length_q'] = question_length\n",
    "    train_data['img_list'] = image_ids\n",
    "    train_data['answers'] = answers\n",
    "\n",
    "    print('Normalizing image feature')\n",
    "    if img_normalize:\n",
    "        tem = np.sqrt(np.sum(np.multiply(img_feature, img_feature)))\n",
    "        img_feature = np.divide(img_feature, np.tile(tem,(1,img_vec_dim)))\n",
    "\n",
    "    return img_feature, train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def get_data_test(args):\n",
    "#     dataset = {}\n",
    "#     test_data = {}\n",
    "#     # load json file\n",
    "#     print('loading json file...')\n",
    "#     with open(args.input_json) as data_file:\n",
    "#         data = json.load(data_file)\n",
    "#     for key in data.keys():\n",
    "#         dataset[key] = data[key]\n",
    "\n",
    "#     # load image feature\n",
    "#     print('loading image feature...')\n",
    "#     img_feature = np.load('img_vectors_sample.npy')\n",
    "    \n",
    "#     # load h5 file\n",
    "#     print('loading h5 file...')\n",
    "#     with h5py.File(args.input_ques_h5,'r') as hf:\n",
    "#         # total number of training data is 215375\n",
    "#         # question is (26, )\n",
    "#         tem = hf.get('ques_test')\n",
    "#         test_data['question'] = test_questions\n",
    "#         # max length is 23\n",
    "#         tem = hf.get('ques_length_test')\n",
    "#         test_data['length_q'] = np.array(tem)\n",
    "#         # total 82460 img\n",
    "#         # -----1~82460-----\n",
    "#         tem = hf.get('img_pos_test')\n",
    "#         # convert into 0~82459\n",
    "#         test_data['img_list'] = np.array(tem)-1\n",
    "#         # quiestion id\n",
    "#         tem = hf.get('question_id_test')\n",
    "#         test_data['ques_id'] = np.array(tem)\n",
    "#     # MC_answer_test\n",
    "#     tem = hf.get('MC_ans_test')\n",
    "#     test_data['MC_ans_test'] = np.array(tem)\n",
    "\n",
    "#     print('Normalizing image feature')\n",
    "#     if img_norm:\n",
    "#         tem =  np.sqrt(np.sum(np.multiply(img_feature, img_feature)))\n",
    "#         img_feature = np.divide(img_feature, np.tile(tem,(1,args.img_vec_dim)))\n",
    "\n",
    "\n",
    "#     # make sure the ans_file is provided\n",
    "#     nb_data_test = len(test_data[u'question'])\n",
    "#     val_all_answers_dict = json.load(open(args.ans_file))\n",
    "#     val_answers = np.zeros(nb_data_test, dtype=np.int32)\n",
    "\n",
    "#     ans_to_ix = {v: k for k, v in dataset[u'ix_to_ans'].items()}\n",
    "#     count_of_not_found = 0\n",
    "#     for i in xrange(nb_data_test):\n",
    "#         qid = test_data[u'ques_id'][i]\n",
    "#         try : \n",
    "#             val_ans_ix =int(ans_to_ix[most_common(val_all_answers_dict[str(qid)])]) -1\n",
    "#         except KeyError:\n",
    "#             count_of_not_found += 1\n",
    "#             val_ans_ix = 480\n",
    "#         val_answers[i] = val_ans_ix\n",
    "#     print(\"Beware: \" + str(count_of_not_found) + \" number of val answers are not really correct\")\n",
    "\n",
    "#     return img_feature, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading image feature...\n",
      "Normalizing image feature\n"
     ]
    }
   ],
   "source": [
    "train_img_feature, train_data = get_train_data()\n",
    "# test_img_feature,  test_data, val_answers = get_test_data(args)\n",
    "\n",
    "train_X = [train_data[u'question'], train_img_feature]\n",
    "# train_X = np.reshape(train_X, (train_X.shape[0], 1, train_X.shape[1]))\n",
    "\n",
    "train_Y = np_utils.to_categorical(train_data[u'answers'], nb_classes)\n",
    "\n",
    "# test_X = [test_data[u'question'], test_img_feature]\n",
    "# test_Y = np_utils.to_categorical(val_answers, args.nb_classes)\n",
    "\n",
    "\n",
    "# model_name = importlib.import_module(\"models.\"+model)\n",
    "\n",
    "model = VQA_MODEL()\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "\n",
    "# model.build(train_img_feature.shape)\n",
    "# model.summary() # prints model layers with weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Error when checking input: expected reshape_2_input to have shape (300,) but got array with shape (1,)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-b60c079b0075>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_X\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_Y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnb_epoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m# history = model.fit(train_X, train_Y, batch_size = args.batch_size, nb_epoch=args.nb_epoch, validation_data=(test_X, test_Y))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1152\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1153\u001b[0m             \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1154\u001b[0;31m             batch_size=batch_size)\n\u001b[0m\u001b[1;32m   1155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1156\u001b[0m         \u001b[0;31m# Prepare validation data.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_standardize_user_data\u001b[0;34m(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)\u001b[0m\n\u001b[1;32m    577\u001b[0m             \u001b[0mfeed_input_shapes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    578\u001b[0m             \u001b[0mcheck_batch_axis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0;31m# Don't enforce the batch size.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 579\u001b[0;31m             exception_prefix='input')\n\u001b[0m\u001b[1;32m    580\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/keras/engine/training_utils.py\u001b[0m in \u001b[0;36mstandardize_input_data\u001b[0;34m(data, names, shapes, check_batch_axis, exception_prefix)\u001b[0m\n\u001b[1;32m    143\u001b[0m                             \u001b[0;34m': expected '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnames\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' to have shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m                             \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' but got array with shape '\u001b[0m \u001b[0;34m+\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m                             str(data_shape))\n\u001b[0m\u001b[1;32m    146\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Error when checking input: expected reshape_2_input to have shape (300,) but got array with shape (1,)"
     ]
    }
   ],
   "source": [
    "history = model.fit(train_X, train_Y, batch_size = batch_size, epochs=nb_epoch)\n",
    "# history = model.fit(train_X, train_Y, batch_size = args.batch_size, nb_epoch=args.nb_epoch, validation_data=(test_X, test_Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_weights(\"./VQA_MODEL_WEIGHTS.hdf5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VQA_weights_file_name   = \"./VQA_MODEL_WEIGHTS.hdf5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_VQA_model(VQA_weights_file_name):\n",
    "    ''' Given the VQA model and its weights, compiles and returns the model '''\n",
    "\n",
    "    from models.VQA.VQA import VQA_MODEL\n",
    "    vqa_model = VQA_MODEL()\n",
    "    vqa_model.load_weights(VQA_weights_file_name)\n",
    "\n",
    "    vqa_model.compile(loss='categorical_crossentropy', optimizer='rmsprop')\n",
    "    return vqa_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vqa_model = get_VQA_model(VQA_weights_file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vqa_model.predict([question_features, image_features])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}