SI 670 Final Project-LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(216930, 7)\n",
      "(4000, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>166403</th>\n",
       "      <td>4536</td>\n",
       "      <td>2004-05-03</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>DOUBLE A, B, Cs</td>\n",
       "      <td>$200</td>\n",
       "      <td>I'm this, you're glue, everything you say boun...</td>\n",
       "      <td>rubber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>4335</td>\n",
       "      <td>2003-06-06</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>MY PLACE?</td>\n",
       "      <td>$200</td>\n",
       "      <td>A Norman could say, \"I'm the king of the motte...</td>\n",
       "      <td>castle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119920</th>\n",
       "      <td>5224</td>\n",
       "      <td>2007-05-03</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>POTPOURRI</td>\n",
       "      <td>$200</td>\n",
       "      <td>Shelley &amp; Eliot would be happy to know that Ap...</td>\n",
       "      <td>poetry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33882</th>\n",
       "      <td>5668</td>\n",
       "      <td>2009-04-08</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>IT'S A COUNTRY THING</td>\n",
       "      <td>$200</td>\n",
       "      <td>Hat dance &amp; jumping bean</td>\n",
       "      <td>Mexican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186569</th>\n",
       "      <td>6247</td>\n",
       "      <td>2011-11-15</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>MELTING POTPOURRI</td>\n",
       "      <td>$200</td>\n",
       "      <td>\"Our actors\", says Prospero, \"were all spirits...</td>\n",
       "      <td>air</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45283</th>\n",
       "      <td>5687</td>\n",
       "      <td>2009-05-05</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ARCHAEOLOGY</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1996 Franck Goddio discovered her palace un...</td>\n",
       "      <td>Cleopatra</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184063</th>\n",
       "      <td>5023</td>\n",
       "      <td>2006-06-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>&amp; TAKIN' NAMES</td>\n",
       "      <td>$200</td>\n",
       "      <td>World poverty fighter, Time magazine Person of...</td>\n",
       "      <td>Bono</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85474</th>\n",
       "      <td>5139</td>\n",
       "      <td>2007-01-04</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>BEAN</td>\n",
       "      <td>$200</td>\n",
       "      <td>This bean that shares the name of a South Amer...</td>\n",
       "      <td>lima</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155694</th>\n",
       "      <td>5853</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>BE TRUE TO YOUR SCHOOL</td>\n",
       "      <td>$200</td>\n",
       "      <td>The benefactor for whom this West Lafayette, I...</td>\n",
       "      <td>Purdue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147690</th>\n",
       "      <td>4293</td>\n",
       "      <td>2003-04-09</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>DECODE THE PERSONAL AD</td>\n",
       "      <td>$200</td>\n",
       "      <td>To start with, S. is for this</td>\n",
       "      <td>single</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Show Number    Air Date      Round                Category Value  \\\n",
       "166403         4536  2004-05-03  Jeopardy!         DOUBLE A, B, Cs  $200   \n",
       "781            4335  2003-06-06  Jeopardy!               MY PLACE?  $200   \n",
       "119920         5224  2007-05-03  Jeopardy!               POTPOURRI  $200   \n",
       "33882          5668  2009-04-08  Jeopardy!    IT'S A COUNTRY THING  $200   \n",
       "186569         6247  2011-11-15  Jeopardy!       MELTING POTPOURRI  $200   \n",
       "45283          5687  2009-05-05  Jeopardy!             ARCHAEOLOGY  $200   \n",
       "184063         5023  2006-06-14  Jeopardy!          & TAKIN' NAMES  $200   \n",
       "85474          5139  2007-01-04  Jeopardy!                    BEAN  $200   \n",
       "155694         5853  2010-02-10  Jeopardy!  BE TRUE TO YOUR SCHOOL  $200   \n",
       "147690         4293  2003-04-09  Jeopardy!  DECODE THE PERSONAL AD  $200   \n",
       "\n",
       "                                                 Question     Answer  \n",
       "166403  I'm this, you're glue, everything you say boun...     rubber  \n",
       "781     A Norman could say, \"I'm the king of the motte...     castle  \n",
       "119920  Shelley & Eliot would be happy to know that Ap...     poetry  \n",
       "33882                            Hat dance & jumping bean    Mexican  \n",
       "186569  \"Our actors\", says Prospero, \"were all spirits...        air  \n",
       "45283   In 1996 Franck Goddio discovered her palace un...  Cleopatra  \n",
       "184063  World poverty fighter, Time magazine Person of...       Bono  \n",
       "85474   This bean that shares the name of a South Amer...       lima  \n",
       "155694  The benefactor for whom this West Lafayette, I...     Purdue  \n",
       "147690                      To start with, S. is for this     single  "
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy_data = pd.read_csv('JEOPARDY_CSV.csv')\n",
    "# remove spaces from column names\n",
    "jeopardy_data.columns = [col.strip() for col in jeopardy_data.columns]\n",
    "print(jeopardy_data.shape)\n",
    "jeopardy_data.head()\n",
    "\n",
    "jeopardy_data_sub = jeopardy_data[jeopardy_data['Round'] == 'Jeopardy!']\n",
    "jeopardy_data_sub = jeopardy_data_sub[jeopardy_data_sub.Answer.str.isalpha() == True]\n",
    "jeopardy_data_sub = jeopardy_data_sub[~jeopardy_data_sub.Question.str.contains(\"<a href=\")]\n",
    "jeopardy_data_sub = jeopardy_data_sub[jeopardy_data_sub['Show Number'] >= 4000]\n",
    "jeopardy_data_sub = jeopardy_data_sub[jeopardy_data_sub['Question'].str.split().str.len() >= 5]\n",
    "\n",
    "import random\n",
    "# get a sample of 1,000 for each Value\n",
    "jeopardy_data_sub_200 = jeopardy_data_sub[jeopardy_data_sub['Value']=='$200'].sample(2000, random_state=670)\n",
    "jeopardy_data_sub_1000 = jeopardy_data_sub[jeopardy_data_sub['Value']=='$1000'].sample(2000, random_state=670)\n",
    "jeopardy_data_sub = pd.concat([jeopardy_data_sub_200, jeopardy_data_sub_1000])\n",
    "\n",
    "\n",
    "print(jeopardy_data_sub.shape)\n",
    "jeopardy_data_sub.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>label</th>\n",
       "      <th>label_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>166403</th>\n",
       "      <td>4536</td>\n",
       "      <td>2004-05-03</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>DOUBLE A, B, Cs</td>\n",
       "      <td>$200</td>\n",
       "      <td>I'm this, you're glue, everything you say boun...</td>\n",
       "      <td>rubber</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>4335</td>\n",
       "      <td>2003-06-06</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>MY PLACE?</td>\n",
       "      <td>$200</td>\n",
       "      <td>A Norman could say, \"I'm the king of the motte...</td>\n",
       "      <td>castle</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119920</th>\n",
       "      <td>5224</td>\n",
       "      <td>2007-05-03</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>POTPOURRI</td>\n",
       "      <td>$200</td>\n",
       "      <td>Shelley &amp; Eliot would be happy to know that Ap...</td>\n",
       "      <td>poetry</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33882</th>\n",
       "      <td>5668</td>\n",
       "      <td>2009-04-08</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>IT'S A COUNTRY THING</td>\n",
       "      <td>$200</td>\n",
       "      <td>Hat dance &amp; jumping bean</td>\n",
       "      <td>Mexican</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186569</th>\n",
       "      <td>6247</td>\n",
       "      <td>2011-11-15</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>MELTING POTPOURRI</td>\n",
       "      <td>$200</td>\n",
       "      <td>\"Our actors\", says Prospero, \"were all spirits...</td>\n",
       "      <td>air</td>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Show Number    Air Date      Round              Category Value  \\\n",
       "166403         4536  2004-05-03  Jeopardy!       DOUBLE A, B, Cs  $200   \n",
       "781            4335  2003-06-06  Jeopardy!             MY PLACE?  $200   \n",
       "119920         5224  2007-05-03  Jeopardy!             POTPOURRI  $200   \n",
       "33882          5668  2009-04-08  Jeopardy!  IT'S A COUNTRY THING  $200   \n",
       "186569         6247  2011-11-15  Jeopardy!     MELTING POTPOURRI  $200   \n",
       "\n",
       "                                                 Question   Answer  label  \\\n",
       "166403  I'm this, you're glue, everything you say boun...   rubber    200   \n",
       "781     A Norman could say, \"I'm the king of the motte...   castle    200   \n",
       "119920  Shelley & Eliot would be happy to know that Ap...   poetry    200   \n",
       "33882                            Hat dance & jumping bean  Mexican    200   \n",
       "186569  \"Our actors\", says Prospero, \"were all spirits...      air    200   \n",
       "\n",
       "        label_id  \n",
       "166403         0  \n",
       "781            0  \n",
       "119920         0  \n",
       "33882          0  \n",
       "186569         0  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy_data_sub['label']= jeopardy_data_sub['Value'].map(lambda x: int(x.replace('$','')))\n",
    "# turn labels to int\n",
    "jeopardy_data_sub['label_id'],group_name = pd.factorize(jeopardy_data_sub['label'])\n",
    "jeopardy_data_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = jeopardy_data_sub.filter(['Question'])\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, jeopardy_data_sub['label_id'], \n",
    "                   stratify=jeopardy_data_sub['label_id'],random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = y_train\n",
    "train_questions = X_train['Question']\n",
    "test_labels = y_test\n",
    "test_questions = X_test['Question']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize & Pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=50000)\n",
    "tokenizer.fit_on_texts(jeopardy_data_sub['Question'])\n",
    "\n",
    "train_sequence = tokenizer.texts_to_sequences(train_questions)\n",
    "test_sequence = tokenizer.texts_to_sequences(test_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000, 40)\n",
      "(1000, 32)\n"
     ]
    }
   ],
   "source": [
    "X_train = pad_sequences(train_sequence)\n",
    "X_test = pad_sequences(test_sequence)\n",
    "\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode labels as counts\n",
    "\n",
    "Unlike Sklearn, Keras requires your labels to be either one-hot-encoded, or encoded using label encoders. For the former, you will need to use a categorical_crossentropy loss when you compile the model, and for the latter you need to use sparse_categorical_crossentropy. We will use the latter for simplicity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000,)\n",
      "(1000,)\n"
     ]
    }
   ],
   "source": [
    "le = LabelEncoder()\n",
    "le.fit(jeopardy_data_sub['label_id'])\n",
    "\n",
    "y_train = le.transform(y_train)\n",
    "y_test = le.transform(y_test)\n",
    "\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building and running the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_words = tokenizer.num_words\n",
    "output_size = len(le.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_5 (Embedding)      (None, 50, 200)           10000000  \n",
      "_________________________________________________________________\n",
      "bidirectional_5 (Bidirection (None, 50, 300)           421200    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_5 (Glob (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 2)                 602       \n",
      "=================================================================\n",
      "Total params: 10,512,102\n",
      "Trainable params: 10,512,102\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Embedding(input_dim=num_words, \n",
    "              output_dim=200, \n",
    "              mask_zero=True, \n",
    "              input_length=50),\n",
    "    Bidirectional(LSTM(150, return_sequences=True)),\n",
    "    GlobalMaxPooling1D(),\n",
    "    Dense(300, activation='relu'),\n",
    "    Dropout(0.5),\n",
    "    Dense(output_size, activation='softmax')\n",
    "    \n",
    "])\n",
    "\n",
    "model.compile('adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_5_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 40).\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_5_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 40).\n",
      "3/3 [==============================] - ETA: 0s - loss: 0.6933 - accuracy: 0.4885WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_5_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 40).\n",
      "3/3 [==============================] - 5s 2s/step - loss: 0.6933 - accuracy: 0.4885 - val_loss: 0.6940 - val_accuracy: 0.4600\n",
      "Epoch 2/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.6904 - accuracy: 0.5096 - val_loss: 0.6942 - val_accuracy: 0.4633\n",
      "Epoch 3/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.6851 - accuracy: 0.5704 - val_loss: 0.6914 - val_accuracy: 0.5300\n",
      "Epoch 4/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.6727 - accuracy: 0.7496 - val_loss: 0.6900 - val_accuracy: 0.5167\n",
      "Epoch 5/10\n",
      "3/3 [==============================] - 6s 2s/step - loss: 0.6403 - accuracy: 0.7993 - val_loss: 0.6873 - val_accuracy: 0.5033\n",
      "Epoch 6/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.5572 - accuracy: 0.8763 - val_loss: 0.7183 - val_accuracy: 0.4900\n",
      "Epoch 7/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.3732 - accuracy: 0.9174 - val_loss: 1.0329 - val_accuracy: 0.5300\n",
      "Epoch 8/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.1713 - accuracy: 0.9507 - val_loss: 2.4758 - val_accuracy: 0.4800\n",
      "Epoch 9/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.0938 - accuracy: 0.9733 - val_loss: 3.1070 - val_accuracy: 0.5133\n",
      "Epoch 10/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.0576 - accuracy: 0.9807 - val_loss: 4.5854 - val_accuracy: 0.5000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x281a2d1c988>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs=10, batch_size=1024, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_5_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 32).\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.51      0.37      0.42       500\n",
      "           1       0.50      0.64      0.56       500\n",
      "\n",
      "    accuracy                           0.50      1000\n",
      "   macro avg       0.50      0.50      0.49      1000\n",
      "weighted avg       0.50      0.50      0.49      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test, batch_size=1024).argmax(axis=1)\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM with Textstat Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(216930, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORY</td>\n",
       "      <td>$200</td>\n",
       "      <td>For the last 8 years of his life, Galileo was ...</td>\n",
       "      <td>Copernicus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ESPN's TOP 10 ALL-TIME ATHLETES</td>\n",
       "      <td>$200</td>\n",
       "      <td>No. 2: 1912 Olympian; football star at Carlisl...</td>\n",
       "      <td>Jim Thorpe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EVERYBODY TALKS ABOUT IT...</td>\n",
       "      <td>$200</td>\n",
       "      <td>The city of Yuma in this state has a record av...</td>\n",
       "      <td>Arizona</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>THE COMPANY LINE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1963, live on \"The Art Linkletter Show\", th...</td>\n",
       "      <td>McDonald's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EPITAPHS &amp; TRIBUTES</td>\n",
       "      <td>$200</td>\n",
       "      <td>Signer of the Dec. of Indep., framer of the Co...</td>\n",
       "      <td>John Adams</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Show Number    Air Date      Round                         Category Value  \\\n",
       "0         4680  2004-12-31  Jeopardy!                          HISTORY  $200   \n",
       "1         4680  2004-12-31  Jeopardy!  ESPN's TOP 10 ALL-TIME ATHLETES  $200   \n",
       "2         4680  2004-12-31  Jeopardy!      EVERYBODY TALKS ABOUT IT...  $200   \n",
       "3         4680  2004-12-31  Jeopardy!                 THE COMPANY LINE  $200   \n",
       "4         4680  2004-12-31  Jeopardy!              EPITAPHS & TRIBUTES  $200   \n",
       "\n",
       "                                            Question      Answer  \n",
       "0  For the last 8 years of his life, Galileo was ...  Copernicus  \n",
       "1  No. 2: 1912 Olympian; football star at Carlisl...  Jim Thorpe  \n",
       "2  The city of Yuma in this state has a record av...     Arizona  \n",
       "3  In 1963, live on \"The Art Linkletter Show\", th...  McDonald's  \n",
       "4  Signer of the Dec. of Indep., framer of the Co...  John Adams  "
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in and preview data\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "jeopardy_data = pd.read_csv('JEOPARDY_CSV.csv')\n",
    "# remove spaces from column names\n",
    "jeopardy_data.columns = [col.strip() for col in jeopardy_data.columns]\n",
    "print(jeopardy_data.shape)\n",
    "jeopardy_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4000, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>166403</th>\n",
       "      <td>4536</td>\n",
       "      <td>2004-05-03</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>DOUBLE A, B, Cs</td>\n",
       "      <td>$200</td>\n",
       "      <td>I'm this, you're glue, everything you say boun...</td>\n",
       "      <td>rubber</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>4335</td>\n",
       "      <td>2003-06-06</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>MY PLACE?</td>\n",
       "      <td>$200</td>\n",
       "      <td>A Norman could say, \"I'm the king of the motte...</td>\n",
       "      <td>castle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119920</th>\n",
       "      <td>5224</td>\n",
       "      <td>2007-05-03</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>POTPOURRI</td>\n",
       "      <td>$200</td>\n",
       "      <td>Shelley &amp; Eliot would be happy to know that Ap...</td>\n",
       "      <td>poetry</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33882</th>\n",
       "      <td>5668</td>\n",
       "      <td>2009-04-08</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>IT'S A COUNTRY THING</td>\n",
       "      <td>$200</td>\n",
       "      <td>Hat dance &amp; jumping bean</td>\n",
       "      <td>Mexican</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186569</th>\n",
       "      <td>6247</td>\n",
       "      <td>2011-11-15</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>MELTING POTPOURRI</td>\n",
       "      <td>$200</td>\n",
       "      <td>\"Our actors\", says Prospero, \"were all spirits...</td>\n",
       "      <td>air</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45283</th>\n",
       "      <td>5687</td>\n",
       "      <td>2009-05-05</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ARCHAEOLOGY</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1996 Franck Goddio discovered her palace un...</td>\n",
       "      <td>Cleopatra</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184063</th>\n",
       "      <td>5023</td>\n",
       "      <td>2006-06-14</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>&amp; TAKIN' NAMES</td>\n",
       "      <td>$200</td>\n",
       "      <td>World poverty fighter, Time magazine Person of...</td>\n",
       "      <td>Bono</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85474</th>\n",
       "      <td>5139</td>\n",
       "      <td>2007-01-04</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>BEAN</td>\n",
       "      <td>$200</td>\n",
       "      <td>This bean that shares the name of a South Amer...</td>\n",
       "      <td>lima</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155694</th>\n",
       "      <td>5853</td>\n",
       "      <td>2010-02-10</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>BE TRUE TO YOUR SCHOOL</td>\n",
       "      <td>$200</td>\n",
       "      <td>The benefactor for whom this West Lafayette, I...</td>\n",
       "      <td>Purdue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147690</th>\n",
       "      <td>4293</td>\n",
       "      <td>2003-04-09</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>DECODE THE PERSONAL AD</td>\n",
       "      <td>$200</td>\n",
       "      <td>To start with, S. is for this</td>\n",
       "      <td>single</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Show Number    Air Date      Round                Category Value  \\\n",
       "166403         4536  2004-05-03  Jeopardy!         DOUBLE A, B, Cs  $200   \n",
       "781            4335  2003-06-06  Jeopardy!               MY PLACE?  $200   \n",
       "119920         5224  2007-05-03  Jeopardy!               POTPOURRI  $200   \n",
       "33882          5668  2009-04-08  Jeopardy!    IT'S A COUNTRY THING  $200   \n",
       "186569         6247  2011-11-15  Jeopardy!       MELTING POTPOURRI  $200   \n",
       "45283          5687  2009-05-05  Jeopardy!             ARCHAEOLOGY  $200   \n",
       "184063         5023  2006-06-14  Jeopardy!          & TAKIN' NAMES  $200   \n",
       "85474          5139  2007-01-04  Jeopardy!                    BEAN  $200   \n",
       "155694         5853  2010-02-10  Jeopardy!  BE TRUE TO YOUR SCHOOL  $200   \n",
       "147690         4293  2003-04-09  Jeopardy!  DECODE THE PERSONAL AD  $200   \n",
       "\n",
       "                                                 Question     Answer  \n",
       "166403  I'm this, you're glue, everything you say boun...     rubber  \n",
       "781     A Norman could say, \"I'm the king of the motte...     castle  \n",
       "119920  Shelley & Eliot would be happy to know that Ap...     poetry  \n",
       "33882                            Hat dance & jumping bean    Mexican  \n",
       "186569  \"Our actors\", says Prospero, \"were all spirits...        air  \n",
       "45283   In 1996 Franck Goddio discovered her palace un...  Cleopatra  \n",
       "184063  World poverty fighter, Time magazine Person of...       Bono  \n",
       "85474   This bean that shares the name of a South Amer...       lima  \n",
       "155694  The benefactor for whom this West Lafayette, I...     Purdue  \n",
       "147690                      To start with, S. is for this     single  "
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract subset of the data using the following criteria:\n",
    "# (1) only 'Jeopardy!' round\n",
    "# (2) no answers with numbers or questions containing images/links\n",
    "# (3) only shows after 4000 (to limit amount of data)\n",
    "# (4) only questions with 5 or more words\n",
    "# (5) only questions with a value of $200 or $1000\n",
    "\n",
    "jeopardy_data_sub = jeopardy_data[jeopardy_data['Round'] == 'Jeopardy!']\n",
    "jeopardy_data_sub = jeopardy_data_sub[jeopardy_data_sub.Answer.str.isalpha() == True]\n",
    "jeopardy_data_sub = jeopardy_data_sub[~jeopardy_data_sub.Question.str.contains(\"<a href=\")]\n",
    "jeopardy_data_sub = jeopardy_data_sub[jeopardy_data_sub['Show Number'] >= 4000]\n",
    "jeopardy_data_sub = jeopardy_data_sub[jeopardy_data_sub['Question'].str.split().str.len() >= 5]\n",
    "\n",
    "import random\n",
    "# get a sample of 1,000 for each Value\n",
    "jeopardy_data_sub_200 = jeopardy_data_sub[jeopardy_data_sub['Value']=='$200'].sample(2000, random_state=670)\n",
    "jeopardy_data_sub_1000 = jeopardy_data_sub[jeopardy_data_sub['Value']=='$1000'].sample(2000, random_state=670)\n",
    "jeopardy_data_sub = pd.concat([jeopardy_data_sub_200, jeopardy_data_sub_1000])\n",
    "\n",
    "\n",
    "print(jeopardy_data_sub.shape)\n",
    "jeopardy_data_sub.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to count the average word length in the sentence \n",
    "import numpy as np\n",
    "\n",
    "def avg_word_length(text):\n",
    "    lens = []\n",
    "    for word in text.split():\n",
    "        lens.append(len(word))\n",
    "    return np.mean(lens)\n",
    "\n",
    "# function to count the max word length in the sentence \n",
    "def longest_word(text):\n",
    "    lens = []\n",
    "    for word in text.split():\n",
    "        lens.append(len(word))\n",
    "    try:\n",
    "        return np.max(lens)\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yashu\\Anaconda3\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Flesch Reading Ease Score</th>\n",
       "      <th>Flesch-Kincaid Grade Level</th>\n",
       "      <th>Longest Word (Question)</th>\n",
       "      <th>Longest Word (Answer)</th>\n",
       "      <th>Average Answer Word Length (Cleaned)</th>\n",
       "      <th>Answer Len (Cleaned)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Value</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>$1000</th>\n",
       "      <td>70.390675</td>\n",
       "      <td>6.98215</td>\n",
       "      <td>9.8085</td>\n",
       "      <td>7.0475</td>\n",
       "      <td>7.060150</td>\n",
       "      <td>0.9975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>$200</th>\n",
       "      <td>72.825295</td>\n",
       "      <td>6.51860</td>\n",
       "      <td>9.5480</td>\n",
       "      <td>6.5100</td>\n",
       "      <td>6.533434</td>\n",
       "      <td>0.9945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Flesch Reading Ease Score  Flesch-Kincaid Grade Level  \\\n",
       "Value                                                          \n",
       "$1000                  70.390675                     6.98215   \n",
       "$200                   72.825295                     6.51860   \n",
       "\n",
       "       Longest Word (Question)  Longest Word (Answer)  \\\n",
       "Value                                                   \n",
       "$1000                   9.8085                 7.0475   \n",
       "$200                    9.5480                 6.5100   \n",
       "\n",
       "       Average Answer Word Length (Cleaned)  Answer Len (Cleaned)  \n",
       "Value                                                              \n",
       "$1000                              7.060150                0.9975  \n",
       "$200                               6.533434                0.9945  "
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import textstat as ts # source: https://pypi.org/project/textstat/\n",
    "\n",
    "# how many words are in the question?\n",
    "jeopardy_data_sub['Question Len'] = jeopardy_data_sub['Question'].apply(lambda x: ts.lexicon_count(x, removepunct=True))\n",
    "# what is the readability of the question?\n",
    "jeopardy_data_sub['Flesch Reading Ease Score'] = jeopardy_data_sub['Question'].apply(lambda text: ts.flesch_reading_ease(text))\n",
    "# what is the grade level associated the question?\n",
    "jeopardy_data_sub['Flesch-Kincaid Grade Level'] = jeopardy_data_sub['Question'].apply(lambda text:ts.flesch_kincaid_grade(text))\n",
    "# longest word in question? \n",
    "jeopardy_data_sub['Longest Word (Question)'] = jeopardy_data_sub['Question'].apply(lambda text:longest_word(text))\n",
    "# longest word in answer?\n",
    "jeopardy_data_sub['Longest Word (Answer)'] = jeopardy_data_sub['Answer'].apply(lambda text:longest_word(text))\n",
    "\n",
    "\n",
    "# clean questions/answers by lowercasing and removing stop words\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "stopwords = stopwords.words('english')\n",
    "\n",
    "jeopardy_data_sub['Question_cleaned'] = jeopardy_data_sub['Question'].apply(lambda x: \" \".join([word.lower() for word in x.split() if word.lower() not in stopwords]))\n",
    "jeopardy_data_sub['Answer_cleaned'] = jeopardy_data_sub['Answer'].apply(lambda x: \" \".join([word.lower() for word in x.split() if word.lower() not in stopwords]))\n",
    "jeopardy_data_sub['Category_cleaned'] = jeopardy_data_sub['Category'].apply(lambda x: \" \".join([word.lower() for word in x.split() if word.lower() not in stopwords]))\n",
    "\n",
    "# what is the average length of a word in the answer?\n",
    "jeopardy_data_sub['Average Answer Word Length (Cleaned)'] = jeopardy_data_sub['Answer_cleaned'].apply(lambda text: avg_word_length(text))\n",
    "# how many words are in the answer?\n",
    "jeopardy_data_sub['Answer Len (Cleaned)'] = jeopardy_data_sub['Answer_cleaned'].apply(lambda x: ts.lexicon_count(x, removepunct=True))\n",
    "\n",
    "\n",
    "jeopardy_data_sub.groupby('Value')[['Question_cleaned','Answer_cleaned','Category_cleaned','Flesch Reading Ease Score', \n",
    "         'Flesch-Kincaid Grade Level', 'Longest Word (Question)', 'Longest Word (Answer)',\n",
    "         'Average Answer Word Length (Cleaned)', 'Answer Len (Cleaned)']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "jeopardy_data_sub['label']= jeopardy_data_sub['Value'].map(lambda x: int(x.replace('$','')))\n",
    "# turn labels to int\n",
    "jeopardy_data_sub['label_id'],group_name = pd.factorize(jeopardy_data_sub['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_sub = ['Flesch Reading Ease Score', \n",
    "         'Flesch-Kincaid Grade Level', 'Longest Word (Question)', 'Longest Word (Answer)',\n",
    "         'Average Answer Word Length (Cleaned)']\n",
    "\n",
    "X = jeopardy_data_sub['Question_cleaned']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, jeopardy_data_sub['label_id'], \n",
    "                   stratify=jeopardy_data_sub['label_id'],random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = y_train\n",
    "train_questions = X_train\n",
    "test_labels = y_test\n",
    "test_questions = X_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize & Pad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words=50000)\n",
    "tokenizer.fit_on_texts(jeopardy_data_sub['Question_cleaned'])\n",
    "\n",
    "train_sequence = tokenizer.texts_to_sequences(train_questions)\n",
    "test_sequence = tokenizer.texts_to_sequences(test_questions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000, 30)\n",
      "(1000, 19)\n"
     ]
    }
   ],
   "source": [
    "X_train = pad_sequences(train_sequence)\n",
    "X_test = pad_sequences(test_sequence)\n",
    "\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode labels as counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3000,)\n",
      "(1000,)\n"
     ]
    }
   ],
   "source": [
    "le = LabelEncoder()\n",
    "le.fit(jeopardy_data_sub['label_id'])\n",
    "\n",
    "y_train = le.transform(y_train)\n",
    "y_test = le.transform(y_test)\n",
    "\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building and running the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_8 (Embedding)      (None, 50, 200)           10000000  \n",
      "_________________________________________________________________\n",
      "bidirectional_8 (Bidirection (None, 50, 300)           421200    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_8 (Glob (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 300)               90300     \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 300)               0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 2)                 602       \n",
      "=================================================================\n",
      "Total params: 10,512,102\n",
      "Trainable params: 10,512,102\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential([\n",
    "    Embedding(input_dim=num_words, \n",
    "              output_dim=200, \n",
    "              mask_zero=True, \n",
    "              input_length=50),\n",
    "    Bidirectional(LSTM(150, return_sequences=True)),\n",
    "    GlobalMaxPooling1D(),\n",
    "    Dense(300, activation='relu'),\n",
    "    Dropout(0.5),\n",
    "    Dense(output_size, activation='softmax')\n",
    "    \n",
    "])\n",
    "\n",
    "model.compile('adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_8_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 30).\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_8_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 30).\n",
      "3/3 [==============================] - ETA: 0s - loss: 0.6932 - accuracy: 0.4985WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_8_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 30).\n",
      "3/3 [==============================] - 5s 2s/step - loss: 0.6932 - accuracy: 0.4985 - val_loss: 0.6950 - val_accuracy: 0.4600\n",
      "Epoch 2/10\n",
      "3/3 [==============================] - 5s 2s/step - loss: 0.6897 - accuracy: 0.5104 - val_loss: 0.6935 - val_accuracy: 0.4700\n",
      "Epoch 3/10\n",
      "3/3 [==============================] - 5s 2s/step - loss: 0.6830 - accuracy: 0.6404 - val_loss: 0.6919 - val_accuracy: 0.5000\n",
      "Epoch 4/10\n",
      "3/3 [==============================] - 4s 1s/step - loss: 0.6677 - accuracy: 0.7926 - val_loss: 0.6897 - val_accuracy: 0.5133\n",
      "Epoch 5/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.6295 - accuracy: 0.8578 - val_loss: 0.6874 - val_accuracy: 0.5200\n",
      "Epoch 6/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.5410 - accuracy: 0.9119 - val_loss: 0.6976 - val_accuracy: 0.5300\n",
      "Epoch 7/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.3722 - accuracy: 0.9511 - val_loss: 0.8155 - val_accuracy: 0.5233\n",
      "Epoch 8/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.1718 - accuracy: 0.9726 - val_loss: 1.4128 - val_accuracy: 0.5133\n",
      "Epoch 9/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.0588 - accuracy: 0.9889 - val_loss: 2.9195 - val_accuracy: 0.5167\n",
      "Epoch 10/10\n",
      "3/3 [==============================] - 3s 1s/step - loss: 0.0231 - accuracy: 0.9944 - val_loss: 4.9300 - val_accuracy: 0.5100\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x281fc2d0408>"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs=10, batch_size=1024, validation_split=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Model was constructed with shape (None, 50) for input Tensor(\"embedding_8_input:0\", shape=(None, 50), dtype=float32), but it was called on an input with incompatible shape (None, 19).\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.51      0.48      0.49       500\n",
      "           1       0.51      0.53      0.52       500\n",
      "\n",
      "    accuracy                           0.51      1000\n",
      "   macro avg       0.51      0.51      0.51      1000\n",
      "weighted avg       0.51      0.51      0.51      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict(X_test, batch_size=1024).argmax(axis=1)\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}