Stanford-Sentiment-Treebank- movie reviews with fine-grained labels.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "from collections import namedtuple\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stanford Sentiment Treebank - movie reviews with fine-grained labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stanford Sentiment Treebank - movie reviews with fine-grained labels\n",
    "# https://nlp.stanford.edu/sentiment/\n",
    "\n",
    "ST_sentence = namedtuple(\"Stanford_Sentiment\", \"id sentence\")\n",
    "ST_score = namedtuple(\"Stanford_Sentiment\", \"id score\")\n",
    "\n",
    "sentences = dict()\n",
    "scores = dict()\n",
    "train = []\n",
    "dev = []\n",
    "test = []\n",
    "\n",
    "for filename in ['datasetSentences.txt','datasetSplit.txt', 'sentiment_labels.txt']:\n",
    "    with open(\"ST/\"+filename,'r') as f_input:\n",
    "        for line in f_input:\n",
    "            \n",
    "            # skip headers\n",
    "            if line.startswith(\"sentence_index\") or line.startswith('phrase id'):\n",
    "                continue\n",
    "            \n",
    "            # load sentences\n",
    "            if filename=='datasetSentences.txt':                                \n",
    "                sent_id, sentence = line.split('\\t', 1)\n",
    "                sentences[sent_id] = sentence.strip()\n",
    "            \n",
    "            # load splits\n",
    "            if filename=='datasetSplit.txt':\n",
    "                sent_id, split = line.split(',', 1)\n",
    "                split = int(split.strip())\n",
    "                if split == 1:                    \n",
    "                    train.append(sent_id)\n",
    "                if split == 2:                                        \n",
    "                    test.append(sent_id)\n",
    "                if split == 3:\n",
    "                    dev.append(sent_id)\n",
    "            \n",
    "            # sentences_id        \n",
    "            if filename=='sentiment_labels.txt':                    \n",
    "                sent_id, sent_score = line.split('|', 1)\n",
    "                #sent_score = float(sent_score.strip())\n",
    "                sample = ST_score(sent_id, float(sent_score.strip()))\n",
    "                scores[sent_id] = sent_score.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Samples and Classes/Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Nr. Samples: 11855\n",
      "Total Nr. Scores : 239232\n",
      "\n",
      "Train  : 8544\n",
      "Dev    : 1101\n",
      "Test   : 2210\n"
     ]
    }
   ],
   "source": [
    "print(\"Total Nr. Samples: {}\".format(len(sentences)))\n",
    "print(\"Total Nr. Scores : {}\".format(len(scores)))\n",
    "print()\n",
    "print(\"Train  : {}\".format(len(train)))\n",
    "print(\"Dev    : {}\".format(len(dev)))\n",
    "print(\"Test   : {}\".format(len(test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# built two lists with sentences and labels\n",
    "x_train_data = [sentences[x] for x in train]\n",
    "y_train_data = [scores[x] for x in train]\n",
    "\n",
    "x_dev_data = [sentences[x] for x in dev]\n",
    "y_dev_data = [scores[x] for x in dev]\n",
    "\n",
    "x_test_data = [sentences[x] for x in test]\n",
    "y_test_data = [scores[x] for x in test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 15337 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "# convert list of tokens/words to indexes\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(x_train_data)\n",
    "sequences_train = tokenizer.texts_to_sequences(x_train_data)\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the max sentence lenght, needed for padding\n",
    "max_input_lenght = max([len(x) for x in sequences_train])\n",
    "max_input_lenght"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pad all the sequences of indexes to the 'max_input_lenght'\n",
    "x_train_data_padded = pad_sequences(sequences_train, maxlen=max_input_lenght, padding='post', truncating='post')\n",
    "x_dev_data_padded = pad_sequences(tokenizer.texts_to_sequences(x_dev_data), maxlen=max_input_lenght, padding='post', truncating='post')\n",
    "x_test_data_padded = pad_sequences(tokenizer.texts_to_sequences(x_test_data), maxlen=max_input_lenght, padding='post', truncating='post')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transform scores to classes as re-labeled by Socher et al. (2013)\n",
    "- 0 - 2.0    : very negative\n",
    "- 2.0 - 4.0  : negative\n",
    "- 4.0 - 6.0  : neutral\n",
    "- 6.0 - 8.0  : negative\n",
    "- 8.0 - 10.0 : very positive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_categories(y_data):\n",
    "    y_categories = []\n",
    "    for score in y_data:\n",
    "        if 0.0<=float(score)<0.2:\n",
    "            y_categories.append('very_negative')\n",
    "        elif 0.2<=float(score)<0.4:\n",
    "            y_categories.append('negative')\n",
    "        elif 0.4<=float(score)<0.6:\n",
    "            y_categories.append('neutral')\n",
    "        elif 0.6<=float(score)<0.8:\n",
    "            y_categories.append('positive')\n",
    "        elif 0.8<=float(score)<=1.0:\n",
    "            y_categories.append('very positive')\n",
    "            \n",
    "    return y_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert from scores to categories\n",
    "y_train_data_categ = convert_to_categories(y_train_data)\n",
    "y_dev_data_categ = convert_to_categories(y_dev_data)\n",
    "y_test_data_categ = convert_to_categories(y_test_data)\n",
    "\n",
    "# Encode the labels, each must be a vector with dim = num. of possible labels\n",
    "le = LabelEncoder()\n",
    "le.fit(y_train_data_categ)\n",
    "\n",
    "labels_encoded_train = le.transform(y_train_data_categ)\n",
    "labels_encoded_dev = le.transform(y_dev_data_categ)\n",
    "labels_encoded_test = le.transform(y_test_data_categ)\n",
    "\n",
    "categorical_labels_train = to_categorical(labels_encoded_train, num_classes=None)\n",
    "categorical_labels_dev = to_categorical(labels_encoded_dev, num_classes=None)\n",
    "categorical_labels_test = to_categorical(labels_encoded_test, num_classes=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8544, 49)\n",
      "(8544, 5)\n"
     ]
    }
   ],
   "source": [
    "print(x_train_data_padded.shape)\n",
    "print(categorical_labels_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1101, 49)\n",
      "(1101, 5)\n"
     ]
    }
   ],
   "source": [
    "print(x_dev_data_padded.shape)\n",
    "print(categorical_labels_dev.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2210, 49)\n",
      "(2210, 5)\n"
     ]
    }
   ],
   "source": [
    "print(x_test_data_padded.shape)\n",
    "print(categorical_labels_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from convnets_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN with random word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_1 = get_cnn_rand(200, len(word_index)+1, max_input_lenght, 5, loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8544 samples, validate on 1101 samples\n",
      "Epoch 1/10\n",
      "8544/8544 [==============================] - 11s 1ms/step - loss: 0.3173 - accuracy: 0.8700 - val_loss: 4.9630 - val_accuracy: 0.3560\n",
      "Epoch 2/10\n",
      "8544/8544 [==============================] - 13s 2ms/step - loss: 0.3178 - accuracy: 0.8697 - val_loss: 5.8125 - val_accuracy: 0.3724\n",
      "Epoch 3/10\n",
      "8544/8544 [==============================] - 13s 2ms/step - loss: 0.3098 - accuracy: 0.8714 - val_loss: 5.4760 - val_accuracy: 0.3769\n",
      "Epoch 4/10\n",
      "8544/8544 [==============================] - 16s 2ms/step - loss: 0.3054 - accuracy: 0.8718 - val_loss: 6.0862 - val_accuracy: 0.3806\n",
      "Epoch 5/10\n",
      "8544/8544 [==============================] - 15s 2ms/step - loss: 0.3068 - accuracy: 0.8704 - val_loss: 5.6802 - val_accuracy: 0.3697\n",
      "Epoch 6/10\n",
      "8544/8544 [==============================] - 13s 1ms/step - loss: 0.2942 - accuracy: 0.8759 - val_loss: 5.7673 - val_accuracy: 0.3833\n",
      "Epoch 7/10\n",
      "8544/8544 [==============================] - 13s 2ms/step - loss: 0.2990 - accuracy: 0.8761 - val_loss: 4.9000 - val_accuracy: 0.3542\n",
      "Epoch 8/10\n",
      "8544/8544 [==============================] - 13s 1ms/step - loss: 0.3049 - accuracy: 0.8737 - val_loss: 5.9475 - val_accuracy: 0.3778\n",
      "Epoch 9/10\n",
      "8544/8544 [==============================] - 13s 2ms/step - loss: 0.2950 - accuracy: 0.8790 - val_loss: 6.1217 - val_accuracy: 0.3824\n",
      "Epoch 10/10\n",
      "8544/8544 [==============================] - 13s 2ms/step - loss: 0.2934 - accuracy: 0.8814 - val_loss: 5.8682 - val_accuracy: 0.3651\n"
     ]
    }
   ],
   "source": [
    "history = model_1.fit(x=x_train_data_padded, y=categorical_labels_train,\n",
    "                      validation_data=(x_dev_data_padded, categorical_labels_dev),\n",
    "                      batch_size=50, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4321267008781433"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss, accuracy = model_1.evaluate(x_test_data_padded, categorical_labels_test, verbose=0)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "     negative       0.22      0.19      0.20       405\n",
      "      neutral       0.53      0.72      0.61      1155\n",
      "     positive       0.19      0.11      0.14       424\n",
      "very positive       0.07      0.02      0.03       112\n",
      "very_negative       0.05      0.01      0.01       114\n",
      "\n",
      "     accuracy                           0.43      2210\n",
      "    macro avg       0.21      0.21      0.20      2210\n",
      " weighted avg       0.36      0.43      0.39      2210\n",
      "\n"
     ]
    }
   ],
   "source": [
    "raw_predictions = model_1.predict(x_test_data_padded)\n",
    "class_predictions = [np.argmax(x) for x in raw_predictions]\n",
    "print(classification_report(y_test_data_categ, le.inverse_transform(class_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN with pre-trained static word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 400000 word vectors.\n",
      "Matrix shape: (15338, 100)\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = load_fasttext_embeddings()\n",
    "embeddings_matrix = create_embeddings_matrix(embeddings_index, word_index, 100)\n",
    "\n",
    "embedding_layer_static = get_embeddings_layer(embeddings_matrix, \n",
    "                                              'embedding_layer_static', \n",
    "                                              max_input_lenght, \n",
    "                                              trainable=False)\n",
    "\n",
    "model_2 = get_cnn_pre_trained_embeddings(embedding_layer_static, max_input_lenght, 5, \n",
    "                                         loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8544 samples, validate on 1101 samples\n",
      "Epoch 1/10\n",
      "8544/8544 [==============================] - 3s 352us/step - loss: 1.4719 - accuracy: 0.4746 - val_loss: 1.4119 - val_accuracy: 0.4569\n",
      "Epoch 2/10\n",
      "8544/8544 [==============================] - 3s 312us/step - loss: 1.2527 - accuracy: 0.5325 - val_loss: 1.4272 - val_accuracy: 0.4569\n",
      "Epoch 3/10\n",
      "8544/8544 [==============================] - 3s 310us/step - loss: 1.1984 - accuracy: 0.5444 - val_loss: 1.4531 - val_accuracy: 0.4569\n",
      "Epoch 4/10\n",
      "8544/8544 [==============================] - 3s 319us/step - loss: 1.1459 - accuracy: 0.5590 - val_loss: 1.4517 - val_accuracy: 0.4559\n",
      "Epoch 5/10\n",
      "8544/8544 [==============================] - 3s 326us/step - loss: 1.0870 - accuracy: 0.5813 - val_loss: 1.4654 - val_accuracy: 0.4550\n",
      "Epoch 6/10\n",
      "8544/8544 [==============================] - 3s 337us/step - loss: 1.0274 - accuracy: 0.6069 - val_loss: 1.5544 - val_accuracy: 0.4559\n",
      "Epoch 7/10\n",
      "8544/8544 [==============================] - 3s 313us/step - loss: 0.9627 - accuracy: 0.6321 - val_loss: 1.4961 - val_accuracy: 0.4450\n",
      "Epoch 8/10\n",
      "8544/8544 [==============================] - 3s 318us/step - loss: 0.8896 - accuracy: 0.6601 - val_loss: 1.5888 - val_accuracy: 0.4541\n",
      "Epoch 9/10\n",
      "8544/8544 [==============================] - 3s 326us/step - loss: 0.8406 - accuracy: 0.6842 - val_loss: 1.5462 - val_accuracy: 0.4369\n",
      "Epoch 10/10\n",
      "8544/8544 [==============================] - 2s 292us/step - loss: 0.7793 - accuracy: 0.7100 - val_loss: 1.6377 - val_accuracy: 0.4223\n"
     ]
    }
   ],
   "source": [
    "history = model_2.fit(x=x_train_data_padded, \n",
    "                      validation_data=(x_dev_data_padded, categorical_labels_dev),\n",
    "                      y=categorical_labels_train, batch_size=50, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4841628968715668"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss, accuracy = model_2.evaluate(x_test_data_padded, categorical_labels_test, verbose=0)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "     negative       0.13      0.03      0.05       405\n",
      "      neutral       0.52      0.91      0.66      1155\n",
      "     positive       0.13      0.03      0.05       424\n",
      "very positive       0.00      0.00      0.00       112\n",
      "very_negative       0.00      0.00      0.00       114\n",
      "\n",
      "     accuracy                           0.48      2210\n",
      "    macro avg       0.15      0.19      0.15      2210\n",
      " weighted avg       0.32      0.48      0.36      2210\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dsbatista/virtual_envs/python37/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "raw_predictions = model_2.predict(x_test_data_padded)\n",
    "class_predictions = [np.argmax(x) for x in raw_predictions]\n",
    "print(classification_report(y_test_data_categ, le.inverse_transform(class_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN with pre-trained dynamic word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer_dynamic = get_embeddings_layer(embeddings_matrix, 'embedding_layer_dynamic', \n",
    "                                               max_input_lenght, trainable=True)\n",
    "\n",
    "model_3 = get_cnn_pre_trained_embeddings(embedding_layer_dynamic, max_input_lenght, 5, \n",
    "                                         loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "8544/8544 [==============================] - 7s 855us/step - loss: 0.7416 - accuracy: 0.7362\n",
      "Epoch 2/10\n",
      "8544/8544 [==============================] - 8s 922us/step - loss: 0.5957 - accuracy: 0.7969\n",
      "Epoch 3/10\n",
      "8544/8544 [==============================] - 6s 745us/step - loss: 0.4728 - accuracy: 0.8449\n",
      "Epoch 4/10\n",
      "8544/8544 [==============================] - 7s 773us/step - loss: 0.3785 - accuracy: 0.8751\n",
      "Epoch 5/10\n",
      "8544/8544 [==============================] - 6s 746us/step - loss: 0.3051 - accuracy: 0.9031\n",
      "Epoch 6/10\n",
      "8544/8544 [==============================] - 6s 758us/step - loss: 0.2419 - accuracy: 0.9281\n",
      "Epoch 7/10\n",
      "8544/8544 [==============================] - 9s 1ms/step - loss: 0.2078 - accuracy: 0.9382\n",
      "Epoch 8/10\n",
      "8544/8544 [==============================] - 8s 897us/step - loss: 0.1767 - accuracy: 0.9480\n",
      "Epoch 9/10\n",
      "8544/8544 [==============================] - 7s 764us/step - loss: 0.1465 - accuracy: 0.9583\n",
      "Epoch 10/10\n",
      "8544/8544 [==============================] - 8s 951us/step - loss: 0.1279 - accuracy: 0.9650\n"
     ]
    }
   ],
   "source": [
    "history = model_3.fit(x=x_train_data_padded, y=categorical_labels_train, batch_size=50, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4325791895389557"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss, accuracy = model_3.evaluate(x_test_data_padded, categorical_labels_test, verbose=0)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "     negative       0.15      0.10      0.12       405\n",
      "      neutral       0.52      0.76      0.61      1155\n",
      "     positive       0.18      0.08      0.11       424\n",
      "very positive       0.12      0.03      0.04       112\n",
      "very_negative       0.04      0.01      0.01       114\n",
      "\n",
      "     accuracy                           0.43      2210\n",
      "    macro avg       0.20      0.20      0.18      2210\n",
      " weighted avg       0.34      0.43      0.37      2210\n",
      "\n"
     ]
    }
   ],
   "source": [
    "raw_predictions = model_3.predict(x_test_data_padded)\n",
    "class_predictions = [np.argmax(x) for x in raw_predictions]\n",
    "print(classification_report(y_test_data_categ, le.inverse_transform(class_predictions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN multichannel with pre-trained dynamic and static word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_4 = get_cnn_multichannel(embedding_layer_static, embedding_layer_dynamic, max_input_lenght, 5,\n",
    "                               loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 5724 samples, validate on 2820 samples\n",
      "Epoch 1/10\n",
      "5724/5724 [==============================] - 6s 1ms/step - loss: 0.4383 - accuracy: 0.8735 - val_loss: 0.9029 - val_accuracy: 0.6798\n",
      "Epoch 2/10\n",
      "5724/5724 [==============================] - 6s 1ms/step - loss: 0.2982 - accuracy: 0.9210 - val_loss: 0.8606 - val_accuracy: 0.6869\n",
      "Epoch 3/10\n",
      "5724/5724 [==============================] - 8s 1ms/step - loss: 0.2176 - accuracy: 0.9457 - val_loss: 0.8563 - val_accuracy: 0.6936\n",
      "Epoch 4/10\n",
      "5724/5724 [==============================] - 12s 2ms/step - loss: 0.1633 - accuracy: 0.9607 - val_loss: 0.8678 - val_accuracy: 0.6915\n",
      "Epoch 5/10\n",
      "5724/5724 [==============================] - 6s 1ms/step - loss: 0.1237 - accuracy: 0.9741 - val_loss: 0.8916 - val_accuracy: 0.6848\n",
      "Epoch 6/10\n",
      "5724/5724 [==============================] - 7s 1ms/step - loss: 0.0960 - accuracy: 0.9804 - val_loss: 0.8924 - val_accuracy: 0.6918\n",
      "Epoch 7/10\n",
      "5724/5724 [==============================] - 8s 1ms/step - loss: 0.0732 - accuracy: 0.9865 - val_loss: 0.9442 - val_accuracy: 0.6851\n",
      "Epoch 8/10\n",
      "5724/5724 [==============================] - 7s 1ms/step - loss: 0.0620 - accuracy: 0.9883 - val_loss: 0.9314 - val_accuracy: 0.6883\n",
      "Epoch 9/10\n",
      "5724/5724 [==============================] - 6s 1ms/step - loss: 0.0561 - accuracy: 0.9886 - val_loss: 0.9494 - val_accuracy: 0.6936\n",
      "Epoch 10/10\n",
      "5724/5724 [==============================] - 8s 1ms/step - loss: 0.0465 - accuracy: 0.9914 - val_loss: 0.9942 - val_accuracy: 0.6826\n"
     ]
    }
   ],
   "source": [
    "history = model_4.fit(x=[x_train_data_padded,x_train_data_padded], y=categorical_labels_train, \n",
    "                      batch_size=50, epochs=10, validation_split=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4058823585510254"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss, accuracy = model_4.evaluate(x=[x_test_data_padded,x_test_data_padded], y=categorical_labels_test, verbose=0)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               precision    recall  f1-score   support\n",
      "\n",
      "     negative       0.16      0.11      0.13       405\n",
      "      neutral       0.52      0.65      0.58      1155\n",
      "     positive       0.23      0.22      0.22       424\n",
      "very positive       0.02      0.01      0.01       112\n",
      "very_negative       0.12      0.03      0.04       114\n",
      "\n",
      "     accuracy                           0.41      2210\n",
      "    macro avg       0.21      0.20      0.20      2210\n",
      " weighted avg       0.35      0.41      0.37      2210\n",
      "\n"
     ]
    }
   ],
   "source": [
    "raw_predictions = model_4.predict(x=[x_test_data_padded,x_test_data_padded])\n",
    "class_predictions = [np.argmax(x) for x in raw_predictions]\n",
    "print(classification_report(y_test_data_categ, le.inverse_transform(class_predictions)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}