Chatbot with glove embeddings and LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "from keras.layers import Input, Embedding, LSTM, TimeDistributed, Dense, Bidirectional\n",
    "from keras.models import Model, load_model\n",
    "import tensorflow as tf\n",
    "from keras.layers import SimpleRNN\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import Input, Dense, LSTM, TimeDistributed\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Global Variables declaration and intitialization\n",
    "INPUT_VECTOR_LENGTH = 20\n",
    "OUTPUT_VECTORLENGTH = 20\n",
    "minimum_length = 2\n",
    "maximum_length = 20\n",
    "sample_size = 30000 \n",
    "WORD_START = 1\n",
    "WORD_PADDING = 0\n",
    "GLOVE_MODEL = \"C:/Users/15712/Downloads/glove.6B/glove.6B.50d.txt\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "exit_words = [\n",
    "        'bye', 'goodbye', 'exit', \n",
    "        'tata','see you','terminate',\n",
    "        'Bye', 'Goodbye', 'Exit',\n",
    "        'Tata','See you','Terminate'\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Mapping the Ids to lines and splitting the lines by using the delimiter.\n",
    "def map_linetoID(movie_lines):\n",
    "    linetoID_mapping = {}\n",
    "    for line in movie_lines:\n",
    "        split_line = line.split(' +++$+++ ')\n",
    "        if len(split_line) == 5:\n",
    "            linetoID_mapping[split_line[0]] = split_line[4]\n",
    "    return linetoID_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Splitting the converstions by the delimiter and creating a list of coversation ID's.\n",
    "def extract_converstionIDs(conversation_lines):\n",
    "    conversations = []\n",
    "    for line in conversation_lines[:-1]:\n",
    "        split_line = line.split(' +++$+++ ')[-1][1:-1].replace(\"'\",\"\").replace(\" \",\"\")\n",
    "        conversations.append(split_line.split(','))\n",
    "    return conversations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function is used to form pairs of questions and answers.\n",
    "def extract_quesans_pairs(linetoID_mapping,conversations):\n",
    "    questions = []\n",
    "    answers = []\n",
    "    for con in conversations:\n",
    "        for i in range(len(con)-1):\n",
    "            questions.append(linetoID_mapping[con[i]])\n",
    "            answers.append(linetoID_mapping[con[i+1]])\n",
    "    return questions,answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function is used to transsfrom the text\n",
    "#For example I'm gets transformed to I am\n",
    "def transform_text(input_text):\n",
    "    input_text = input_text.lower()\n",
    "    input_text = re.sub(r\"I'm\", \"I am\", input_text)\n",
    "    input_text = re.sub(r\"he's\", \"he is\", input_text)\n",
    "    input_text = re.sub(r\"she's\", \"she is\", input_text)\n",
    "    input_text = re.sub(r\"it's\", \"it is\", input_text)\n",
    "    input_text = re.sub(r\"that's\", \"that is\", input_text)\n",
    "    input_text = re.sub(r\"what's\", \"that is\", input_text)\n",
    "    input_text = re.sub(r\"where's\", \"where is\", input_text)\n",
    "    input_text = re.sub(r\"how's\", \"how is\", input_text)\n",
    "    input_text = re.sub(r\"\\'ll\", \" will\", input_text)\n",
    "    input_text = re.sub(r\"\\'ve\", \" have\", input_text)\n",
    "    input_text = re.sub(r\"\\'re\", \" are\", input_text)\n",
    "    input_text = re.sub(r\"\\'d\", \" would\", input_text)\n",
    "    input_text = re.sub(r\"\\'re\", \" are\", input_text)\n",
    "    input_text = re.sub(r\"won't\", \"will not\", input_text)\n",
    "    input_text = re.sub(r\"can't\", \"cannot\", input_text)\n",
    "    input_text = re.sub(r\"n't\", \" not\", input_text)\n",
    "    input_text = re.sub(r\"'til\", \"until\", input_text)\n",
    "    input_text = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|]\", \"\", input_text)\n",
    "    input_text = \" \".join(input_text.split())\n",
    "    return input_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter the questions and answer. The minimum length is 2 and \n",
    "#maximum is 20\n",
    "def filter_ques_ans(clean_questions,clean_answers):\n",
    "    # Filter out the questions that are too short/long\n",
    "    short_questions_temp = []\n",
    "    short_answers_temp = []\n",
    "    for i, question in enumerate(clean_questions):\n",
    "        if len(question.split()) >= minimum_length and len(question.split()) <= maximum_length:\n",
    "            short_questions_temp.append(question)\n",
    "            short_answers_temp.append(clean_answers[i])\n",
    "    short_questions = []\n",
    "    short_answers = []\n",
    "    for i, answer in enumerate(short_answers_temp):\n",
    "        if len(answer.split()) >= minimum_length and len(answer.split()) <= maximum_length:\n",
    "            short_answers.append(answer)\n",
    "            short_questions.append(short_questions_temp[i])\n",
    "    return short_questions,short_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calculate the word count \n",
    "def create_vocabulary(tokenized_ques,tokenized_ans):\n",
    "    vocabulary = {}\n",
    "    for question in tokenized_ques:\n",
    "        for word in question:\n",
    "            if word not in vocabulary:\n",
    "                vocabulary[word] = 1\n",
    "            else:\n",
    "                vocabulary[word] += 1\n",
    "    for answer in tokenized_ans:\n",
    "        for word in answer:\n",
    "            if word not in vocabulary:\n",
    "                vocabulary[word] = 1\n",
    "            else:\n",
    "                vocabulary[word] += 1  \n",
    "    return vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create the encodings and decodings by assigning unique \n",
    "#index to the words.\n",
    "def create_encoding_decoding(vocabulary):\n",
    "    threshold = 15\n",
    "    count = 0\n",
    "    for k,v in vocabulary.items():\n",
    "        if v >= threshold:\n",
    "            count += 1\n",
    "    vocab_size  = 2 \n",
    "    encoding = {}\n",
    "    decoding = {1: 'START'}\n",
    "    for word, count in vocabulary.items():\n",
    "        if count >= threshold:\n",
    "            encoding[word] = vocab_size \n",
    "            decoding[vocab_size ] = word\n",
    "            vocab_size += 1\n",
    "    return encoding,decoding,vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert the training and validation data into vectors\n",
    "def transform(encoding, data, vector_size=20):\n",
    "    transformed_data = np.zeros(shape=(len(data), vector_size))\n",
    "    for i in range(len(data)):\n",
    "        for j in range(min(len(data[i]), vector_size)):\n",
    "            try:\n",
    "                transformed_data[i][j] = encoding[data[i][j]]\n",
    "            except:\n",
    "                transformed_data[i][j] = encoding['<UNKNOWN>']\n",
    "    return transformed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create glove embeddings form the pre-trained glove model.\n",
    "def create_gloveEmbeddings(encoding,size):\n",
    "    file = open(GLOVE_MODEL, mode='rt', encoding='utf8')\n",
    "    words = set()\n",
    "    word_to_vec_map = {}\n",
    "    for line in file:\n",
    "        line = line.strip().split()\n",
    "        word = line[0]\n",
    "        words.add(word)\n",
    "        word_to_vec_map[word] = np.array(line[1:], dtype=np.float64)\n",
    "    embedding_matrix = np.zeros((size, 50))\n",
    "    for word,index in encoding.items():\n",
    "        try:\n",
    "            embedding_matrix[index, :] = word_to_vec_map[word.lower()]\n",
    "        except: continue\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating the LSTM model\n",
    "def create_model(dict_size,embed_layer,hidden_dim):\n",
    "    \n",
    "    encoder_inputs = Input(shape=(maximum_length, ), dtype='int32',)\n",
    "    encoder_embedding = embed_layer(encoder_inputs)\n",
    "    encoder_LSTM = LSTM(hidden_dim, return_state=True)\n",
    "    encoder_outputs, state_h, state_c = encoder_LSTM(encoder_embedding)\n",
    "    decoder_inputs = Input(shape=(maximum_length, ), dtype='int32',)\n",
    "    decoder_embedding = embed_layer(decoder_inputs)\n",
    "    decoder_LSTM = LSTM(hidden_dim, return_state=True, return_sequences=True)\n",
    "    decoder_outputs, _, _ = decoder_LSTM(decoder_embedding, initial_state=[state_h, state_c])\n",
    "    outputs = TimeDistributed(Dense(dict_size, activation='softmax'))(decoder_outputs)\n",
    "    model = Model([encoder_inputs, decoder_inputs], outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predicting the answer to the question\n",
    "#and returning the output vectors.\n",
    "def prediction_answer(user_input,model):\n",
    "    transformed_input = transform_text(user_input)\n",
    "    input_tokens = [nltk.word_tokenize(transformed_input)]\n",
    "    input_tokens = [input_tokens[0][::-1]]  #reverseing input seq\n",
    "    encoder_input = transform(encoding, input_tokens, 20)\n",
    "    decoder_input = np.zeros(shape=(len(encoder_input), OUTPUT_VECTORLENGTH))\n",
    "    decoder_input[:,0] = WORD_START\n",
    "    for i in range(1, OUTPUT_VECTORLENGTH):\n",
    "        pred_output = model.predict([encoder_input, decoder_input]).argmax(axis=2)\n",
    "        decoder_input[:,i] = pred_output[:,i]\n",
    "    return pred_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#decoding the vectors.\n",
    "def decode_answer(decoding, ans_vec):\n",
    "    ans = ''\n",
    "    for i in ans_vec:\n",
    "        if i == 0:\n",
    "            break\n",
    "        ans += ' '\n",
    "        ans += decoding[i]\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Samples used for training: 24000\n",
      "Number of samples in the validation: 6000\n",
      "Length of vocabulary: 16570\n",
      "The size of the dictionary: 1963\n",
      "The size of encoding: 1961\n",
      "The size of decoding: 1962\n",
      "Shape of Encoded Training Input (24000, 20)\n",
      "Shape of Encoded Training Output (24000, 20)\n",
      "Shape of Encoded validation Input (6000, 20)\n",
      "Shape of Encoded validation Output (6000, 20)\n",
      "(1963, 50)\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 20)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 20)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 20, 50)       98150       input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, 300), (None, 421200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, 20, 300), (N 421200      embedding_1[1][0]                \n",
      "                                                                 lstm_1[0][1]                     \n",
      "                                                                 lstm_1[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 20, 1963)     590863      lstm_2[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 1,531,413\n",
      "Trainable params: 1,531,413\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "linetoID_mapping={}\n",
    "conversations=[]\n",
    "#Reading the conversational data\n",
    "movie_lines = open('C:/Users/15712/Downloads/cornell_movie_dialogs_corpus/cornell movie-dialogs corpus/movie_lines.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "conversation_lines = open('C:/Users/15712/Downloads/cornell_movie_dialogs_corpus/cornell movie-dialogs corpus/movie_conversations.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "#calling map_linetoID()\n",
    "linetoID_mapping=map_linetoID(movie_lines)\n",
    "    \n",
    "#calling extract_converstions()\n",
    "conversations=extract_converstionIDs(conversation_lines)\n",
    "    \n",
    "#extracting question answer pairs\n",
    "questions,answers=extract_quesans_pairs(linetoID_mapping,conversations)\n",
    "transformed_ques = []\n",
    "for question in questions:\n",
    "    transformed_ques.append( transform_text(question))\n",
    "transformed_answers = []    \n",
    "for answer in answers:\n",
    "     transformed_answers.append(transform_text(answer))\n",
    "    \n",
    "#Limiting the length of questionas and answers\n",
    "filtered_questions=[]\n",
    "filtered_answers=[]\n",
    "filtered_questions,filtered_answers=filter_ques_ans(transformed_ques,transformed_answers)\n",
    "    \n",
    "#Tokeninzing\n",
    "filtered_questions = filtered_questions[:sample_size]\n",
    "filtered_answers = filtered_answers[:sample_size]\n",
    "#tokenizing the questions and answers\n",
    "tokenized_ques = [nltk.word_tokenize(sent) for sent in filtered_questions]\n",
    "tokenized_ans = [nltk.word_tokenize(sent) for sent in filtered_answers]\n",
    "    \n",
    "#Splitting the data into training and validation datasets\n",
    "size = len(tokenized_ques)\n",
    "training_input  = tokenized_ques[:round(size*(80/100))]\n",
    "training_input  = [tr_input[::-1] for tr_input in training_input] #reverseing input seq for better performance\n",
    "training_output = tokenized_ans[:round(size*(80/100))]\n",
    "\n",
    "# We will use the remaining for validation\n",
    "validation_input = tokenized_ques[round(size*(80/100)):]\n",
    "validation_input  = [val_input[::-1] for val_input in validation_input] #reverseing input seq for better performance\n",
    "validation_output = tokenized_ans[round(size*(80/100)):]\n",
    "\n",
    "print('Number of Samples used for training:', len(training_input))\n",
    "print('Number of samples in the validation:', len(validation_input))\n",
    "    \n",
    "#creating vacabulary\n",
    "vocabulary={}\n",
    "vocabulary=create_vocabulary(tokenized_ques,tokenized_ans)\n",
    "print(\"Length of vocabulary:\", len(vocabulary))\n",
    "    \n",
    "#creating encodings and decodings\n",
    "dict_size=0\n",
    "encoding={}\n",
    "decoding={}\n",
    "encoding,decoding,dict_size=create_encoding_decoding(vocabulary)\n",
    "dict_size=dict_size+1\n",
    "decoding[len(encoding)+2] = '<UNKNOWN>'\n",
    "encoding['<UNKNOWN>'] = len(encoding)+2\n",
    "print(\"The size of the dictionary:\",dict_size)\n",
    "print(\"The size of encoding:\",len(encoding))\n",
    "print(\"The size of decoding:\",len(decoding))\n",
    "    \n",
    "    \n",
    "#Function call to the transform function\n",
    "encoded_training_input = transform(\n",
    "encoding, training_input, vector_size=INPUT_VECTOR_LENGTH)\n",
    "encoded_training_output = transform(\n",
    "encoding, training_output, vector_size=OUTPUT_VECTORLENGTH)\n",
    "print('Shape of Encoded Training Input', encoded_training_input.shape)\n",
    "print('Shape of Encoded Training Output', encoded_training_output.shape)\n",
    "    \n",
    "#For Validation data \n",
    "encoded_validation_input = transform(\n",
    "encoding, validation_input, vector_size=INPUT_VECTOR_LENGTH)\n",
    "encoded_validation_output = transform(\n",
    "encoding, validation_output, vector_size=OUTPUT_VECTORLENGTH)\n",
    "print('Shape of Encoded validation Input', encoded_validation_input.shape)\n",
    "print('Shape of Encoded validation Output', encoded_validation_output.shape)\n",
    "    \n",
    "#Create the glove embedding which will be used as weights for the embedding layer.\n",
    "tf.keras.backend.clear_session()\n",
    "embedding_matrix = np.zeros((dict_size, 50))\n",
    "embedding_matrix= create_gloveEmbeddings(encoding,dict_size)\n",
    "print(embedding_matrix.shape)\n",
    "    \n",
    "#forming th embedding layer\n",
    "embed_layer = Embedding(input_dim=dict_size, output_dim=50, trainable=True,)\n",
    "embed_layer.build((None,))\n",
    "embed_layer.set_weights([embedding_matrix])\n",
    "    \n",
    "#creating model\n",
    "hidden_dim=300\n",
    "lstm_model = create_model(dict_size,embed_layer,hidden_dim)\n",
    "#getting the summary of model\n",
    "lstm_model.summary()\n",
    "    \n",
    "#compiling the model\n",
    "lstm_model.compile(optimizer='adam', loss ='categorical_crossentropy', metrics = ['accuracy'])\n",
    "    \n",
    "training_encoder_input = encoded_training_input\n",
    "training_decoder_input = np.zeros_like(encoded_training_output)\n",
    "training_decoder_input[:, 1:] = encoded_training_output[:,:-1]\n",
    "training_decoder_input[:, 0] = WORD_START\n",
    "training_decoder_output = np.eye(dict_size)[encoded_training_output.astype('int32')]\n",
    "\n",
    "validation_encoder_input = encoded_validation_input\n",
    "validation_decoder_input = np.zeros_like(encoded_validation_output)\n",
    "validation_decoder_input[:, 1:] = encoded_validation_output[:,:-1]\n",
    "validation_decoder_input[:, 0] = WORD_START\n",
    "validation_decoder_output = np.eye(dict_size)[encoded_validation_output.astype('int32')]\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "   \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\15712\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 24000 samples, validate on 6000 samples\n",
      "Epoch 1/100\n",
      "24000/24000 [==============================] - 135s 6ms/step - loss: 2.6099 - accuracy: 0.5640 - val_loss: 2.3026 - val_accuracy: 0.5974\n",
      "Epoch 2/100\n",
      "24000/24000 [==============================] - 142s 6ms/step - loss: 2.1739 - accuracy: 0.6089 - val_loss: 2.1605 - val_accuracy: 0.6070\n",
      "Epoch 3/100\n",
      "24000/24000 [==============================] - 141s 6ms/step - loss: 2.0650 - accuracy: 0.6194 - val_loss: 2.0866 - val_accuracy: 0.6179\n",
      "Epoch 4/100\n",
      "24000/24000 [==============================] - 145s 6ms/step - loss: 1.9997 - accuracy: 0.6263 - val_loss: 2.0451 - val_accuracy: 0.6225\n",
      "Epoch 5/100\n",
      "24000/24000 [==============================] - 150s 6ms/step - loss: 1.9559 - accuracy: 0.6295 - val_loss: 2.0221 - val_accuracy: 0.6250\n",
      "Epoch 6/100\n",
      "24000/24000 [==============================] - 146s 6ms/step - loss: 1.9216 - accuracy: 0.6320 - val_loss: 2.0041 - val_accuracy: 0.6256\n",
      "Epoch 7/100\n",
      "24000/24000 [==============================] - 148s 6ms/step - loss: 1.8925 - accuracy: 0.6339 - val_loss: 1.9904 - val_accuracy: 0.6270\n",
      "Epoch 8/100\n",
      "24000/24000 [==============================] - 153s 6ms/step - loss: 1.8668 - accuracy: 0.6357 - val_loss: 1.9802 - val_accuracy: 0.6283\n",
      "Epoch 9/100\n",
      "24000/24000 [==============================] - 146s 6ms/step - loss: 1.8434 - accuracy: 0.6376 - val_loss: 1.9723 - val_accuracy: 0.6300\n",
      "Epoch 10/100\n",
      "24000/24000 [==============================] - 148s 6ms/step - loss: 1.8216 - accuracy: 0.6388 - val_loss: 1.9658 - val_accuracy: 0.6299\n",
      "Epoch 11/100\n",
      "24000/24000 [==============================] - 149s 6ms/step - loss: 1.8005 - accuracy: 0.6406 - val_loss: 1.9625 - val_accuracy: 0.6309\n",
      "Epoch 12/100\n",
      "24000/24000 [==============================] - 146s 6ms/step - loss: 1.7809 - accuracy: 0.6418 - val_loss: 1.9630 - val_accuracy: 0.6311\n",
      "Epoch 13/100\n",
      "24000/24000 [==============================] - 151s 6ms/step - loss: 1.7615 - accuracy: 0.6431 - val_loss: 1.9584 - val_accuracy: 0.6317\n",
      "Epoch 14/100\n",
      "24000/24000 [==============================] - 148s 6ms/step - loss: 1.7432 - accuracy: 0.6447 - val_loss: 1.9580 - val_accuracy: 0.6316\n",
      "Epoch 15/100\n",
      "24000/24000 [==============================] - 148s 6ms/step - loss: 1.7248 - accuracy: 0.6460 - val_loss: 1.9614 - val_accuracy: 0.6319\n",
      "Epoch 16/100\n",
      "24000/24000 [==============================] - 152s 6ms/step - loss: 1.7068 - accuracy: 0.6474 - val_loss: 1.9627 - val_accuracy: 0.6318\n",
      "Epoch 17/100\n",
      "24000/24000 [==============================] - 151s 6ms/step - loss: 1.6883 - accuracy: 0.6489 - val_loss: 1.9641 - val_accuracy: 0.6328\n",
      "Epoch 18/100\n",
      "24000/24000 [==============================] - 149s 6ms/step - loss: 1.6703 - accuracy: 0.6502 - val_loss: 1.9676 - val_accuracy: 0.6320\n",
      "Epoch 19/100\n",
      "24000/24000 [==============================] - 148s 6ms/step - loss: 1.6521 - accuracy: 0.6516 - val_loss: 1.9719 - val_accuracy: 0.6321\n",
      "Epoch 20/100\n",
      "24000/24000 [==============================] - 145s 6ms/step - loss: 1.6337 - accuracy: 0.6536 - val_loss: 1.9798 - val_accuracy: 0.6314\n",
      "Epoch 21/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.6144 - accuracy: 0.6554 - val_loss: 1.9828 - val_accuracy: 0.6306\n",
      "Epoch 22/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.5956 - accuracy: 0.6571 - val_loss: 1.9900 - val_accuracy: 0.6312\n",
      "Epoch 23/100\n",
      "24000/24000 [==============================] - 152s 6ms/step - loss: 1.5754 - accuracy: 0.6591 - val_loss: 1.9954 - val_accuracy: 0.6303\n",
      "Epoch 24/100\n",
      "24000/24000 [==============================] - 141s 6ms/step - loss: 1.5553 - accuracy: 0.6613 - val_loss: 2.0058 - val_accuracy: 0.6279\n",
      "Epoch 25/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 1.5343 - accuracy: 0.6644 - val_loss: 2.0129 - val_accuracy: 0.6295\n",
      "Epoch 26/100\n",
      "24000/24000 [==============================] - 136s 6ms/step - loss: 1.5130 - accuracy: 0.6669 - val_loss: 2.0241 - val_accuracy: 0.6285\n",
      "Epoch 27/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 1.4910 - accuracy: 0.6697 - val_loss: 2.0342 - val_accuracy: 0.6278\n",
      "Epoch 28/100\n",
      "24000/24000 [==============================] - 139s 6ms/step - loss: 1.4683 - accuracy: 0.6729 - val_loss: 2.0479 - val_accuracy: 0.6269\n",
      "Epoch 29/100\n",
      "24000/24000 [==============================] - 134s 6ms/step - loss: 1.4453 - accuracy: 0.6762 - val_loss: 2.0568 - val_accuracy: 0.6253\n",
      "Epoch 30/100\n",
      "24000/24000 [==============================] - 139s 6ms/step - loss: 1.4220 - accuracy: 0.6795 - val_loss: 2.0731 - val_accuracy: 0.6230\n",
      "Epoch 31/100\n",
      "24000/24000 [==============================] - 137s 6ms/step - loss: 1.3986 - accuracy: 0.6833 - val_loss: 2.0871 - val_accuracy: 0.6218\n",
      "Epoch 32/100\n",
      "24000/24000 [==============================] - 139s 6ms/step - loss: 1.3748 - accuracy: 0.6871 - val_loss: 2.1002 - val_accuracy: 0.6221\n",
      "Epoch 33/100\n",
      "24000/24000 [==============================] - 139s 6ms/step - loss: 1.3505 - accuracy: 0.6912 - val_loss: 2.1185 - val_accuracy: 0.6207\n",
      "Epoch 34/100\n",
      "24000/24000 [==============================] - 138s 6ms/step - loss: 1.3266 - accuracy: 0.6950 - val_loss: 2.1331 - val_accuracy: 0.6185\n",
      "Epoch 35/100\n",
      "24000/24000 [==============================] - 135s 6ms/step - loss: 1.3027 - accuracy: 0.6995 - val_loss: 2.1542 - val_accuracy: 0.6154\n",
      "Epoch 36/100\n",
      "24000/24000 [==============================] - 136s 6ms/step - loss: 1.2789 - accuracy: 0.7036 - val_loss: 2.1693 - val_accuracy: 0.6178\n",
      "Epoch 37/100\n",
      "24000/24000 [==============================] - 142s 6ms/step - loss: 1.2551 - accuracy: 0.7080 - val_loss: 2.1872 - val_accuracy: 0.6167\n",
      "Epoch 38/100\n",
      "24000/24000 [==============================] - 139s 6ms/step - loss: 1.2316 - accuracy: 0.7126 - val_loss: 2.2046 - val_accuracy: 0.6147\n",
      "Epoch 39/100\n",
      "24000/24000 [==============================] - 138s 6ms/step - loss: 1.2083 - accuracy: 0.7168 - val_loss: 2.2280 - val_accuracy: 0.6115\n",
      "Epoch 40/100\n",
      "24000/24000 [==============================] - 157s 7ms/step - loss: 1.1851 - accuracy: 0.7212 - val_loss: 2.2444 - val_accuracy: 0.6118\n",
      "Epoch 41/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.1616 - accuracy: 0.7258 - val_loss: 2.2663 - val_accuracy: 0.6098\n",
      "Epoch 42/100\n",
      "24000/24000 [==============================] - 140s 6ms/step - loss: 1.1398 - accuracy: 0.7301 - val_loss: 2.2875 - val_accuracy: 0.6087\n",
      "Epoch 43/100\n",
      "24000/24000 [==============================] - 138s 6ms/step - loss: 1.1184 - accuracy: 0.7349 - val_loss: 2.3076 - val_accuracy: 0.6090\n",
      "Epoch 44/100\n",
      "24000/24000 [==============================] - 153s 6ms/step - loss: 1.0959 - accuracy: 0.7394 - val_loss: 2.3290 - val_accuracy: 0.6052\n",
      "Epoch 45/100\n",
      "24000/24000 [==============================] - 155s 6ms/step - loss: 1.0743 - accuracy: 0.7438 - val_loss: 2.3479 - val_accuracy: 0.6050\n",
      "Epoch 46/100\n",
      "24000/24000 [==============================] - 149s 6ms/step - loss: 1.0533 - accuracy: 0.7478 - val_loss: 2.3669 - val_accuracy: 0.6041\n",
      "Epoch 47/100\n",
      "24000/24000 [==============================] - 145s 6ms/step - loss: 1.0328 - accuracy: 0.7526 - val_loss: 2.3915 - val_accuracy: 0.6031\n",
      "Epoch 48/100\n",
      "24000/24000 [==============================] - 154s 6ms/step - loss: 1.0127 - accuracy: 0.7572 - val_loss: 2.4097 - val_accuracy: 0.6026\n",
      "Epoch 49/100\n",
      "24000/24000 [==============================] - 155s 6ms/step - loss: 0.9939 - accuracy: 0.7612 - val_loss: 2.4365 - val_accuracy: 0.6001\n",
      "Epoch 50/100\n",
      "24000/24000 [==============================] - 158s 7ms/step - loss: 0.9736 - accuracy: 0.7655 - val_loss: 2.4586 - val_accuracy: 0.6000\n",
      "Epoch 51/100\n",
      "24000/24000 [==============================] - 162s 7ms/step - loss: 0.9557 - accuracy: 0.7692 - val_loss: 2.4780 - val_accuracy: 0.5983\n",
      "Epoch 52/100\n",
      "24000/24000 [==============================] - 146s 6ms/step - loss: 0.9367 - accuracy: 0.7734 - val_loss: 2.5002 - val_accuracy: 0.5981\n",
      "Epoch 53/100\n",
      "24000/24000 [==============================] - 152s 6ms/step - loss: 0.9190 - accuracy: 0.7779 - val_loss: 2.5194 - val_accuracy: 0.5968\n",
      "Epoch 54/100\n",
      "24000/24000 [==============================] - 155s 6ms/step - loss: 0.9014 - accuracy: 0.7818 - val_loss: 2.5451 - val_accuracy: 0.5956\n",
      "Epoch 55/100\n",
      "24000/24000 [==============================] - 152s 6ms/step - loss: 0.8843 - accuracy: 0.7857 - val_loss: 2.5615 - val_accuracy: 0.5964\n",
      "Epoch 56/100\n",
      "24000/24000 [==============================] - 164s 7ms/step - loss: 0.8680 - accuracy: 0.7891 - val_loss: 2.5855 - val_accuracy: 0.5954\n",
      "Epoch 57/100\n",
      "24000/24000 [==============================] - 165s 7ms/step - loss: 0.8517 - accuracy: 0.7931 - val_loss: 2.6085 - val_accuracy: 0.5947\n",
      "Epoch 58/100\n",
      "24000/24000 [==============================] - 156s 7ms/step - loss: 0.8356 - accuracy: 0.7967 - val_loss: 2.6270 - val_accuracy: 0.5915\n",
      "Epoch 59/100\n",
      "24000/24000 [==============================] - 154s 6ms/step - loss: 0.8195 - accuracy: 0.8000 - val_loss: 2.6541 - val_accuracy: 0.5932\n",
      "Epoch 60/100\n",
      "24000/24000 [==============================] - 2943s 123ms/step - loss: 0.8056 - accuracy: 0.8034 - val_loss: 2.6724 - val_accuracy: 0.5904\n",
      "Epoch 61/100\n",
      "24000/24000 [==============================] - 119s 5ms/step - loss: 0.7907 - accuracy: 0.8069 - val_loss: 2.6978 - val_accuracy: 0.5913\n",
      "Epoch 62/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 0.7760 - accuracy: 0.8103 - val_loss: 2.7208 - val_accuracy: 0.5894\n",
      "Epoch 63/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 0.7610 - accuracy: 0.8139 - val_loss: 2.7428 - val_accuracy: 0.5879\n",
      "Epoch 64/100\n",
      "24000/24000 [==============================] - 138s 6ms/step - loss: 0.7481 - accuracy: 0.8174 - val_loss: 2.7609 - val_accuracy: 0.5883\n",
      "Epoch 65/100\n",
      "24000/24000 [==============================] - 134s 6ms/step - loss: 0.7343 - accuracy: 0.8199 - val_loss: 2.7840 - val_accuracy: 0.5897\n",
      "Epoch 66/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 0.7224 - accuracy: 0.8230 - val_loss: 2.8072 - val_accuracy: 0.5864\n",
      "Epoch 67/100\n",
      "24000/24000 [==============================] - 165s 7ms/step - loss: 0.7108 - accuracy: 0.8254 - val_loss: 2.8295 - val_accuracy: 0.5871\n",
      "Epoch 68/100\n",
      "24000/24000 [==============================] - 162s 7ms/step - loss: 0.6979 - accuracy: 0.8287 - val_loss: 2.8521 - val_accuracy: 0.5854\n",
      "Epoch 69/100\n",
      "24000/24000 [==============================] - 164s 7ms/step - loss: 0.6842 - accuracy: 0.8321 - val_loss: 2.8690 - val_accuracy: 0.5850\n",
      "Epoch 70/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 0.6726 - accuracy: 0.8343 - val_loss: 2.8850 - val_accuracy: 0.5854\n",
      "Epoch 71/100\n",
      "24000/24000 [==============================] - 136s 6ms/step - loss: 0.6615 - accuracy: 0.8374 - val_loss: 2.9173 - val_accuracy: 0.5825\n",
      "Epoch 72/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 0.6558 - accuracy: 0.8380 - val_loss: 2.9310 - val_accuracy: 0.5842\n",
      "Epoch 73/100\n",
      "24000/24000 [==============================] - 132s 5ms/step - loss: 0.6420 - accuracy: 0.8417 - val_loss: 2.9515 - val_accuracy: 0.5842\n",
      "Epoch 74/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 0.6284 - accuracy: 0.8450 - val_loss: 2.9737 - val_accuracy: 0.5815\n",
      "Epoch 75/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 0.6215 - accuracy: 0.8467 - val_loss: 2.9907 - val_accuracy: 0.5823\n",
      "Epoch 76/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 0.6106 - accuracy: 0.8496 - val_loss: 3.0142 - val_accuracy: 0.5829\n",
      "Epoch 77/100\n",
      "24000/24000 [==============================] - 120s 5ms/step - loss: 0.6001 - accuracy: 0.8518 - val_loss: 3.0355 - val_accuracy: 0.5828\n",
      "Epoch 78/100\n",
      "24000/24000 [==============================] - 132s 6ms/step - loss: 0.5904 - accuracy: 0.8544 - val_loss: 3.0566 - val_accuracy: 0.5811\n",
      "Epoch 79/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 0.5820 - accuracy: 0.8559 - val_loss: 3.0743 - val_accuracy: 0.5803\n",
      "Epoch 80/100\n",
      "24000/24000 [==============================] - 133s 6ms/step - loss: 0.5714 - accuracy: 0.8589 - val_loss: 3.0938 - val_accuracy: 0.5794\n",
      "Epoch 81/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 0.5648 - accuracy: 0.8599 - val_loss: 3.1102 - val_accuracy: 0.5786\n",
      "Epoch 82/100\n",
      "24000/24000 [==============================] - 138s 6ms/step - loss: 0.5560 - accuracy: 0.8621 - val_loss: 3.1305 - val_accuracy: 0.5787\n",
      "Epoch 83/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 0.5449 - accuracy: 0.8652 - val_loss: 3.1509 - val_accuracy: 0.5798\n",
      "Epoch 84/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 0.5371 - accuracy: 0.8670 - val_loss: 3.1722 - val_accuracy: 0.5794\n",
      "Epoch 85/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 0.5306 - accuracy: 0.8686 - val_loss: 3.1858 - val_accuracy: 0.5788\n",
      "Epoch 86/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 0.5259 - accuracy: 0.8692 - val_loss: 3.2052 - val_accuracy: 0.5780\n",
      "Epoch 87/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 0.5175 - accuracy: 0.8714 - val_loss: 3.2269 - val_accuracy: 0.5777\n",
      "Epoch 88/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 0.5064 - accuracy: 0.8744 - val_loss: 3.2429 - val_accuracy: 0.5787\n",
      "Epoch 89/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 0.5009 - accuracy: 0.8757 - val_loss: 3.2603 - val_accuracy: 0.5777\n",
      "Epoch 90/100\n",
      "24000/24000 [==============================] - 136s 6ms/step - loss: 0.4922 - accuracy: 0.8779 - val_loss: 3.2829 - val_accuracy: 0.5780\n",
      "Epoch 91/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 0.4887 - accuracy: 0.8784 - val_loss: 3.2951 - val_accuracy: 0.5752\n",
      "Epoch 92/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 0.4854 - accuracy: 0.8787 - val_loss: 3.3147 - val_accuracy: 0.5759\n",
      "Epoch 93/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 0.4753 - accuracy: 0.8815 - val_loss: 3.3321 - val_accuracy: 0.5758\n",
      "Epoch 94/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 0.4716 - accuracy: 0.8822 - val_loss: 3.3420 - val_accuracy: 0.5774\n",
      "Epoch 95/100\n",
      "24000/24000 [==============================] - 134s 6ms/step - loss: 0.4606 - accuracy: 0.8852 - val_loss: 3.3669 - val_accuracy: 0.5753\n",
      "Epoch 96/100\n",
      "24000/24000 [==============================] - 133s 6ms/step - loss: 0.4585 - accuracy: 0.8854 - val_loss: 3.3801 - val_accuracy: 0.5761\n",
      "Epoch 97/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 0.4540 - accuracy: 0.8866 - val_loss: 3.3962 - val_accuracy: 0.5740\n",
      "Epoch 98/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 0.4427 - accuracy: 0.8894 - val_loss: 3.4143 - val_accuracy: 0.5743\n",
      "Epoch 99/100\n",
      "24000/24000 [==============================] - 149s 6ms/step - loss: 0.4335 - accuracy: 0.8923 - val_loss: 3.4348 - val_accuracy: 0.5751\n",
      "Epoch 100/100\n",
      "24000/24000 [==============================] - 135s 6ms/step - loss: 0.4321 - accuracy: 0.8918 - val_loss: 3.4487 - val_accuracy: 0.5750\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x2558ba9a2b0>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fitting the model\n",
    "lstm_model.fit(x=[training_encoder_input, training_decoder_input], y=[training_decoder_output],\n",
    "    validation_data=([validation_encoder_input, validation_decoder_input], [validation_decoder_output]),\n",
    "          batch_size=64, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_model.save('lstm_model_glove_embeddings.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\15712\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "lstm_model=load_model('lstm_model_glove_embeddings.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: well, i thought we would start with pronunciation, if that is okay with you.\n",
      "Answer:  not no general\n",
      "Question: not the hacking and gagging and spitting part. please.\n",
      "Answer:  okay tell sir\n",
      "Question: you are asking me out. that is so cute. that is your name again?\n",
      "Answer:  forget . it it\n",
      "Question: gosh, if only we could find kat a boyfriend...\n",
      "Answer:  let you i\n",
      "Question: c'esc ma tete. this is my head\n",
      "Answer:  right there later\n",
      "Question: that is because it is such a nice one.\n",
      "Answer:  forget . lord\n",
      "Question: how is our little find the wench a date plan progressing?\n",
      "Answer:  well i i\n",
      "Question: you have my word. as a gentleman\n",
      "Answer:  you are know\n",
      "Question: how do you get your hair to look like that?\n",
      "Answer:  <UNKNOWN> quite it\n",
      "Question: sure have.\n",
      "Answer:  i just , do should\n",
      "Question: i really, really, really wanna go, but i cannot. not unless my sister goes.\n",
      "Answer:  i i ,\n",
      "Question: so that is the kind of guy she likes? pretty ones?\n",
      "Answer:  who i yes course\n",
      "Question: you know chastity?\n",
      "Answer:  i i i you\n",
      "Question: i looked for you back at the party, but you always seemed to be occupied.\n",
      "Answer:  i i i\n",
      "Question: i was?\n",
      "Answer:  you one do\n",
      "Question: well, no...\n",
      "Answer:  all no that\n",
      "Question: do you listen to this crap?\n",
      "Answer:  what baby no\n",
      "Question: what crap?\n",
      "Answer:  my i sir i\n",
      "Question: me. this endless ...blonde babble. i'm like, boring myself.\n",
      "Answer:  thank course <UNKNOWN>\n",
      "Question: i figured you would get to the good stuff eventually.\n",
      "Answer:  what i i by\n"
     ]
    }
   ],
   "source": [
    "#Printing the sample Questions and answers.\n",
    "for i in range(20):\n",
    "    output = prediction_answer(filtered_questions[i],lstm_model)\n",
    "    print ('Question:', filtered_questions[i])\n",
    "    print ('Answer:', decode_answer(decoding, output[0]))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello! I am a Generative Chatbot Trained on Cornell Movie Corpus Data\n",
      "~Hello\n",
      " when not ,\n",
      "~How are you doing?\n",
      " not that all\n",
      "~Where are you going?\n",
      " i 'm got\n",
      "~Should we go out?\n",
      " no not . i i i\n",
      "~Why not?\n",
      " i this ...\n",
      "~I want some food\n",
      " i do harry\n",
      "~Where is the pen\n",
      " i . there\n",
      "~when is the flight ?\n",
      " the hotel ya\n",
      "~shall we go out tomorrow?\n",
      " yes , ,\n",
      "~Are you not well?\n",
      " yes i do yes\n",
      "~Let us party today.\n",
      " oh do do\n",
      "~I want to go to paris.\n",
      " thank who then\n",
      "~can you come with me?\n",
      " i i do\n",
      "~Let us go then\n",
      " right wait hello\n",
      "~pack your bags.\n",
      " you baby you i\n",
      "~Who all are coming?\n",
      " pardon 'm i\n",
      "~bye\n"
     ]
    }
   ],
   "source": [
    "print(\"Hello! I am a Generative Chatbot Trained on Cornell Movie Corpus Data\")\n",
    "user_input = input(\"~\")\n",
    "predicted_seq =  prediction_answer(user_input,lstm_model)\n",
    "print (decode_answer(decoding, predicted_seq[0]))\n",
    "while user_input not in exit_words:\n",
    "    try:\n",
    "        user_input = input(\"~\")\n",
    "        if user_input not in exit_words:\n",
    "            decode_seq = prediction_answer(user_input,lstm_model)\n",
    "            print (decode_answer(decoding, decode_seq[0]))\n",
    "        else:\n",
    "            break\n",
    "    except EOFError:\n",
    "        print(\"Bye\")\n",
    "        exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}