Chatbot with skipgram embeddings and LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "from keras.layers import Input, Embedding, LSTM, TimeDistributed, Dense, Bidirectional\n",
    "from keras.models import Model, load_model\n",
    "import tensorflow as tf\n",
    "from keras.layers import SimpleRNN\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import Input, Dense, LSTM, TimeDistributed\n",
    "from keras.models import Model\n",
    "from gensim.models import Word2Vec\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Global Variables declaration and intitialization\n",
    "INPUT_VECTOR_LENGTH = 20\n",
    "OUTPUT_VECTORLENGTH = 20\n",
    "minimum_length = 2\n",
    "maximum_length = 20\n",
    "sample_size = 30000 \n",
    "WORD_START = 1\n",
    "WORD_PADDING = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "exit_words = [\n",
    "        'bye', 'goodbye', 'exit', \n",
    "        'tata','see you','terminate',\n",
    "        'Bye', 'Goodbye', 'Exit',\n",
    "        'Tata','See you','Terminate'\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Mapping the Ids to lines and splitting the lines by using the delimiter.\n",
    "def map_linetoID(movie_lines):\n",
    "    linetoID_mapping = {}\n",
    "    for line in movie_lines:\n",
    "        split_line = line.split(' +++$+++ ')\n",
    "        if len(split_line) == 5:\n",
    "            linetoID_mapping[split_line[0]] = split_line[4]\n",
    "    return linetoID_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Splitting the converstions by the delimiter and creating a list of coversation ID's.\n",
    "def extract_converstionIDs(conversation_lines):\n",
    "    conversations = []\n",
    "    for line in conversation_lines[:-1]:\n",
    "        split_line = line.split(' +++$+++ ')[-1][1:-1].replace(\"'\",\"\").replace(\" \",\"\")\n",
    "        conversations.append(split_line.split(','))\n",
    "    return conversations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function is used to form pairs of questions and answers.\n",
    "def extract_quesans_pairs(linetoID_mapping,conversations):\n",
    "    questions = []\n",
    "    answers = []\n",
    "    for con in conversations:\n",
    "        for i in range(len(con)-1):\n",
    "            questions.append(linetoID_mapping[con[i]])\n",
    "            answers.append(linetoID_mapping[con[i+1]])\n",
    "    return questions,answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function is used to transsfrom the text\n",
    "#For example I'm gets transformed to I am\n",
    "def transform_text(input_text):\n",
    "    input_text = input_text.lower()\n",
    "    input_text = re.sub(r\"I'm\", \"I am\", input_text)\n",
    "    input_text = re.sub(r\"he's\", \"he is\", input_text)\n",
    "    input_text = re.sub(r\"she's\", \"she is\", input_text)\n",
    "    input_text = re.sub(r\"it's\", \"it is\", input_text)\n",
    "    input_text = re.sub(r\"that's\", \"that is\", input_text)\n",
    "    input_text = re.sub(r\"what's\", \"that is\", input_text)\n",
    "    input_text = re.sub(r\"where's\", \"where is\", input_text)\n",
    "    input_text = re.sub(r\"how's\", \"how is\", input_text)\n",
    "    input_text = re.sub(r\"\\'ll\", \" will\", input_text)\n",
    "    input_text = re.sub(r\"\\'ve\", \" have\", input_text)\n",
    "    input_text = re.sub(r\"\\'re\", \" are\", input_text)\n",
    "    input_text = re.sub(r\"\\'d\", \" would\", input_text)\n",
    "    input_text = re.sub(r\"\\'re\", \" are\", input_text)\n",
    "    input_text = re.sub(r\"won't\", \"will not\", input_text)\n",
    "    input_text = re.sub(r\"can't\", \"cannot\", input_text)\n",
    "    input_text = re.sub(r\"n't\", \" not\", input_text)\n",
    "    input_text = re.sub(r\"'til\", \"until\", input_text)\n",
    "    input_text = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|]\", \"\", input_text)\n",
    "    input_text = \" \".join(input_text.split())\n",
    "    return input_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Filter the questions and answer. The minimum length is 2 and \n",
    "#maximum is 20\n",
    "def filter_ques_ans(clean_questions,clean_answers):\n",
    "    short_questions_temp = []\n",
    "    short_answers_temp = []\n",
    "    for i, question in enumerate(clean_questions):\n",
    "        if len(question.split()) >= minimum_length and len(question.split()) <= maximum_length:\n",
    "            short_questions_temp.append(question)\n",
    "            short_answers_temp.append(clean_answers[i])\n",
    "    short_questions = []\n",
    "    short_answers = []\n",
    "    for i, answer in enumerate(short_answers_temp):\n",
    "        if len(answer.split()) >= minimum_length and len(answer.split()) <= maximum_length:\n",
    "            short_answers.append(answer)\n",
    "            short_questions.append(short_questions_temp[i])\n",
    "    return short_questions,short_answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Calculate the word count \n",
    "def create_vocabulary(tokenized_ques,tokenized_ans):\n",
    "    vocabulary = {}\n",
    "    for question in tokenized_ques:\n",
    "        for word in question:\n",
    "            if word not in vocabulary:\n",
    "                vocabulary[word] = 1\n",
    "            else:\n",
    "                vocabulary[word] += 1\n",
    "    for answer in tokenized_ans:\n",
    "        for word in answer:\n",
    "            if word not in vocabulary:\n",
    "                vocabulary[word] = 1\n",
    "            else:\n",
    "                vocabulary[word] += 1  \n",
    "    return vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create the encodings and decodings by assigning unique \n",
    "#index to the words.\n",
    "def create_encoding_decoding(vocabulary):\n",
    "    threshold = 15\n",
    "    count = 0\n",
    "    encoding_skipgram=[]\n",
    "    for k,v in vocabulary.items():\n",
    "        if v >= threshold:\n",
    "            count += 1\n",
    "    vocab_size  = 2 \n",
    "    encoding = {}\n",
    "    decoding = {1: 'START'}\n",
    "    for word, count in vocabulary.items():\n",
    "        if count >= threshold:\n",
    "            encoding[word] = vocab_size \n",
    "            decoding[vocab_size ] = word\n",
    "            encoding_skipgram.append(word)\n",
    "            vocab_size += 1\n",
    "    return encoding,decoding,vocab_size,encoding_skipgram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert the training and validation data into vectors\n",
    "def transform(encoding, data, vector_size=20):\n",
    "    transformed_data = np.zeros(shape=(len(data), vector_size))\n",
    "    for i in range(len(data)):\n",
    "        for j in range(min(len(data[i]), vector_size)):\n",
    "            try:\n",
    "                transformed_data[i][j] = encoding[data[i][j]]\n",
    "            except:\n",
    "                transformed_data[i][j] = encoding['<UNKNOWN>']\n",
    "    return transformed_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create skip gram model and apply it on the encoding data.\n",
    "def create_skipgramEmbeddings(encoding,size,encoding_skipgram):\n",
    "    skipgram_model = Word2Vec(encoding_skipgram,sg=1)\n",
    "    embedding_matrix = np.zeros((size, 100))\n",
    "    for word,index in encoding.items():\n",
    "        try:\n",
    "            extractedword=word.lower()\n",
    "            embedding_matrix[index, :] = skipgram_model.wv[extractedword]\n",
    "        except: continue\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating the LSTM model\n",
    "def create_model(dict_size,embed_layer,hidden_dim):\n",
    "    \n",
    "    encoder_inputs = Input(shape=(maximum_length, ), dtype='int32',)\n",
    "    encoder_embedding = embed_layer(encoder_inputs)\n",
    "    encoder_LSTM = LSTM(hidden_dim, return_state=True)\n",
    "    encoder_outputs, state_h, state_c = encoder_LSTM(encoder_embedding)\n",
    "    decoder_inputs = Input(shape=(maximum_length, ), dtype='int32',)\n",
    "    decoder_embedding = embed_layer(decoder_inputs)\n",
    "    decoder_LSTM = LSTM(hidden_dim, return_state=True, return_sequences=True)\n",
    "    decoder_outputs, _, _ = decoder_LSTM(decoder_embedding, initial_state=[state_h, state_c])\n",
    "    outputs = TimeDistributed(Dense(dict_size, activation='softmax'))(decoder_outputs)\n",
    "    model = Model([encoder_inputs, decoder_inputs], outputs)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predicting the answer to the question\n",
    "#and returning the output vectors.\n",
    "def prediction_answer(user_input,model):\n",
    "    transformed_input = transform_text(user_input)\n",
    "    input_tokens = [nltk.word_tokenize(transformed_input)]\n",
    "    input_tokens = [input_tokens[0][::-1]]  #reverseing input seq\n",
    "    encoder_input = transform(encoding, input_tokens, 20)\n",
    "    decoder_input = np.zeros(shape=(len(encoder_input), OUTPUT_VECTORLENGTH))\n",
    "    decoder_input[:,0] = WORD_START\n",
    "    for i in range(1, OUTPUT_VECTORLENGTH):\n",
    "        pred_output = model.predict([encoder_input, decoder_input]).argmax(axis=2)\n",
    "        decoder_input[:,i] = pred_output[:,i]\n",
    "    return pred_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#decoding the vectors.\n",
    "def decode_answer(decoding, ans_vec):\n",
    "    ans = ''\n",
    "    for i in ans_vec:\n",
    "        if i == 0:\n",
    "            break\n",
    "        ans += ' '\n",
    "        ans += decoding[i]\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Samples used for training: 24000\n",
      "Number of samples in the validation: 6000\n",
      "Length of vocabulary: 16570\n",
      "The size of the dictionary: 1963\n",
      "The size of encoding: 1961\n",
      "The size of decoding: 1962\n",
      "Shape of Encoded Training Input (24000, 20)\n",
      "Shape of Encoded Training Output (24000, 20)\n",
      "Shape of Encoded validation Input (6000, 20)\n",
      "Shape of Encoded validation Output (6000, 20)\n",
      "(1963, 100)\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 20)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            (None, 20)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 20, 100)      196300      input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, 300), (None, 481200      embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   [(None, 20, 300), (N 481200      embedding_1[1][0]                \n",
      "                                                                 lstm_1[0][1]                     \n",
      "                                                                 lstm_1[0][2]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 20, 1963)     590863      lstm_2[0][0]                     \n",
      "==================================================================================================\n",
      "Total params: 1,749,563\n",
      "Trainable params: 1,749,563\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "linetoID_mapping={}\n",
    "conversations=[]\n",
    "#Reading the conversational data\n",
    "movie_lines = open('C:/Users/15712/Downloads/cornell_movie_dialogs_corpus/cornell movie-dialogs corpus/movie_lines.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "conversation_lines = open('C:/Users/15712/Downloads/cornell_movie_dialogs_corpus/cornell movie-dialogs corpus/movie_conversations.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "#calling map_linetoID()\n",
    "linetoID_mapping=map_linetoID(movie_lines)\n",
    "    \n",
    "#calling extract_converstions()\n",
    "conversations=extract_converstionIDs(conversation_lines)\n",
    "    \n",
    "#extracting question answer pairs\n",
    "questions,answers=extract_quesans_pairs(linetoID_mapping,conversations)\n",
    "transformed_ques = []\n",
    "for question in questions:\n",
    "    transformed_ques.append( transform_text(question))\n",
    "transformed_answers = []    \n",
    "for answer in answers:\n",
    "     transformed_answers.append(transform_text(answer))\n",
    "    \n",
    "#Limiting the length of questionas and answers\n",
    "filtered_questions=[]\n",
    "filtered_answers=[]\n",
    "filtered_questions,filtered_answers=filter_ques_ans(transformed_ques,transformed_answers)\n",
    "    \n",
    "#Tokeninzing\n",
    "filtered_questions = filtered_questions[:sample_size]\n",
    "filtered_answers = filtered_answers[:sample_size]\n",
    "#tokenizing the questions and answers\n",
    "tokenized_ques = [nltk.word_tokenize(sent) for sent in filtered_questions]\n",
    "tokenized_ans = [nltk.word_tokenize(sent) for sent in filtered_answers]\n",
    "    \n",
    "#Splitting the data into training and validation datasets\n",
    "size = len(tokenized_ques)\n",
    "training_input  = tokenized_ques[:round(size*(80/100))]\n",
    "training_input  = [tr_input[::-1] for tr_input in training_input] #reverseing input seq for better performance\n",
    "training_output = tokenized_ans[:round(size*(80/100))]\n",
    "\n",
    "# We will use the remaining for validation\n",
    "validation_input = tokenized_ques[round(size*(80/100)):]\n",
    "validation_input  = [val_input[::-1] for val_input in validation_input] #reverseing input seq for better performance\n",
    "validation_output = tokenized_ans[round(size*(80/100)):]\n",
    "\n",
    "print('Number of Samples used for training:', len(training_input))\n",
    "print('Number of samples in the validation:', len(validation_input))\n",
    "    \n",
    "#creating vacabulary\n",
    "vocabulary={}\n",
    "vocabulary=create_vocabulary(tokenized_ques,tokenized_ans)\n",
    "print(\"Length of vocabulary:\", len(vocabulary))\n",
    "    \n",
    "#creating encodings and decodings\n",
    "dict_size=0\n",
    "encoding={}\n",
    "decoding={}\n",
    "encoding,decoding,dict_size,encoding_skipgram=create_encoding_decoding(vocabulary)\n",
    "dict_size=dict_size+1\n",
    "decoding[len(encoding)+2] = '<UNKNOWN>'\n",
    "encoding['<UNKNOWN>'] = len(encoding)+2\n",
    "print(\"The size of the dictionary:\",dict_size)\n",
    "print(\"The size of encoding:\",len(encoding))\n",
    "print(\"The size of decoding:\",len(decoding))\n",
    "    \n",
    "    \n",
    "#Function call to the transform function\n",
    "encoded_training_input = transform(\n",
    "encoding, training_input, vector_size=INPUT_VECTOR_LENGTH)\n",
    "encoded_training_output = transform(\n",
    "encoding, training_output, vector_size=OUTPUT_VECTORLENGTH)\n",
    "print('Shape of Encoded Training Input', encoded_training_input.shape)\n",
    "print('Shape of Encoded Training Output', encoded_training_output.shape)\n",
    "    \n",
    "#For Validation data \n",
    "encoded_validation_input = transform(\n",
    "encoding, validation_input, vector_size=INPUT_VECTOR_LENGTH)\n",
    "encoded_validation_output = transform(\n",
    "encoding, validation_output, vector_size=OUTPUT_VECTORLENGTH)\n",
    "print('Shape of Encoded validation Input', encoded_validation_input.shape)\n",
    "print('Shape of Encoded validation Output', encoded_validation_output.shape)\n",
    "    \n",
    "#Create the skip-gram embedding which will be used as weights for the embedding layer.\n",
    "tf.keras.backend.clear_session()\n",
    "embedding_matrix = np.zeros((dict_size, 100))\n",
    "embedding_matrix= create_skipgramEmbeddings(encoding,dict_size,encoding_skipgram)\n",
    "print(embedding_matrix.shape)\n",
    "    \n",
    "#forming th embedding layer\n",
    "embed_layer = Embedding(input_dim=dict_size, output_dim=100, trainable=True,)\n",
    "embed_layer.build((None,))\n",
    "embed_layer.set_weights([embedding_matrix])\n",
    "    \n",
    "#creating model\n",
    "hidden_dim=300\n",
    "lstm_model = create_model(dict_size,embed_layer,hidden_dim)\n",
    "#getting the summary of model\n",
    "lstm_model.summary()\n",
    "    \n",
    "#compiling the model\n",
    "lstm_model.compile(optimizer='adam', loss ='categorical_crossentropy', metrics = ['accuracy'])\n",
    "    \n",
    "training_encoder_input = encoded_training_input\n",
    "training_decoder_input = np.zeros_like(encoded_training_output)\n",
    "training_decoder_input[:, 1:] = encoded_training_output[:,:-1]\n",
    "training_decoder_input[:, 0] = WORD_START\n",
    "training_decoder_output = np.eye(dict_size)[encoded_training_output.astype('int32')]\n",
    "\n",
    "validation_encoder_input = encoded_validation_input\n",
    "validation_decoder_input = np.zeros_like(encoded_validation_output)\n",
    "validation_decoder_input[:, 1:] = encoded_validation_output[:,:-1]\n",
    "validation_decoder_input[:, 0] = WORD_START\n",
    "validation_decoder_output = np.eye(dict_size)[encoded_validation_output.astype('int32')]\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "   \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\15712\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\framework\\indexed_slices.py:424: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 24000 samples, validate on 6000 samples\n",
      "Epoch 1/100\n",
      "24000/24000 [==============================] - 135s 6ms/step - loss: 2.7687 - accuracy: 0.5504 - val_loss: 2.3952 - val_accuracy: 0.5838\n",
      "Epoch 2/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 2.2771 - accuracy: 0.5983 - val_loss: 2.2393 - val_accuracy: 0.5994\n",
      "Epoch 3/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 2.1387 - accuracy: 0.6115 - val_loss: 2.1465 - val_accuracy: 0.6105\n",
      "Epoch 4/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 2.0601 - accuracy: 0.6207 - val_loss: 2.0955 - val_accuracy: 0.6186\n",
      "Epoch 5/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 2.0077 - accuracy: 0.6264 - val_loss: 2.0615 - val_accuracy: 0.6224\n",
      "Epoch 6/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.9684 - accuracy: 0.6295 - val_loss: 2.0376 - val_accuracy: 0.6245\n",
      "Epoch 7/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 1.9356 - accuracy: 0.6320 - val_loss: 2.0195 - val_accuracy: 0.6265\n",
      "Epoch 8/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 1.9078 - accuracy: 0.6334 - val_loss: 2.0060 - val_accuracy: 0.6268\n",
      "Epoch 9/100\n",
      "24000/24000 [==============================] - 125s 5ms/step - loss: 1.8837 - accuracy: 0.6353 - val_loss: 1.9984 - val_accuracy: 0.6268\n",
      "Epoch 10/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.8626 - accuracy: 0.6369 - val_loss: 1.9900 - val_accuracy: 0.6284\n",
      "Epoch 11/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.8431 - accuracy: 0.6384 - val_loss: 1.9833 - val_accuracy: 0.6300\n",
      "Epoch 12/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.8238 - accuracy: 0.6398 - val_loss: 1.9818 - val_accuracy: 0.6292\n",
      "Epoch 13/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 1.8064 - accuracy: 0.6410 - val_loss: 1.9778 - val_accuracy: 0.6309\n",
      "Epoch 14/100\n",
      "24000/24000 [==============================] - 132s 5ms/step - loss: 1.7898 - accuracy: 0.6422 - val_loss: 1.9747 - val_accuracy: 0.6308\n",
      "Epoch 15/100\n",
      "24000/24000 [==============================] - 135s 6ms/step - loss: 1.7740 - accuracy: 0.6430 - val_loss: 1.9755 - val_accuracy: 0.6310\n",
      "Epoch 16/100\n",
      "24000/24000 [==============================] - 132s 6ms/step - loss: 1.7590 - accuracy: 0.6443 - val_loss: 1.9775 - val_accuracy: 0.6310\n",
      "Epoch 17/100\n",
      "24000/24000 [==============================] - 132s 6ms/step - loss: 1.7440 - accuracy: 0.6455 - val_loss: 1.9767 - val_accuracy: 0.6318\n",
      "Epoch 18/100\n",
      "24000/24000 [==============================] - 136s 6ms/step - loss: 1.7295 - accuracy: 0.6466 - val_loss: 1.9791 - val_accuracy: 0.6312\n",
      "Epoch 19/100\n",
      "24000/24000 [==============================] - 136s 6ms/step - loss: 1.7155 - accuracy: 0.6474 - val_loss: 1.9824 - val_accuracy: 0.6314\n",
      "Epoch 20/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 1.7016 - accuracy: 0.6482 - val_loss: 1.9856 - val_accuracy: 0.6306\n",
      "Epoch 21/100\n",
      "24000/24000 [==============================] - 183s 8ms/step - loss: 1.6883 - accuracy: 0.6494 - val_loss: 1.9869 - val_accuracy: 0.6318\n",
      "Epoch 22/100\n",
      "24000/24000 [==============================] - 133s 6ms/step - loss: 1.6747 - accuracy: 0.6506 - val_loss: 1.9913 - val_accuracy: 0.6316\n",
      "Epoch 23/100\n",
      "24000/24000 [==============================] - 125s 5ms/step - loss: 1.6617 - accuracy: 0.6519 - val_loss: 1.9962 - val_accuracy: 0.6310\n",
      "Epoch 24/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.6488 - accuracy: 0.6523 - val_loss: 1.9991 - val_accuracy: 0.6318\n",
      "Epoch 25/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 1.6353 - accuracy: 0.6535 - val_loss: 2.0030 - val_accuracy: 0.6320\n",
      "Epoch 26/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 1.6232 - accuracy: 0.6547 - val_loss: 2.0064 - val_accuracy: 0.6312\n",
      "Epoch 27/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 1.6102 - accuracy: 0.6560 - val_loss: 2.0122 - val_accuracy: 0.6308\n",
      "Epoch 28/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.5978 - accuracy: 0.6570 - val_loss: 2.0209 - val_accuracy: 0.6312\n",
      "Epoch 29/100\n",
      "24000/24000 [==============================] - 120s 5ms/step - loss: 1.5856 - accuracy: 0.6582 - val_loss: 2.0217 - val_accuracy: 0.6302\n",
      "Epoch 30/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 1.5732 - accuracy: 0.6595 - val_loss: 2.0257 - val_accuracy: 0.6300\n",
      "Epoch 31/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.5608 - accuracy: 0.6609 - val_loss: 2.0307 - val_accuracy: 0.6302\n",
      "Epoch 32/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.5490 - accuracy: 0.6624 - val_loss: 2.0374 - val_accuracy: 0.6293\n",
      "Epoch 33/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.5372 - accuracy: 0.6634 - val_loss: 2.0433 - val_accuracy: 0.6292\n",
      "Epoch 34/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.5258 - accuracy: 0.6645 - val_loss: 2.0489 - val_accuracy: 0.6293\n",
      "Epoch 35/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.5140 - accuracy: 0.6659 - val_loss: 2.0540 - val_accuracy: 0.6286\n",
      "Epoch 36/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 1.5026 - accuracy: 0.6674 - val_loss: 2.0616 - val_accuracy: 0.6284\n",
      "Epoch 37/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 1.4913 - accuracy: 0.6689 - val_loss: 2.0671 - val_accuracy: 0.6283\n",
      "Epoch 38/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.4801 - accuracy: 0.6703 - val_loss: 2.0739 - val_accuracy: 0.6279\n",
      "Epoch 39/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.4685 - accuracy: 0.6721 - val_loss: 2.0785 - val_accuracy: 0.6270\n",
      "Epoch 40/100\n",
      "24000/24000 [==============================] - 132s 5ms/step - loss: 1.4579 - accuracy: 0.6736 - val_loss: 2.0846 - val_accuracy: 0.6266\n",
      "Epoch 41/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.4469 - accuracy: 0.6750 - val_loss: 2.0908 - val_accuracy: 0.6253\n",
      "Epoch 42/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 1.4363 - accuracy: 0.6764 - val_loss: 2.0988 - val_accuracy: 0.6256\n",
      "Epoch 43/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.4256 - accuracy: 0.6779 - val_loss: 2.1074 - val_accuracy: 0.6262\n",
      "Epoch 44/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 1.4145 - accuracy: 0.6797 - val_loss: 2.1104 - val_accuracy: 0.6252\n",
      "Epoch 45/100\n",
      "24000/24000 [==============================] - 125s 5ms/step - loss: 1.4041 - accuracy: 0.6815 - val_loss: 2.1165 - val_accuracy: 0.6245\n",
      "Epoch 46/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 1.3934 - accuracy: 0.6832 - val_loss: 2.1254 - val_accuracy: 0.6235\n",
      "Epoch 47/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 1.3835 - accuracy: 0.6850 - val_loss: 2.1326 - val_accuracy: 0.6248\n",
      "Epoch 48/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.3731 - accuracy: 0.6867 - val_loss: 2.1403 - val_accuracy: 0.6224\n",
      "Epoch 49/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.3628 - accuracy: 0.6885 - val_loss: 2.1478 - val_accuracy: 0.6223\n",
      "Epoch 50/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 1.3531 - accuracy: 0.6901 - val_loss: 2.1536 - val_accuracy: 0.6232\n",
      "Epoch 51/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.3430 - accuracy: 0.6920 - val_loss: 2.1615 - val_accuracy: 0.6204\n",
      "Epoch 52/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.3329 - accuracy: 0.6936 - val_loss: 2.1687 - val_accuracy: 0.6210\n",
      "Epoch 53/100\n",
      "24000/24000 [==============================] - 121s 5ms/step - loss: 1.3225 - accuracy: 0.6956 - val_loss: 2.1745 - val_accuracy: 0.6205\n",
      "Epoch 54/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 1.3131 - accuracy: 0.6972 - val_loss: 2.1846 - val_accuracy: 0.6203\n",
      "Epoch 55/100\n",
      "24000/24000 [==============================] - 125s 5ms/step - loss: 1.3033 - accuracy: 0.6992 - val_loss: 2.1908 - val_accuracy: 0.6189\n",
      "Epoch 56/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.2941 - accuracy: 0.7006 - val_loss: 2.1983 - val_accuracy: 0.6201\n",
      "Epoch 57/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.2840 - accuracy: 0.7026 - val_loss: 2.2065 - val_accuracy: 0.6204\n",
      "Epoch 58/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 1.2744 - accuracy: 0.7045 - val_loss: 2.2152 - val_accuracy: 0.6188\n",
      "Epoch 59/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.2643 - accuracy: 0.7063 - val_loss: 2.2238 - val_accuracy: 0.6171\n",
      "Epoch 60/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 1.2552 - accuracy: 0.7083 - val_loss: 2.2324 - val_accuracy: 0.6171\n",
      "Epoch 61/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 1.2465 - accuracy: 0.7100 - val_loss: 2.2390 - val_accuracy: 0.6166\n",
      "Epoch 62/100\n",
      "24000/24000 [==============================] - 132s 5ms/step - loss: 1.2368 - accuracy: 0.7115 - val_loss: 2.2509 - val_accuracy: 0.6161\n",
      "Epoch 63/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 1.2268 - accuracy: 0.7135 - val_loss: 2.2585 - val_accuracy: 0.6154\n",
      "Epoch 64/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.2178 - accuracy: 0.7157 - val_loss: 2.2639 - val_accuracy: 0.6168\n",
      "Epoch 65/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 1.2087 - accuracy: 0.7173 - val_loss: 2.2760 - val_accuracy: 0.6150\n",
      "Epoch 66/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 1.1992 - accuracy: 0.7196 - val_loss: 2.2844 - val_accuracy: 0.6140\n",
      "Epoch 67/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 1.1904 - accuracy: 0.7213 - val_loss: 2.2945 - val_accuracy: 0.6132\n",
      "Epoch 68/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.1812 - accuracy: 0.7232 - val_loss: 2.3004 - val_accuracy: 0.6120\n",
      "Epoch 69/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 1.1714 - accuracy: 0.7253 - val_loss: 2.3114 - val_accuracy: 0.6110\n",
      "Epoch 70/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 1.1631 - accuracy: 0.7270 - val_loss: 2.3202 - val_accuracy: 0.6116\n",
      "Epoch 71/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.1535 - accuracy: 0.7291 - val_loss: 2.3277 - val_accuracy: 0.6119\n",
      "Epoch 72/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.1439 - accuracy: 0.7307 - val_loss: 2.3382 - val_accuracy: 0.6109\n",
      "Epoch 73/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.1354 - accuracy: 0.7328 - val_loss: 2.3492 - val_accuracy: 0.6094\n",
      "Epoch 74/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.1262 - accuracy: 0.7348 - val_loss: 2.3564 - val_accuracy: 0.6104\n",
      "Epoch 75/100\n",
      "24000/24000 [==============================] - 134s 6ms/step - loss: 1.1175 - accuracy: 0.7365 - val_loss: 2.3671 - val_accuracy: 0.6088\n",
      "Epoch 76/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.1086 - accuracy: 0.7382 - val_loss: 2.3742 - val_accuracy: 0.6094\n",
      "Epoch 77/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.0997 - accuracy: 0.7405 - val_loss: 2.3874 - val_accuracy: 0.6086\n",
      "Epoch 78/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 1.0907 - accuracy: 0.7426 - val_loss: 2.3980 - val_accuracy: 0.6079\n",
      "Epoch 79/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 1.0816 - accuracy: 0.7443 - val_loss: 2.4082 - val_accuracy: 0.6061\n",
      "Epoch 80/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.0728 - accuracy: 0.7462 - val_loss: 2.4177 - val_accuracy: 0.6060\n",
      "Epoch 81/100\n",
      "24000/24000 [==============================] - 130s 5ms/step - loss: 1.0639 - accuracy: 0.7484 - val_loss: 2.4274 - val_accuracy: 0.6070\n",
      "Epoch 82/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 1.0552 - accuracy: 0.7504 - val_loss: 2.4386 - val_accuracy: 0.6050\n",
      "Epoch 83/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 1.0464 - accuracy: 0.7522 - val_loss: 2.4469 - val_accuracy: 0.6062\n",
      "Epoch 84/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 1.0375 - accuracy: 0.7541 - val_loss: 2.4573 - val_accuracy: 0.6051\n",
      "Epoch 85/100\n",
      "24000/24000 [==============================] - 132s 6ms/step - loss: 1.0289 - accuracy: 0.7561 - val_loss: 2.4711 - val_accuracy: 0.6046\n",
      "Epoch 86/100\n",
      "24000/24000 [==============================] - 122s 5ms/step - loss: 1.0211 - accuracy: 0.7577 - val_loss: 2.4812 - val_accuracy: 0.6031\n",
      "Epoch 87/100\n",
      "24000/24000 [==============================] - 137s 6ms/step - loss: 1.0124 - accuracy: 0.7595 - val_loss: 2.4904 - val_accuracy: 0.6037\n",
      "Epoch 88/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 1.0038 - accuracy: 0.7618 - val_loss: 2.4991 - val_accuracy: 0.6027\n",
      "Epoch 89/100\n",
      "24000/24000 [==============================] - 129s 5ms/step - loss: 0.9955 - accuracy: 0.7634 - val_loss: 2.5128 - val_accuracy: 0.6031\n",
      "Epoch 90/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 0.9865 - accuracy: 0.7652 - val_loss: 2.5225 - val_accuracy: 0.6017\n",
      "Epoch 91/100\n",
      "24000/24000 [==============================] - 125s 5ms/step - loss: 0.9790 - accuracy: 0.7671 - val_loss: 2.5334 - val_accuracy: 0.6004\n",
      "Epoch 92/100\n",
      "24000/24000 [==============================] - 133s 6ms/step - loss: 0.9709 - accuracy: 0.7690 - val_loss: 2.5462 - val_accuracy: 0.5986\n",
      "Epoch 93/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 0.9619 - accuracy: 0.7709 - val_loss: 2.5547 - val_accuracy: 0.6006\n",
      "Epoch 94/100\n",
      "24000/24000 [==============================] - 128s 5ms/step - loss: 0.9540 - accuracy: 0.7730 - val_loss: 2.5695 - val_accuracy: 0.5993\n",
      "Epoch 95/100\n",
      "24000/24000 [==============================] - 124s 5ms/step - loss: 0.9461 - accuracy: 0.7746 - val_loss: 2.5780 - val_accuracy: 0.5994\n",
      "Epoch 96/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 0.9375 - accuracy: 0.7767 - val_loss: 2.5896 - val_accuracy: 0.5977\n",
      "Epoch 97/100\n",
      "24000/24000 [==============================] - 127s 5ms/step - loss: 0.9298 - accuracy: 0.7785 - val_loss: 2.6049 - val_accuracy: 0.5986\n",
      "Epoch 98/100\n",
      "24000/24000 [==============================] - 126s 5ms/step - loss: 0.9229 - accuracy: 0.7800 - val_loss: 2.6122 - val_accuracy: 0.5969\n",
      "Epoch 99/100\n",
      "24000/24000 [==============================] - 131s 5ms/step - loss: 0.9141 - accuracy: 0.7820 - val_loss: 2.6220 - val_accuracy: 0.5973\n",
      "Epoch 100/100\n",
      "24000/24000 [==============================] - 123s 5ms/step - loss: 0.9070 - accuracy: 0.7834 - val_loss: 2.6353 - val_accuracy: 0.5966\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x27e79deab70>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fitting the model\n",
    "lstm_model.fit(x=[training_encoder_input, training_decoder_input], y=[training_decoder_output],\n",
    "    validation_data=([validation_encoder_input, validation_decoder_input], [validation_decoder_output]),\n",
    "          batch_size=64, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_model.save('lstm_model_skipgram_embeddings.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: well, i thought we would start with pronunciation, if that is okay with you.\n",
      "Answer:  i why\n",
      "Question: not the hacking and gagging and spitting part. please.\n",
      "Answer:  that is <UNKNOWN>\n",
      "Question: you are asking me out. that is so cute. that is your name again?\n",
      "Answer:  i why\n",
      "Question: gosh, if only we could find kat a boyfriend...\n",
      "Answer:  i <UNKNOWN>\n",
      "Question: c'esc ma tete. this is my head\n",
      "Answer:  oh is can\n",
      "Question: that is because it is such a nice one.\n",
      "Answer:  <UNKNOWN> <UNKNOWN>\n",
      "Question: how is our little find the wench a date plan progressing?\n",
      "Answer:  no <UNKNOWN>\n",
      "Question: you have my word. as a gentleman\n",
      "Answer:  you\n",
      "Question: how do you get your hair to look like that?\n",
      "Answer:  it is <UNKNOWN>\n",
      "Question: sure have.\n",
      "Answer:  i is <UNKNOWN>\n",
      "Question: i really, really, really wanna go, but i cannot. not unless my sister goes.\n",
      "Answer:  i is\n",
      "Question: so that is the kind of guy she likes? pretty ones?\n",
      "Answer:  no when\n",
      "Question: you know chastity?\n",
      "Answer:  i , ,\n",
      "Question: i looked for you back at the party, but you always seemed to be occupied.\n",
      "Answer:  i <UNKNOWN>\n",
      "Question: i was?\n",
      "Answer:  that when\n",
      "Question: well, no...\n",
      "Answer:  you is <UNKNOWN>\n",
      "Question: do you listen to this crap?\n",
      "Answer:  no\n",
      "Question: what crap?\n",
      "Answer:  he when\n",
      "Question: me. this endless ...blonde babble. i'm like, boring myself.\n",
      "Answer:  i is <UNKNOWN>\n",
      "Question: i figured you would get to the good stuff eventually.\n",
      "Answer:  i <UNKNOWN>\n"
     ]
    }
   ],
   "source": [
    "#Printing the sample Questions and answers.\n",
    "for i in range(20):\n",
    "    output = prediction_answer(filtered_questions[i],lstm_model)\n",
    "    print ('Question:', filtered_questions[i])\n",
    "    print ('Answer:', decode_answer(decoding, output[0]))\n",
    "    i=i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello! I am a Generative Chatbot Trained on Cornell Movie Corpus Data\n",
      "~Hey\n",
      " yeah is can\n",
      "~Let us go out.\n",
      " i is are\n",
      "~What?\n",
      " that is <UNKNOWN>\n",
      "~Can we go out?\n",
      " yeah is did\n",
      "~What are u doing?\n",
      " i how\n",
      "~Are you not understanding?\n",
      " no when\n",
      "~What do you understand?\n",
      " yes why\n",
      "~When is the flight?\n",
      " i of\n",
      "~Lets us pack out bags.\n",
      " i why\n",
      "~Because we need to go out.\n",
      " i <UNKNOWN>\n",
      "~Do you understand?\n",
      " yes why\n",
      "~Is is allowed to go out?\n",
      " that is ten\n",
      "~What is ten?\n",
      " we is ten\n",
      "~We?\n",
      " no is did\n",
      "~Let us go shopping.\n",
      " i <UNKNOWN>\n",
      "~You always seem to be occupied.\n",
      " i is <UNKNOWN>\n",
      "~You need to learn alot\n",
      " ... you glory\n",
      "~bye\n"
     ]
    }
   ],
   "source": [
    "print(\"Hello! I am a Generative Chatbot Trained on Cornell Movie Corpus Data\")\n",
    "user_input = input(\"~\")\n",
    "predicted_seq =  prediction_answer(user_input,lstm_model)\n",
    "print (decode_answer(decoding, predicted_seq[0]))\n",
    "while user_input not in exit_words:\n",
    "    try:\n",
    "        user_input = input(\"~\")\n",
    "        if user_input not in exit_words:\n",
    "            decode_seq = prediction_answer(user_input,lstm_model)\n",
    "            print (decode_answer(decoding, decode_seq[0]))\n",
    "        else:\n",
    "            break\n",
    "    except EOFError:\n",
    "        print(\"Bye\")\n",
    "        exit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}