Text_Generation_with_Neural_Networks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6"
    },
    "colab": {
      "name": "Text-Generation-with-Neural-Networks.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shantanu9326/Data-Science-Portfolio/blob/master/Text_Generation_with_Neural_Networks.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mXRWETNvp-1U",
        "colab_type": "text"
      },
      "source": [
        "# Text Generation with Neural Networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oG39BOxMp-1V",
        "colab_type": "text"
      },
      "source": [
        "## Functions for Processing Text\n",
        "\n",
        "### Reading in files as a string text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p2Izh3oTp-1W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_file(filepath):\n",
        "    \n",
        "    with open(filepath) as f:\n",
        "        str_text = f.read()\n",
        "    \n",
        "    return str_text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vCn9hJCcp-1Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "read_file('moby_dick_four_chapters.txt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": true,
        "id": "vvDbRFD_p-1a",
        "colab_type": "text"
      },
      "source": [
        "### Tokenize and Clean Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QcqYeq_Qp-1a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en',disable=['parser', 'tagger','ner'])\n",
        "\n",
        "nlp.max_length = 1198623"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8k1DqRlCp-1c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def separate_punc(doc_text):\n",
        "    return [token.text.lower() for token in nlp(doc_text) if token.text not in '\\n\\n \\n\\n\\n!\"-#$%&()--.*+,-/:;<=>?@[\\\\]^_`{|}~\\t\\n ']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1v9-yY4Mp-1e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "d = read_file('melville-moby_dick.txt')\n",
        "tokens = separate_punc(d)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K0o8nH3Pp-1f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokens"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KdjIXcep-1h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(tokens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d6ccQdY4p-1j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "4431/25"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IYXc57_Tp-1m",
        "colab_type": "text"
      },
      "source": [
        "## Create Sequences of Tokens"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YkK8dLRUp-1n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# organize into sequences of tokens\n",
        "train_len = 25+1 # 50 training words , then one target word\n",
        "\n",
        "# Empty list of sequences\n",
        "text_sequences = []\n",
        "\n",
        "for i in range(train_len, len(tokens)):\n",
        "    \n",
        "    # Grab train_len# amount of characters\n",
        "    seq = tokens[i-train_len:i]\n",
        "    \n",
        "    # Add to list of sequences\n",
        "    text_sequences.append(seq)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4bpALGXVp-1o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "' '.join(text_sequences[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HrYABSvdp-1q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "' '.join(text_sequences[1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "72xaV4-Vp-1r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "' '.join(text_sequences[2])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sAy5wlwvp-1s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "len(text_sequences)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fJ08ocJ0p-1u",
        "colab_type": "text"
      },
      "source": [
        "# Keras"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Rs_KXMBp-1u",
        "colab_type": "text"
      },
      "source": [
        "### Keras Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gVJ2jV4Sp-1v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3DHc_Wz3p-1w",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# integer encode sequences of words\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(text_sequences)\n",
        "sequences = tokenizer.texts_to_sequences(text_sequences)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Owy258o0p-1y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ksEUcX27p-1z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer.index_word"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qCL4K6ETp-11",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i in sequences[0]:\n",
        "    print(f'{i} : {tokenizer.index_word[i]}')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ocX9iXRYp-13",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer.word_counts"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytfWmmd5p-16",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "vocabulary_size = len(tokenizer.word_counts)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Q8Ls1BIp-17",
        "colab_type": "text"
      },
      "source": [
        "### Convert to Numpy Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bf0Bv5eTp-18",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gx_fCFjtp-19",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences = np.array(sequences)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcQEf7pDp-2A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RFAVEVHbp-2B",
        "colab_type": "text"
      },
      "source": [
        "# Creating an LSTM based model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HiekDgfOp-2C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import keras\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense,LSTM,Embedding"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7OPZUN9dp-2D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_model(vocabulary_size, seq_len):\n",
        "    model = Sequential()\n",
        "    model.add(Embedding(vocabulary_size, 25, input_length=seq_len))\n",
        "    model.add(LSTM(150, return_sequences=True))\n",
        "    model.add(LSTM(150))\n",
        "    model.add(Dense(150, activation='relu'))\n",
        "\n",
        "    model.add(Dense(vocabulary_size, activation='softmax'))\n",
        "    \n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "   \n",
        "    model.summary()\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61AVruktp-2E",
        "colab_type": "text"
      },
      "source": [
        "### Train / Test Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "gluECLSvp-2F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.utils import to_categorical"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "_LZriQsrp-2G",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sequences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "ob1dzz95p-2I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# First 49 words\n",
        "sequences[:,:-1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "lHXs1BkRp-2K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# last Word\n",
        "sequences[:,-1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "cYhnq8Csp-2L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = sequences[:,:-1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "kHhzNOkbp-2N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = sequences[:,-1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "5m8Uw5P9p-2P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y = to_categorical(y, num_classes=vocabulary_size+1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "S08ERCEGp-2R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seq_len = X.shape[1]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "w3KXjGlip-2S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seq_len"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9J75-5btp-2T",
        "colab_type": "text"
      },
      "source": [
        "### Training the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "6GDSus9Fp-2U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# define model\n",
        "model = create_model(vocabulary_size+1, seq_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MMqsmjWhp-2V",
        "colab_type": "text"
      },
      "source": [
        "---\n",
        "\n",
        "----"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "zazcuoXkp-2V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pickle import dump,load"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "pP_Fd2y0p-2W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# fit model\n",
        "model.fit(X, y, batch_size=128, epochs=300,verbose=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "scrolled": true,
        "id": "ZRA4RvWhp-2Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# save the model to file\n",
        "model.save('epochBIG.h5')\n",
        "# save the tokenizer\n",
        "dump(tokenizer, open('epochBIG', 'wb'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X_djmagrp-2a",
        "colab_type": "text"
      },
      "source": [
        "# Generating New Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Of8VWIzyp-2a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from random import randint\n",
        "from pickle import load\n",
        "from keras.models import load_model\n",
        "from keras.preprocessing.sequence import pad_sequences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mf_r7MY0p-2b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_text(model, tokenizer, seq_len, seed_text, num_gen_words):\n",
        "    '''\n",
        "    INPUTS:\n",
        "    model : model that was trained on text data\n",
        "    tokenizer : tokenizer that was fit on text data\n",
        "    seq_len : length of training sequence\n",
        "    seed_text : raw string text to serve as the seed\n",
        "    num_gen_words : number of words to be generated by model\n",
        "    '''\n",
        "    \n",
        "    # Final Output\n",
        "    output_text = []\n",
        "    \n",
        "    # Intial Seed Sequence\n",
        "    input_text = seed_text\n",
        "    \n",
        "    # Create num_gen_words\n",
        "    for i in range(num_gen_words):\n",
        "        \n",
        "        # Take the input text string and encode it to a sequence\n",
        "        encoded_text = tokenizer.texts_to_sequences([input_text])[0]\n",
        "        \n",
        "        # Pad sequences to our trained rate (50 words in the video)\n",
        "        pad_encoded = pad_sequences([encoded_text], maxlen=seq_len, truncating='pre')\n",
        "        \n",
        "        # Predict Class Probabilities for each word\n",
        "        pred_word_ind = model.predict_classes(pad_encoded, verbose=0)[0]\n",
        "        \n",
        "        # Grab word\n",
        "        pred_word = tokenizer.index_word[pred_word_ind] \n",
        "        \n",
        "        # Update the sequence of input text (shifting one over with the new word)\n",
        "        input_text += ' ' + pred_word\n",
        "        \n",
        "        output_text.append(pred_word)\n",
        "        \n",
        "    # Make it look like a sentence.\n",
        "    return ' '.join(output_text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SNJiCy9Hp-2d",
        "colab_type": "text"
      },
      "source": [
        "### Grab a random seed sequence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RDfFUYRCp-2d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text_sequences[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RWH6pPeEp-2e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "random.seed(101)\n",
        "random_pick = random.randint(0,len(text_sequences))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_Y5s1Ytp-2h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "random_seed_text = text_sequences[random_pick]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kjxyuhi1p-2j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "random_seed_text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qZl3iyYyp-2k",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed_text = ' '.join(random_seed_text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "smGsrMgap-2l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seed_text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8P4RahB8p-2m",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "generate_text(model,tokenizer,seq_len,seed_text=seed_text,num_gen_words=50)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "28_uX2Lzp-2n",
        "colab_type": "text"
      },
      "source": [
        "### Exploring Generated Sequence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KsJwQAUVp-2o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "full_text = read_file('moby_dick_four_chapters.txt')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IcsXT0nip-2p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for i,word in enumerate(full_text.split()):\n",
        "    if word == 'inkling':\n",
        "        print(' '.join(full_text.split()[i-20:i+20]))\n",
        "        print('\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "etIos-pHp-2q",
        "colab_type": "text"
      },
      "source": [
        "# Great Job!"
      ]
    }
  ]
}