TD7_Text_Generation_With_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [],
   "source": [
    "import github_command as gt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 732,
   "metadata": {},
   "outputs": [],
   "source": [
    "gt.push(file_to_transfer=\"TD7_Text_Generation_With_LSTM.ipynb\",\n",
    "       message=\"beam search\",\n",
    "       repos=\"TDs_ESILV.git\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /Users/lucbertin/anaconda3/lib/python3.6/site-packages (3.3)\r\n",
      "Requirement already satisfied: six in /Users/lucbertin/anaconda3/lib/python3.6/site-packages (from nltk) (1.11.0)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.text import Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package gutenberg to\n",
      "[nltk_data]     /Users/lucbertin/nltk_data...\n",
      "[nltk_data]   Package gutenberg is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('gutenberg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "alice = nltk.corpus.gutenberg.words('carroll-alice.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Characters: 150118\n",
      "Total Vocab: 46\n",
      "Total Patterns: 150018\n"
     ]
    }
   ],
   "source": [
    "# load ascii text and covert to lowercase\n",
    "#filename = \"wonderland.txt\"\n",
    "#raw_text = open(filename, 'r', encoding='utf-8').read()\n",
    "#raw_text = raw_text.lower()\n",
    "raw_text = \" \".join(alice).lower()\n",
    "\n",
    "# create mapping of unique chars to integers\n",
    "chars = sorted(list(set(raw_text)))\n",
    "char_to_int = dict((c, i) for i, c in enumerate(chars))\n",
    "\n",
    "# summarize the loaded data\n",
    "n_chars = len(raw_text)\n",
    "n_vocab = len(chars)\n",
    "print(\"Total Characters: {}\".format(n_chars))\n",
    "print(\"Total Vocab: {}\".format(n_vocab))\n",
    "\n",
    "# prepare the dataset of input to output pairs encoded as integers\n",
    "seq_length = 100\n",
    "dataX = []\n",
    "dataY = []\n",
    "\n",
    "for i in range(0, n_chars - seq_length, 1):\n",
    "    seq_in = raw_text[i:i + seq_length]\n",
    "    #print(seq_in)\n",
    "    seq_out = raw_text[i + seq_length]\n",
    "    #print(seq_out)\n",
    "    dataX.append([char_to_int[char] for char in seq_in])\n",
    "    #print(dataX)\n",
    "    dataY.append(char_to_int[seq_out])\n",
    "    #print(dataY)\n",
    "    \n",
    "\n",
    "n_patterns = len(dataX)\n",
    "print(\"Total Patterns: {}\".format(n_patterns))\n",
    "\n",
    "### \n",
    "# reshape X to be [samples, time steps, features]\n",
    "X = numpy.reshape(dataX, (n_patterns, seq_length, 1))\n",
    "\n",
    "# normalize\n",
    "X = X / float(n_vocab)\n",
    "\n",
    "# one hot encode the output variable\n",
    "y = np_utils.to_categorical(dataY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the LSTM model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((150018, 100, 1), (150018, 46))"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X.shape[1], X.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(y.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_2 (LSTM)                (None, 256)               264192    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 46)                11822     \n",
      "=================================================================\n",
      "Total params: 276,014\n",
      "Trainable params: 276,014\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150018, 100, 1)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath=\"weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "  5000/150018 [..............................] - ETA: 5:06 - loss: 2.8943"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-fcb084521a77>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckpoint\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "hist = model.fit(X, y, epochs=1, batch_size=500, callbacks=[checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_to_char = dict((i, c) for i, c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" d concluded the banquet --] ' what is the use of repeating all that stuff ,' the mock turtle interru \"\n",
      " , ' io the toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe toe \n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# pick a random seed\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print(\"Seed:\")\n",
    "print( \"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
    "# generate characters\n",
    "for i in range(1000):\n",
    "    x = numpy.reshape(pattern, (1, len(pattern), 1))\n",
    "    x = x / float(n_vocab)\n",
    "    prediction = model.predict(x, verbose=0)\n",
    "    index = numpy.argmax(prediction)\n",
    "    result = int_to_char[index]\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    sys.stdout.write(result)\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print(\"\\nDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" looking at everything about her , to pass away the time . alice had never been in a court of justice \"\n",
      " , ' io ' s '  said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t ' v aen ' t ' ve toe taad ,' shi hatter sard to herself , ' in ' s ' m ' t ' v aen ' t ' ve toe taad ,' said the cat . ' io ' s ' m ' t '\n",
      "Done.\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights_file = \"weights-improvement-01-2.4514.hdf5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model(weights_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_2 (LSTM)                (None, 256)               264192    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 46)                11822     \n",
      "=================================================================\n",
      "Total params: 276,014\n",
      "Trainable params: 276,014\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling from the Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_from_softmax(preds):\n",
    "    import numpy as np\n",
    "    preds = np.reshape(preds, -1)\n",
    "    probas = np.random.multinomial(1, preds, 1)\n",
    "    return np.argmax(probas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_from_softmax([0,0.2,0.8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" their slates , ' she doesn ' t believe there ' s an atom of meaning in it ,' but none of them attemp \"\n",
      " . ' to faereis oneec . shue , aed ni lameey qo .' shid  ltec : ' tfu d tael ? why tant ,l eami , ro ihdeh ''aatelts oe ,  navdri blice : ' teacr jo oeeciest ,  'reld the cflpesu tone woe maah a''-wo mnsteadtg teon !'aou vreoonet a autdenan \"' ihee thi ln lhuil ih opgx ti dnady . sou mtdoglt g '    wh d solh b oerr ihggy a- 'tdg mpeee , '   ' * * * ( * * * * * * * * *sd tout ih a'' niseee tfrlrsg c ''n souu ,'d noeh sflpeemy !  saad mhe goeghnt . ' ie   ie*tgia g'romeeec ch soon , t ! ahd ,  'abnd iiot afdeg io sou tey loonynl lone po hypt ; iheeee as cuos diacp soeh b'mifdcgu ohcsendl lo sopcdoo fhhtel ieitha oaop lo a ' ih nlee veu --y thee rfe yoa wis ,tt io maa luueoa toiiee ! 'si , cutkehl , and the aioy soo mien , 'oeiwe mamd haemm bi mhet areihef dnrloude-c . ' neud wh  tand ,   ' '   b * * vhn , ilt go vhir vro ('r wai toan seoy ho tolyey wn sidt ,  she taoh t eryc.ahln the siyl doopr dyodige b ru yhs bet ; ihdc cetea '   anr soedlg auitedi '' ' ay woey ii reoe rasn the gidn ,'\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# pick a random seed\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print(\"Seed:\")\n",
    "print( \"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
    "# generate characters\n",
    "for i in range(1000):\n",
    "    # take the sequence ( <=> pattern), reshape\n",
    "    x = numpy.reshape(pattern, (1, len(pattern), 1))\n",
    "    # normalize\n",
    "    x = x / float(n_vocab)\n",
    "    # predict next character\n",
    "    prediction = model.predict(x, verbose=0)\n",
    "    # sample from softmax output for a little bit of variance\n",
    "    index = sample_from_softmax(prediction)\n",
    "    # transform index to char from dict\n",
    "    result = int_to_char[index]\n",
    "    # show back the pattern to the user\n",
    "    seq_in = [int_to_char[value] for value in pattern]\n",
    "    sys.stdout.write(result)\n",
    "    pattern.append(index)\n",
    "    # entry sequence must have same lenght so drop the first character\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print(\"\\nDone.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Beam Search "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[9.19839218e-02, 2.41464382e-04, 3.33781245e-05, 7.52547130e-05,\n",
       "        6.78745637e-05, 1.55610578e-05, 1.09534645e-04, 4.16072464e-04,\n",
       "        1.63882403e-04, 4.79996816e-05, 2.81081475e-06, 4.95848099e-06,\n",
       "        4.67444715e-06, 3.57902263e-06, 2.63577749e-05, 2.62667854e-05,\n",
       "        3.35156328e-05, 1.49173457e-05, 9.46370164e-06, 1.11620529e-05,\n",
       "        4.68286984e-02, 3.21025820e-03, 3.85166146e-02, 8.32669660e-02,\n",
       "        2.32710198e-01, 8.15854128e-03, 1.31016383e-02, 4.32418324e-02,\n",
       "        3.38758342e-02, 1.89665428e-04, 7.29463622e-03, 1.36435134e-02,\n",
       "        9.28971730e-03, 5.13990596e-02, 1.91199742e-02, 1.37465633e-02,\n",
       "        2.27772645e-04, 2.84235794e-02, 3.73835564e-02, 1.27047941e-01,\n",
       "        2.95352563e-02, 4.50058654e-03, 1.32524297e-02, 6.22327439e-04,\n",
       "        4.79218252e-02, 1.98288282e-04]], dtype=float32)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([23, 39,  0])"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = np.tile([2,32], (3,1))\n",
    "B = np.c_[A, best_k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = np.random.randint(size=(3,), low=0, high=5)\n",
    "D = np.random.randint(size=(3,3), low=0, high=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = np.tile(c, (3,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3, 4, 1],\n",
       "       [3, 4, 1],\n",
       "       [3, 4, 1]])"
      ]
     },
     "execution_count": 446,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4, 4, 3],\n",
       "       [0, 3, 3],\n",
       "       [2, 1, 4]])"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[7, 8, 4],\n",
       "       [3, 7, 4],\n",
       "       [5, 5, 5]])"
      ]
     },
     "execution_count": 441,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c + D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {},
   "outputs": [],
   "source": [
    "e = np.argsort(c+D, None)[-k:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 4, 5])"
      ]
     },
     "execution_count": 426,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argsort(np.multiply( c, D.T).T, None)[-3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 4, 5])"
      ]
     },
     "execution_count": 417,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = np.array([1, 4, 5])\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4, 4, 1])"
      ]
     },
     "execution_count": 447,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.reshape(-1)[test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0, 34, 25, 25,  0, 42, 28, 39, 27,  0, 27, 28, 38,  0, 27, 24,\n",
       "        20, 23,  0,  1,  2,  3,  0,  3,  0, 27, 34, 42,  0, 23, 37, 24,\n",
       "        20, 23, 25, 40, 31, 31, 44,  0, 38, 20, 41, 20, 26, 24,  0,  1,\n",
       "         3,  0, 24, 43, 22, 31, 20, 28, 32, 24, 23,  0, 20, 31, 28, 22,\n",
       "        24,  0,  9,  0,  3,  0, 20, 33, 23,  0, 24, 41, 24, 37,  0, 38,\n",
       "        28, 33, 22, 24,  0, 39, 27, 20, 39,  0,  7,  3,  0, 39, 27, 24,\n",
       "         0, 27, 20],\n",
       "       [ 0, 34, 25, 25,  0, 42, 28, 39, 27,  0, 27, 28, 38,  0, 27, 24,\n",
       "        20, 23,  0,  1,  2,  3,  0,  3,  0, 27, 34, 42,  0, 23, 37, 24,\n",
       "        20, 23, 25, 40, 31, 31, 44,  0, 38, 20, 41, 20, 26, 24,  0,  1,\n",
       "         3,  0, 24, 43, 22, 31, 20, 28, 32, 24, 23,  0, 20, 31, 28, 22,\n",
       "        24,  0,  9,  0,  3,  0, 20, 33, 23,  0, 24, 41, 24, 37,  0, 38,\n",
       "        28, 33, 22, 24,  0, 39, 27, 20, 39,  0,  7,  3,  0, 39, 27, 24,\n",
       "         0, 27, 20],\n",
       "       [ 0, 34, 25, 25,  0, 42, 28, 39, 27,  0, 27, 28, 38,  0, 27, 24,\n",
       "        20, 23,  0,  1,  2,  3,  0,  3,  0, 27, 34, 42,  0, 23, 37, 24,\n",
       "        20, 23, 25, 40, 31, 31, 44,  0, 38, 20, 41, 20, 26, 24,  0,  1,\n",
       "         3,  0, 24, 43, 22, 31, 20, 28, 32, 24, 23,  0, 20, 31, 28, 22,\n",
       "        24,  0,  9,  0,  3,  0, 20, 33, 23,  0, 24, 41, 24, 37,  0, 38,\n",
       "        28, 33, 22, 24,  0, 39, 27, 20, 39,  0,  7,  3,  0, 39, 27, 24,\n",
       "         0, 27, 20]])"
      ]
     },
     "execution_count": 453,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 746,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\"  timid voice at her side . she was walking by the white rabbit , who was peeping anxiously into her  \"\n",
      "best_k\n",
      " [39 38 28]\n",
      "best_k_after\n",
      " [[24 34 27]\n",
      " [24 34 27]\n",
      " [24 34 33]]\n",
      "best_k_scores\n",
      " [-2.5306926 -2.519325  -2.39208  ]\n",
      "best_k_scores_after\n",
      " [[-2.1252801 -1.7170271 -1.2686133]\n",
      " [-2.1002257 -1.7165658 -1.3603643]\n",
      " [-2.0852182 -1.9948661 -1.9028726]]\n",
      "best_combis_k\n",
      " [4 5 2]\n",
      "av\n",
      " [38 28 28]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[39, 28, 32, 28, 23,  0, 41, 34, 28, 22, 24,  0, 20, 39,  0, 27,\n",
       "        24, 37,  0, 38, 28, 23, 24,  0,  9,  0, 38, 27, 24,  0, 42, 20,\n",
       "        38,  0, 42, 20, 31, 30, 28, 33, 26,  0, 21, 44,  0, 39, 27, 24,\n",
       "         0, 42, 27, 28, 39, 24,  0, 37, 20, 21, 21, 28, 39,  0,  7,  0,\n",
       "        42, 27, 34,  0, 42, 20, 38,  0, 35, 24, 24, 35, 28, 33, 26,  0,\n",
       "        20, 33, 43, 28, 34, 40, 38, 31, 44,  0, 28, 33, 39, 34,  0, 27,\n",
       "        24, 37,  0, 38],\n",
       "       [39, 28, 32, 28, 23,  0, 41, 34, 28, 22, 24,  0, 20, 39,  0, 27,\n",
       "        24, 37,  0, 38, 28, 23, 24,  0,  9,  0, 38, 27, 24,  0, 42, 20,\n",
       "        38,  0, 42, 20, 31, 30, 28, 33, 26,  0, 21, 44,  0, 39, 27, 24,\n",
       "         0, 42, 27, 28, 39, 24,  0, 37, 20, 21, 21, 28, 39,  0,  7,  0,\n",
       "        42, 27, 34,  0, 42, 20, 38,  0, 35, 24, 24, 35, 28, 33, 26,  0,\n",
       "        20, 33, 43, 28, 34, 40, 38, 31, 44,  0, 28, 33, 39, 34,  0, 27,\n",
       "        24, 37,  0, 28],\n",
       "       [39, 28, 32, 28, 23,  0, 41, 34, 28, 22, 24,  0, 20, 39,  0, 27,\n",
       "        24, 37,  0, 38, 28, 23, 24,  0,  9,  0, 38, 27, 24,  0, 42, 20,\n",
       "        38,  0, 42, 20, 31, 30, 28, 33, 26,  0, 21, 44,  0, 39, 27, 24,\n",
       "         0, 42, 27, 28, 39, 24,  0, 37, 20, 21, 21, 28, 39,  0,  7,  0,\n",
       "        42, 27, 34,  0, 42, 20, 38,  0, 35, 24, 24, 35, 28, 33, 26,  0,\n",
       "        20, 33, 43, 28, 34, 40, 38, 31, 44,  0, 28, 33, 39, 34,  0, 27,\n",
       "        24, 37,  0, 28]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after update:\n",
      "\n",
      "best_k\n",
      " [34 27 27]\n",
      "best_k_scores\n",
      " [-1.7165658 -1.3603643 -1.2686133]\n",
      "\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# pick a random seed\n",
    "start = numpy.random.randint(0, len(dataX)-1)\n",
    "pattern = dataX[start]\n",
    "print(\"Seed:\")\n",
    "print( \"\\\"\", ''.join([int_to_char[value] for value in pattern]), \"\\\"\")\n",
    "k = 3\n",
    "# take the sequence ( <=> pattern), reshape (batch=1, len seq, features)\n",
    "x = np.tile(pattern, (k, 1))\n",
    "x = np.reshape(x, (k, len(pattern), 1))\n",
    "#best_k_scores = np.ones((k,1))\n",
    "# generate characters\n",
    "for i in range(1000):\n",
    "    if i==0:\n",
    "        # 1st prediction for next character\n",
    "        predictions = np.log(model.predict(x/float(n_vocab), batch_size=k))\n",
    "        # takes k best\n",
    "        best_k = np.argsort(predictions)[:,-k:][1]\n",
    "        best_k_scores = np.sort(predictions)[:,-k:][1]\n",
    "    \n",
    "    # Append to x the best k and 2nd prediction for each k sequence\n",
    "    x = np.c_[x[...,0], best_k][:,1:]\n",
    "    \n",
    "    predictions = np.log(model.predict(x[..., np.newaxis]/float(n_vocab), batch_size=k))\n",
    "    # takes k best FOR EACH k proposals\n",
    "    best_k_after = np.argsort(predictions)[:,-k:]\n",
    "    best_k_scores_after = np.sort(predictions)[:,-k:]\n",
    "\n",
    "    # takes best combi \n",
    "    best_combis_k = np.argsort(best_k_scores + best_k_scores_after, None)[-k:]\n",
    "\n",
    "    # update best_k\n",
    "    av = np.tile(best_k, (k,1)).reshape(-1)[best_combis_k]\n",
    "\n",
    "    print(\"best_k\\n\", best_k)\n",
    "    print(\"best_k_after\\n\", best_k_after)\n",
    "    print(\"best_k_scores\\n\", best_k_scores)\n",
    "    print(\"best_k_scores_after\\n\", best_k_scores_after)\n",
    "    print(\"best_combis_k\\n\", best_combis_k)\n",
    "    print(\"av\\n\", av)\n",
    "\n",
    "    x = np.c_[x[:,:-1], av] #(<=> take the last k stored, and put the best ones, leave first index)\n",
    "    display(x)\n",
    "\n",
    "    ### best_k becomes best_k_after for next iteration, to be refined\n",
    "    best_k = best_k_after.reshape(-1)[best_combis_k]\n",
    "    best_k_scores = best_k_scores_after.reshape(-1)[best_combis_k]\n",
    "\n",
    "    print('after update:\\n')\n",
    "    print(\"best_k\\n\", best_k)\n",
    "    print(\"best_k_scores\\n\", best_k_scores)\n",
    "    \n",
    "    \n",
    "    break\n",
    "    # show back the pattern to the user\n",
    "    #seq_in = [int_to_char[value] for value in pattern]\n",
    "    #sys.stdout.write(result)\n",
    "    #pattern.append(index)\n",
    "print(\"\\nDone.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beam_search_decoder(sequence, k, model):\n",
    "    # probabilities\n",
    "    probs  = model.predict(x, verbose=0)\n",
    "    # select k best\n",
    "    best_k = np.argsort(probs)[-k:]\n",
    "    # append to original sequence as different proposal new sequences\n",
    "    proposals = [sequence + [elem] for elem in best_k]\n",
    "    # append \n",
    "    # \n",
    "    #\n",
    "    \n",
    "    sequences = [[list(), 0.0]]\n",
    "    # walk over each step in sequence\n",
    "    for row in data:\n",
    "        all_candidates = list()\n",
    "        # expand each current candidate\n",
    "        for i in range(len(sequences)):\n",
    "            seq, score = sequences[i]\n",
    "#             for j in range(len(row)): # instead of exploring all the labels, explore only k best at the current time\n",
    "            \n",
    "            # explore k best\n",
    "            for j in best_k:\n",
    "                candidate = [seq + [j], score + tf.math.log(row[j])]\n",
    "                all_candidates.append(candidate)\n",
    "        # order all candidates by score\n",
    "        ordered = sorted(all_candidates, key=lambda tup:tup[1], reverse=True)\n",
    "        # select k best\n",
    "        sequences = ordered[:k]\n",
    "    return sequences"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}