LSTM_ver1_self_generated.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Couldn't import dot_parser, loading of dot files will not be possible.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import sys\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Dropout\n",
    "from keras.layers import LSTM\n",
    "from keras.layers.convolutional import Convolution1D, MaxPooling1D\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s=10\n",
    "n=10\n",
    "# Generate n seeds of size s\n",
    "seed_set = []\n",
    "for _ in range (n):\n",
    "    seed = ''\n",
    "    for i in range(s):\n",
    "        seed +=str(np.random.randint(0, 2))\n",
    "    seed_set.append(seed)    \n",
    "\n",
    "\n",
    "# Generate bit sequence by randomly picking one of n seeds in seed_set\n",
    "bit_seq = ''\n",
    "for i in range(35000):\n",
    "        bit_seq += seed_set[np.random.randint(0, n)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training length: 329999\n",
      "Test length 20000\n"
     ]
    }
   ],
   "source": [
    "#path = './data/self_generator_random_bin_350000_10_10.txt'\n",
    "#text = open(path).read().lower()\n",
    "#text = text.strip('\\n')\n",
    "test = bit_seq[-20001:-1]\n",
    "text = bit_seq[:-20001]\n",
    "print('Training length:', len(text))\n",
    "print ('Test length', len(test))\n",
    "del bit_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def chunkstring(string, length):\n",
    "    '''\n",
    "    Cut given \"string\" into chunks of size \"length\"\n",
    "    '''\n",
    "    return [string[0+i:length+i] for i in range(0, len(string), length)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Cut the bit string into single bit chunks\n",
    "chunk_len = 1\n",
    "chunk = chunkstring(text,chunk_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['0', '1']\n",
      "Total words: 2\n"
     ]
    }
   ],
   "source": [
    "# We are considering each bit as one \"word\"\n",
    "chars = sorted(list(set(chunk)))\n",
    "print(chars)\n",
    "print('Total words:', len(chars))\n",
    "\n",
    "char_indices = dict((c, i) for i, c in enumerate(chars))\n",
    "indices_char = dict((i, c) for i, c in enumerate(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (109989, 32, 2) (109989, 2)\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preparing training set. Each sample consists of \"maxlen\" bits.\n",
    "The next bit is assigned as the label.\n",
    "\"step\" variable is used to decide how much overlap between samples.\n",
    "'''\n",
    "maxlen = 32\n",
    "step = 3\n",
    "sentences = []\n",
    "next_chars = []\n",
    "for i in range(0, len(text)/chunk_len - maxlen, step):\n",
    "    sentences.append(text[i*chunk_len: (i + maxlen)*chunk_len])\n",
    "    next_chars.append(text[(i + maxlen)*chunk_len:(i + maxlen + 1)*chunk_len])\n",
    "\n",
    "X = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)\n",
    "y = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n",
    "for i, sentence in enumerate(sentences):\n",
    "    #print ('sentence', sentence)\n",
    "    for t, char in enumerate(chunkstring(sentence,chunk_len)):        \n",
    "        X[i, t, char_indices[char]] = 1\n",
    "    y[i, char_indices[next_chars[i]]] = 1\n",
    "\n",
    "print ('Training set', X.shape, y.shape)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_1 (LSTM)                (None, 32, 256)           265216    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 32, 256)           0         \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 128)               197120    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 258       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 479,106\n",
      "Trainable params: 479,106\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "LSTM_activation = 'relu'\n",
    "\n",
    "model = Sequential()\n",
    "model.add(LSTM(256, return_sequences=True, activation=LSTM_activation,\n",
    "               input_shape=(maxlen, len(chars))))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(LSTM(128, return_sequences=False, activation=LSTM_activation))\n",
    "model.add(Dense(128, activation='sigmoid'))\n",
    "model.add(Dense(len(chars)))\n",
    "model.add(Activation('softmax'))\n",
    "optimizer = RMSprop(lr=0.0005)\n",
    "model.compile(loss='categorical_crossentropy', optimizer=optimizer, \n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print (model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 87991 samples, validate on 21998 samples\n",
      "Epoch 1/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.6590 - acc: 0.6189Epoch 00000: val_loss improved from inf to 0.65138, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 112s - loss: 0.6590 - acc: 0.6189 - val_loss: 0.6514 - val_acc: 0.6226\n",
      "Epoch 2/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.6376 - acc: 0.6453Epoch 00001: val_loss did not improve\n",
      "87991/87991 [==============================] - 122s - loss: 0.6375 - acc: 0.6453 - val_loss: 0.6594 - val_acc: 0.6544\n",
      "Epoch 3/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.5741 - acc: 0.6985Epoch 00002: val_loss improved from 0.65138 to 0.52818, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 121s - loss: 0.5741 - acc: 0.6985 - val_loss: 0.5282 - val_acc: 0.7307\n",
      "Epoch 4/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.4314 - acc: 0.7867Epoch 00003: val_loss improved from 0.52818 to 0.39132, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 122s - loss: 0.4313 - acc: 0.7868 - val_loss: 0.3913 - val_acc: 0.8177\n",
      "Epoch 5/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.2929 - acc: 0.8474Epoch 00004: val_loss improved from 0.39132 to 0.26865, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 122s - loss: 0.2929 - acc: 0.8474 - val_loss: 0.2686 - val_acc: 0.8572\n",
      "Epoch 6/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.2549 - acc: 0.8624Epoch 00005: val_loss improved from 0.26865 to 0.24151, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 122s - loss: 0.2549 - acc: 0.8624 - val_loss: 0.2415 - val_acc: 0.8622\n",
      "Epoch 7/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.2449 - acc: 0.8668Epoch 00006: val_loss improved from 0.24151 to 0.24001, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 121s - loss: 0.2449 - acc: 0.8668 - val_loss: 0.2400 - val_acc: 0.8685\n",
      "Epoch 8/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.2412 - acc: 0.8673Epoch 00007: val_loss improved from 0.24001 to 0.23190, saving model to weights_LSTM_self_v1.hdf5\n",
      "87991/87991 [==============================] - 122s - loss: 0.2412 - acc: 0.8672 - val_loss: 0.2319 - val_acc: 0.8701\n",
      "Epoch 9/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.2385 - acc: 0.8688Epoch 00008: val_loss did not improve\n",
      "87991/87991 [==============================] - 121s - loss: 0.2384 - acc: 0.8688 - val_loss: 0.2341 - val_acc: 0.8703\n",
      "Epoch 10/10\n",
      "87936/87991 [============================>.] - ETA: 0s - loss: 0.2374 - acc: 0.8688Epoch 00009: val_loss did not improve\n",
      "87991/87991 [==============================] - 122s - loss: 0.2374 - acc: 0.8689 - val_loss: 0.2470 - val_acc: 0.8678\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7ff024694fd0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "early_stopping = EarlyStopping(monitor='val_loss', patience=4, verbose=1)\n",
    "monitoring = ModelCheckpoint('weights_LSTM_self_v1.hdf5', monitor='val_loss', \n",
    "                             verbose=1, save_best_only=True)\n",
    "model.fit(X, y, batch_size=128, epochs=10, validation_split=0.2, verbose=1,\n",
    "         callbacks=[early_stopping,monitoring])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set (19968, 32, 2) (19968, 2)\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Preparing test set.\n",
    "'''\n",
    "step = 1\n",
    "sentences = []\n",
    "next_chars = []\n",
    "for i in range(0, len(test)/chunk_len - maxlen, step):\n",
    "    sentences.append(test[i*chunk_len: (i + maxlen)*chunk_len])\n",
    "    next_chars.append(test[(i + maxlen)*chunk_len:(i + maxlen + 1)*chunk_len])\n",
    "\n",
    "Xt = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)\n",
    "yt = np.zeros((len(sentences), len(chars)), dtype=np.bool)\n",
    "for i, sentence in enumerate(sentences):    \n",
    "    for t, char in enumerate(chunkstring(sentence,1)):        \n",
    "        Xt[i, t, char_indices[char]] = 1\n",
    "    yt[i, char_indices[next_chars[i]]] = 1\n",
    "    \n",
    "print ('Test set', Xt.shape, yt.shape)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted 0 next bits, 0 correct\n",
      "Predicted 2000 next bits, 1724 correct\n",
      "Predicted 4000 next bits, 3460 correct\n",
      "Predicted 6000 next bits, 5195 correct\n",
      "Predicted 8000 next bits, 6943 correct\n",
      "Predicted 10000 next bits, 8697 correct\n",
      "Predicted 12000 next bits, 10437 correct\n",
      "Predicted 14000 next bits, 12167 correct\n",
      "Predicted 16000 next bits, 13910 correct\n",
      "Predicted 18000 next bits, 15627 correct\n",
      "Predicted 19968 next bits in total, 17332 correct. Accuracy is 0.86799.\n"
     ]
    }
   ],
   "source": [
    "n_true = 0\n",
    "y_true = []\n",
    "y_pred = []\n",
    "\n",
    "batch_size = 1000\n",
    "nb_batch = Xt.shape[0]/batch_size+1\n",
    "\n",
    "for i in range(nb_batch):\n",
    "    if i % 2 == 0:\n",
    "        print (\"Predicted %d next bits, %d correct\" % (i*batch_size,n_true))\n",
    "    x = Xt[i*batch_size:(i+1)*batch_size]\n",
    "    preds = model.predict(x, verbose=0)\n",
    "    pred_next_indexes = list(np.argmax(preds,axis=1))\n",
    "    pred_next_chars = [indices_char[next_index] for next_index in pred_next_indexes]        \n",
    "    y_pred += pred_next_chars\n",
    "\n",
    "    true_next_indexes = list(np.argmax(yt[i*batch_size:(i+1)*batch_size],axis=1))\n",
    "    true_next_chars = [indices_char[next_index] for next_index in true_next_indexes]\n",
    "    y_true += true_next_chars\n",
    "\n",
    "    n_true += np.sum(np.array(pred_next_chars)==np.array(true_next_chars))\n",
    "\n",
    "y_true = map(int,y_true)\n",
    "y_pred = map(int,y_pred)\n",
    "y_true = np.asarray(y_true)\n",
    "y_pred = np.asarray(y_pred)\n",
    "print (\"Predicted %d next bits in total, %d correct. Accuracy is %.5f.\" \n",
    "       % (yt.shape[0],n_true,(float(n_true)/yt.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}