lstm_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n",
      "Data size 100000000\n",
      "(99999000, 'ons anarchists advocate social relations based upon voluntary association of autonomous individuals mutual aid and self governance while anarchism is most easily defined by what it is against anarchists also offer positive visions of what they believe to be a truly free society however ideas about how an anarchist society might work vary considerably especially with respect to economics there is also disagreement about how a free society might be brought about origins and predecessors kropotkin and others argue that before recorded history human society was organized on anarchist principles most anthropologists follow kropotkin and engels in believing that hunter gatherer bands were egalitarian and lacked division of labour accumulated wealth or decreed law and had equal access to resources william godwin anarchists including the the anarchy organisation and rothbard find anarchist attitudes in taoism from ancient china kropotkin found similar ideas in stoic zeno of citium according to kropotkin zeno repudiat')\n",
      "(1000, ' anarchism originated as a term of abuse first used against early working class radicals including t')\n",
      "Unexpected character: ï\n",
      "(1, 26, 0, 0)\n",
      "('a', 'z', ' ')\n",
      "[1, 781243, 1562485, 2343727, 3124969, 3906211, 4687453, 5468695, 6249937, 7031179, 7812421, 8593663, 9374905, 10156147, 10937389, 11718631, 12499873, 13281115, 14062357, 14843599, 15624841, 16406083, 17187325, 17968567, 18749809, 19531051, 20312293, 21093535, 21874777, 22656019, 23437261, 24218503, 24999745, 25780987, 26562229, 27343471, 28124713, 28905955, 29687197, 30468439, 31249681, 32030923, 32812165, 33593407, 34374649, 35155891, 35937133, 36718375, 37499617, 38280859, 39062101, 39843343, 40624585, 41405827, 42187069, 42968311, 43749553, 44530795, 45312037, 46093279, 46874521, 47655763, 48437005, 49218247, 49999489, 50780731, 51561973, 52343215, 53124457, 53905699, 54686941, 55468183, 56249425, 57030667, 57811909, 58593151, 59374393, 60155635, 60936877, 61718119, 62499361, 63280603, 64061845, 64843087, 65624329, 66405571, 67186813, 67968055, 68749297, 69530539, 70311781, 71093023, 71874265, 72655507, 73436749, 74217991, 74999233, 75780475, 76561717, 77342959, 78124201, 78905443, 79686685, 80467927, 81249169, 82030411, 82811653, 83592895, 84374137, 85155379, 85936621, 86717863, 87499105, 88280347, 89061589, 89842831, 90624073, 91405315, 92186557, 92967799, 93749041, 94530283, 95311525, 96092767, 96874009, 97655251, 98436493, 99217735]\n",
      "['o', 'n', 'w', ' ', 'l', 'r', ' ', 's', 'm', 's', 'h', 'a', 'y', ' ', 'a', 's', 't', 'a', 'm', ' ', 'n', 's', 'h', 's', 'e', 'l', 'e', ' ', 'o', 'l', 'y', 'o', 'o', 'r', 'a', ' ', ' ', 'a', 'a', 'e', 'i', 'e', ' ', 't', 't', 'h', 'd', 'e', 'f', 'f', 'a', 't', 'e', 'i', 'e', 'a', 'a', 'c', 'r', 'g', 'i', ' ', 'o', 'r', 'a', 'f', 'g', 'g', 'i', 'a', 'r', 's', 'c', 'f', 'a', 'e', ' ', 'a', 'm', 'f', 't', 'o', 'u', 'e', 'e', 't', 'o', 'n', 'o', 'e', 's', 'e', 'k', ' ', 'e', ' ', 'w', 'l', 'e', 'e', 't', ' ', 'e', 'n', ' ', 'l', 'i', 'e', 't', 'n', 'd', 's', 't', 'e', 'e', 'r', 'f', 'e', 'd', 'i', 't', 'n', 'a', 'e', 'a', 'r', 's', 'n']\n",
      "['ons anarchists advocate social relations based upon vol', 'nomination gore s endorsement of dean was helpful to th', 'when military governments failed to revive the economy ', ' three nine one six zero two zero zero one census peter', 'lleria arches national park photographic virtual tour o', 'reviated as dr mr and mrs respectively they are also fr', ' abbeys and monasteries index sacred destinations abbey', 'shing the right of appeal to the judicial committee of ', 'married urraca princess of castile daughter of alfonso ', 'sity upset the devils which cost the school its nationa', 'hel and richard baer h provided a detailed description ', 'ased in the st family here they are in rough chronologi', 'y and liturgical language among jews mandaeans and some', ' disgust because of the relationship between the anus a', 'ay opened for passengers in december one nine zero two ', 'society and that this neglect is the true cause of the ', 'tion from the national media and from presidential cand', 'ago based chess records label the influence of blues on', 'migration took place during the one nine eight zero s w', ' zero zero five yaniv shaked and avishai wool published', 'new york other well known manufacturers of bass amplifi', 'short subject college humor one nine three three too mu', 'he boeing seven six seven a widebody jet was introduced', 'sgow two young white men whose murderers were asian and', 'e listed with a gloss covering some of their deeds a si', 'lt during this period however the iran iraq war of the ', 'eber has probably been one of the most influential user', ' not dead naturally and hangs herself upon hearing the ', 'o be made to recognize single acts of merit or meritori', 'll s enthusiastic backing darwin read his first paper t', 'yer who received the first card from the deal may be kn', 'operates three submarines based in talcahuano air force', 'ore significant than in jersey and guernsey has maintai', 'rmines security of the system provided that there is no', 'a fierce critic of the poverty and social stratificatio', ' fuel extracted from the ground by underground mining o', ' two six eight in signs of humanity vol three michel ba', 'ature that was attacking his livestock it was later det', 'aristotle s uncaused cause so aquinas comes to the same', 'e dragas constantine i of imereti constantine iii of ro', 'ity can be lost as in denaturalization and gained as in', 'ecombinant region and the diode becomes conductive whic', ' and intracellular ice formation solution effects are c', 'tensive manufacturing sectors the question of why the m', 'tion of the size of the input usually measured in bits ', 'he attack from hyrsyl northwards and reached petrozavod', 'dy to pass him a stick to pull him out but she refuses ', 'ed to bring good fortune to those who carried them owne', 'f certain drugs confusion inability to orient oneself l', 'french jansenist theologian b one six three four one se', 'at it will take to complete an operation cannot be boun', 'tion from euclidean geometry and analysis such as gradi', 'e convince the priest of the mistakes of a pious life t', 'ither spontaneously or have employed in their daily wri', 'ent told him to name it fort des moines the original or', 'argest partner of the uk has also made it a destination', 'ampaign and barred attempts by his opponents to run cam', 'ce in a special cell named down s cell the cell is conn', 'rver side standard formats for mailboxes include maildi', 'gain the amplified signal from q one is directly fed to', 'ious texts such as esoteric christianity and the work o', ' assignment of numbers to positions a player who is not', 'o capitalize on the growing popularity of disco with th', 'rettas francis poulenc jean philippe rameau maurice rav', 'a duplicate of the original document fax machines with ', 'former is widely used in the internal combustion engine', 'gh ann es d hiver one nine eight zero one nine eight si', 'g the series to a seventh and deciding game the next ni', 'ine january eight march eight listing of all days days ', 'ar it is possible the tradition was carried on by the l', 'ross zero the lead character lieutenant shin kudo playe', 's worldwide after english german is the second largest ', 'cal theories classical mechanics and special relativity', 'f an inch they were fit together so perfectly that the ', 'ast instance the non gm comparison maize crop had also ', 'e phantom appear on stage is pepper s ghost technique i', ' dimensional analysis fundamental applications of proba', 'as pi approx frac the algorithm has second order conver', 'most holy mormons believe the configuration of the cont', 'fice did not become formalized for some decades charlem', 't s support or at least not parliament s opposition a s', 'olunteers originally hand picked who used to do the job', 'u is still disagreed upon by historians and linguists i', 'e end of world war ii and the near extermination of eur', 'e oscillating system example rlc circuit full mathemati', 'tain its surface area over time and potassium which inc', 'o eight subtypes based on the whole genome that are eac', 'normal course of study was abolished the academic cours', 'of italy languages the official language of italy is st', 'e of the leadership according to the law all presidenti', 's the tower commission at this point president reagan s', 'e icj reports seven see charney j compromissory clauses', 'klahoma press one nine three two one one th printing on', ' most of the indictees are serbs the tribunal exclusive', 'erprise linux suse linux enterprise server debian and t', ' running of the household until her death in one seven ', 'ws becomes the first daily college newspaper in the uni', 'lization about the performance of java programs in gene', 'et in a nazi concentration camp lewis has explained why', 'ey accepted a proposal from their relative johann matth', 'the fabian society nehru wished the economy of india to', ' help when the queen left he supposedly noticed her foo', 'etchy to relatively stiff from flat to tightly curled a', 'ng bombs with global positioning system satellite guida', ' sharman networks sharman s sydney based boss nikki hem', 'laces for karaoke and terms of karaoke for a descriptio', 'ised emperor hirohito to begin negotiations to end worl', 'erican football and also called an onside pass in canad', 'ting in political initiatives the lesotho congress for ', 'nd has been awarded the two zero zero six polar music p', 'd neo latin most of these authors wrote in their variou', 'seco the a s took an early lead in game one on a grand ', 'th risky riskerdoo ricky ricardo this classic includes ', 'er donna troy who is destined to play a critical role i', 'encyclopedic overview of mathematics presented in clear', 'rall definition of spheres of influence in the area abs', 'fense the air component of arm is represented by the co', 'ecause of communist regime languages khalkha mongol nin', 'duating from acnm accredited programs must pass the sam', 'ifferent focal planes called a z stack plus the knowled', 'treet grid centerline external links bbc on this day ma', 'n militant union leaderships additionally thatcher s ri', 'ations more than any other state modern day montana bec', 'etal compounds such as cp two ba or structures with ben', 'appeal of devotional buddhism especially represented by', 'rs larry and james one older sister delores and a young', 'si have made such devices possible the systemic advanta', 'ncluding employees of many companies who had replaced t']\n",
      "['luntary association of autonomous individuals mutual ai', 'he latter in legitimizing him in the eyes of the establ', ' and suppress escalating terrorism in the late one nine', 'rhead is the largest town in aberdeenshire the principa', 'of arches national park archaeological sites in the uni', 'requently written as in canada and the u s as dr mr and', 'ys of france sacred destinations abbeys art history the', ' the privy council an evolving independence thus the in', ' viii king of castile and leonora of aquitaine in one t', 'al ranking the wins over washington state and washingto', ' of the camp s workings during his interrogations after', 'ical order after the original five two zero st five two', 'e christians and is still spoken by small isolated comm', 'and feces however it is not uncommon for the rectum to ', ' on the night of friday one four th january one nine th', ' poverty and misery experienced by the residents of tho', 'didate john f kennedy despite this incident atlanta s p', 'n mainstream american popular music was huge in the fif', 'with the arrival of thousands of refugees from guatemal', 'd the paper cracking the bluetooth pin one which shows ', 'iers or loudspeakers include accugroove loudpeakers acm', 'uch harmony one nine three three please one nine three ', 'd at around the same time as the seven five seven its n', 'd whose murders the bnp maintains were hate crimes the ', 'ignificance is attached to the thirty and the three all', ' one nine eight zero s was a difficult time for the cit', 'rs of the word in its social science sense he is well k', ' news discovery and translation modern day picture of t', 'ious service the required achievement or service while ', 'to the geological society of london on four january one', 'nown as eldest hand or as forehand the set of cards dea', 'e fach gen osvaldo sarabia heads a force of one two fiv', 'ined light industry as a higher proportion of its econo', 'o analytic attack i e a structural weakness in the algo', 'on of victorian society throughout his works dickens re', 'or open pit mining strip mining it is a readily combust', 'alat and janice deledalle rhodes eds g rard deledalle g', 'termined to be a canine most likely a coyote of some so', 'e conclusion that god exists whether there was a first ', 'ome constantine mavrocordato constantine nicolaievich c', 'n naturalization supranational citizenship in recent ye', 'ch allows electrons to flow though the diode from the c', 'caused by concentration of solutes in non frozen soluti', 'maritimes fell from being a centre of canadian manufact', ' using the most efficient algorithm to understand this ', 'dsk railroad and the main road the next day from there ', ' unless he declare his devotion to god almighty hume ac', 'ership was restricted among various castes by color wit', 'later signs lethargy decreased ability to perform simpl', 'even two three philip ii duke of orl ans regent of fran', 'nded in advance see unbounded nondeterminism scalabilit', 'ient of a function divergence length of curves and so o', 'the novel the one two zero days of sodom written in one', 'iting to explore themselves and their experience of the', 'rigin of the name des moines is uncertain it could have', 'n for economic migrants from scotland wales and norther', 'mpaign advertisements for this reason many countries en', 'nected to a battery allowing electrons migration from t', 'ir and mbox several prominent e mail clients use their ', 'o the second stage q three which provides further ampli', 'of g i gurdjieff a variety of past traditions could be ', 't wearing a number that corresponds to an eligible rece', 'he album discovery or disco very as he has been quoted ', 'vel claude joseph rouget de lisle composer of la marsei', ' additional electronic features can connect to computer', 'e while both are used in power generation nuclear fuels', 'ix and cartographies schizoanalytique one nine eight ni', 'ight which was won by cincinnati duffy s cliff from one', ' february nine is the four zero th day of the year in t', 'local community after the travellers had left or that l', 'ed by kenichi suzumura is a qualified f one four pilot ', ' of the germanic languages german is the language with ', 'y classical mechanics and special relativity are lumped', ' tip of a knife cannot be inserted between the joints a', ' been treated with environmentally damaging pesticides ', 'in asian horror cinema the ghost stories often include ', 'ability and statistics nine specialized topics nuclear ', 'rgent nature which essentially means that the number of', 'tinents was different before the great flood and that t', 'magne went on to adopt the title augustus from earlier ', 'subtle but important difference it also gives parliamen', 'bs of scouts as well prior to the creation of peer revi', 'it is generally accepted as having originally been a pe', 'ropean jewry by the nazis international support for jew', 'ical definition most harmonic oscillators at least appr', 'creases the electron density of the catalyst and so imp', 'ch geographically distinct the most prevalent are subty', 'se became standard across the student body in one nine ', 'tandard italian a direct descendant of latin some seven', 'ial candidates must be approved by the council of guard', 'said he had not been informed of the operation the towe', 's and the jurisdiction of the international court of ju', 'ne nine eight nine isbn zero eight zero six one one one', 'ely uses translators who speak bosnian and croatian lan', 'the version sgi offers on their altix machines in jan t', ' two nine it was in weimar that two musically significa', 'ited states one eight eight seven in a snowstorm at for', 'eral because run time performance is affected much more', 'y the film hasn t been released by suggesting litigatio', 'hias franck the schoolmaster and choirmaster in hainbur', 'o be partially capitalist but with the state occupying ', 'otprint in the floor plaster of his workplace even thou', 'and so on process a modern knitting machine in action a', 'ance devices that are immune to bad weather also althou', 'mming and associate kevin bermeister had knowingly allo', 'on of karaoke boxes in two zero zero four daisuke inoue', 'ld war ii after the beginning of the american occupatio', 'dian football is a sideways or rearward throwing of the', ' democracy lcd won the majority in parliament in the tw', 'prize in rolling stone magazine s tabulation of the one', 'us vernaculars as well as in latin but each produced a ', ' slam by canseco and led four three in the bottom of th', ' lucy winding the cello s tuning peg as if it were a wa', 'in infinite crisis he s also been carefully surveilling', 'r simple language hazewinkel michiel ed encyclopaedia o', 'solute french control over madagascar was established b', 'ommand of military aviation and air defense of the repu', 'ne zero turkic qazaq tuvin russian literacy definition ', 'me certifying exam administered by the american midwife', 'dge of the psf which can be either derived experimental', 'ay two seven may two nine april two eight june two eigh', 'ight to buy scheme to buy policy whereby council housin', 'came montana territory in one eight six four by appoint', 'nt aromatic rings such as found in manganocene or titan', 'y the pure land this rich cosmography also allowed maha', 'ger sister roslyn he married juanita jordan in septembe', 'ages of mpls such as the ability to support multiple se', 'their pension systems with annuities purchased from the']\n",
      "[' a']\n",
      "['an']\n"
     ]
    }
   ],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "\n",
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "def maybe_download(filename, expected_bytes):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if not os.path.exists(filename):\n",
    "    filename, _ = urlretrieve(url + filename, filename)\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print('Found and verified %s' % filename)\n",
    "  else:\n",
    "    print(statinfo.st_size)\n",
    "    raise Exception(\n",
    "      'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)\n",
    "\n",
    "def read_data(filename):\n",
    "  f = zipfile.ZipFile(filename)\n",
    "  for name in f.namelist():\n",
    "    return tf.compat.as_str(f.read(name))\n",
    "  f.close()\n",
    "  \n",
    "text = read_data(filename)\n",
    "print('Data size %d' % len(text))\n",
    "\n",
    "valid_size = 1000\n",
    "valid_text = text[:valid_size]\n",
    "train_text = text[valid_size:]\n",
    "train_size = len(train_text)\n",
    "print(train_size, train_text[:1024])\n",
    "print(valid_size, valid_text[:100])\n",
    "\n",
    "vocabulary_size = len(string.ascii_lowercase) + 1 # [a-z] + ' '\n",
    "first_letter = ord(string.ascii_lowercase[0])\n",
    "\n",
    "def char2id(char):\n",
    "  if char in string.ascii_lowercase:\n",
    "    return ord(char) - first_letter + 1\n",
    "  elif char == ' ':\n",
    "    return 0\n",
    "  else:\n",
    "    print('Unexpected character: %s' % char)\n",
    "    return 0\n",
    "  \n",
    "def id2char(dictid):\n",
    "  if dictid > 0:\n",
    "    return chr(dictid + first_letter - 1)\n",
    "  else:\n",
    "    return ' '\n",
    "\n",
    "print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'))\n",
    "print(id2char(1), id2char(26), id2char(0))\n",
    "\n",
    "class BatchGenerator(object):\n",
    "  def __init__(self, text, batch_size, num_unrollings):\n",
    "    self._text = text\n",
    "    self._text_size = len(text)\n",
    "    self._batch_size = batch_size\n",
    "    self._num_unrollings = num_unrollings\n",
    "    segment = self._text_size // batch_size\n",
    "    self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "    self._last_batch = self._next_batch()\n",
    "  \n",
    "  def _next_batch(self):\n",
    "    \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "    batch = np.zeros(shape=(self._batch_size, vocabulary_size), dtype=np.float)\n",
    "    #print 'batch idx %i' % \n",
    "    for b in range(self._batch_size):\n",
    "      batch[b, char2id(self._text[self._cursor[b]])] = 1.0\n",
    "      self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "    return batch\n",
    "  \n",
    "  def next(self):\n",
    "    \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "    the last batch of the previous array, followed by num_unrollings new ones.\n",
    "    \"\"\"\n",
    "    batches = [self._last_batch]\n",
    "    for step in range(self._num_unrollings):\n",
    "      batches.append(self._next_batch())\n",
    "    self._last_batch = batches[-1]\n",
    "    return batches\n",
    "\n",
    "def characters(probabilities):\n",
    "  \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "  characters back into its (mostl likely) character representation.\"\"\"\n",
    "  return [id2char(c) for c in np.argmax(probabilities, 1)]\n",
    "\n",
    "def batches2string(batches):\n",
    "  \"\"\"Convert a sequence of batches back into their (most likely) string\n",
    "  representation.\"\"\"\n",
    "  s = [''] * batches[0].shape[0]\n",
    "  for b in batches:\n",
    "    s = [''.join(x) for x in zip(s, characters(b))]\n",
    "  return s\n",
    "\n",
    "train_batches = BatchGenerator(train_text, 128, 54)\n",
    "valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "print train_batches._cursor\n",
    "\n",
    "batch = train_batches.next()\n",
    "print batches2string([batch[0]])\n",
    "print(batches2string(batch))\n",
    "print(batches2string(train_batches.next()))\n",
    "print(batches2string(valid_batches.next()))\n",
    "print(batches2string(valid_batches.next()))\n",
    "\n",
    "def logprob(predictions, labels):\n",
    "  \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n",
    "  predictions[predictions < 1e-10] = 1e-10\n",
    "  return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]\n",
    "\n",
    "def sample_distribution(distribution):\n",
    "  \"\"\"Sample one element from a distribution assumed to be an array of normalized\n",
    "  probabilities.\n",
    "  \"\"\"\n",
    "  r = random.uniform(0, 1)\n",
    "  s = 0\n",
    "  for i in range(len(distribution)):\n",
    "    s += distribution[i]\n",
    "    if s >= r:\n",
    "      return i\n",
    "  return len(distribution) - 1\n",
    "\n",
    "def sample(prediction, size=vocabulary_size):\n",
    "  \"\"\"Turn a (column) prediction into 1-hot encoded samples.\"\"\"\n",
    "  p = np.zeros(shape=[1, size], dtype=np.float)\n",
    "  p[0, sample_distribution(prediction[0])] = 1.0\n",
    "  return p\n",
    "\n",
    "def random_distribution(size=vocabulary_size):\n",
    "  \"\"\"Generate a random column of probabilities.\"\"\"\n",
    "  b = np.random.uniform(0.0, 1.0, size=[1, size])\n",
    "  return b/np.sum(b, 1)[:,None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Two layers character bases LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x7efe6ed77190>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.ops.Graph at 0x7efe6f603350>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_lstm_graph_bm(num_nodes1, num_nodes2, num_unrollings, batch_size):\n",
    "    with tf.Graph().as_default() as g:\n",
    "        # cell variables - input, memory, biases        \n",
    "        lstm1_x = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes1*4], -0.1, 0.1), name='lstm1_x')\n",
    "        lstm1_m = tf.Variable(tf.truncated_normal([num_nodes1, num_nodes1*4], -0.1, 0.1), name='lstm1_m')\n",
    "        lstm1_b = tf.Variable(tf.zeros([1, num_nodes1*4]), name='lstm1_b')\n",
    "        lstm2_x = tf.Variable(tf.truncated_normal([num_nodes1, num_nodes2*4], -0.1, 0.1), name='lstm2_x')\n",
    "        lstm2_m = tf.Variable(tf.truncated_normal([num_nodes2, num_nodes2*4], -0.1, 0.1), name='lstm2_m')\n",
    "        lstm2_b = tf.Variable(tf.zeros([1, num_nodes2*4]), name='lstm2_b')\n",
    "        # Variables saving state across unrollings.\n",
    "        lstm1_saved_output = tf.Variable(tf.zeros([batch_size, num_nodes1]), trainable=False)\n",
    "        lstm2_saved_output = tf.Variable(tf.zeros([batch_size, num_nodes2]), trainable=False)\n",
    "        lstm1_saved_state = tf.Variable(tf.zeros([batch_size, num_nodes1]), trainable=False)\n",
    "        lstm2_saved_state = tf.Variable(tf.zeros([batch_size, num_nodes2]), trainable=False)\n",
    "        # Classifier weights and biases.\n",
    "        w = tf.Variable(tf.truncated_normal([num_nodes2, vocabulary_size], -0.1, 0.1))\n",
    "        b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "        # Definition of the cell computation.\n",
    "        def lstm_cell(i, o, state, x, m, b, num_nodes):                                    \n",
    "            #multiply in one operation then split matrix between gates            \n",
    "            mult = tf.matmul(i, x) + tf.matmul(o, m) + b\n",
    "            input_gate = tf.sigmoid(mult[:,:num_nodes])\n",
    "            forget_gate = tf.sigmoid(mult[:,num_nodes:num_nodes*2])\n",
    "            update = mult[:,num_nodes*3:num_nodes*4]\n",
    "            state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "            output_gate = tf.sigmoid(mult[:,num_nodes*3:])\n",
    "            return output_gate * tf.tanh(state), state\n",
    "\n",
    "        # Input data. [num_unrollings, batch_size, vocabulary_size]\n",
    "        tf_train_data = tf.placeholder(tf.float32, shape=[None, None, vocabulary_size], name='tf_train_data')\n",
    "        train_data = list()\n",
    "        for i in tf.split(0, num_unrollings + 1, tf_train_data):\n",
    "            train_data.append(tf.squeeze(i))\n",
    "        train_inputs = train_data[:num_unrollings]\n",
    "        train_labels = train_data[1:]  # labels are inputs shifted by one time step.\n",
    "\n",
    "        # Unrolled LSTM loop.\n",
    "        outputs = list()\n",
    "        lstm1_output = lstm1_saved_output\n",
    "        lstm2_output = lstm2_saved_output\n",
    "        lstm1_state = lstm1_saved_state\n",
    "        lstm2_state = lstm2_saved_state\n",
    "        #python loop used: tensorflow does not support sequential operations yet\n",
    "        for i in train_inputs: # having a loop simulates having time\n",
    "            lstm1_output, lstm1_state = lstm_cell(i, lstm1_output, lstm1_state, lstm1_x, lstm1_m, lstm1_b, \n",
    "                                                 num_nodes1)\n",
    "            lstm2_output, lstm2_state = lstm_cell(lstm1_output, lstm2_output, lstm2_state, lstm2_x, lstm2_m,\n",
    "                                                  lstm2_b, num_nodes2)\n",
    "            outputs.append(lstm2_output)\n",
    "\n",
    "        # State saving across unrollings, control_dependencies makes sure that output and state are computed\n",
    "        with tf.control_dependencies([lstm1_saved_output.assign(lstm1_output), lstm1_saved_state.assign(lstm1_state), \n",
    "                                     lstm2_saved_output.assign(lstm2_output), lstm2_saved_state.assign(lstm2_state)]):\n",
    "            # Classifier.\n",
    "            logits = tf.nn.xw_plus_b(tf.concat(0, outputs), w, b)\n",
    "            loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf.concat(0, train_labels)),\n",
    "                                 name='loss')\n",
    "\n",
    "        # Optimizer.\n",
    "        global_step = tf.Variable(0, name='global_step')\n",
    "        learning_rate = tf.train.exponential_decay(10.0, global_step, 5000, 0.1, staircase=True, name='learning_rate')\n",
    "        optimizer = tf.train.GradientDescentOptimizer(learning_rate, name='optimizer')\n",
    "        print optimizer\n",
    "        gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "        gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "        optimizer = optimizer.apply_gradients(zip(gradients, v), global_step=global_step)\n",
    "\n",
    "        # Predictions.\n",
    "        train_prediction = tf.nn.softmax(logits, name='train_prediction')\n",
    "\n",
    "        # Sampling and validation eval: batch 1, no unrolling.\n",
    "        sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size], name='sample_input')\n",
    "        lstm1_saved_sample_output = tf.Variable(tf.zeros([1, num_nodes1]))\n",
    "        lstm2_saved_sample_output = tf.Variable(tf.zeros([1, num_nodes2]))\n",
    "        lstm1_saved_sample_state = tf.Variable(tf.zeros([1, num_nodes1]))\n",
    "        lstm2_saved_sample_state = tf.Variable(tf.zeros([1, num_nodes2]))\n",
    "        reset_sample_state = tf.group( lstm1_saved_sample_output.assign(tf.zeros([1, num_nodes1])), \n",
    "                                      lstm2_saved_sample_output.assign(tf.zeros([1, num_nodes2])),\n",
    "                                      lstm1_saved_sample_state.assign(tf.zeros([1, num_nodes1])), \n",
    "                                      lstm2_saved_sample_state.assign(tf.zeros([1, num_nodes2])),\n",
    "                                      name='reset_sample_state')\n",
    "        lstm1_sample_output, lstm1_sample_state = lstm_cell(sample_input, lstm1_saved_sample_output,\n",
    "                                                             lstm1_saved_sample_state, lstm1_x, lstm1_m, lstm1_b,\n",
    "                                                            num_nodes1)\n",
    "        lstm2_sample_output, lstm2_sample_state = lstm_cell(lstm1_sample_output, lstm2_saved_sample_output,\n",
    "                                                             lstm2_saved_sample_state, lstm2_x, lstm2_m, lstm2_b,\n",
    "                                                             num_nodes2)\n",
    "        \n",
    "        with tf.control_dependencies([lstm1_saved_sample_output.assign(lstm1_sample_output),\n",
    "                                      lstm2_saved_sample_output.assign(lstm2_sample_output), \n",
    "                                     lstm1_saved_sample_state.assign(lstm1_sample_state),\n",
    "                                      lstm2_saved_sample_state.assign(lstm2_sample_state)]):\n",
    "            sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(lstm2_sample_output, w, b), name='sample_prediction')\n",
    "\n",
    "        return g\n",
    "\n",
    "#test graph\n",
    "create_lstm_graph_bm(128, 64, 10, 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(g, num_steps, summary_frequency, num_unrollings, batch_size):\n",
    "    #initalize batch generators\n",
    "    train_batches = BatchGenerator(train_text, batch_size, num_unrollings)\n",
    "    valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "    optimizer = g.get_tensor_by_name('optimizer:0')\n",
    "    #print optimizer\n",
    "    loss = g.get_tensor_by_name('loss:0')\n",
    "    train_prediction = g.get_tensor_by_name('train_prediction:0')\n",
    "    learning_rate = g.get_tensor_by_name('learning_rate:0')\n",
    "    tf_train_data = g.get_tensor_by_name('tf_train_data:0')\n",
    "    sample_prediction = g.get_tensor_by_name('sample_prediction:0')\n",
    "    reset_sample_state = g.get_operation_by_name('reset_sample_state')\n",
    "    sample_input = g.get_tensor_by_name('sample_input:0')\n",
    "    with tf.Session(graph=g) as session:\n",
    "        tf.initialize_all_variables().run()\n",
    "        print('Initialized')    \n",
    "        mean_loss = 0\n",
    "        for step in range(num_steps):\n",
    "            batches = train_batches.next()            \n",
    "            _, l, predictions, lr = session.run([optimizer, loss, train_prediction, learning_rate], \n",
    "                                                feed_dict={ tf_train_data: batches})\n",
    "            mean_loss += 1\n",
    "            if step % summary_frequency == 0:\n",
    "                if step > 0:\n",
    "                    mean_loss = mean_loss / summary_frequency\n",
    "                # The mean loss is an estimate of the loss over the last few batches.\n",
    "                #print mean_loss\n",
    "                #print type(mean_loss)\n",
    "                print 'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr)\n",
    "                mean_loss = 0\n",
    "                labels = np.concatenate(list(batches)[1:])\n",
    "                print('Minibatch perplexity: %.2f' % float(np.exp(logprob(predictions, labels))))\n",
    "                if step % (summary_frequency * 10) == 0:\n",
    "                    # Generate some samples.\n",
    "                    print('=' * 80)\n",
    "                    for _ in range(5):\n",
    "                        feed = sample(random_distribution())\n",
    "                        sentence = characters(feed)[0]\n",
    "                        reset_sample_state.run()\n",
    "                        for _ in range(79):\n",
    "                            prediction = sample_prediction.eval({sample_input: feed})\n",
    "                            feed = sample(prediction)\n",
    "                            sentence += characters(feed)[0]\n",
    "                        print(sentence)\n",
    "                    print('=' * 80)\n",
    "                # Measure validation set perplexity.\n",
    "                reset_sample_state.run()\n",
    "                valid_logprob = 0\n",
    "                for _ in range(valid_size):\n",
    "                    b = valid_batches.next()\n",
    "                    predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "                    valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "                print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "                    valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x7efe6d62f410>\n",
      "Initialized\n",
      "Average loss at step 0: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 27.04\n",
      "================================================================================\n",
      "t lncadi oiml zrag b   tg fxg fau ar koeehcnwnni  sajiarunvgpu   isqm  elwgno bc\n",
      "zrp nrltgbaa ldzexslageeydpc  vrok x il osiwwtidpegp er tx moi n  exn t pihttp  \n",
      "eadliavmtrza m kor   ik t ec yznaneoffs ibuoky zfvinm whoo img  o ntm ere ssomeg\n",
      "qil ge e upd tv hini ltvs gcllg pilf hix selodysdieslarl blxai  idyt ess ktnv i \n",
      "ekzn isb ynn  xlaqfn gonjoaubdhmp o it  lcls wngey  shbld uib xonfngoc vtcie etc\n",
      "================================================================================\n",
      "Validation set perplexity: 20.94\n",
      "Average loss at step 100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 14.18\n",
      "Validation set perplexity: 14.49\n",
      "Average loss at step 200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 10.27\n",
      "Validation set perplexity: 11.41\n",
      "Average loss at step 300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 10.35\n",
      "Validation set perplexity: 10.14\n",
      "Average loss at step 400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 8.38\n",
      "Validation set perplexity: 9.45\n",
      "Average loss at step 500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 9.09\n",
      "Validation set perplexity: 9.92\n",
      "Average loss at step 600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.56\n",
      "Validation set perplexity: 8.33\n",
      "Average loss at step 700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 8.08\n",
      "Validation set perplexity: 7.81\n",
      "Average loss at step 800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.48\n",
      "Validation set perplexity: 7.44\n",
      "Average loss at step 900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.80\n",
      "Validation set perplexity: 7.30\n",
      "Average loss at step 1000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.84\n",
      "================================================================================\n",
      "caidy on veseral u evir meave amerats of se mbenvetail lameut entends he the oth\n",
      "fir seveto lencas is kut were cas the film jepbliqes los nee the sevemh atso pea\n",
      "d rechovers of unisan cleabure the arryic mishs fichlcots messute wam dis seceri\n",
      "zerection treatic end sak of adeamia wigho b nary vu poil scodine of i gases clh\n",
      "cunt testenratod mot cort i ussapt uspich vonace jicas newters to the recitices \n",
      "================================================================================\n",
      "Validation set perplexity: 6.66\n",
      "Average loss at step 1100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.96\n",
      "Validation set perplexity: 6.57\n",
      "Average loss at step 1200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.72\n",
      "Validation set perplexity: 6.14\n",
      "Average loss at step 1300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.16\n",
      "Validation set perplexity: 6.09\n",
      "Average loss at step 1400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.45\n",
      "Validation set perplexity: 5.83\n",
      "Average loss at step 1500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.65\n",
      "Validation set perplexity: 5.76\n",
      "Average loss at step 1600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.45\n",
      "Validation set perplexity: 5.72\n",
      "Average loss at step 1700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.04\n",
      "Validation set perplexity: 5.43\n",
      "Average loss at step 1800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.30\n",
      "Validation set perplexity: 5.36\n",
      "Average loss at step 1900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.33\n",
      "Validation set perplexity: 5.30\n",
      "Average loss at step 2000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.83\n",
      "================================================================================\n",
      "list is one eight from alrore hie fow emperor moghles work in heecondect of chou\n",
      "qunce bit a actrohsive indied actor wible to pains of tlanecty granck of the one\n",
      "klands for a lange the longe three co sectine secale stor scult perfoor atlerg d\n",
      "x the nascung c teniler at iting alonal gocied to jetist of soveed hound yector \n",
      "f leve into heat the ffisae heaghat evivitions has one nine zero zero zero zero \n",
      "================================================================================\n",
      "Validation set perplexity: 5.21\n",
      "Average loss at step 2100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.29\n",
      "Validation set perplexity: 5.24\n",
      "Average loss at step 2200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.12\n",
      "Validation set perplexity: 5.49\n",
      "Average loss at step 2300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.22\n",
      "Validation set perplexity: 5.02\n",
      "Average loss at step 2400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.64\n",
      "Validation set perplexity: 4.82\n",
      "Average loss at step 2500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.76\n",
      "Validation set perplexity: 4.91\n",
      "Average loss at step 2600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.59\n",
      "Validation set perplexity: 4.76\n",
      "Average loss at step 2700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.75\n",
      "Validation set perplexity: 4.79\n",
      "Average loss at step 2800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.20\n",
      "Validation set perplexity: 4.71\n",
      "Average loss at step 2900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.65\n",
      "Validation set perplexity: 4.71\n",
      "Average loss at step 3000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.09\n",
      "================================================================================\n",
      "ur liwra frence resuitecalo bur ims three one eight nine six five kde becomes th\n",
      "ricourcle is cal two one eim in chabriet descized succe of ko lihest otrators so\n",
      "ly disschhomors runge was vear jo be words muman clennine are analcesian license\n",
      "men somissing erenker stawls arre of boocj oulty bway weren tradition one nine s\n",
      "y or idepritor mosn the come mansaftay the spegen in end or who the calh accuse \n",
      "================================================================================\n",
      "Validation set perplexity: 4.64\n",
      "Average loss at step 3100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.85\n",
      "Validation set perplexity: 4.63\n",
      "Average loss at step 3200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.62\n",
      "Average loss at step 3300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.07\n",
      "Validation set perplexity: 4.58\n",
      "Average loss at step 3400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.74\n",
      "Validation set perplexity: 4.75\n",
      "Average loss at step 3500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.04\n",
      "Validation set perplexity: 4.66\n",
      "Average loss at step 3600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.72\n",
      "Validation set perplexity: 4.59\n",
      "Average loss at step 3700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.39\n",
      "Validation set perplexity: 4.52\n",
      "Average loss at step 3800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.24\n",
      "Validation set perplexity: 4.43\n",
      "Average loss at step 3900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.96\n",
      "Validation set perplexity: 4.47\n",
      "Average loss at step 4000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.54\n",
      "================================================================================\n",
      "ditist onporty the caganier one nine zero one nine pales chikier vole of attenci\n",
      "zy and the late then one nine zero zero th attracter in clason ingers wasing est\n",
      "gy the regulition can emperorie bort untima ordeving given mitary perial inmiced\n",
      "cond from one eight yeals that calms intereal on the gumb wiple and lame from wr\n",
      "jy and day by speed warndribations and putros was it one two formers half dan la\n",
      "================================================================================\n",
      "Validation set perplexity: 4.48\n",
      "Average loss at step 4100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.29\n",
      "Validation set perplexity: 4.51\n",
      "Average loss at step 4200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.50\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-703a59d005a6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcreate_lstm_graph_bm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m128\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m64\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m70001\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-21-2986af67cc2d>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(g, num_steps, summary_frequency, num_unrollings, batch_size)\u001b[0m\n\u001b[0;32m     49\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m                     \u001b[0mb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalid_batches\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m                     \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msample_prediction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0msample_input\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m                     \u001b[0mvalid_logprob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalid_logprob\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlogprob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m                 print('Validation set perplexity: %.2f' % float(np.exp(\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36meval\u001b[1;34m(self, feed_dict, session)\u001b[0m\n\u001b[0;32m    458\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    459\u001b[0m     \"\"\"\n\u001b[1;32m--> 460\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_eval_using_default_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    461\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    462\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36m_eval_using_default_session\u001b[1;34m(tensors, feed_dict, graph, session)\u001b[0m\n\u001b[0;32m   2908\u001b[0m                        \u001b[1;34m\"the tensor's graph is different from the session's \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2909\u001b[0m                        \"graph.\")\n\u001b[1;32m-> 2910\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2911\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2912\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict)\u001b[0m\n\u001b[0;32m    366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m     \u001b[1;31m# Run request and get response.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 368\u001b[1;33m     \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munique_fetch_targets\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m     \u001b[1;31m# User may have fetched the same tensor multiple times, but we\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, target_list, fetch_list, feed_dict)\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    427\u001b[0m       return tf_session.TF_Run(self._session, feed_dict, fetch_list,\n\u001b[1;32m--> 428\u001b[1;33m                                target_list)\n\u001b[0m\u001b[0;32m    429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    430\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStatusNotOK\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "g = create_lstm_graph_bm(128, 64, 10, 32)\n",
    "train(g, 70001, 100, 10, 32)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}