How to get the last hidden vector of rnns properly.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll see how to get the last hidden states of Rnns in Tensorflow and PyTorch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "__author__ = \"kyubyong\"\n",
    "__address__ = \"https://github.com/kyubyong/nlp_made_easy\"\n",
    "__email__ = \"kbpark.linguist@gmail.com\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.14.0'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.2.0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Uni-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rnn(x, bidirectional=False, seqlens=None, reuse=False):\n",
    "    if not bidirectional:\n",
    "        with tf.variable_scope(\"rnn\", reuse=reuse):\n",
    "            cell = tf.contrib.rnn.GRUCell(1)\n",
    "            outputs, last_hidden = tf.nn.dynamic_rnn(cell, x, sequence_length=seqlens, dtype=tf.float32)\n",
    "    else: \n",
    "        with tf.variable_scope(\"birnn\", reuse=reuse):\n",
    "            cell = tf.contrib.rnn.GRUCell(1)\n",
    "            cell_bw = tf.contrib.rnn.GRUCell(1)\n",
    "            outputs, last_hidden = tf.nn.bidirectional_dynamic_rnn(cell, cell_bw, x, sequence_length=seqlens, dtype=tf.float32)\n",
    "            outputs, last_hidden = tf.concat(outputs,-1), tf.concat(last_hidden, -1)\n",
    "\n",
    "    return outputs, last_hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onehot(arry, size):\n",
    "    '''\n",
    "    arry: 2-d array of n, t\n",
    "    size: output dimensions\n",
    "    \n",
    "    returns\n",
    "    3-d array of (n, t, size)\n",
    "    '''\n",
    "    labels_one_hot = (arry.ravel()[np.newaxis] == np.arange(size)[:, np.newaxis]).T\n",
    "    labels_one_hot.shape = arry.shape + (size,)\n",
    "    return labels_one_hot.astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "_x1 = np.array([1, 2, 3], np.int32)\n",
    "_x1 = onehot(np.expand_dims(_x1, 0), 4)\n",
    "\n",
    "_x2 = np.array([1, 2, 3, 0], np.int32) # 0 means padding\n",
    "_x2 = onehot(np.expand_dims(_x2, 0), 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-6-cf3f6a89e607>:4: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From <ipython-input-6-cf3f6a89e607>:5: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\rnn_cell_impl.py:564: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\rnn_cell_impl.py:574: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA3473780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA3473780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA3473780>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA3473780>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "# 1. no padding\n",
    "x1 = tf.convert_to_tensor(_x1)\n",
    "outputs1, last_hidden1 = rnn(x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6090390>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6090390>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6090390>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6090390>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "# 2. zero padding, no seqlens\n",
    "x2 = tf.convert_to_tensor(_x2)\n",
    "outputs2, last_hidden2 = rnn(x2, reuse=True) # We want to sync the variables up to compare the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60FC7B8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60FC7B8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60FC7B8>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60FC7B8>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:From C:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\rnn.py:244: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "# 3. zero padding with explicit seqlens\n",
    "outputs3, last_hidden3 = rnn(x2, seqlens=[3,], reuse=True) # Real sequence length is 3 as the last 0 is a padding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Session\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-0.31044602],\n",
       "        [-0.1496363 ],\n",
       "        [ 0.0102744 ]]], dtype=float32)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-0.31044602],\n",
       "        [-0.1496363 ],\n",
       "        [ 0.0102744 ],\n",
       "        [-0.2648457 ]]], dtype=float32)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs2.eval() # the last step has non-zero outputs. This is not we usually want."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-0.31044602],\n",
       "        [-0.1496363 ],\n",
       "        [ 0.0102744 ],\n",
       "        [ 0.        ]]], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs3.eval() # the last step is masked to zeros. This is usually correct."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0102744]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.2648457]], dtype=float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden2.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.0102744]], dtype=float32)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden3.eval() # Now we have the same results as # 1."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "△ Comment: Paddings are mostly added to construct mini-batches from multiples samples of variable lengths. Therefore typically we want to get the same results as the case we treat them individually and do not pad. To that end, when you add paddings, you should add `seqlens`. Paddings are masked to zeros."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bi-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-cf3f6a89e607>:10: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64D5B00>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64D5B00>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64D5B00>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64D5B00>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64B9C18>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64B9C18>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64B9C18>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA64B9C18>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "# 1. no padding\n",
    "x1 = tf.convert_to_tensor(_x1)\n",
    "outputs1, last_hidden1 = rnn(x1, bidirectional=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6497A20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6497A20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6497A20>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6497A20>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60DA080>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60DA080>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60DA080>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA60DA080>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "# 2. zero padding, no seqlens\n",
    "x2 = tf.convert_to_tensor(_x2)\n",
    "outputs2, last_hidden2 = rnn(x2, bidirectional=True, reuse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6463A90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6463A90>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6463A90>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BA6463A90>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING:tensorflow:Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BC94CE6A0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BC94CE6A0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n",
      "WARNING: Entity <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BC94CE6A0>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method GRUCell.call of <tensorflow.python.ops.rnn_cell_impl.GRUCell object at 0x0000017BC94CE6A0>>: AssertionError: Bad argument number for Name: 3, expecting 4\n"
     ]
    }
   ],
   "source": [
    "# 3. zero padding with explicit seqlens\n",
    "outputs3, last_hidden3 = rnn(x2, bidirectional=True, seqlens=[3,], reuse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py:1735: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "# Session\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess = tf.InteractiveSession()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.03057988, -0.36812052],\n",
       "        [ 0.11028282, -0.25675917],\n",
       "        [ 0.03307126,  0.01558547]]], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.03057988, -0.36047477],\n",
       "        [ 0.11028282, -0.24538502],\n",
       "        [ 0.03307126,  0.03696197],\n",
       "        [ 0.1797748 ,  0.02438327]]], dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs2.eval() # Again, this is not we want."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 0.03057988, -0.36812052],\n",
       "        [ 0.11028282, -0.25675917],\n",
       "        [ 0.03307126,  0.01558547],\n",
       "        [ 0.        ,  0.        ]]], dtype=float32)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs3.eval() # Again, note that the last step is masked to zeros."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.03307126, -0.36812052]], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden1.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.1797748 , -0.36047477]], dtype=float32)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden2.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.03307126, -0.36812052]], dtype=float32)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden3.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "△ Note that in bidirectional rnns, the last_hidden state of the forward rnn (=0.03307126) is from the rightmost step in the sequence, while the last hidden state of the backward rnn (=-0.36812052) is from the leftmost step."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Uni-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Rnn(torch.nn.Module):\n",
    "    def __init__(self, bidirectional=False):\n",
    "        super().__init__()\n",
    "        self.rnn = nn.GRU(4, 1, batch_first=True, bidirectional=bidirectional)\n",
    "        \n",
    "    def forward(self, x, seqlens=None):\n",
    "        if seqlens is not None:\n",
    "            # packing -> rnn -> unpacking -> position recovery\n",
    "            packed_input = pack_padded_sequence(x, seqlens, batch_first=True, enforce_sorted=False)   \n",
    "            outputs, last_hidden = self.rnn(packed_input)\n",
    "            outputs, _ = pad_packed_sequence(outputs, batch_first=True, total_length=x.size()[1])\n",
    "        else:\n",
    "            outputs, last_hidden = self.rnn(x)\n",
    "        last_hidden = last_hidden.permute(1, 2, 0) # to (batch, hidden, num_directions)\n",
    "        last_hidden = last_hidden.view(last_hidden.size()[0], -1) # to (batch, hidden*num_directions)\n",
    "        \n",
    "        return outputs, last_hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. no padding\n",
    "x1 = torch.from_numpy(_x1)\n",
    "model1 = Rnn()\n",
    "outputs1, last_hidden1 = model1(x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. zero padding, no seqlens\n",
    "x2 = torch.from_numpy(_x2)\n",
    "model2 = Rnn()\n",
    "for p1, p2 in zip(model1.parameters(), model2.parameters()): # sync up the variables\n",
    "    p2.data = p1.data\n",
    "outputs2, last_hidden2 = model2(x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. zero padding with explicit seqlens\n",
    "model3 = Rnn()\n",
    "for p1, p3 in zip(model1.parameters(), model3.parameters()):\n",
    "    p3.data = p1.data\n",
    "outputs3, last_hidden3 = model3(x2, seqlens=[3,])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0022],\n",
       "         [0.1187],\n",
       "         [0.2030]]], grad_fn=<TransposeBackward1>)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0022],\n",
       "         [ 0.1187],\n",
       "         [ 0.2030],\n",
       "         [-0.0466]]], grad_fn=<TransposeBackward1>)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0022],\n",
       "         [0.1187],\n",
       "         [0.2030],\n",
       "         [0.0000]]], grad_fn=<IndexSelectBackward>)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2030]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0466]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2030]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "△ Since there's no such argument as seqlens in pytorch, a trick was used. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bi-directional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. no padding\n",
    "model1 = Rnn(bidirectional=True)\n",
    "outputs1, last_hidden1 = model1(x1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. zero padding without seqlens\n",
    "model2 = Rnn(bidirectional=True)\n",
    "for p1, p2 in zip(model1.parameters(), model2.parameters()):\n",
    "    p2.data = p1.data\n",
    "outputs2, last_hidden2 = model2(x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3. zero padding with explicit seqlens\n",
    "model3 = Rnn(bidirectional=True)\n",
    "for p1, p3 in zip(model1.parameters(), model3.parameters()):\n",
    "    p3.data = p1.data\n",
    "outputs3, last_hidden3 = model3(x2, seqlens=[3,])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1541,  0.1542],\n",
       "         [ 0.1018,  0.1953],\n",
       "         [ 0.3250, -0.2467]]], grad_fn=<TransposeBackward1>)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1541,  0.1687],\n",
       "         [ 0.1018,  0.2188],\n",
       "         [ 0.3250, -0.1759],\n",
       "         [ 0.2979,  0.0983]]], grad_fn=<TransposeBackward1>)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.1541,  0.1542],\n",
       "         [ 0.1018,  0.1953],\n",
       "         [ 0.3250, -0.2467],\n",
       "         [ 0.0000,  0.0000]]], grad_fn=<IndexSelectBackward>)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outputs3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3250, 0.1542]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2979, 0.1687]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.3250, 0.1542]], grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_hidden3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "△ Same here."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}