3.Neural_Network/TensorFlow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/x-val/anaconda3/envs/tensorflow/lib/python3.6/site-packages/IPython/html.py:14: ShimWarning: The `IPython.html` package has been deprecated since IPython 4.0. You should import from `notebook` instead. `IPython.html.widgets` has moved to `ipywidgets`.\n",
      "  \"`IPython.html.widgets` has moved to `ipywidgets`.\", ShimWarning)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import seaborn; seaborn.set()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# XOR\n",
    "\n",
    "num_train = 64000\n",
    "num_test = 10000\n",
    "\n",
    "def make_dataset(X, y):\n",
    "    train_X = np.array([])\n",
    "    train_y = np.array([], dtype=int)\n",
    "\n",
    "    for i in range(int(num_train/4)):\n",
    "        train_X = np.append(train_X, X)\n",
    "        train_y = np.append(train_y, y)\n",
    "\n",
    "    train_X = train_X.reshape((num_train, 2))\n",
    "    train_y = train_y.reshape((num_train, 2))\n",
    "\n",
    "    test_indices = np.random.choice(num_train, num_test, replace=True)\n",
    "    test_X = train_X[test_indices]\n",
    "    test_y = train_y[test_indices]\n",
    "\n",
    "    print(train_X.shape, train_y.shape)\n",
    "    print(test_X.shape, test_y.shape)\n",
    "\n",
    "    return train_X, train_y, test_X, test_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XOR\n",
    "- 0, 0 = 0\n",
    "- 0, 1 = 1\n",
    "- 1, 0 = 1\n",
    "- 1, 1 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64000, 2) (64000, 2)\n",
      "(10000, 2) (10000, 2)\n",
      "[[ 0.  0.]\n",
      " [ 0.  1.]\n",
      " [ 1.  0.]\n",
      " [ 1.  1.]\n",
      " [ 0.  0.]\n",
      " [ 0.  1.]\n",
      " [ 1.  0.]\n",
      " [ 1.  1.]\n",
      " [ 0.  0.]\n",
      " [ 0.  1.]]\n",
      "[[1 0]\n",
      " [0 1]\n",
      " [0 1]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [0 1]\n",
      " [0 1]\n",
      " [1 0]\n",
      " [1 0]\n",
      " [0 1]]\n"
     ]
    }
   ],
   "source": [
    "xor_X = [[0, 0], [0, 1], [1, 0], [1, 1]]\n",
    "xor_y = [[1, 0], [0, 1], [0, 1], [1, 0]] # One Hot\n",
    "\n",
    "train_X, train_y, test_X, test_y = make_dataset(xor_X, xor_y)\n",
    "\n",
    "print(train_X[:10])\n",
    "print(train_y[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "learning_rate = 0.0001\n",
    "num_epochs = 5\n",
    "batch_size = 64\n",
    "\n",
    "input_size = 2\n",
    "num_classes = 2\n",
    "\n",
    "num_h1 = 256\n",
    "num_h2 = 512\n",
    "num_h3 = 1024"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, [None, input_size])\n",
    "y = tf.placeholder(tf.float32, [None, num_classes])\n",
    "\n",
    "W1 = tf.get_variable('W1', [input_size, num_h1], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b1 = tf.get_variable('b1', [num_h1], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "W2 = tf.get_variable('W2', [num_h1, num_h2], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b2 = tf.get_variable('b2', [num_h2], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "W3 = tf.get_variable('W3', [num_h2, num_h3], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b3 = tf.get_variable('b3', [num_h3], initializer=tf.constant_initializer(0.0))\n",
    "\n",
    "W4 = tf.get_variable('W4', [num_h3, num_classes], initializer=tf.contrib.layers.xavier_initializer())\n",
    "b4 = tf.get_variable('b4', [num_classes], initializer=tf.constant_initializer(0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Layer\n",
    "input_layer = tf.add(tf.matmul(X, W1), b1)\n",
    "input_layer = tf.nn.relu(input_layer)\n",
    "\n",
    "hidden_layer1 = tf.add(tf.matmul(input_layer, W2), b2)\n",
    "hidden_layer1 = tf.nn.relu(hidden_layer1)\n",
    "\n",
    "hidden_layer2 = tf.add(tf.matmul(hidden_layer1, W3), b3)\n",
    "hidden_layer2 = tf.nn.relu(hidden_layer2)\n",
    "\n",
    "output_layer = tf.add(tf.matmul(hidden_layer2, W4), b4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Construct model\n",
    "pred = output_layer\n",
    "\n",
    "# Define loss and optimizer\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average trainig loss for Epoch 1  (200): 0.214880046877\n",
      "Average trainig loss for Epoch 1  (400): 0.108745440566\n",
      "Average trainig loss for Epoch 1  (600): 0.072642528154\n",
      "Average trainig loss for Epoch 1  (800): 0.0545207863064\n",
      "Average trainig loss for Epoch 2  (200): 3.80547518307e-05\n",
      "Average trainig loss for Epoch 2  (400): 2.92624546455e-05\n",
      "Average trainig loss for Epoch 2  (600): 2.3581103654e-05\n",
      "Average trainig loss for Epoch 2  (800): 1.96799277347e-05\n",
      "Average trainig loss for Epoch 3  (200): 4.12686565824e-06\n",
      "Average trainig loss for Epoch 3  (400): 3.62179314948e-06\n",
      "Average trainig loss for Epoch 3  (600): 3.22495199763e-06\n",
      "Average trainig loss for Epoch 3  (800): 2.9037461934e-06\n",
      "Average trainig loss for Epoch 4  (200): 1.28596914863e-06\n",
      "Average trainig loss for Epoch 4  (400): 1.17614775206e-06\n",
      "Average trainig loss for Epoch 4  (600): 1.08395936905e-06\n",
      "Average trainig loss for Epoch 4  (800): 1.00236320158e-06\n",
      "Average trainig loss for Epoch 5  (200): 5.66541966975e-07\n",
      "Average trainig loss for Epoch 5  (400): 5.11184181846e-07\n",
      "Average trainig loss for Epoch 5  (600): 4.68790398903e-07\n",
      "Average trainig loss for Epoch 5  (800): 4.36939180908e-07\n",
      "Accuracy: 1.0\n"
     ]
    }
   ],
   "source": [
    "num_steps = num_train // batch_size\n",
    "traininig_losses = []\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        traininig_loss = 0\n",
    "        for steps in range(num_steps):\n",
    "            feed_dict = {X: train_X[steps*batch_size:(steps+1)*batch_size], y: train_y[steps*batch_size:(steps+1)*batch_size]}\n",
    "            _, l = sess.run([optimizer, cost], feed_dict=feed_dict)\n",
    "            traininig_loss += l\n",
    "            \n",
    "            if steps != 0 and steps % 200 == 0:\n",
    "                print(\"Average trainig loss for Epoch\", epoch+1, \" (\" + str(steps) + \"):\", traininig_loss/steps)\n",
    "\n",
    "    # Test\n",
    "    test_accuarcy = 0;\n",
    "    for i in range(0, num_test, 50):\n",
    "        correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
    "        test_accuarcy += accuracy.eval({X: test_X[i:i+50], y: test_y[i:i+50]})\n",
    "    print(\"Accuracy:\", test_accuarcy/(num_test/50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.6 (Tensorflow)",
   "language": "python",
   "name": "myenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}