Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81f45912",
   "metadata": {},
   "source": [
    "### Forward Propagation without any hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "44f9334a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "90f36e47",
   "metadata": {},
   "outputs": [],
   "source": [
    "# AND gate\n",
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y = np.array([[0,0,0,1]]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f3f27607",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4, 2), (4, 1))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape , Y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c741e8fe",
   "metadata": {},
   "source": [
    "###### sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d490e746",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sig(z):\n",
    "    return 1/(1+np.exp(-z))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b78d334f",
   "metadata": {},
   "source": [
    "###### derivative of sigmoid function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "708fbb66",
   "metadata": {},
   "outputs": [],
   "source": [
    "def derivativeSig(z):\n",
    "    return sig(z)*(1-sig(z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c9ee00a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.0312625 ],\n",
       "       [-0.24819102]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights = 2*np.random.random((2,1)) - 1\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "363e4d05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.27281806])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bias = 2*np.random.random(1)-1\n",
    "bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f296f451",
   "metadata": {},
   "outputs": [],
   "source": [
    "output0 = X\n",
    "output = sig(np.dot(output0,weights)+ bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0e67c1cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.5677846 ],\n",
       "       [0.50615645],\n",
       "       [0.57543974],\n",
       "       [0.51396875]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e35a68ea",
   "metadata": {},
   "source": [
    "### Forward Propagation with hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "995d34af",
   "metadata": {},
   "outputs": [],
   "source": [
    "wh = 2*np.random.random((2,2)) - 1\n",
    "bh = 2*np.random.random((1,2)) - 1\n",
    "wo = 2*np.random.random((2,1)) - 1\n",
    "bo = 2*np.random.random((1,1)) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "837813e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "output0 = X\n",
    "outputHidden = sig(np.dot(output0,wh)+bh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "832b3714",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = sig(np.dot(outputHidden,wo)+bo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "87542b0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.70105584],\n",
       "       [0.70756646],\n",
       "       [0.72482236],\n",
       "       [0.73248872]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34535e9e",
   "metadata": {},
   "source": [
    "# Neural Netwrok implementation without Hidden Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "708d824a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0]\n",
      " [0 1]\n",
      " [1 0]\n",
      " [1 1]]\n",
      "[[0]\n",
      " [0]\n",
      " [0]\n",
      " [1]]\n",
      "[-0.52407075]\n",
      "[[0.91377251]\n",
      " [0.07176564]]\n",
      "[[0.37190085]\n",
      " [0.38881285]\n",
      " [0.5962109 ]\n",
      " [0.61336223]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "# AND gate\n",
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y = np.array([[0,0,0,1]]).T\n",
    "lr = 0.1\n",
    "weights = 2*np.random.random((2,1)) - 1\n",
    "bias = 2*np.random.random(1)-1\n",
    "output0 = X\n",
    "output = sig(np.dot(output0,weights)+ bias)\n",
    "print(X)\n",
    "print(Y)\n",
    "print(bias)\n",
    "print(weights)\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1f5cfd5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4, 1), (4, 1), (4, 1))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_term = output - Y\n",
    "input_for_last_layer = np.dot(output0,weights) + bias\n",
    "second_term = derivativeSig(input_for_last_layer)\n",
    "first_two = first_term*second_term\n",
    "first_two.shape,second_term.shape,first_two.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cf2e98a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.05184311],\n",
       "       [0.00070572]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "changes = np.array([[0.0],[0.0]])\n",
    "for i in range(2):\n",
    "    for j in range(4):\n",
    "        changes[i][0]+=first_two[j][0]*output0[j][i]\n",
    "changes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "436ac68a",
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = weights - lr*changes\n",
    "bias_change = 0.0\n",
    "for j in range(4):\n",
    "    bias_change += first_two[j][0]*1\n",
    "bias = bias -lr*bias_change"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "46304567",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[5.48372067],\n",
       "        [5.48372064]]),\n",
       " array([-8.31825404]),\n",
       " array([[2.43962013e-04],\n",
       "        [5.54863342e-02],\n",
       "        [5.54863354e-02],\n",
       "        [9.33960881e-01]]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for iter in range(10000):\n",
    "    output0 = X\n",
    "    output = sig(np.dot(output0,weights)+bias)\n",
    "    \n",
    "    first_term = output-Y\n",
    "    input_for_last_layer = np.dot(output0,weights)+bias\n",
    "    second_term = derivativeSig(input_for_last_layer)\n",
    "    first_two = first_term*second_term\n",
    "    \n",
    "    changes = np.array([[0.0],[0.0]])\n",
    "    \n",
    "    for i in range(2):\n",
    "        for j in range(4):\n",
    "            changes[i][0] += first_two[j][0]*output0[j][i]\n",
    "    weights=weights - lr*changes\n",
    "    bias_change = 0.0\n",
    "    for j in range(4):\n",
    "        bias_change += first_two[j][0]*1\n",
    "    bias = bias - lr*bias_change\n",
    "output = sig(np.dot(X,weights)+bias)\n",
    "weights,bias,output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b17f2b4",
   "metadata": {},
   "source": [
    "### Optimizing code using vector operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "27b058d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OR gate\n",
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y = np.array([[0,1,1,1]]).T\n",
    "lr = 0.1\n",
    "weights = 2*np.random.random((2,1)) - 1\n",
    "bias = 2*np.random.random(1)-1\n",
    "output0 = X\n",
    "output = sig(np.dot(output0,weights)+ bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "38d5af1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[6.17509873],\n",
       "        [6.17530702]]),\n",
       " array([-2.84139527]),\n",
       " array([[0.05512781],\n",
       "        [0.96557404],\n",
       "        [0.96556711],\n",
       "        [0.99992583]]))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for iter in range(10000):\n",
    "    output0 = X\n",
    "    output = sig(np.dot(output0,weights)+bias)\n",
    "    \n",
    "    first_term = output-Y\n",
    "    input_for_last_layer = np.dot(output0,weights)+bias\n",
    "    second_term = derivativeSig(input_for_last_layer)\n",
    "    first_two = first_term*second_term\n",
    "    \n",
    "    changes = np.dot(output0.T,first_two)\n",
    "    weights=weights - lr*changes\n",
    "    bias_change = np.sum(first_two)\n",
    "    bias = bias - lr*bias_change\n",
    "output = sig(np.dot(X,weights)+bias)\n",
    "weights,bias,output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "add76fd3",
   "metadata": {},
   "source": [
    "# Neural Netwrok implementation with one hidden layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "41eb7ec9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4, 2), (4, 1))"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
    "Y = np.array([[0,1,1,0]]).T\n",
    "X.shape,Y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "87da948c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# one hidden layer weighst and bias\n",
    "wh = 2*np.random.random((2,2)) - 1\n",
    "bh = 2*np.random.random((1,2)) - 1\n",
    "wo = 2*np.random.random((2,1)) - 1\n",
    "bo = 2*np.random.random((1,1)) - 1\n",
    "lr = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a367f3ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.47686028],\n",
       "       [0.46796646],\n",
       "       [0.42690004],\n",
       "       [0.41943082]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# forward propagation with one huidden layer\n",
    "output0 = X\n",
    "inputHidden = np.dot(output0,wh)+bh\n",
    "outputHidden = sig(inputHidden)\n",
    "inputForOutputLayer = np.dot(outputHidden,wo)+bo\n",
    "output = sig(inputForOutputLayer)\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e10c01ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "first_term_output_layer = output - Y\n",
    "second_term_output_layer = derivativeSig(inputForOutputLayer)\n",
    "first_two_output_layer = first_term_output_layer*second_term_output_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1b4ca301",
   "metadata": {},
   "outputs": [],
   "source": [
    "first_term_hidden_layer = np.dot(first_two_output_layer,wo.T)\n",
    "second_term_hidden_layer = derivativeSig(inputHidden)\n",
    "first_two_hidden_layer = first_term_hidden_layer*second_term_hidden_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "db045e9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "changes_output = np.dot(outputHidden.T,first_two_output_layer)\n",
    "changes_output_bias = np.sum(first_two_output_layer,axis=0,keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "600fa388",
   "metadata": {},
   "outputs": [],
   "source": [
    "changes_hidden = np.dot(output0.T,first_two_hidden_layer)\n",
    "changes_hidden_bias = np.sum(first_two_hidden_layer,axis=0,keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ff6a0854",
   "metadata": {},
   "outputs": [],
   "source": [
    "wo = wo -lr*changes_output\n",
    "bo = bo -lr*changes_output_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cf60b3b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "wh = wh -lr*changes_hidden\n",
    "bh = bh -lr*changes_hidden_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "7f0367bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "for iter in range(10000):\n",
    "    output0 = X\n",
    "    inputHidden = np.dot(output0,wh)+bh\n",
    "    outputHidden = sig(inputHidden)\n",
    "    inputForOutputLayer = np.dot(outputHidden,wo)+bo\n",
    "    output = sig(inputForOutputLayer)\n",
    "    \n",
    "    first_term_output_layer = output - Y\n",
    "    second_term_output_layer = derivativeSig(inputForOutputLayer)\n",
    "    first_two_output_layer = first_term_output_layer*second_term_output_layer\n",
    "    \n",
    "    first_term_hidden_layer = np.dot(first_two_output_layer,wo.T)\n",
    "    second_term_hidden_layer = derivativeSig(inputHidden)\n",
    "    first_two_hidden_layer = first_term_hidden_layer*second_term_hidden_layer\n",
    "    \n",
    "    changes_output = np.dot(outputHidden.T,first_two_output_layer)\n",
    "    changes_output_bias = np.sum(first_two_output_layer,axis=0,keepdims=True)\n",
    "    \n",
    "    changes_hidden = np.dot(output0.T,first_two_hidden_layer)\n",
    "    changes_hidden_bias = np.sum(first_two_hidden_layer,axis=0,keepdims=True)\n",
    "    \n",
    "    wo = wo -lr*changes_output\n",
    "    bo = bo -lr*changes_output_bias\n",
    "    \n",
    "    wh = wh -lr*changes_hidden\n",
    "    bh = bh -lr*changes_hidden_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4b5de2ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.01984259],\n",
       "        [0.98293593],\n",
       "        [0.98293498],\n",
       "        [0.01744344]]),\n",
       " array([[6.04018615, 5.07845562],\n",
       "        [6.0413082 , 5.07895184]]),\n",
       " array([[-2.67178874, -7.78863288]]),\n",
       " array([[ 9.49428434],\n",
       "        [-9.85877775]]),\n",
       " array([[-4.50968664]]))"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output0 = X\n",
    "inputHidden = np.dot(output0,wh)+bh\n",
    "outputHidden = sig(inputHidden)\n",
    "inputForOutputLayer = np.dot(outputHidden,wo)+bo\n",
    "output = sig(inputForOutputLayer)\n",
    "output, wh,bh,wo,bo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b010f3a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e689248",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}