Keras Convolutianl Networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-23T01:29:12.253816Z",
     "start_time": "2018-10-23T01:29:12.248813Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import MaxPooling2D,Activation,BatchNormalization,Dense,Flatten,Conv2D,Dropout\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.initializers import glorot_uniform\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.utils import  to_categorical\n",
    "from  IPython import display\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-23T01:29:24.413572Z",
     "start_time": "2018-10-23T01:29:24.308996Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 3462 images belonging to 5 classes.\n",
      "Found 861 images belonging to 5 classes.\n"
     ]
    }
   ],
   "source": [
    "img_gen_train = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1./255,validation_split=0.2)\n",
    "\n",
    "train_data = img_gen_train.flow_from_directory('../data/flowers',\n",
    "                          batch_size = 8,\n",
    "                                  seed =42,\n",
    "                                  target_size = (128,128),\n",
    "                        subset = 'training' \n",
    "                                 )\n",
    "val_data = img_gen_train.flow_from_directory('../data/flowers',\n",
    "                          batch_size = 8,\n",
    "                                  seed =42,\n",
    "                                  target_size = (128,128),\n",
    "                                subset = 'validation' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-10-23T01:29:46.946694Z",
     "start_time": "2018-10-23T01:29:46.869140Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_simple_cov():    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', input_shape=(128, 128, 3)))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    \n",
    "    model.add(Conv2D(8, (2, 2), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(100, activation='relu'))\n",
    "    model.add(Dense(5, activation='softmax'))\n",
    "    return model\n",
    "\n",
    "model = create_simple_cov()\n",
    "    # Compile model\n",
    "model.compile(optimizer = 'adam',\n",
    "             loss ='categorical_crossentropy',metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-10-23T01:29:58.209Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "433/433 - 92s - loss: 1.2453 - accuracy: 0.4899 - val_loss: 1.0993 - val_accuracy: 0.5470\n",
      "Epoch 2/3\n",
      "433/433 - 36s - loss: 0.9800 - accuracy: 0.6251 - val_loss: 1.1187 - val_accuracy: 0.5470\n",
      "Epoch 3/3\n",
      "433/433 - 38s - loss: 0.7967 - accuracy: 0.6938 - val_loss: 1.1709 - val_accuracy: 0.5343\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1f36006fd48>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_data, validation_data=val_data, epochs=3, batch_size=16, verbose=2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### انتخاب نرخ یادگیری"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_schedule  = tf.keras.optimizers.schedules.InverseTimeDecay(\n",
    "    initial_learning_rate=0.001,\n",
    "    decay_steps=1000,\n",
    "    decay_rate = 1,\n",
    "    staircase= False\n",
    "\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1f35d7865c8>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3yV5f3/8dcnJyQQhLACBgIkbCMywxChuFCccaBCRVGxaN2jA3/9tv1+ra21raJVcIKrKFBUTHGggBMVCIjIJuwww94h4/r9cW7akIbkQBLunJP38/HgwTnXue7rfC5uHnnn3uacQ0RE5KgovwsQEZGqRcEgIiLHUDCIiMgxFAwiInIMBYOIiBwj2u8CKkKjRo1ccnKy32WIiISVefPmbXfOJRRvj4hgSE5OJjMz0+8yRETCipmtK6ldu5JEROQYCgYRETmGgkFERI6hYBARkWMoGERE5BghBYOZDTSz5WaWZWYjS/g81swmep/PNrPkIp894rUvN7OLi7SPM7NtZrao2FgNzOxTM1vp/V3/5KcnIiInqsxgMLMAMBq4BEgFhphZarFuw4Fdzrk2wCjgCW/ZVGAwcCYwEBjjjQfwmtdW3EhghnOuLTDDey8iIqdIKFsMPYEs59xq59wRYAKQXqxPOvC693oycIGZmdc+wTmX65xbA2R54+Gc+xLYWcL3FR3rdeCqE5jPCZmxdCuT5m6orOFFRMJSKMHQDCj60zPbayuxj3MuH9gDNAxx2eKaOOc2e2NtBhqX1MnMRphZppll5uTkhDCNYznneGv2en4z5Ufmrdt1wsuLiESqUILBSmgr/nSf4/UJZdmT4px7yTmX5pxLS0j4ryu6y2RmPHV9FxLja3HX+Hls23e4IsoSEQl7oQRDNtC8yPskYNPx+phZNBBPcDdRKMsWt9XMEr2xEoFtIdR4UuLjavDiTd3ZcyiPe8Z/T15BYWV9lYhI2AglGOYCbc0sxcxiCB5MzijWJwMY5r0eBMx0wWeGZgCDvbOWUoC2wJwyvq/oWMOA90Oo8aSdkViXJ67txJy1O3n8w2WV+VUiImGhzGDwjhncA0wDlgKTnHOLzexRM7vS6zYWaGhmWcBDeGcSOecWA5OAJcDHwN3OuQIAM3sb+BZob2bZZjbcG+vPwAAzWwkM8N5XqvQuzbj1nGTGzVrD+ws2VvbXiYhUaRb8xT68paWlufLeXTWvoJAbX57Nwo27ee+uczgjsW4FVSciUjWZ2TznXFrxdl357KkRiOK5G7tSt2YN7vzHPPYczPO7JBERXygYimhcpybPD+3Gpt2HeGDi9xQWhv/WlIjIiVIwFNO9ZQN+d3kqny3P4enpK/wuR0TklFMwlGBo75Zc1z2Jv8/M4oOFm/0uR0TklFIwlMDMeOzqjnRvWZ+H/7mARRv3+F2SiMgpo2A4jtjoAC8M7U79uBhGvJFJzr5cv0sSETklFAylSKgTy8s3p7Hz4BHu/Mc8cvML/C5JRKTSKRjK0LFZPE9e14V563bx2ymLiITrPkRESqNgCMFlnRK57/w2TMrM5tVZa/0uR0SkUikYQvTAhe24+MwmPPbBEr5cceK3+RYRCRcKhhBFRQVv092uSR3ueWs+q3L2+12SiEilUDCcgNqx0bx8cxox0VHc+upcduzXmUoiEnkUDCeoeYM4Xro5ja17DzPizXkcztOZSiISWRQMJ6Fbi/qMuiF4ptIvJy/UPZVEJKIoGE7SpWcl8uuBHfjXD5sYpXsqiUgEifa7gHB2Z/9WrNtxgGdnZtGiQRzXpTUveyERkSpOwVAOZsYfrupI9q5D/L/3fqRZ/Vr0ad3I77JERMpFu5LKqUYgitE3diO5YW3ufHMeWdt0GquIhDcFQwWIr1WDcbf0CJ7G+toctus0VhEJYwqGCtK8QRwv35xGzr5cbnttLgdy8/0uSUTkpCgYKlDXFvV5bkg3Fm3cw8/HzyevoNDvkkRETpiCoYJdmNqEP119Fl+uyOHX7yzU3VhFJOzorKRKMLhnC7buzWXU9BWcXrcmvxrYwe+SRERCpmCoJPdd0IYtew8z5vNVNKlbk2F9kv0uSUQkJAqGSmJm/CH9TLbvz+V//7WYhDqxXHpWot9liYiUSccYKlF0IIpnh3SlW4v6PDBxAbNX7/C7JBGRMikYKlnNGgHGDkujef1a3P5GJks37/W7JBGRUikYToF6cTG8MbwXp8VGc9PYOazZfsDvkkREjkvBcIo0q1eLN4f3otA5hr4ym817DvldkohIiRQMp1Cbxqfxxm092Xsoj6GvzNYT4ESkSlIwnGIdm8XzyrA0sncd4pZX57LvcJ7fJYmIHCOkYDCzgWa23MyyzGxkCZ/HmtlE7/PZZpZc5LNHvPblZnZxWWOa2flmNt/MFpnZ62YWcafU9mrVkBeGdmfp5r0Mfz1TjwcVkSqlzGAwswAwGrgESAWGmFlqsW7DgV3OuTbAKOAJb9lUYDBwJjAQGGNmgeONaWZRwOvAYOdcR2AdMKz806x6zuvQmFE3dGHu2p3cpfsqiUgVEsoWQ08gyzm32jl3BJgApBfrk07wBzrAZOACMzOvfYJzLtc5twbI8sY73pgNgVzn3NFnZX4KXHvy06varujclD9edRYzl23joUk/UKBnR4tIFRBKMDQDNhR5n+21ldjHOZcP7CH4Q/54yx6vfTtQw8zSvPZBQInPyzSzEWaWaWaZOTk5IUyjavpprxaMvCT47OjfvPcjhQoHEfFZKPvvrYS24j+9jtfneO0lBZJzzjkzGwyMMrNY4BOgxAcbOOdeAl4CSEtLC+ufpnf2b82B3HyenZlFjUAUj6afSXCDS0Tk1AslGLI59rf2JGDTcfpkeweL44GdZSxbYrtz7lugH4CZXQS0C2Ui4e6hAe04UlDIi1+sJjpg/O7yVIWDiPgilF1Jc4G2ZpZiZjEEDyZnFOuTwX8OEg8CZrrggwgygMHeWUspQFtgTmljmllj7+9Y4NfAC+WZYLgwM0YO7MBt56Tw6qy1/PmjZXqWg4j4oswtBudcvpndA0wDAsA459xiM3sUyHTOZQBjgTfNLIvglsJgb9nFZjYJWEJwl9DdzrkCgJLG9L7yl2Z2OcHQet45N7MC51ulmRm/vfwM8goKefHL1cRER/HwRe39LktEqhmLhN9K09LSXGZmpt9lVJjCQsf/e+9HJszdwEMD2nHfBW39LklEIpCZzXPOpRVvj7iLxyJBVJTxp6vPIq/A8dSnK4gOGHed28bvskSkmlAwVFFRUcZfBnUiv7CQv3y8nJhAFLf3a+V3WSJSDSgYqrBAlPHkdZ3JL3A89sFSCgodd/Rv7XdZIhLhFAxVXHQgiqcHd8EMHv9oGfmFjrvP024lEak8CoYwUCMQxdM3dCE6yvjrtOXkFzjuv1AHpEWkcigYwkR0IIonr+9CdCCKUdNXkF9YyEMD2ukiOBGpcAqGMBKIMv5ybSeio4xnZ2aRV+D49cD2CgcRqVAKhjBz9FTW6IDxwheryC8o5DeXnaFwEJEKo2AIQ1FRxh/SOxIdFcUrX68hv9Dx+yt0byURqRgKhjBlZvz+ilQCUcbYr9eQm1/IY1d1JBClcBCR8lEwhDEz438uO4PY6CjGfL6K/bn5PHV9Z2oE9ChvETl5CoYwZ2b8amAH6tSswRMfL+NAbj5jbuxGzRoBv0sTkTClXy0jxM/Pbc1jV3Xks+XbGDZuDvsO5/ldkoiEKQVDBBnauyVP39CFzHW7GPrKbHYdOOJ3SSIShhQMESa9SzNeHNqdpVv2cf2L37J172G/SxKRMKNgiEAXpjbhtVt7sGn3Ia574Vs27Dzod0kiEkYUDBGqT+tGjP9Zb/YcymPQC9+wbMtev0sSkTChYIhgXZrXY9IdZwNw3QvfMnv1Dp8rEpFwoGCIcO1Pr8M7P+9D4zqx3DRuDh/9uNnvkkSkilMwVANJ9eOYfGcfOjaty11vzefN79b5XZKIVGEKhmqifu0Yxt/em/PbN+a3Uxbx5CfLcc75XZaIVEEKhmqkVkyAF2/qzg1pzXl2ZhYj3/mR/IJCv8sSkSpGt8SoZqIDUfz52rNoXDeWZ2dmseNALs8O6UatGN1CQ0SCtMVQDZkZD1/Unj+kn8mMZdu48ZXv2LE/1++yRKSKUDBUYzedncyYn3Zj8aa9XPP8N6zK2e93SSJSBSgYqrlLzkrk7RG92X84n2vGfKNrHUREwSDQrUV93rvrHBqeFsNNY+cw5fuNfpckIj5SMAgALRrG8e7P+9C1RT0emLiAv89YqdNZRaopBYP8W724GN4c3otrujbjqU9X8It/LuRIvk5nFaludLqqHCMmOoonr+9Mi4ZxPD19JZt2H+KFod2Jj6vhd2kicoqEtMVgZgPNbLmZZZnZyBI+jzWzid7ns80suchnj3jty83s4rLGNLMLzGy+mS0ws6/NrE35pignysx44MJ2PHV9ZzLX7eSa52exZvsBv8sSkVOkzGAwswAwGrgESAWGmFlqsW7DgV3OuTbAKOAJb9lUYDBwJjAQGGNmgTLGfB640TnXBXgL+J/yTVFO1jXdknhzeC92HjjCVaNnMStru98licgpEMoWQ08gyzm32jl3BJgApBfrkw687r2eDFxgZua1T3DO5Trn1gBZ3niljemAut7reGDTyU1NKkLvVg15/+6+NKkby83j5vDGt2t1UFokwoUSDM2ADUXeZ3ttJfZxzuUDe4CGpSxb2pi3Ax+aWTZwE/DnkooysxFmlmlmmTk5OSFMQ05Wi4ZxvPPzPpzbLoHfvb+Y/5myiDzdY0kkYoUSDFZCW/FfGY/X50TbAR4ELnXOJQGvAk+VVJRz7iXnXJpzLi0hIaHEwqXi1KlZg5duTuPO/q0ZP3s9N42dza4DR/wuS0QqQSjBkA00L/I+if/evfPvPmYWTXAX0M5Sli2x3cwSgM7Oudle+0SgT0gzkUoXiDJGXtKBUTd0Zv763aSPnsWKrfv8LktEKlgowTAXaGtmKWYWQ/BgckaxPhnAMO/1IGCmC+6IzgAGe2ctpQBtgTmljLkLiDezdt5YA4ClJz89qQxXd01i4ojeHMor4Jox3zBj6Va/SxKRClRmMHjHDO4BphH8IT3JObfYzB41syu9bmOBhmaWBTwEjPSWXQxMApYAHwN3O+cKjjem1/4z4B0z+4HgMYZfVtx0paJ0bVGfjHvOIblRHLe/kcmzM1ZSWKiD0iKRwCLhDJO0tDSXmZnpdxnV0qEjBTzy7kKmLNjEgNQmPHV9Z+rU1MVwIuHAzOY559KKt+uWGFIutWICjLqhC7+7PJWZy7aRPnoWWdt03EEknCkYpNzMjNv6pjD+9l7sPZRH+nOz+HjRFr/LEpGTpGCQCtO7VUP+dW9f2japw53/mMdfpy2jQMcdRMKOgkEqVGJ8LSbe0ZshPZsz+rNV3PraXHYf1PUOIuFEwSAVLjY6wOPXdOLxa87iu1U7uOK5r1m0cY/fZYlIiBQMUmmG9GzBxDt6k1/guOb5bxg/e53usyQSBhQMUqm6tqjPB/f1o3erhvzmvUU8OHEBB3Lz/S5LREqhYJBK16B2DK/d0oOHB7Qj44dNupWGSBWnYJBTIirKuPeCtvxjeC92Hwye0vru/Gy/yxKREigY5JTq06YRH97Xl05J8Tw06QdGvrOQw3kFfpclIkUoGOSUa1y3JuNv78Xd57VmwtwNXD3mGz06VKQKUTCIL6IDUfzy4g68emsPNu85xOV//4op32/0uywRQcEgPjuvfWM+vK8fZzaN54GJC3ho0gL266wlEV8pGMR3TevV4q2f9eKBC9sy5fuNXP73r1iYvdvvskSqLQWDVAnRgSgeuLAdE0aczZH8Qq59/hte/nK1nvEg4gMFg1QpPVMa8OH9/Ti/Q2P++OFSbnltLjn7cv0uS6RaUTBIlVMvLoYXhnbnsas6Mnv1Di555ku+WJHjd1ki1YaCQaokM2No75Zk3NOXBrVjGDZuDo9NXaJrHkROAQWDVGntT69Dxj19Gdq7Ba98vYarRs9i6ea9fpclEtEUDFLl1awR4LGrzuLVW3qwff8R0p+bxYtfrNJDgEQqiYJBwsZ5HRrzyYM/4bwOCTz+0TKGvPwdG3Ye9LsskYijYJCw0qB28MD0367rzJJNe7nkma94Z162nvMgUoEUDBJ2zIxB3ZP46P5+pCbW5eF//sBd4+ez84AeISpSERQMEraaN4jj7RG9eeSSDkxfupWLn/6Sz5Zt87sskbCnYJCwFogy7ujfmvfv7kuDuBhufW0uv5r8A3sO5fldmkjYUjBIREhtWpeMe8/hrnNbM3leNheP+pLPlmvrQeRkKBgkYsRGB/jVwA68d9c51KkZza2vautB5GQoGCTidG5ej6n39eXu81rzzvyN2noQOUEKBolIsdEBfnlxB967qw91awW3Hn75T209iIRCwSARrVNSPf51b3Dr4d3vva0HnbkkUioFg0S8/9p6eG0uD0z4nh37dTtvkZKEFAxmNtDMlptZlpmNLOHzWDOb6H0+28ySi3z2iNe+3MwuLmtMM/vKzBZ4fzaZ2ZTyTVEk6OjWw/0XtOWDHzdz4VNf6KppkRKUGQxmFgBGA5cAqcAQM0st1m04sMs51wYYBTzhLZsKDAbOBAYCY8wsUNqYzrl+zrkuzrkuwLfAu+WfpkhQbHSABwe044P7+pHSqDYP//MHbh43h/U7dM8lkaNC2WLoCWQ551Y7544AE4D0Yn3Sgde915OBC8zMvPYJzrlc59waIMsbr8wxzawOcD6gLQapcO2a1GHynX34Q/qZfL9+Nxc9/QUvfbmK/IJCv0sT8V0owdAM2FDkfbbXVmIf51w+sAdoWMqyoYx5NTDDOVfizffNbISZZZpZZk6Onu4lJy4qyrjp7GQ+fegn9G2TwJ8+XMZVY2axaOMev0sT8VUowWAltBXfKXu8PifaXtQQ4O3jFeWce8k5l+acS0tISDheN5EyJcbX4uWbuzPmxm5s3ZtL+uhZ/OnDpRw6oqfFSfUUSjBkA82LvE8CNh2vj5lFA/HAzlKWLXVMM2tIcHfTB6FMQqS8zIxLz0pk+oP9ua57Ei99uZoLn/qC6Uu2+l2ayCkXSjDMBdqaWYqZxRA8mJxRrE8GMMx7PQiY6YKnemQAg72zllKAtsCcEMa8DpjqnDt8shMTORnxcTX487WdmDiiN3ExAW5/I5PbX88ke5cOTkv1UWYweMcM7gGmAUuBSc65xWb2qJld6XUbCzQ0syzgIWCkt+xiYBKwBPgYuNs5V3C8MYt87WBK2Y0kUtl6tWrIh/f3Y+QlHZiVtZ0Ln/qCMZ9ncSRfB6cl8lkknMOdlpbmMjMz/S5DItTG3Yd49F+LmbZ4K20an8Yf0jtyduuGfpclUm5mNs85l1a8XVc+i5ShWb1avHhTGuNuSSM3v4AhL3/HgxMXkLNPV05LZFIwiITo/A5N+OSB/tx7fhumLtzE+U9+zpvfrqWgMPy3ukWKUjCInIBaMQEevqg9Hz/wEzolxfPb9xdzxbNfM2fNTr9LE6kwCgaRk9A64TT+MbwXz/20K7sPHuH6F7/l3re/Z9PuQ36XJlJuCgaRk2RmXN6pKTMePpf7LmjLJ4u3cMGTX/DczJUcztPFcRK+FAwi5VQrJsBDA9ox/aH+nNs+gb99soIBo75g2uItunOrhCUFg0gFad4gjueHduet23tRq0aAO96cx01j57By6z6/SxM5IQoGkQrWp00jPryvH/97RSoLs3cz8Jmv+L9/LWbPQT1WVMKDgkGkEkQHorjlnBQ++8W5XJ/WnNe+WUv/v33GuK/X6OppqfIUDCKVqOFpsTx+zVl8cG8/OjaN59GpS7ho1Bd8vEjHH6TqUjCInAKpTevy5vCevHpLD6IDUdz5j3nc8NJ3LMze7XdpIv9FwSByipgZ53VozMf39+Oxqzqyatt+rnxuFg9M+J6Nuv5BqhDdRE/EJ/sO5zHm81WM/XoNBtzeL4Wfn9uG02Kj/S5NqgndRE+kiqlTswa/HtiBmQ/3Z2DH0xn92SrO/etnvPndOvL07GnxkYJBxGdJ9eN4ZnBXptx9Dq0ancZvpyxiwFNf8K8fNlGoG/SJDxQMIlVEl+b1mHhHb8YOSyM2OsC9b3/PlaO/5quVOX6XJtWMgkGkCjEzLjijCR/e34+nru/MrgN53DR2Dje+ojOY5NRRMIhUQYEo45puScz8RX9+d3kqSzfv48rnZnH3+Pmsztnvd3kS4XRWkkgY2Hc4j5e/WsMrX60mN7+Q69Oa88CFbWlSt6bfpUkYO95ZSQoGkTCSsy+X52au5K056wlEGTefncwdP2lFw9Ni/S5NwpCCQSSCrN9xkKenr2DKgo3UqhHglnOS+Vm/VtSLi/G7NAkjCgaRCJS1bR9PT1/J1IWbqRMbzfB+KdzWN4W6NWv4XZqEAQWDSARbtmUvoz5dwbTFW4mvVYMRP2nFLX2Sqa2rqKUUCgaRamDRxj2M+nQFM5Zto0HtGO7s34qbeidTKybgd2lSBSkYRKqR79fv4qlPV/DVyu00Oi2Wu85tzU97taBmDQWE/IeCQaQamrt2J09+spzvVu8koU4sI/q14sbeLYiL0S4mUTCIVGvfrd7BszNXMitrBw1qxzC8bwo3n92SOjpIXa0pGESEeet28ezMlXy+PIf4WjW49Zxkbu2TQnycAqI6UjCIyL8tzN7NszOz+HTJVurERjOsTzK39U2hQW1dB1GdKBhE5L8s2bSX0Z9l8eGizdSqEeCm3i25vV8rEuroSurqoFwP6jGzgWa23MyyzGxkCZ/HmtlE7/PZZpZc5LNHvPblZnZxWWNa0B/NbIWZLTWz+050siISmtSmdRl9Yzc+eeAnDEhtwstfrabvEzP5/fuL2LDzoN/liU/K3GIwswCwAhgAZANzgSHOuSVF+twFdHLO3Wlmg4GrnXM3mFkq8DbQE2gKTAfaeYuVOKaZ3QqcB9zinCs0s8bOuW2l1agtBpGKsWb7AcZ8lsWUBRspdHBl56bc0b8VHU6v63dpUgnKs8XQE8hyzq12zh0BJgDpxfqkA697rycDF5iZee0TnHO5zrk1QJY3Xmlj/hx41DlXCFBWKIhIxUlpVJu/XteZL391Hrf2SWba4i0MfPorbnttLnPW7CQSdj1L2UIJhmbAhiLvs722Evs45/KBPUDDUpYtbczWwA1mlmlmH5lZ25KKMrMRXp/MnBw94UqkIiXG1+J/Lk/lm5Hn8/CAdizYsJvrX/yWQS98y6dLtuqRoxEulGCwEtqK/684Xp8TbQeIBQ57mzcvA+NKKso595JzLs05l5aQkFBi4SJSPvXiYrj3grbM+vX5PJp+Jlv3HuZnb2Ry8dNfMnleNnkFhX6XKJUglGDIBpoXeZ8EbDpeHzOLBuKBnaUsW9qY2cA73uv3gE4h1CgilahWTICbz07m81+cyzODuxCIMn7xzx/o/5fPGPv1Gvbn5vtdolSgUIJhLtDWzFLMLAYYDGQU65MBDPNeDwJmuuDOyAxgsHfWUgrQFphTxphTgPO91/0JHqQWkSogOhBFepdmfHR/P169tQdJDeL4w9QlnP34DB7/cCmbdh/yu0SpAGXeMMU5l29m9wDTgAAwzjm32MweBTKdcxnAWOBNM8siuKUw2Ft2sZlNApYA+cDdzrkCgJLG9L7yz8B4M3sQ2A/cXnHTFZGKYGac174x57VvzIINu3nlq9W88vUaXvl6DZedlcjt/VLolFTP7zLlJOkCNxGpENm7DvL6N2uZMGcD+3Lz6ZncgOH9UrjwjCYEoko6rCh+05XPInJK7Ducx6TMbMZ9vYaNuw+R3DCO2/qmMKh7ku7qWsUoGETklMovKGTa4q28/NVqFmzYTXytGvy0VwuGnZ3M6fE1/S5PUDCIiI/mrdvF2K9X8/GiLUSZMbDj6dzSJ5nuLesTvBZW/HC8YNB2nYhUuu4t69O9ZXc27Aweh5iYuYGpCzdzZtO6DOuTzJWdm+rpclWIthhE5JQ7eCSf977fyOvfrGXF1v3Uj6vB4J4tGNq7Jc3q1fK7vGpDu5JEpMpxzvHt6h28/s1aPl2yFYCLUk9nWJ9kerdqoN1MlUy7kkSkyjEz+rRuRJ/WjcjedZB/fLeeCXPX8/HiLbRvUodhfZK5qmtTnc10immLQUSqlMN5BWQs2MRr36xlyea91K0ZzbXdk7ixVwvaNK7jd3kRRbuSRCSsOOfIXLeL179Zy7TFW8grcPRu1YAbe7Xk4jNPJyY6pOeMSSm0K0lEwoqZ0SO5AT2SG7B9fy6TMjfw1uz13Pv29zQ6LYbr0prz054taN4gzu9SI462GEQkbBQWOr5cmcP42euZsXQrDvhJ2wSG9m7Jee0TiA5oK+JEaFeSiESUTbsPMXHuBibMXc/WvbkkxtdkcI8W3NCjua6sDpGCQUQiUn5BIdOXbmP87HV8tXI7gSjj/A6NGdyjOf3baSuiNDrGICIRKToQxcCOpzOw4+ms23GAt+as55152Xy6ZCtN6sYyqHsS16c1p2XD2n6XGja0xSAiESevoJAZS7cxKXMDny/fRqGD3q0aMLhHCwZ2PF233/BoV5KIVEtb9hxm8rwNTMrMZv3Og9StGc1VXZtxfVpzOjaL97s8XykYRKRaKyx0fLdmBxPnbuCjRVs4kl/ImU3rMrhHc67s0oz4WjX8LvGUUzCIiHj2HMzj/R82MmHOBpZs3ktsdPA4xbXdkjinTaNq88Q5BYOISAkWbdzDhLnryViwib2H8zm9bk2u6tqMQd2bRfwtOBQMIiKlOJxXwIyl23hnfjZfrMihoNDROSmea7sncUWnptSvHeN3iRVOwSAiEqKcfbm8v2Aj78zfyNLNe6kRCF4bcW23JM7r0JgaEXJthIJBROQkLNm0l3fmZ/P+go1s33+EBrVjuLJzUwZ1T+LMpnXD+pkRCgYRkXLIKyjkyxU5vDM/m+lLtnGkoJD2TeqQ3rUpV3ZuSlL98LuZn4JBRKSC7D54hKkLN/Pu/Gzmr98NQI/k+qR3acalZyXSIEyORygYREQqwfodB8n4YSNTFmwia9t+oqOM/u0SuLJLUwakNqnST59TMIiIVCLnHEs37+P9BRvJ+OBTP+8AAAe4SURBVGETm/ccJi4mwEWpTUjv2oy+bRpVuYPWCgYRkVOksNAxZ+1O3l+wiQ9/3MyeQ3k0rB3DZZ0SSe/SlG4t6leJg9YKBhERH+TmF/Dliu1MWbCR6Uu2kptfSFL9WlzWKZErOjX19cwmBYOIiM/25+YzbdEWMn7YxKys7eQXOlo2jOOysxK5rFMiqYmnNiQUDCIiVciuA0f4ZMkWpi7czDerdlBQ6GjVqDaXdQqGRPsmdSo9JBQMIiJV1M4DR/h40RY++HET367aQaGD1gm1uaxTUy7vlEi7JpVzz6ZyBYOZDQSeAQLAK865Pxf7PBZ4A+gO7ABucM6t9T57BBgOFAD3OeemlTammb0G9Af2eMPf4pxbUFp9CgYRiRTb9+fy8aItTF24idlrduIctG18Gpd1SuTyTokVemO/kw4GMwsAK4ABQDYwFxjinFtSpM9dQCfn3J1mNhi42jl3g5mlAm8DPYGmwHSgnbdYiWN6wTDVOTc51MkpGEQkEm3bd9gLic3MXRsMiXZNTuOSjolcelYi7ZqcVq7dTeV55nNPIMs5t9obaAKQDiwp0icd+F/v9WTgOQtWmw5McM7lAmvMLMsbjxDGFBGp1hrXqcnNZydz89nJbN17mA9/3MxHi7bw95kreWbGSlol1OaFod0rfFdTKMHQDNhQ5H020Ot4fZxz+Wa2B2jotX9XbNlm3uvSxvyjmf0OmAGM9ILlGGY2AhgB0KJFixCmISISvprUrcmt56Rw6zkpbNt3mGmLtzJ9yVaa1atV4d8VymV4JW2nFN//dLw+J9oO8AjQAegBNAB+XVJRzrmXnHNpzrm0hISEkrqIiESkxnVqclPvlrx+W09qx1b8LTdCCYZsoHmR90nApuP1MbNoIB7YWcqyxx3TObfZBeUCr/KfXU8iInIKhBIMc4G2ZpZiZjHAYCCjWJ8MYJj3ehAw0wWPamcAg80s1sxSgLbAnNLGNLNE728DrgIWlWeCIiJyYsrcBvGOGdwDTCN4auk459xiM3sUyHTOZQBjgTe9g8s7Cf6gx+s3ieBB5XzgbudcAUBJY3pfOd7MEgjubloA3Flx0xURkbLoAjcRkWrqeKerVq17wIqIiO8UDCIicgwFg4iIHEPBICIix4iIg89mlgOsO8nFGwHbK7CccKA5Vw+ac/VQnjm3dM791xXCEREM5WFmmSUdlY9kmnP1oDlXD5UxZ+1KEhGRYygYRETkGAoGeMnvAnygOVcPmnP1UOFzrvbHGERE5FjaYhARkWMoGERE5BjVOhjMbKCZLTezLDMb6Xc9FcHMmpvZZ2a21MwWm9n9XnsDM/vUzFZ6f9f32s3M/u79Gyw0s27+zuDkmVnAzL43s6ne+xQzm+3NeaJ3i3e828BP9OY828yS/az7ZJlZPTObbGbLvPV9dqSvZzN70Pt/vcjM3jazmpG2ns1snJltM7NFRdpOeL2a2TCv/0ozG1bSdx1PtQ0GMwsAo4FLgFRgiJml+ltVhcgHHnbOnQH0Bu725jUSmOGca4v3yFSv/yUEn5PRluCjUp8/9SVXmPuBpUXePwGM8ua8CxjutQ8Hdjnn2gCjvH7h6BngY+dcB6AzwblH7Ho2s2bAfUCac64jwVv2Dyby1vNrwMBibSe0Xs2sAfB7go9M7gn8/miYhMQ5Vy3/AGcD04q8fwR4xO+6KmGe7wMDgOVAoteWCCz3Xr8IDCnS/9/9wukPwacAzgDOB6YSfJ7HdiC6+Pom+ByQs73X0V4/83sOJzjfusCa4nVH8nrmP8+Wb+Ctt6nAxZG4noFkYNHJrldgCPBikfZj+pX1p9puMfCf/2RHZXttEcPbdO4KzAaaOOc2Q/DxqUBjr1uk/Ds8DfwKKPTeNwR2O+fyvfdF5/XvOXuf7/H6h5NWQA7wqrf77BUzq00Er2fn3Ebgb8B6YDPB9TaPyF7PR53oei3X+q7OwWAltEXMubtmdhrwDvCAc25vaV1LaAurfwczuxzY5pybV7S5hK4uhM/CRTTQDXjeOdcVOMB/di+UJOzn7O0KSQdSgKZAbYK7UoqLpPVcluPNsVxzr87BkA00L/I+CdjkUy0VysxqEAyF8c65d73mrUWep50IbPPaI+Hf4RzgSjNbC0wguDvpaaCemR19fG3Ref17zt7n8QQfSRtOsoFs59xs7/1kgkERyev5QmCNcy7HOZcHvAv0IbLX81Enul7Ltb6rczDMBdp6ZzTEEDyIleFzTeVmZkbwGdxLnXNPFfkoAzh6ZsIwgscejrbf7J3d0BvYc3STNVw45x5xziU555IJrseZzrkbgc+AQV634nM++m8xyOsfVr9JOue2ABvMrL3XdAHBZ6tH7HomuAupt5nFef/Pj845YtdzESe6XqcBF5lZfW9L6yKvLTR+H2Tx+QDPpcAKYBXwG7/rqaA59SW4ybgQWOD9uZTgvtUZwErv7wZefyN4dtYq4EeCZ3z4Po9yzP9cYKr3uhUwB8gC/gnEeu01vfdZ3uet/K77JOfaBcj01vUUoH6kr2fg/4BlwCLgTSA20tYz8DbBYyh5BH/zH34y6xW4zZt7FnDridSgW2KIiMgxqvOuJBERKYGCQUREjqFgEBGRYygYRETkGAoGERE5hoJBRESOoWAQEZFj/H/zqs3Iuh3+NgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(0,1000),lr_schedule(range(0,1000)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "model = create_simple_cov()\n",
    "# Compile model\n",
    "model.compile(optimizer = tf.keras.optimizers.Adam(lr_schedule),\n",
    "             loss ='categorical_crossentropy',\n",
    "              metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "  2/433 [..............................] - ETA: 1:39 - loss: 1.5407 - accuracy: 0.3750WARNING:tensorflow:Method (on_train_batch_end) is slow compared to the batch update (0.193924). Check your callbacks.\n",
      "433/433 [==============================] - 39s 90ms/step - loss: 1.2678 - accuracy: 0.4682 - val_loss: 1.2441 - val_accuracy: 0.4739\n",
      "Epoch 2/10\n",
      "433/433 [==============================] - 40s 92ms/step - loss: 0.9914 - accuracy: 0.6023 - val_loss: 1.2252 - val_accuracy: 0.5192\n",
      "Epoch 3/10\n",
      "433/433 [==============================] - 37s 85ms/step - loss: 0.8140 - accuracy: 0.6837 - val_loss: 1.1184 - val_accuracy: 0.5587\n",
      "Epoch 4/10\n",
      "433/433 [==============================] - 38s 88ms/step - loss: 0.6711 - accuracy: 0.7565 - val_loss: 1.0917 - val_accuracy: 0.5738\n",
      "Epoch 5/10\n",
      "433/433 [==============================] - 39s 90ms/step - loss: 0.5240 - accuracy: 0.8140 - val_loss: 1.1440 - val_accuracy: 0.5738\n",
      "Epoch 6/10\n",
      "433/433 [==============================] - 39s 90ms/step - loss: 0.3985 - accuracy: 0.8689 - val_loss: 1.2694 - val_accuracy: 0.5842\n",
      "Epoch 7/10\n",
      "433/433 [==============================] - 39s 90ms/step - loss: 0.3105 - accuracy: 0.9053 - val_loss: 1.3848 - val_accuracy: 0.5598\n",
      "Epoch 8/10\n",
      "433/433 [==============================] - 39s 89ms/step - loss: 0.2386 - accuracy: 0.9350 - val_loss: 1.4552 - val_accuracy: 0.5738\n",
      "Epoch 9/10\n",
      "433/433 [==============================] - 37s 86ms/step - loss: 0.1766 - accuracy: 0.9523 - val_loss: 1.6061 - val_accuracy: 0.5738\n",
      "Epoch 10/10\n",
      "433/433 [==============================] - 39s 90ms/step - loss: 0.1369 - accuracy: 0.9665 - val_loss: 1.7377 - val_accuracy: 0.5633\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1f35d861b08>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_data, validation_data=val_data, epochs=10, batch_size=16,\n",
    "          callbacks=[tf.keras.callbacks.TensorBoard('logs')],\n",
    "          verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ERROR: Timed out waiting for TensorBoard to start. It may still be running as pid 27556."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load the TensorBoard notebook extension\n",
    "%load_ext tensorboard\n",
    "\n",
    "# Open an embedded TensorBoard viewer\n",
    "%tensorboard --logdir logs/sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"800px\"\n",
       "            src=\"https://tensorboard.dev/experiment/vW7jmmF9TmKmy3rbheMQpw/#scalars&_smoothingWeight=0.97\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x1f35da9eb08>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display.IFrame(\n",
    "    src=\"https://tensorboard.dev/experiment/vW7jmmF9TmKmy3rbheMQpw/#scalars&_smoothingWeight=0.97\",\n",
    "    width=\"100%\", height=\"800px\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('models/custom_conv_mnist.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### تنظیم سازی"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### تنظیم سازی وزن نورون های شبکه عصبی\n",
    "\n",
    "##### اضافه کردن لایه حذف تصادفی"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_simple_cov():    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', input_shape=(128, 128, 3),kernel_regularizer = tf.keras.regularizers.l2(0.001)))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    model.add(Conv2D(8, (2, 2), activation='relu',kernel_regularizer = tf.keras.regularizers.l2(0.001)))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(100, activation='relu'))\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(5, activation='softmax'))\n",
    "    return model\n",
    "\n",
    "model = create_simple_cov()\n",
    "\n",
    "# Compile model\n",
    "model.compile(optimizer = tf.keras.optimizers.Adam(lr_schedule),\n",
    "             loss ='categorical_crossentropy',\n",
    "              metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "433/433 [==============================] - 37s 85ms/step - loss: 1.3755 - accuracy: 0.4209 - val_loss: 1.2805 - val_accuracy: 0.4762\n",
      "Epoch 2/10\n",
      "433/433 [==============================] - 27s 63ms/step - loss: 1.1090 - accuracy: 0.5633 - val_loss: 1.1238 - val_accuracy: 0.5563\n",
      "Epoch 3/10\n",
      "433/433 [==============================] - 30s 68ms/step - loss: 0.9453 - accuracy: 0.6453 - val_loss: 1.0657 - val_accuracy: 0.5854\n",
      "Epoch 4/10\n",
      "433/433 [==============================] - 26s 59ms/step - loss: 0.7540 - accuracy: 0.7302 - val_loss: 1.0127 - val_accuracy: 0.6051\n",
      "Epoch 5/10\n",
      "433/433 [==============================] - 25s 57ms/step - loss: 0.6010 - accuracy: 0.7955 - val_loss: 1.0502 - val_accuracy: 0.5993\n",
      "Epoch 6/10\n",
      "433/433 [==============================] - 24s 56ms/step - loss: 0.4775 - accuracy: 0.8339 - val_loss: 1.1714 - val_accuracy: 0.5923\n",
      "Epoch 7/10\n",
      "433/433 [==============================] - 22s 52ms/step - loss: 0.3683 - accuracy: 0.8876 - val_loss: 1.3719 - val_accuracy: 0.5726\n",
      "Epoch 8/10\n",
      "433/433 [==============================] - 28s 64ms/step - loss: 0.2998 - accuracy: 0.9041 - val_loss: 1.2192 - val_accuracy: 0.5970\n",
      "Epoch 9/10\n",
      "433/433 [==============================] - 28s 65ms/step - loss: 0.2351 - accuracy: 0.9344 - val_loss: 1.3490 - val_accuracy: 0.6074\n",
      "Epoch 10/10\n",
      "433/433 [==============================] - 25s 57ms/step - loss: 0.2042 - accuracy: 0.9486 - val_loss: 1.4281 - val_accuracy: 0.5889\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x28b8c43d608>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_data, validation_data=val_data, epochs=10, batch_size=16,\n",
    "          callbacks=[tf.keras.callbacks.TensorBoard('logs')],\n",
    "          verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### مقداردهی اولیه به وزن های شبکه\n",
    "\n",
    "به صورت پیشفرض کراس از روش مقداردهی  Glorot برای مقداردهی وزن های شبکه استفاده می کند. می توانیم این رفتار را با آرگومان `kernel_initializer` تغییر دهیم.\n",
    "\n",
    "\n",
    "به طور مثال می توانیم وزن های شبکه را با روش He تعیین کنیم.\n",
    "\n",
    "`kernel_initializer = 'he_normal'`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_simple_cov():    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(16, (5, 5), activation='relu', input_shape=(128, 128, 3),kernel_regularizer = tf.keras.regularizers.l2(0.001),kernel_initializer = 'he_normal'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    model.add(Conv2D(8, (2, 2), activation='relu',kernel_regularizer = tf.keras.regularizers.l2(0.001),kernel_initializer = 'he_normal'\n",
    "                    ))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(100, activation='relu'),kernel_initializer = 'he_normal')\n",
    "    model.add(Dropout(0.3))\n",
    "    model.add(Dense(5, activation='softmax'))\n",
    "    return model\n",
    "\n",
    "model = create_simple_cov()\n",
    "\n",
    "# Compile model\n",
    "model.compile(optimizer = tf.keras.optimizers.Adam(lr_schedule),\n",
    "             loss ='categorical_crossentropy',\n",
    "              metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "توابع فعال سازی ReLu فاقد مشکل نیستند. \n",
    "\n",
    "مثلا یک مشکل رایج توابع فعال سازی ReLu نورون های در حال مرگ است. یعنی برخی از نورون ها در طول فرآیند آموزش تغییری پیدا نمی کنند و خروجی آن ها به صوت ثابت برابر با 0 خواهد بود و دیگر گرادیان نزولی وزن های آن ها را نمی تواند به روز رسانی کند. برای حل این مشکل می توانیم از سایر توابع فعال سازی variant ReLU مثل Leaky ReLU استفاده کنیم.\n",
    "\n",
    "در حقیقت طبق یافته بعضی از محققین یادگیری عمیق، این  توابع فعال سازی بهتر از ReLU عمل می کنند.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import ReLU,ELU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####\n",
    "\n",
    "To recap: here are the most common ways to prevent overfitting in neural networks:\n",
    "\n",
    "Get more training data.\n",
    "Reduce the capacity of the network.\n",
    "Add weight regularization.\n",
    "Add dropout.\n",
    "Two important approaches not covered in this guide are:\n",
    "\n",
    "data-augmentation\n",
    "batch normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### شبکه Resnet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://i.imgur.com/XwcnU5x.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](https://i.imgur.com/ZyseLci.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def shortcut_block(X):\n",
    "    X = conv_bn_relu(X)\n",
    "    X = MaxPooling2D((2,2))(X)\n",
    "def conv_bn_relu(X):\n",
    "    X = Conv2D(filter_size,\n",
    "                    kernel_size = (3,3),\n",
    "                    strides = (1,1),\n",
    "                    padding = (1,1),\n",
    "                    kernel_initializer = \"he_normal\",\n",
    "              kernel_regularizer = l2(1.e-4))(X)\n",
    "    X = BatchNormalization(axis = 3)(X) #Channel axis\n",
    "    X = Activation('relu')(X)\n",
    "    return X\n",
    "def residual_block(X,repetition)    \n",
    "residual  = conv_bn_relu(X)\n",
    "def resnet_blcoks(X,filter_size,stage):\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_resnet_18(input_shape, num_outputs):\n",
    "    for block in range(num_blocks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"resnet50\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 224, 224, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)       (None, 230, 230, 3)  0           input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1_conv (Conv2D)             (None, 112, 112, 64) 9472        conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1_bn (BatchNormalization)   (None, 112, 112, 64) 256         conv1_conv[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1_relu (Activation)         (None, 112, 112, 64) 0           conv1_bn[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "pool1_pad (ZeroPadding2D)       (None, 114, 114, 64) 0           conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "pool1_pool (MaxPooling2D)       (None, 56, 56, 64)   0           pool1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_1_conv (Conv2D)    (None, 56, 56, 64)   4160        pool1_pool[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_1_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_1_relu (Activation (None, 56, 56, 64)   0           conv2_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_2_conv (Conv2D)    (None, 56, 56, 64)   36928       conv2_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_2_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_2_relu (Activation (None, 56, 56, 64)   0           conv2_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_0_conv (Conv2D)    (None, 56, 56, 256)  16640       pool1_pool[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_3_conv (Conv2D)    (None, 56, 56, 256)  16640       conv2_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_0_bn (BatchNormali (None, 56, 56, 256)  1024        conv2_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_3_bn (BatchNormali (None, 56, 56, 256)  1024        conv2_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_add (Add)          (None, 56, 56, 256)  0           conv2_block1_0_bn[0][0]          \n",
      "                                                                 conv2_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_out (Activation)   (None, 56, 56, 256)  0           conv2_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_1_conv (Conv2D)    (None, 56, 56, 64)   16448       conv2_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_1_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_1_relu (Activation (None, 56, 56, 64)   0           conv2_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_2_conv (Conv2D)    (None, 56, 56, 64)   36928       conv2_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_2_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_2_relu (Activation (None, 56, 56, 64)   0           conv2_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_3_conv (Conv2D)    (None, 56, 56, 256)  16640       conv2_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_3_bn (BatchNormali (None, 56, 56, 256)  1024        conv2_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_add (Add)          (None, 56, 56, 256)  0           conv2_block1_out[0][0]           \n",
      "                                                                 conv2_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_out (Activation)   (None, 56, 56, 256)  0           conv2_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_1_conv (Conv2D)    (None, 56, 56, 64)   16448       conv2_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_1_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_1_relu (Activation (None, 56, 56, 64)   0           conv2_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_2_conv (Conv2D)    (None, 56, 56, 64)   36928       conv2_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_2_bn (BatchNormali (None, 56, 56, 64)   256         conv2_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_2_relu (Activation (None, 56, 56, 64)   0           conv2_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_3_conv (Conv2D)    (None, 56, 56, 256)  16640       conv2_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_3_bn (BatchNormali (None, 56, 56, 256)  1024        conv2_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_add (Add)          (None, 56, 56, 256)  0           conv2_block2_out[0][0]           \n",
      "                                                                 conv2_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_out (Activation)   (None, 56, 56, 256)  0           conv2_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_1_conv (Conv2D)    (None, 28, 28, 128)  32896       conv2_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_1_relu (Activation (None, 28, 28, 128)  0           conv3_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_2_conv (Conv2D)    (None, 28, 28, 128)  147584      conv3_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_2_relu (Activation (None, 28, 28, 128)  0           conv3_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_0_conv (Conv2D)    (None, 28, 28, 512)  131584      conv2_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_0_bn (BatchNormali (None, 28, 28, 512)  2048        conv3_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_3_bn (BatchNormali (None, 28, 28, 512)  2048        conv3_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_add (Add)          (None, 28, 28, 512)  0           conv3_block1_0_bn[0][0]          \n",
      "                                                                 conv3_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_out (Activation)   (None, 28, 28, 512)  0           conv3_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_1_conv (Conv2D)    (None, 28, 28, 128)  65664       conv3_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_1_relu (Activation (None, 28, 28, 128)  0           conv3_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_2_conv (Conv2D)    (None, 28, 28, 128)  147584      conv3_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_2_relu (Activation (None, 28, 28, 128)  0           conv3_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_3_bn (BatchNormali (None, 28, 28, 512)  2048        conv3_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_add (Add)          (None, 28, 28, 512)  0           conv3_block1_out[0][0]           \n",
      "                                                                 conv3_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_out (Activation)   (None, 28, 28, 512)  0           conv3_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_1_conv (Conv2D)    (None, 28, 28, 128)  65664       conv3_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_1_relu (Activation (None, 28, 28, 128)  0           conv3_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_2_conv (Conv2D)    (None, 28, 28, 128)  147584      conv3_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_2_relu (Activation (None, 28, 28, 128)  0           conv3_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_3_bn (BatchNormali (None, 28, 28, 512)  2048        conv3_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_add (Add)          (None, 28, 28, 512)  0           conv3_block2_out[0][0]           \n",
      "                                                                 conv3_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_out (Activation)   (None, 28, 28, 512)  0           conv3_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_1_conv (Conv2D)    (None, 28, 28, 128)  65664       conv3_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_1_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block4_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_1_relu (Activation (None, 28, 28, 128)  0           conv3_block4_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_2_conv (Conv2D)    (None, 28, 28, 128)  147584      conv3_block4_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_2_bn (BatchNormali (None, 28, 28, 128)  512         conv3_block4_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_2_relu (Activation (None, 28, 28, 128)  0           conv3_block4_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_3_conv (Conv2D)    (None, 28, 28, 512)  66048       conv3_block4_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_3_bn (BatchNormali (None, 28, 28, 512)  2048        conv3_block4_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_add (Add)          (None, 28, 28, 512)  0           conv3_block3_out[0][0]           \n",
      "                                                                 conv3_block4_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_out (Activation)   (None, 28, 28, 512)  0           conv3_block4_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_1_conv (Conv2D)    (None, 14, 14, 256)  131328      conv3_block4_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_1_relu (Activation (None, 14, 14, 256)  0           conv4_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_2_conv (Conv2D)    (None, 14, 14, 256)  590080      conv4_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_2_relu (Activation (None, 14, 14, 256)  0           conv4_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_0_conv (Conv2D)    (None, 14, 14, 1024) 525312      conv3_block4_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_0_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_3_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_add (Add)          (None, 14, 14, 1024) 0           conv4_block1_0_bn[0][0]          \n",
      "                                                                 conv4_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_out (Activation)   (None, 14, 14, 1024) 0           conv4_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_1_conv (Conv2D)    (None, 14, 14, 256)  262400      conv4_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_1_relu (Activation (None, 14, 14, 256)  0           conv4_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_2_conv (Conv2D)    (None, 14, 14, 256)  590080      conv4_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_2_relu (Activation (None, 14, 14, 256)  0           conv4_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_3_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_add (Add)          (None, 14, 14, 1024) 0           conv4_block1_out[0][0]           \n",
      "                                                                 conv4_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_out (Activation)   (None, 14, 14, 1024) 0           conv4_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_1_conv (Conv2D)    (None, 14, 14, 256)  262400      conv4_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_1_relu (Activation (None, 14, 14, 256)  0           conv4_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_2_conv (Conv2D)    (None, 14, 14, 256)  590080      conv4_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_2_relu (Activation (None, 14, 14, 256)  0           conv4_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_3_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_add (Add)          (None, 14, 14, 1024) 0           conv4_block2_out[0][0]           \n",
      "                                                                 conv4_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_out (Activation)   (None, 14, 14, 1024) 0           conv4_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_1_conv (Conv2D)    (None, 14, 14, 256)  262400      conv4_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block4_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_1_relu (Activation (None, 14, 14, 256)  0           conv4_block4_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_2_conv (Conv2D)    (None, 14, 14, 256)  590080      conv4_block4_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block4_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_2_relu (Activation (None, 14, 14, 256)  0           conv4_block4_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block4_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_3_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block4_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_add (Add)          (None, 14, 14, 1024) 0           conv4_block3_out[0][0]           \n",
      "                                                                 conv4_block4_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_out (Activation)   (None, 14, 14, 1024) 0           conv4_block4_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_1_conv (Conv2D)    (None, 14, 14, 256)  262400      conv4_block4_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block5_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_1_relu (Activation (None, 14, 14, 256)  0           conv4_block5_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_2_conv (Conv2D)    (None, 14, 14, 256)  590080      conv4_block5_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block5_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_2_relu (Activation (None, 14, 14, 256)  0           conv4_block5_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block5_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_3_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block5_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_add (Add)          (None, 14, 14, 1024) 0           conv4_block4_out[0][0]           \n",
      "                                                                 conv4_block5_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_out (Activation)   (None, 14, 14, 1024) 0           conv4_block5_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_1_conv (Conv2D)    (None, 14, 14, 256)  262400      conv4_block5_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_1_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block6_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_1_relu (Activation (None, 14, 14, 256)  0           conv4_block6_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_2_conv (Conv2D)    (None, 14, 14, 256)  590080      conv4_block6_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_2_bn (BatchNormali (None, 14, 14, 256)  1024        conv4_block6_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_2_relu (Activation (None, 14, 14, 256)  0           conv4_block6_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_3_conv (Conv2D)    (None, 14, 14, 1024) 263168      conv4_block6_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_3_bn (BatchNormali (None, 14, 14, 1024) 4096        conv4_block6_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_add (Add)          (None, 14, 14, 1024) 0           conv4_block5_out[0][0]           \n",
      "                                                                 conv4_block6_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_out (Activation)   (None, 14, 14, 1024) 0           conv4_block6_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_1_conv (Conv2D)    (None, 7, 7, 512)    524800      conv4_block6_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_1_relu (Activation (None, 7, 7, 512)    0           conv5_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_2_conv (Conv2D)    (None, 7, 7, 512)    2359808     conv5_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_2_relu (Activation (None, 7, 7, 512)    0           conv5_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_0_conv (Conv2D)    (None, 7, 7, 2048)   2099200     conv4_block6_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_0_bn (BatchNormali (None, 7, 7, 2048)   8192        conv5_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_3_bn (BatchNormali (None, 7, 7, 2048)   8192        conv5_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_add (Add)          (None, 7, 7, 2048)   0           conv5_block1_0_bn[0][0]          \n",
      "                                                                 conv5_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_out (Activation)   (None, 7, 7, 2048)   0           conv5_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_1_conv (Conv2D)    (None, 7, 7, 512)    1049088     conv5_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_1_relu (Activation (None, 7, 7, 512)    0           conv5_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_2_conv (Conv2D)    (None, 7, 7, 512)    2359808     conv5_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_2_relu (Activation (None, 7, 7, 512)    0           conv5_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_3_bn (BatchNormali (None, 7, 7, 2048)   8192        conv5_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_add (Add)          (None, 7, 7, 2048)   0           conv5_block1_out[0][0]           \n",
      "                                                                 conv5_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_out (Activation)   (None, 7, 7, 2048)   0           conv5_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_conv (Conv2D)    (None, 7, 7, 512)    1049088     conv5_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_relu (Activation (None, 7, 7, 512)    0           conv5_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_conv (Conv2D)    (None, 7, 7, 512)    2359808     conv5_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_bn (BatchNormali (None, 7, 7, 512)    2048        conv5_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_relu (Activation (None, 7, 7, 512)    0           conv5_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_3_conv (Conv2D)    (None, 7, 7, 2048)   1050624     conv5_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_3_bn (BatchNormali (None, 7, 7, 2048)   8192        conv5_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_add (Add)          (None, 7, 7, 2048)   0           conv5_block2_out[0][0]           \n",
      "                                                                 conv5_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_out (Activation)   (None, 7, 7, 2048)   0           conv5_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "avg_pool (GlobalAveragePooling2 (None, 2048)         0           conv5_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "predictions (Dense)             (None, 1000)         2049000     avg_pool[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 25,636,712\n",
      "Trainable params: 25,583,592\n",
      "Non-trainable params: 53,120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.applications import ResNet50\n",
    "\n",
    "model = ResNet50()\n",
    "model.summary()    X  = conv_bn_relu(X)\n",
    "    X_main = Conv2D(filter_size,\n",
    "                    kernel_size = (3,3),\n",
    "                    strides = (1,1),\n",
    "                    padding = (1,1),\n",
    "                    kernel_initializer = \"he_normal\")(X)\n",
    "                   \n",
    "    X_main = BatchNormalization(axis = 3)(X_main)\n",
    "    X_main = Activation('relu')(X_main)\n",
    "    \n",
    "    \n",
    "    X_main = Conv2D(filter_size,\n",
    "                    kernel_size = (3,3),\n",
    "                    strides = (1,1),\n",
    "                    padding = (1,1),\n",
    "                    kernel_initializer = glorot_uniform(seed = 0))(X_main)\n",
    "                   \n",
    "    X_main = BatchNormalization(axis = 3)(X_main)\n",
    "    X_main = Activation('relu')(X_main)\n",
    "    \n",
    "     X_main = Conv2D(filter_size,\n",
    "                    kernel_size = (3,3),\n",
    "                    strides = (1,1),\n",
    "                    padding = 'same',\n",
    "                    kernel_initializer = glorot_uniform(seed = 0))(X_main)\n",
    "                   \n",
    "    X_main = BatchNormalization(axis = 3)(X_main)\n",
    "    X_main = Activation('relu')(X_main)\n",
    "    \n",
    "    \n",
    "    X_shortcut = Conv2D()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### نمایش شبکه عصبی ساخته شده"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting netron\n",
      "  Downloading https://files.pythonhosted.org/packages/50/a3/ca00f329295971406b8d9f2e8db9943d597c08efbef1ff92bccaddeda6cd/netron-4.4.8-py2.py3-none-any.whl (1.3MB)\n",
      "Installing collected packages: netron\n",
      "Successfully installed netron-4.4.8\n"
     ]
    }
   ],
   "source": [
    "!pip install netron"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/lutzroeder/netron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import netron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Serving 'models/custom_conv_mnist.h5' at http://localhost:8080\n"
     ]
    }
   ],
   "source": [
    "netron.start('models/custom_conv_mnist.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### منابع\n",
    "\n",
    "https://towardsdatascience.com/an-overview-of-resnet-and-its-variants-5281e2f56035"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}