test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorlayer as tl\n",
    "import tensorflow as tf\n",
    "import rawpy\n",
    "\n",
    "import model\n",
    "import utils\n",
    "\n",
    "from PIL import Image\n",
    "import numpy as np\n",
    "import glob\n",
    "import importlib\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## read data from file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = rawpy.imread('test_data/' + train_data_list[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.raw_image_visible "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "importlib.reload(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] [!] training_dir exists ...\n",
      "[TL] [!] checkpoint exists ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import config as _config\n",
    "importlib.reload(_config)\n",
    "from config import config, log_config\n",
    "\n",
    "###====================== HYPER-PARAMETERS ===========================###\n",
    "## Adam\n",
    "batch_size = config.TRAIN.batch_size\n",
    "lr_init = config.TRAIN.lr_init\n",
    "beta1 = config.TRAIN.beta1\n",
    "## initialize G\n",
    "n_epoch_init = config.TRAIN.n_epoch_init\n",
    "## adversarial learning (SRGAN)\n",
    "n_epoch = config.TRAIN.n_epoch\n",
    "lr_decay = config.TRAIN.lr_decay\n",
    "decay_every = config.TRAIN.decay_every\n",
    "training_dir = config.TRAIN.training_dir\n",
    "pretrain_checkpoint = config.TRAIN.pretrain_checkpoint\n",
    "train_data_dir = config.TRAIN.train_data_dir\n",
    "\n",
    "crop_num = config.TRAIN.crop_num\n",
    "\n",
    "tl.files.exists_or_mkdir(training_dir)\n",
    "tl.files.exists_or_mkdir(pretrain_checkpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import model as _model\n",
    "importlib.reload(_model)\n",
    "import utils as _utils\n",
    "importlib.reload(_utils)\n",
    "from model import *\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] InputLayer  SRGAN_g/in: (?, 96, 96, 3)\n",
      "[TL] Conv2d SRGAN_g/n64s1/c: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/0: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/0: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/0: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/1: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/1: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/1: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/2: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/2: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/2: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/3: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/3: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/4: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/4: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/4: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/5: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/5: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/5: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/6: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/6: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/6: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/7: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/7: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/7: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/8: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/8: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/8: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/9: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/9: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/9: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/10: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/10: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/10: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/11: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/11: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/11: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/12: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/12: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/12: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/13: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/13: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/13: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/14: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/14: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/14: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/15: decay: 0.900000 epsilon: 0.000010 act: relu is_train: True\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/15: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/15: size: (?, 96, 96, 64) fn: add\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] Conv2d SRGAN_g/n64s1/c/m: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b/m: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_g/add3: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n256s1/1_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/1_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/n256s1/2_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/2_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/out_new: n_filter: 1 filter_size: (1, 1) strides: (1, 1) pad: SAME act: tanh\n",
      "[TL] WARNING: Function: `tensorlayer.layers.utils.set_name_reuse` (in file: /home/hwa125/.local/lib/python3.6/site-packages/tensorlayer/layers/utils.py) is deprecated and will be removed after 2018-06-30.\n",
      "Instructions for updating: TensorLayer relies on TensorFlow to check name reusing\n",
      "\n",
      "[TL] WARNING: this method is DEPRECATED and has no effect, please remove it from your code.\n",
      "[TL] InputLayer  SRGAN_d/input/images: (?, 384, 384, 1)\n",
      "[TL] Conv2d SRGAN_d/h0/c: n_filter: 64 filter_size: (4, 4) strides: (2, 2) pad: SAME act: <lambda>\n",
      "[TL] WARNING: Function: `tensorlayer.activation.leaky_relu` (in file: /home/hwa125/.local/lib/python3.6/site-packages/tensorlayer/activation.py) is deprecated and will be removed after 2018-09-30.\n",
      "Instructions for updating: This API is deprecated. Please use as `tf.nn.leaky_relu`\n",
      "\n",
      "[TL] Conv2d SRGAN_d/h1/c: n_filter: 128 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h1/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h2/c: n_filter: 256 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h2/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h3/c: n_filter: 512 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h3/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h4/c: n_filter: 1024 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h4/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h5/c: n_filter: 2048 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h5/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h6/c: n_filter: 1024 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h6/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h7/c: n_filter: 512 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h7/bn: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c: n_filter: 128 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c2: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn2: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_d/res/add: size: (?, 6, 6, 512) fn: add\n",
      "[TL] FlattenLayer SRGAN_d/ho/flatten: 18432\n",
      "[TL] DenseLayer  SRGAN_d/ho/dense: 1 No Activation\n",
      "[TL] WARNING: this method is DEPRECATED and has no effect, please remove it from your code.\n",
      "[TL] InputLayer  SRGAN_d/input/images: (?, 384, 384, 1)\n",
      "[TL] Conv2d SRGAN_d/h0/c: n_filter: 64 filter_size: (4, 4) strides: (2, 2) pad: SAME act: <lambda>\n",
      "[TL] Conv2d SRGAN_d/h1/c: n_filter: 128 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h1/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h2/c: n_filter: 256 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h2/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h3/c: n_filter: 512 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h3/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h4/c: n_filter: 1024 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h4/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h5/c: n_filter: 2048 filter_size: (4, 4) strides: (2, 2) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h5/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h6/c: n_filter: 1024 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h6/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/h7/c: n_filter: 512 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/h7/bn: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c: n_filter: 128 filter_size: (1, 1) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c2: n_filter: 128 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn2: decay: 0.900000 epsilon: 0.000010 act: <lambda> is_train: True\n",
      "[TL] Conv2d SRGAN_d/res/c3: n_filter: 512 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_d/res/bn3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: True\n",
      "[TL] ElementwiseLayer SRGAN_d/res/add: size: (?, 6, 6, 512) fn: add\n",
      "[TL] FlattenLayer SRGAN_d/ho/flatten: 18432\n",
      "[TL] DenseLayer  SRGAN_d/ho/dense: 1 No Activation\n",
      "[TL]   param   0: SRGAN_g/n64s1/c/kernel:0 (3, 3, 3, 64)      float32_ref\n",
      "[TL]   param   1: SRGAN_g/n64s1/c/bias:0 (64,)              float32_ref\n",
      "[TL]   param   2: SRGAN_g/n64s1/c1/0/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param   3: SRGAN_g/n64s1/b1/0/beta:0 (64,)              float32_ref\n",
      "[TL]   param   4: SRGAN_g/n64s1/b1/0/gamma:0 (64,)              float32_ref\n",
      "[TL]   param   5: SRGAN_g/n64s1/b1/0/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param   6: SRGAN_g/n64s1/b1/0/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param   7: SRGAN_g/n64s1/c2/0/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param   8: SRGAN_g/n64s1/b2/0/beta:0 (64,)              float32_ref\n",
      "[TL]   param   9: SRGAN_g/n64s1/b2/0/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  10: SRGAN_g/n64s1/b2/0/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  11: SRGAN_g/n64s1/b2/0/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  12: SRGAN_g/n64s1/c1/1/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  13: SRGAN_g/n64s1/b1/1/beta:0 (64,)              float32_ref\n",
      "[TL]   param  14: SRGAN_g/n64s1/b1/1/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  15: SRGAN_g/n64s1/b1/1/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  16: SRGAN_g/n64s1/b1/1/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  17: SRGAN_g/n64s1/c2/1/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  18: SRGAN_g/n64s1/b2/1/beta:0 (64,)              float32_ref\n",
      "[TL]   param  19: SRGAN_g/n64s1/b2/1/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  20: SRGAN_g/n64s1/b2/1/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  21: SRGAN_g/n64s1/b2/1/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  22: SRGAN_g/n64s1/c1/2/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  23: SRGAN_g/n64s1/b1/2/beta:0 (64,)              float32_ref\n",
      "[TL]   param  24: SRGAN_g/n64s1/b1/2/gamma:0 (64,)              float32_ref\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL]   param  25: SRGAN_g/n64s1/b1/2/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  26: SRGAN_g/n64s1/b1/2/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  27: SRGAN_g/n64s1/c2/2/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  28: SRGAN_g/n64s1/b2/2/beta:0 (64,)              float32_ref\n",
      "[TL]   param  29: SRGAN_g/n64s1/b2/2/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  30: SRGAN_g/n64s1/b2/2/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  31: SRGAN_g/n64s1/b2/2/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  32: SRGAN_g/n64s1/c1/3/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  33: SRGAN_g/n64s1/b1/3/beta:0 (64,)              float32_ref\n",
      "[TL]   param  34: SRGAN_g/n64s1/b1/3/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  35: SRGAN_g/n64s1/b1/3/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  36: SRGAN_g/n64s1/b1/3/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  37: SRGAN_g/n64s1/c2/3/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  38: SRGAN_g/n64s1/b2/3/beta:0 (64,)              float32_ref\n",
      "[TL]   param  39: SRGAN_g/n64s1/b2/3/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  40: SRGAN_g/n64s1/b2/3/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  41: SRGAN_g/n64s1/b2/3/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  42: SRGAN_g/n64s1/c1/4/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  43: SRGAN_g/n64s1/b1/4/beta:0 (64,)              float32_ref\n",
      "[TL]   param  44: SRGAN_g/n64s1/b1/4/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  45: SRGAN_g/n64s1/b1/4/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  46: SRGAN_g/n64s1/b1/4/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  47: SRGAN_g/n64s1/c2/4/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  48: SRGAN_g/n64s1/b2/4/beta:0 (64,)              float32_ref\n",
      "[TL]   param  49: SRGAN_g/n64s1/b2/4/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  50: SRGAN_g/n64s1/b2/4/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  51: SRGAN_g/n64s1/b2/4/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  52: SRGAN_g/n64s1/c1/5/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  53: SRGAN_g/n64s1/b1/5/beta:0 (64,)              float32_ref\n",
      "[TL]   param  54: SRGAN_g/n64s1/b1/5/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  55: SRGAN_g/n64s1/b1/5/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  56: SRGAN_g/n64s1/b1/5/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  57: SRGAN_g/n64s1/c2/5/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  58: SRGAN_g/n64s1/b2/5/beta:0 (64,)              float32_ref\n",
      "[TL]   param  59: SRGAN_g/n64s1/b2/5/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  60: SRGAN_g/n64s1/b2/5/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  61: SRGAN_g/n64s1/b2/5/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  62: SRGAN_g/n64s1/c1/6/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  63: SRGAN_g/n64s1/b1/6/beta:0 (64,)              float32_ref\n",
      "[TL]   param  64: SRGAN_g/n64s1/b1/6/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  65: SRGAN_g/n64s1/b1/6/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  66: SRGAN_g/n64s1/b1/6/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  67: SRGAN_g/n64s1/c2/6/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  68: SRGAN_g/n64s1/b2/6/beta:0 (64,)              float32_ref\n",
      "[TL]   param  69: SRGAN_g/n64s1/b2/6/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  70: SRGAN_g/n64s1/b2/6/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  71: SRGAN_g/n64s1/b2/6/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  72: SRGAN_g/n64s1/c1/7/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  73: SRGAN_g/n64s1/b1/7/beta:0 (64,)              float32_ref\n",
      "[TL]   param  74: SRGAN_g/n64s1/b1/7/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  75: SRGAN_g/n64s1/b1/7/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  76: SRGAN_g/n64s1/b1/7/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  77: SRGAN_g/n64s1/c2/7/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  78: SRGAN_g/n64s1/b2/7/beta:0 (64,)              float32_ref\n",
      "[TL]   param  79: SRGAN_g/n64s1/b2/7/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  80: SRGAN_g/n64s1/b2/7/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  81: SRGAN_g/n64s1/b2/7/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  82: SRGAN_g/n64s1/c1/8/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  83: SRGAN_g/n64s1/b1/8/beta:0 (64,)              float32_ref\n",
      "[TL]   param  84: SRGAN_g/n64s1/b1/8/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  85: SRGAN_g/n64s1/b1/8/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  86: SRGAN_g/n64s1/b1/8/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  87: SRGAN_g/n64s1/c2/8/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  88: SRGAN_g/n64s1/b2/8/beta:0 (64,)              float32_ref\n",
      "[TL]   param  89: SRGAN_g/n64s1/b2/8/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  90: SRGAN_g/n64s1/b2/8/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  91: SRGAN_g/n64s1/b2/8/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  92: SRGAN_g/n64s1/c1/9/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  93: SRGAN_g/n64s1/b1/9/beta:0 (64,)              float32_ref\n",
      "[TL]   param  94: SRGAN_g/n64s1/b1/9/gamma:0 (64,)              float32_ref\n",
      "[TL]   param  95: SRGAN_g/n64s1/b1/9/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param  96: SRGAN_g/n64s1/b1/9/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param  97: SRGAN_g/n64s1/c2/9/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param  98: SRGAN_g/n64s1/b2/9/beta:0 (64,)              float32_ref\n",
      "[TL]   param  99: SRGAN_g/n64s1/b2/9/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 100: SRGAN_g/n64s1/b2/9/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 101: SRGAN_g/n64s1/b2/9/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 102: SRGAN_g/n64s1/c1/10/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 103: SRGAN_g/n64s1/b1/10/beta:0 (64,)              float32_ref\n",
      "[TL]   param 104: SRGAN_g/n64s1/b1/10/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 105: SRGAN_g/n64s1/b1/10/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 106: SRGAN_g/n64s1/b1/10/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 107: SRGAN_g/n64s1/c2/10/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 108: SRGAN_g/n64s1/b2/10/beta:0 (64,)              float32_ref\n",
      "[TL]   param 109: SRGAN_g/n64s1/b2/10/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 110: SRGAN_g/n64s1/b2/10/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 111: SRGAN_g/n64s1/b2/10/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 112: SRGAN_g/n64s1/c1/11/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 113: SRGAN_g/n64s1/b1/11/beta:0 (64,)              float32_ref\n",
      "[TL]   param 114: SRGAN_g/n64s1/b1/11/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 115: SRGAN_g/n64s1/b1/11/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 116: SRGAN_g/n64s1/b1/11/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 117: SRGAN_g/n64s1/c2/11/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 118: SRGAN_g/n64s1/b2/11/beta:0 (64,)              float32_ref\n",
      "[TL]   param 119: SRGAN_g/n64s1/b2/11/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 120: SRGAN_g/n64s1/b2/11/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 121: SRGAN_g/n64s1/b2/11/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 122: SRGAN_g/n64s1/c1/12/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 123: SRGAN_g/n64s1/b1/12/beta:0 (64,)              float32_ref\n",
      "[TL]   param 124: SRGAN_g/n64s1/b1/12/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 125: SRGAN_g/n64s1/b1/12/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 126: SRGAN_g/n64s1/b1/12/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 127: SRGAN_g/n64s1/c2/12/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 128: SRGAN_g/n64s1/b2/12/beta:0 (64,)              float32_ref\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL]   param 129: SRGAN_g/n64s1/b2/12/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 130: SRGAN_g/n64s1/b2/12/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 131: SRGAN_g/n64s1/b2/12/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 132: SRGAN_g/n64s1/c1/13/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 133: SRGAN_g/n64s1/b1/13/beta:0 (64,)              float32_ref\n",
      "[TL]   param 134: SRGAN_g/n64s1/b1/13/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 135: SRGAN_g/n64s1/b1/13/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 136: SRGAN_g/n64s1/b1/13/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 137: SRGAN_g/n64s1/c2/13/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 138: SRGAN_g/n64s1/b2/13/beta:0 (64,)              float32_ref\n",
      "[TL]   param 139: SRGAN_g/n64s1/b2/13/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 140: SRGAN_g/n64s1/b2/13/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 141: SRGAN_g/n64s1/b2/13/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 142: SRGAN_g/n64s1/c1/14/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 143: SRGAN_g/n64s1/b1/14/beta:0 (64,)              float32_ref\n",
      "[TL]   param 144: SRGAN_g/n64s1/b1/14/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 145: SRGAN_g/n64s1/b1/14/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 146: SRGAN_g/n64s1/b1/14/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 147: SRGAN_g/n64s1/c2/14/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 148: SRGAN_g/n64s1/b2/14/beta:0 (64,)              float32_ref\n",
      "[TL]   param 149: SRGAN_g/n64s1/b2/14/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 150: SRGAN_g/n64s1/b2/14/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 151: SRGAN_g/n64s1/b2/14/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 152: SRGAN_g/n64s1/c1/15/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 153: SRGAN_g/n64s1/b1/15/beta:0 (64,)              float32_ref\n",
      "[TL]   param 154: SRGAN_g/n64s1/b1/15/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 155: SRGAN_g/n64s1/b1/15/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 156: SRGAN_g/n64s1/b1/15/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 157: SRGAN_g/n64s1/c2/15/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 158: SRGAN_g/n64s1/b2/15/beta:0 (64,)              float32_ref\n",
      "[TL]   param 159: SRGAN_g/n64s1/b2/15/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 160: SRGAN_g/n64s1/b2/15/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 161: SRGAN_g/n64s1/b2/15/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 162: SRGAN_g/n64s1/c/m/kernel:0 (3, 3, 64, 64)     float32_ref\n",
      "[TL]   param 163: SRGAN_g/n64s1/b/m/beta:0 (64,)              float32_ref\n",
      "[TL]   param 164: SRGAN_g/n64s1/b/m/gamma:0 (64,)              float32_ref\n",
      "[TL]   param 165: SRGAN_g/n64s1/b/m/moving_mean:0 (64,)              float32_ref\n",
      "[TL]   param 166: SRGAN_g/n64s1/b/m/moving_variance:0 (64,)              float32_ref\n",
      "[TL]   param 167: SRGAN_g/n256s1/1_new/kernel:0 (3, 3, 64, 256)    float32_ref\n",
      "[TL]   param 168: SRGAN_g/n256s1/1_new/bias:0 (256,)             float32_ref\n",
      "[TL]   param 169: SRGAN_g/n256s1/2_new/kernel:0 (3, 3, 64, 256)    float32_ref\n",
      "[TL]   param 170: SRGAN_g/n256s1/2_new/bias:0 (256,)             float32_ref\n",
      "[TL]   param 171: SRGAN_g/out_new/kernel:0 (1, 1, 64, 1)      float32_ref\n",
      "[TL]   param 172: SRGAN_g/out_new/bias:0 (1,)               float32_ref\n",
      "[TL]   num of params: 1522241\n",
      "[TL]   param   0: SRGAN_d/h0/c/kernel:0 (4, 4, 1, 64)      float32_ref\n",
      "[TL]   param   1: SRGAN_d/h0/c/bias:0  (64,)              float32_ref\n",
      "[TL]   param   2: SRGAN_d/h1/c/kernel:0 (4, 4, 64, 128)    float32_ref\n",
      "[TL]   param   3: SRGAN_d/h1/bn/beta:0 (128,)             float32_ref\n",
      "[TL]   param   4: SRGAN_d/h1/bn/gamma:0 (128,)             float32_ref\n",
      "[TL]   param   5: SRGAN_d/h1/bn/moving_mean:0 (128,)             float32_ref\n",
      "[TL]   param   6: SRGAN_d/h1/bn/moving_variance:0 (128,)             float32_ref\n",
      "[TL]   param   7: SRGAN_d/h2/c/kernel:0 (4, 4, 128, 256)    float32_ref\n",
      "[TL]   param   8: SRGAN_d/h2/bn/beta:0 (256,)             float32_ref\n",
      "[TL]   param   9: SRGAN_d/h2/bn/gamma:0 (256,)             float32_ref\n",
      "[TL]   param  10: SRGAN_d/h2/bn/moving_mean:0 (256,)             float32_ref\n",
      "[TL]   param  11: SRGAN_d/h2/bn/moving_variance:0 (256,)             float32_ref\n",
      "[TL]   param  12: SRGAN_d/h3/c/kernel:0 (4, 4, 256, 512)    float32_ref\n",
      "[TL]   param  13: SRGAN_d/h3/bn/beta:0 (512,)             float32_ref\n",
      "[TL]   param  14: SRGAN_d/h3/bn/gamma:0 (512,)             float32_ref\n",
      "[TL]   param  15: SRGAN_d/h3/bn/moving_mean:0 (512,)             float32_ref\n",
      "[TL]   param  16: SRGAN_d/h3/bn/moving_variance:0 (512,)             float32_ref\n",
      "[TL]   param  17: SRGAN_d/h4/c/kernel:0 (4, 4, 512, 1024)    float32_ref\n",
      "[TL]   param  18: SRGAN_d/h4/bn/beta:0 (1024,)            float32_ref\n",
      "[TL]   param  19: SRGAN_d/h4/bn/gamma:0 (1024,)            float32_ref\n",
      "[TL]   param  20: SRGAN_d/h4/bn/moving_mean:0 (1024,)            float32_ref\n",
      "[TL]   param  21: SRGAN_d/h4/bn/moving_variance:0 (1024,)            float32_ref\n",
      "[TL]   param  22: SRGAN_d/h5/c/kernel:0 (4, 4, 1024, 2048)    float32_ref\n",
      "[TL]   param  23: SRGAN_d/h5/bn/beta:0 (2048,)            float32_ref\n",
      "[TL]   param  24: SRGAN_d/h5/bn/gamma:0 (2048,)            float32_ref\n",
      "[TL]   param  25: SRGAN_d/h5/bn/moving_mean:0 (2048,)            float32_ref\n",
      "[TL]   param  26: SRGAN_d/h5/bn/moving_variance:0 (2048,)            float32_ref\n",
      "[TL]   param  27: SRGAN_d/h6/c/kernel:0 (1, 1, 2048, 1024)    float32_ref\n",
      "[TL]   param  28: SRGAN_d/h6/bn/beta:0 (1024,)            float32_ref\n",
      "[TL]   param  29: SRGAN_d/h6/bn/gamma:0 (1024,)            float32_ref\n",
      "[TL]   param  30: SRGAN_d/h6/bn/moving_mean:0 (1024,)            float32_ref\n",
      "[TL]   param  31: SRGAN_d/h6/bn/moving_variance:0 (1024,)            float32_ref\n",
      "[TL]   param  32: SRGAN_d/h7/c/kernel:0 (1, 1, 1024, 512)    float32_ref\n",
      "[TL]   param  33: SRGAN_d/h7/bn/beta:0 (512,)             float32_ref\n",
      "[TL]   param  34: SRGAN_d/h7/bn/gamma:0 (512,)             float32_ref\n",
      "[TL]   param  35: SRGAN_d/h7/bn/moving_mean:0 (512,)             float32_ref\n",
      "[TL]   param  36: SRGAN_d/h7/bn/moving_variance:0 (512,)             float32_ref\n",
      "[TL]   param  37: SRGAN_d/res/c/kernel:0 (1, 1, 512, 128)    float32_ref\n",
      "[TL]   param  38: SRGAN_d/res/bn/beta:0 (128,)             float32_ref\n",
      "[TL]   param  39: SRGAN_d/res/bn/gamma:0 (128,)             float32_ref\n",
      "[TL]   param  40: SRGAN_d/res/bn/moving_mean:0 (128,)             float32_ref\n",
      "[TL]   param  41: SRGAN_d/res/bn/moving_variance:0 (128,)             float32_ref\n",
      "[TL]   param  42: SRGAN_d/res/c2/kernel:0 (3, 3, 128, 128)    float32_ref\n",
      "[TL]   param  43: SRGAN_d/res/bn2/beta:0 (128,)             float32_ref\n",
      "[TL]   param  44: SRGAN_d/res/bn2/gamma:0 (128,)             float32_ref\n",
      "[TL]   param  45: SRGAN_d/res/bn2/moving_mean:0 (128,)             float32_ref\n",
      "[TL]   param  46: SRGAN_d/res/bn2/moving_variance:0 (128,)             float32_ref\n",
      "[TL]   param  47: SRGAN_d/res/c3/kernel:0 (3, 3, 128, 512)    float32_ref\n",
      "[TL]   param  48: SRGAN_d/res/bn3/beta:0 (512,)             float32_ref\n",
      "[TL]   param  49: SRGAN_d/res/bn3/gamma:0 (512,)             float32_ref\n",
      "[TL]   param  50: SRGAN_d/res/bn3/moving_mean:0 (512,)             float32_ref\n",
      "[TL]   param  51: SRGAN_d/res/bn3/moving_variance:0 (512,)             float32_ref\n",
      "[TL]   param  52: SRGAN_d/ho/dense/W:0 (18432, 1)         float32_ref\n",
      "[TL]   param  53: SRGAN_d/ho/dense/b:0 (1,)               float32_ref\n",
      "[TL]   num of params: 48164417\n",
      "[TL] InputLayer  SRGAN_g/in: (?, 96, 96, 3)\n",
      "[TL] Conv2d SRGAN_g/n64s1/c: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: relu\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/0: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/0: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/0: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/0: size: (?, 96, 96, 64) fn: add\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] Conv2d SRGAN_g/n64s1/c1/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/1: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/1: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/1: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/1: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/2: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/2: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/2: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/2: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/3: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/3: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/3: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/3: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/4: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/4: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/4: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/4: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/5: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/5: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/5: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/5: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/6: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/6: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/6: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/6: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/7: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/7: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/7: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/7: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/8: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/8: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/8: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/8: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/9: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/9: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/9: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/9: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/10: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/10: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/10: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/10: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/11: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/11: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/11: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/11: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/12: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/12: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/12: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/12: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/13: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/13: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/13: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/13: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/14: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/14: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/14: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/14: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c1/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b1/15: decay: 0.900000 epsilon: 0.000010 act: relu is_train: False\n",
      "[TL] Conv2d SRGAN_g/n64s1/c2/15: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b2/15: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/b_residual_add/15: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n64s1/c/m: n_filter: 64 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] BatchNormLayer SRGAN_g/n64s1/b/m: decay: 0.900000 epsilon: 0.000010 act: No Activation is_train: False\n",
      "[TL] ElementwiseLayer SRGAN_g/add3: size: (?, 96, 96, 64) fn: add\n",
      "[TL] Conv2d SRGAN_g/n256s1/1_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n",
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/1_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/n256s1/2_new: n_filter: 256 filter_size: (3, 3) strides: (1, 1) pad: SAME act: No Activation\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] SubpixelConv2d  SRGAN_g/pixelshufflerx2/2_new: scale: 2 n_out_channel: 64 act: relu\n",
      "[TL] Conv2d SRGAN_g/out_new: n_filter: 1 filter_size: (1, 1) strides: (1, 1) pad: SAME act: tanh\n"
     ]
    }
   ],
   "source": [
    "t_image = tf.placeholder('float32', [None, 96, 96, 3], name='t_image_input_to_SRGAN_generator')\n",
    "t_target_image = tf.placeholder('float32', [None, 384, 384, 1], name='t_target_image')\n",
    "\n",
    "net_g = SRGAN_g(t_image, is_train=True, reuse=False)\n",
    "net_d, logits_real = SRGAN_d(t_target_image, is_train=True, reuse=False)\n",
    "_, logits_fake = SRGAN_d(net_g.outputs, is_train=True, reuse=True)\n",
    "\n",
    "# net_g.print_params(False)￼\n",
    "\n",
    "net_g.print_params(False)\n",
    "# net_g.print_layers()\n",
    "net_d.print_params(False)\n",
    "# net_d.print_layers()\n",
    "\n",
    "## test inference\n",
    "net_g_test = SRGAN_g(t_image, is_train=False, reuse=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define training op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL]   [*] geting variables with SRGAN_g\n",
      "[TL]   got   0: SRGAN_g/n64s1/c/kernel:0   (3, 3, 3, 64)\n",
      "[TL]   got   1: SRGAN_g/n64s1/c/bias:0   (64,)\n",
      "[TL]   got   2: SRGAN_g/n64s1/c1/0/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got   3: SRGAN_g/n64s1/b1/0/beta:0   (64,)\n",
      "[TL]   got   4: SRGAN_g/n64s1/b1/0/gamma:0   (64,)\n",
      "[TL]   got   5: SRGAN_g/n64s1/c2/0/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got   6: SRGAN_g/n64s1/b2/0/beta:0   (64,)\n",
      "[TL]   got   7: SRGAN_g/n64s1/b2/0/gamma:0   (64,)\n",
      "[TL]   got   8: SRGAN_g/n64s1/c1/1/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got   9: SRGAN_g/n64s1/b1/1/beta:0   (64,)\n",
      "[TL]   got  10: SRGAN_g/n64s1/b1/1/gamma:0   (64,)\n",
      "[TL]   got  11: SRGAN_g/n64s1/c2/1/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  12: SRGAN_g/n64s1/b2/1/beta:0   (64,)\n",
      "[TL]   got  13: SRGAN_g/n64s1/b2/1/gamma:0   (64,)\n",
      "[TL]   got  14: SRGAN_g/n64s1/c1/2/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  15: SRGAN_g/n64s1/b1/2/beta:0   (64,)\n",
      "[TL]   got  16: SRGAN_g/n64s1/b1/2/gamma:0   (64,)\n",
      "[TL]   got  17: SRGAN_g/n64s1/c2/2/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  18: SRGAN_g/n64s1/b2/2/beta:0   (64,)\n",
      "[TL]   got  19: SRGAN_g/n64s1/b2/2/gamma:0   (64,)\n",
      "[TL]   got  20: SRGAN_g/n64s1/c1/3/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  21: SRGAN_g/n64s1/b1/3/beta:0   (64,)\n",
      "[TL]   got  22: SRGAN_g/n64s1/b1/3/gamma:0   (64,)\n",
      "[TL]   got  23: SRGAN_g/n64s1/c2/3/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  24: SRGAN_g/n64s1/b2/3/beta:0   (64,)\n",
      "[TL]   got  25: SRGAN_g/n64s1/b2/3/gamma:0   (64,)\n",
      "[TL]   got  26: SRGAN_g/n64s1/c1/4/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  27: SRGAN_g/n64s1/b1/4/beta:0   (64,)\n",
      "[TL]   got  28: SRGAN_g/n64s1/b1/4/gamma:0   (64,)\n",
      "[TL]   got  29: SRGAN_g/n64s1/c2/4/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  30: SRGAN_g/n64s1/b2/4/beta:0   (64,)\n",
      "[TL]   got  31: SRGAN_g/n64s1/b2/4/gamma:0   (64,)\n",
      "[TL]   got  32: SRGAN_g/n64s1/c1/5/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  33: SRGAN_g/n64s1/b1/5/beta:0   (64,)\n",
      "[TL]   got  34: SRGAN_g/n64s1/b1/5/gamma:0   (64,)\n",
      "[TL]   got  35: SRGAN_g/n64s1/c2/5/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  36: SRGAN_g/n64s1/b2/5/beta:0   (64,)\n",
      "[TL]   got  37: SRGAN_g/n64s1/b2/5/gamma:0   (64,)\n",
      "[TL]   got  38: SRGAN_g/n64s1/c1/6/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  39: SRGAN_g/n64s1/b1/6/beta:0   (64,)\n",
      "[TL]   got  40: SRGAN_g/n64s1/b1/6/gamma:0   (64,)\n",
      "[TL]   got  41: SRGAN_g/n64s1/c2/6/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  42: SRGAN_g/n64s1/b2/6/beta:0   (64,)\n",
      "[TL]   got  43: SRGAN_g/n64s1/b2/6/gamma:0   (64,)\n",
      "[TL]   got  44: SRGAN_g/n64s1/c1/7/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  45: SRGAN_g/n64s1/b1/7/beta:0   (64,)\n",
      "[TL]   got  46: SRGAN_g/n64s1/b1/7/gamma:0   (64,)\n",
      "[TL]   got  47: SRGAN_g/n64s1/c2/7/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  48: SRGAN_g/n64s1/b2/7/beta:0   (64,)\n",
      "[TL]   got  49: SRGAN_g/n64s1/b2/7/gamma:0   (64,)\n",
      "[TL]   got  50: SRGAN_g/n64s1/c1/8/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  51: SRGAN_g/n64s1/b1/8/beta:0   (64,)\n",
      "[TL]   got  52: SRGAN_g/n64s1/b1/8/gamma:0   (64,)\n",
      "[TL]   got  53: SRGAN_g/n64s1/c2/8/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  54: SRGAN_g/n64s1/b2/8/beta:0   (64,)\n",
      "[TL]   got  55: SRGAN_g/n64s1/b2/8/gamma:0   (64,)\n",
      "[TL]   got  56: SRGAN_g/n64s1/c1/9/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  57: SRGAN_g/n64s1/b1/9/beta:0   (64,)\n",
      "[TL]   got  58: SRGAN_g/n64s1/b1/9/gamma:0   (64,)\n",
      "[TL]   got  59: SRGAN_g/n64s1/c2/9/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  60: SRGAN_g/n64s1/b2/9/beta:0   (64,)\n",
      "[TL]   got  61: SRGAN_g/n64s1/b2/9/gamma:0   (64,)\n",
      "[TL]   got  62: SRGAN_g/n64s1/c1/10/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  63: SRGAN_g/n64s1/b1/10/beta:0   (64,)\n",
      "[TL]   got  64: SRGAN_g/n64s1/b1/10/gamma:0   (64,)\n",
      "[TL]   got  65: SRGAN_g/n64s1/c2/10/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  66: SRGAN_g/n64s1/b2/10/beta:0   (64,)\n",
      "[TL]   got  67: SRGAN_g/n64s1/b2/10/gamma:0   (64,)\n",
      "[TL]   got  68: SRGAN_g/n64s1/c1/11/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  69: SRGAN_g/n64s1/b1/11/beta:0   (64,)\n",
      "[TL]   got  70: SRGAN_g/n64s1/b1/11/gamma:0   (64,)\n",
      "[TL]   got  71: SRGAN_g/n64s1/c2/11/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  72: SRGAN_g/n64s1/b2/11/beta:0   (64,)\n",
      "[TL]   got  73: SRGAN_g/n64s1/b2/11/gamma:0   (64,)\n",
      "[TL]   got  74: SRGAN_g/n64s1/c1/12/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  75: SRGAN_g/n64s1/b1/12/beta:0   (64,)\n",
      "[TL]   got  76: SRGAN_g/n64s1/b1/12/gamma:0   (64,)\n",
      "[TL]   got  77: SRGAN_g/n64s1/c2/12/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  78: SRGAN_g/n64s1/b2/12/beta:0   (64,)\n",
      "[TL]   got  79: SRGAN_g/n64s1/b2/12/gamma:0   (64,)\n",
      "[TL]   got  80: SRGAN_g/n64s1/c1/13/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  81: SRGAN_g/n64s1/b1/13/beta:0   (64,)\n",
      "[TL]   got  82: SRGAN_g/n64s1/b1/13/gamma:0   (64,)\n",
      "[TL]   got  83: SRGAN_g/n64s1/c2/13/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  84: SRGAN_g/n64s1/b2/13/beta:0   (64,)\n",
      "[TL]   got  85: SRGAN_g/n64s1/b2/13/gamma:0   (64,)\n",
      "[TL]   got  86: SRGAN_g/n64s1/c1/14/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  87: SRGAN_g/n64s1/b1/14/beta:0   (64,)\n",
      "[TL]   got  88: SRGAN_g/n64s1/b1/14/gamma:0   (64,)\n",
      "[TL]   got  89: SRGAN_g/n64s1/c2/14/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  90: SRGAN_g/n64s1/b2/14/beta:0   (64,)\n",
      "[TL]   got  91: SRGAN_g/n64s1/b2/14/gamma:0   (64,)\n",
      "[TL]   got  92: SRGAN_g/n64s1/c1/15/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  93: SRGAN_g/n64s1/b1/15/beta:0   (64,)\n",
      "[TL]   got  94: SRGAN_g/n64s1/b1/15/gamma:0   (64,)\n",
      "[TL]   got  95: SRGAN_g/n64s1/c2/15/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  96: SRGAN_g/n64s1/b2/15/beta:0   (64,)\n",
      "[TL]   got  97: SRGAN_g/n64s1/b2/15/gamma:0   (64,)\n",
      "[TL]   got  98: SRGAN_g/n64s1/c/m/kernel:0   (3, 3, 64, 64)\n",
      "[TL]   got  99: SRGAN_g/n64s1/b/m/beta:0   (64,)\n",
      "[TL]   got 100: SRGAN_g/n64s1/b/m/gamma:0   (64,)\n",
      "[TL]   got 101: SRGAN_g/n256s1/1_new/kernel:0   (3, 3, 64, 256)\n",
      "[TL]   got 102: SRGAN_g/n256s1/1_new/bias:0   (256,)\n",
      "[TL]   got 103: SRGAN_g/n256s1/2_new/kernel:0   (3, 3, 64, 256)\n",
      "[TL]   got 104: SRGAN_g/n256s1/2_new/bias:0   (256,)\n",
      "[TL]   got 105: SRGAN_g/out_new/kernel:0   (1, 1, 64, 1)\n",
      "[TL]   got 106: SRGAN_g/out_new/bias:0   (1,)\n",
      "[TL]   [*] geting variables with SRGAN_d\n",
      "[TL]   got   0: SRGAN_d/h0/c/kernel:0   (4, 4, 1, 64)\n",
      "[TL]   got   1: SRGAN_d/h0/c/bias:0   (64,)\n",
      "[TL]   got   2: SRGAN_d/h1/c/kernel:0   (4, 4, 64, 128)\n",
      "[TL]   got   3: SRGAN_d/h1/bn/beta:0   (128,)\n",
      "[TL]   got   4: SRGAN_d/h1/bn/gamma:0   (128,)\n",
      "[TL]   got   5: SRGAN_d/h2/c/kernel:0   (4, 4, 128, 256)\n",
      "[TL]   got   6: SRGAN_d/h2/bn/beta:0   (256,)\n",
      "[TL]   got   7: SRGAN_d/h2/bn/gamma:0   (256,)\n",
      "[TL]   got   8: SRGAN_d/h3/c/kernel:0   (4, 4, 256, 512)\n",
      "[TL]   got   9: SRGAN_d/h3/bn/beta:0   (512,)\n",
      "[TL]   got  10: SRGAN_d/h3/bn/gamma:0   (512,)\n",
      "[TL]   got  11: SRGAN_d/h4/c/kernel:0   (4, 4, 512, 1024)\n",
      "[TL]   got  12: SRGAN_d/h4/bn/beta:0   (1024,)\n",
      "[TL]   got  13: SRGAN_d/h4/bn/gamma:0   (1024,)\n",
      "[TL]   got  14: SRGAN_d/h5/c/kernel:0   (4, 4, 1024, 2048)\n",
      "[TL]   got  15: SRGAN_d/h5/bn/beta:0   (2048,)\n",
      "[TL]   got  16: SRGAN_d/h5/bn/gamma:0   (2048,)\n",
      "[TL]   got  17: SRGAN_d/h6/c/kernel:0   (1, 1, 2048, 1024)\n",
      "[TL]   got  18: SRGAN_d/h6/bn/beta:0   (1024,)\n",
      "[TL]   got  19: SRGAN_d/h6/bn/gamma:0   (1024,)\n",
      "[TL]   got  20: SRGAN_d/h7/c/kernel:0   (1, 1, 1024, 512)\n",
      "[TL]   got  21: SRGAN_d/h7/bn/beta:0   (512,)\n",
      "[TL]   got  22: SRGAN_d/h7/bn/gamma:0   (512,)\n",
      "[TL]   got  23: SRGAN_d/res/c/kernel:0   (1, 1, 512, 128)\n",
      "[TL]   got  24: SRGAN_d/res/bn/beta:0   (128,)\n",
      "[TL]   got  25: SRGAN_d/res/bn/gamma:0   (128,)\n",
      "[TL]   got  26: SRGAN_d/res/c2/kernel:0   (3, 3, 128, 128)\n",
      "[TL]   got  27: SRGAN_d/res/bn2/beta:0   (128,)\n",
      "[TL]   got  28: SRGAN_d/res/bn2/gamma:0   (128,)\n",
      "[TL]   got  29: SRGAN_d/res/c3/kernel:0   (3, 3, 128, 512)\n",
      "[TL]   got  30: SRGAN_d/res/bn3/beta:0   (512,)\n",
      "[TL]   got  31: SRGAN_d/res/bn3/gamma:0   (512,)\n",
      "[TL]   got  32: SRGAN_d/ho/dense/W:0   (18432, 1)\n",
      "[TL]   got  33: SRGAN_d/ho/dense/b:0   (1,)\n"
     ]
    }
   ],
   "source": [
    "d_loss1 = tl.cost.sigmoid_cross_entropy(logits_real, tf.ones_like(logits_real), name='d1')\n",
    "d_loss2 = tl.cost.sigmoid_cross_entropy(logits_fake, tf.zeros_like(logits_fake), name='d2')\n",
    "d_loss = d_loss1 + d_loss2\n",
    "\n",
    "g_gan_loss = 1e-3 * tl.cost.sigmoid_cross_entropy(logits_fake, tf.ones_like(logits_fake), name='g')\n",
    "mse_loss = tl.cost.mean_squared_error(net_g.outputs, t_target_image, is_mean=True)\n",
    "# vgg_loss = 2e-6 * tl.cost.mean_squared_error(vgg_predict_emb.outputs, vgg_target_emb.outputs, is_mean=True)\n",
    "\n",
    "# g_loss = mse_loss + vgg_loss + g_gan_loss\n",
    "g_loss = mse_loss + g_gan_loss\n",
    "\n",
    "g_vars = tl.layers.get_variables_with_name('SRGAN_g', True, True)\n",
    "d_vars = tl.layers.get_variables_with_name('SRGAN_d', True, True)\n",
    "\n",
    "with tf.variable_scope('learning_rate'):\n",
    "    lr_v = tf.Variable(lr_init, trainable=False)\n",
    "## Pretrain\n",
    "g_optim_init = tf.train.AdamOptimizer(lr_v, beta1=beta1).minimize(mse_loss, var_list=g_vars)\n",
    "## SRGAN\n",
    "g_optim = tf.train.AdamOptimizer(lr_v, beta1=beta1).minimize(g_loss, var_list=g_vars)\n",
    "d_optim = tf.train.AdamOptimizer(lr_v, beta1=beta1).minimize(d_loss, var_list=d_vars)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## restore model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True,\n",
    "                                        log_device_placement=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] WARNING: Function: `tensorlayer.layers.utils.initialize_global_variables` (in file: /home/hwa125/.local/lib/python3.6/site-packages/tensorlayer/layers/utils.py) is deprecated and will be removed after 2018-09-30.\n",
      "Instructions for updating: This API is deprecated in favor of `tf.global_variables_initializer`\n",
      "\n",
      "[TL] ERROR: file training_dir/g_srgan.npz doesn't exist.\n",
      "[TL] [*] Load checkpoint/g_srgan.npz SUCCESS!\n",
      "[TL] ERROR: file training_dir/d_srgan.npz doesn't exist.\n"
     ]
    }
   ],
   "source": [
    "tl.layers.initialize_global_variables(sess)\n",
    "if tl.files.load_and_assign_npz(sess=sess, name=training_dir + '/g_srgan.npz', network=net_g) is None:\n",
    "    load_pretrain_model(sess=sess, npz_file=pretrain_checkpoint + '/g_srgan.npz', network=net_g)\n",
    "tl.files.load_and_assign_npz(sess=sess, name=training_dir + '/d_srgan.npz', network=net_d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training\n",
    "### initialize G"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ** fixed learning rate: 0.000100 (for init G)\n"
     ]
    }
   ],
   "source": [
    "sess.run(tf.assign(lr_v, lr_init))\n",
    "print(\" ** fixed learning rate: %f (for init G)\" % lr_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_list = tl.files.load_file_list(train_data_dir, regx = '.*.arw', printable = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [ 0/100]    0 time: 5.1405s, mse: 0.00431612 \n",
      "Epoch [ 0/100]    1 time: 0.9554s, mse: 0.00110007 \n",
      "Epoch [ 0/100]    2 time: 0.9247s, mse: 0.00241852 \n",
      "Epoch [ 0/100]    3 time: 0.9285s, mse: 0.00706428 \n",
      "[*] Epoch: [ 0/100] time: 7.9493s, mse: 0.00372475\n",
      "Epoch [ 1/100]    0 time: 0.9258s, mse: 0.00024928 \n",
      "Epoch [ 1/100]    1 time: 0.9266s, mse: 0.02691215 \n",
      "Epoch [ 1/100]    2 time: 0.9395s, mse: 0.00140690 \n",
      "Epoch [ 1/100]    3 time: 0.8895s, mse: 0.03431595 \n",
      "[*] Epoch: [ 1/100] time: 3.6818s, mse: 0.01572107\n",
      "Epoch [ 2/100]    0 time: 0.9324s, mse: 0.00060681 \n",
      "Epoch [ 2/100]    1 time: 0.9282s, mse: 0.00479323 \n",
      "Epoch [ 2/100]    2 time: 0.9430s, mse: 0.00100325 \n",
      "Epoch [ 2/100]    3 time: 0.8869s, mse: 0.00275586 \n",
      "[*] Epoch: [ 2/100] time: 3.6907s, mse: 0.00228978\n",
      "Epoch [ 3/100]    0 time: 0.9547s, mse: 0.00097979 \n",
      "Epoch [ 3/100]    1 time: 0.9330s, mse: 0.00444230 \n",
      "Epoch [ 3/100]    2 time: 0.9270s, mse: 0.01230310 \n",
      "Epoch [ 3/100]    3 time: 0.8923s, mse: 0.00263789 \n",
      "[*] Epoch: [ 3/100] time: 3.7072s, mse: 0.00509077\n",
      "Epoch [ 4/100]    0 time: 0.9378s, mse: 0.00136259 \n",
      "Epoch [ 4/100]    1 time: 0.9468s, mse: 0.00237773 \n",
      "Epoch [ 4/100]    2 time: 0.9351s, mse: 0.00139935 \n",
      "Epoch [ 4/100]    3 time: 0.9110s, mse: 0.00345145 \n",
      "[*] Epoch: [ 4/100] time: 3.7310s, mse: 0.00214778\n",
      "Epoch [ 5/100]    0 time: 0.9315s, mse: 0.00333788 \n",
      "Epoch [ 5/100]    1 time: 0.9605s, mse: 0.01476618 \n",
      "Epoch [ 5/100]    2 time: 0.9260s, mse: 0.00439561 \n",
      "Epoch [ 5/100]    3 time: 0.9055s, mse: 0.00319005 \n",
      "[*] Epoch: [ 5/100] time: 3.7238s, mse: 0.00642243\n",
      "Epoch [ 6/100]    0 time: 0.9276s, mse: 0.00205716 \n",
      "Epoch [ 6/100]    1 time: 0.9399s, mse: 0.00223336 \n",
      "Epoch [ 6/100]    2 time: 0.9258s, mse: 0.00203476 \n",
      "Epoch [ 6/100]    3 time: 0.8773s, mse: 0.00344323 \n",
      "[*] Epoch: [ 6/100] time: 3.6708s, mse: 0.00244212\n",
      "Epoch [ 7/100]    0 time: 0.9215s, mse: 0.00243551 \n",
      "Epoch [ 7/100]    1 time: 0.9420s, mse: 0.01214830 \n",
      "Epoch [ 7/100]    2 time: 0.9295s, mse: 0.00166949 \n",
      "Epoch [ 7/100]    3 time: 0.8870s, mse: 0.00371925 \n",
      "[*] Epoch: [ 7/100] time: 3.6802s, mse: 0.00499314\n",
      "Epoch [ 8/100]    0 time: 0.9224s, mse: 0.00316008 \n",
      "Epoch [ 8/100]    1 time: 0.9323s, mse: 0.02368838 \n",
      "Epoch [ 8/100]    2 time: 0.9333s, mse: 0.03914604 \n",
      "Epoch [ 8/100]    3 time: 0.8785s, mse: 0.00366525 \n",
      "[*] Epoch: [ 8/100] time: 3.6667s, mse: 0.01741494\n",
      "Epoch [ 9/100]    0 time: 0.9237s, mse: 0.00227143 \n",
      "Epoch [ 9/100]    1 time: 0.9299s, mse: 0.00339318 \n",
      "Epoch [ 9/100]    2 time: 0.9469s, mse: 0.01869759 \n",
      "Epoch [ 9/100]    3 time: 0.9077s, mse: 0.00251504 \n",
      "[*] Epoch: [ 9/100] time: 3.7084s, mse: 0.00671931\n",
      "Epoch [10/100]    0 time: 0.9313s, mse: 0.00413507 \n",
      "Epoch [10/100]    1 time: 0.9321s, mse: 0.00298011 \n",
      "Epoch [10/100]    2 time: 0.9373s, mse: 0.00279575 \n",
      "Epoch [10/100]    3 time: 0.8926s, mse: 0.00506397 \n",
      "[*] Epoch: [10/100] time: 3.6936s, mse: 0.00374372\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [11/100]    0 time: 0.9573s, mse: 0.00296322 \n",
      "Epoch [11/100]    1 time: 0.9322s, mse: 0.00487471 \n",
      "Epoch [11/100]    2 time: 0.9373s, mse: 0.00279927 \n",
      "Epoch [11/100]    3 time: 0.8864s, mse: 0.00409214 \n",
      "[*] Epoch: [11/100] time: 3.7134s, mse: 0.00368234\n",
      "Epoch [12/100]    0 time: 0.9451s, mse: 0.00289592 \n",
      "Epoch [12/100]    1 time: 0.9563s, mse: 0.01574961 \n",
      "Epoch [12/100]    2 time: 0.9796s, mse: 0.00115169 \n",
      "Epoch [12/100]    3 time: 0.9099s, mse: 0.00133578 \n",
      "[*] Epoch: [12/100] time: 3.7912s, mse: 0.00528325\n",
      "Epoch [13/100]    0 time: 0.9477s, mse: 0.00157484 \n",
      "Epoch [13/100]    1 time: 0.9661s, mse: 0.00076694 \n",
      "Epoch [13/100]    2 time: 0.9358s, mse: 0.00052562 \n",
      "Epoch [13/100]    3 time: 0.8933s, mse: 0.00101889 \n",
      "[*] Epoch: [13/100] time: 3.7431s, mse: 0.00097157\n",
      "Epoch [14/100]    0 time: 0.9364s, mse: 0.00085707 \n",
      "Epoch [14/100]    1 time: 0.9491s, mse: 0.00131102 \n",
      "Epoch [14/100]    2 time: 0.9574s, mse: 0.00143711 \n",
      "Epoch [14/100]    3 time: 0.9157s, mse: 0.00197621 \n",
      "[*] Epoch: [14/100] time: 3.7589s, mse: 0.00139535\n",
      "Epoch [15/100]    0 time: 0.9873s, mse: 0.00068088 \n",
      "Epoch [15/100]    1 time: 0.9753s, mse: 0.00071886 \n",
      "Epoch [15/100]    2 time: 0.9316s, mse: 0.00176052 \n",
      "Epoch [15/100]    3 time: 0.9023s, mse: 0.00130403 \n",
      "[*] Epoch: [15/100] time: 3.7968s, mse: 0.00111607\n",
      "Epoch [16/100]    0 time: 0.9503s, mse: 0.00067179 \n",
      "Epoch [16/100]    1 time: 0.9567s, mse: 0.00079262 \n",
      "Epoch [16/100]    2 time: 0.9490s, mse: 0.00458829 \n",
      "Epoch [16/100]    3 time: 0.9044s, mse: 0.00115596 \n",
      "[*] Epoch: [16/100] time: 3.7607s, mse: 0.00180217\n",
      "Epoch [17/100]    0 time: 0.9485s, mse: 0.00124550 \n",
      "Epoch [17/100]    1 time: 0.9568s, mse: 0.00064598 \n",
      "Epoch [17/100]    2 time: 0.9341s, mse: 0.00096776 \n",
      "Epoch [17/100]    3 time: 0.9312s, mse: 0.01224908 \n",
      "[*] Epoch: [17/100] time: 3.7708s, mse: 0.00377708\n",
      "Epoch [18/100]    0 time: 0.9512s, mse: 0.00050048 \n",
      "Epoch [18/100]    1 time: 0.9500s, mse: 0.01581523 \n",
      "Epoch [18/100]    2 time: 0.9496s, mse: 0.01248962 \n",
      "Epoch [18/100]    3 time: 0.9088s, mse: 0.00117646 \n",
      "[*] Epoch: [18/100] time: 3.7604s, mse: 0.00749545\n",
      "Epoch [19/100]    0 time: 0.9392s, mse: 0.00146816 \n",
      "Epoch [19/100]    1 time: 0.9672s, mse: 0.00255619 \n",
      "Epoch [19/100]    2 time: 0.9691s, mse: 0.00679309 \n",
      "Epoch [19/100]    3 time: 0.9066s, mse: 0.00171387 \n",
      "[*] Epoch: [19/100] time: 3.7823s, mse: 0.00313283\n",
      "Epoch [20/100]    0 time: 0.9614s, mse: 0.00187951 \n",
      "Epoch [20/100]    1 time: 0.9576s, mse: 0.00247239 \n",
      "Epoch [20/100]    2 time: 0.9426s, mse: 0.00318595 \n",
      "Epoch [20/100]    3 time: 0.9640s, mse: 0.00421596 \n",
      "[*] Epoch: [20/100] time: 3.8259s, mse: 0.00293845\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [21/100]    0 time: 0.9520s, mse: 0.00161723 \n",
      "Epoch [21/100]    1 time: 0.9588s, mse: 0.00151898 \n",
      "Epoch [21/100]    2 time: 0.9472s, mse: 0.00164665 \n",
      "Epoch [21/100]    3 time: 0.9078s, mse: 0.03797131 \n",
      "[*] Epoch: [21/100] time: 3.7660s, mse: 0.01068854\n",
      "Epoch [22/100]    0 time: 0.9500s, mse: 0.00102045 \n",
      "Epoch [22/100]    1 time: 0.9268s, mse: 0.00201411 \n",
      "Epoch [22/100]    2 time: 0.9330s, mse: 0.00118948 \n",
      "Epoch [22/100]    3 time: 0.8891s, mse: 0.04140424 \n",
      "[*] Epoch: [22/100] time: 3.6991s, mse: 0.01140707\n",
      "Epoch [23/100]    0 time: 0.9314s, mse: 0.00132648 \n",
      "Epoch [23/100]    1 time: 0.9398s, mse: 0.00308575 \n",
      "Epoch [23/100]    2 time: 0.9350s, mse: 0.00120621 \n",
      "Epoch [23/100]    3 time: 0.8807s, mse: 0.00263616 \n",
      "[*] Epoch: [23/100] time: 3.6871s, mse: 0.00206365\n",
      "Epoch [24/100]    0 time: 0.9318s, mse: 0.00154386 \n",
      "Epoch [24/100]    1 time: 0.9289s, mse: 0.00342254 \n",
      "Epoch [24/100]    2 time: 0.9314s, mse: 0.00109112 \n",
      "Epoch [24/100]    3 time: 0.9167s, mse: 0.00222914 \n",
      "[*] Epoch: [24/100] time: 3.7091s, mse: 0.00207166\n",
      "Epoch [25/100]    0 time: 0.9526s, mse: 0.00156218 \n",
      "Epoch [25/100]    1 time: 0.9522s, mse: 0.00824255 \n",
      "Epoch [25/100]    2 time: 0.9326s, mse: 0.00124715 \n",
      "Epoch [25/100]    3 time: 0.8859s, mse: 0.00235183 \n",
      "[*] Epoch: [25/100] time: 3.7236s, mse: 0.00335093\n",
      "Epoch [26/100]    0 time: 0.9331s, mse: 0.00117189 \n",
      "Epoch [26/100]    1 time: 0.9320s, mse: 0.00161603 \n",
      "Epoch [26/100]    2 time: 0.9398s, mse: 0.01421403 \n",
      "Epoch [26/100]    3 time: 0.8906s, mse: 0.01365774 \n",
      "[*] Epoch: [26/100] time: 3.6958s, mse: 0.00766492\n",
      "Epoch [27/100]    0 time: 0.9376s, mse: 0.00086523 \n",
      "Epoch [27/100]    1 time: 0.9781s, mse: 0.00893492 \n",
      "Epoch [27/100]    2 time: 0.9575s, mse: 0.00097607 \n",
      "Epoch [27/100]    3 time: 0.9413s, mse: 0.00371037 \n",
      "[*] Epoch: [27/100] time: 3.8149s, mse: 0.00362164\n",
      "Epoch [28/100]    0 time: 0.9956s, mse: 0.00259748 \n",
      "Epoch [28/100]    1 time: 0.9946s, mse: 0.00543663 \n",
      "Epoch [28/100]    2 time: 0.9284s, mse: 0.00257833 \n",
      "Epoch [28/100]    3 time: 0.8964s, mse: 0.00294535 \n",
      "[*] Epoch: [28/100] time: 3.8153s, mse: 0.00338945\n",
      "Epoch [29/100]    0 time: 0.9334s, mse: 0.00525065 \n",
      "Epoch [29/100]    1 time: 0.9476s, mse: 0.00150968 \n",
      "Epoch [29/100]    2 time: 0.9401s, mse: 0.02415923 \n",
      "Epoch [29/100]    3 time: 0.8844s, mse: 0.00334081 \n",
      "[*] Epoch: [29/100] time: 3.7058s, mse: 0.00856509\n",
      "Epoch [30/100]    0 time: 0.9364s, mse: 0.00160213 \n",
      "Epoch [30/100]    1 time: 0.9588s, mse: 0.00536332 \n",
      "Epoch [30/100]    2 time: 0.9396s, mse: 0.01107072 \n",
      "Epoch [30/100]    3 time: 0.9138s, mse: 0.00132658 \n",
      "[*] Epoch: [30/100] time: 3.7488s, mse: 0.00484069\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TL] [*] Saved\n",
      "Epoch [31/100]    0 time: 0.9989s, mse: 0.00182154 \n",
      "Epoch [31/100]    1 time: 0.9518s, mse: 0.00174835 \n",
      "Epoch [31/100]    2 time: 0.9240s, mse: 0.04317007 \n",
      "Epoch [31/100]    3 time: 0.8883s, mse: 0.00708913 \n",
      "[*] Epoch: [31/100] time: 3.7633s, mse: 0.01345727\n",
      "Epoch [32/100]    0 time: 0.9336s, mse: 0.00536372 \n",
      "Epoch [32/100]    1 time: 0.9470s, mse: 0.00283436 \n",
      "Epoch [32/100]    2 time: 0.9261s, mse: 0.00244184 \n",
      "Epoch [32/100]    3 time: 0.9189s, mse: 0.00173305 \n",
      "[*] Epoch: [32/100] time: 3.7261s, mse: 0.00309324\n",
      "Epoch [33/100]    0 time: 0.9451s, mse: 0.00123014 \n",
      "Epoch [33/100]    1 time: 0.9611s, mse: 0.00139862 \n",
      "Epoch [33/100]    2 time: 1.0318s, mse: 0.00157274 \n",
      "Epoch [33/100]    3 time: 0.9161s, mse: 0.00120164 \n",
      "[*] Epoch: [33/100] time: 3.8543s, mse: 0.00135078\n",
      "Epoch [34/100]    0 time: 1.0102s, mse: 0.00101302 \n",
      "Epoch [34/100]    1 time: 0.9683s, mse: 0.00084487 \n",
      "Epoch [34/100]    2 time: 0.9365s, mse: 0.01576240 \n",
      "Epoch [34/100]    3 time: 0.9229s, mse: 0.00139163 \n",
      "[*] Epoch: [34/100] time: 3.8381s, mse: 0.00475298\n",
      "Epoch [35/100]    0 time: 0.9279s, mse: 0.00100500 \n",
      "Epoch [35/100]    1 time: 0.9511s, mse: 0.00520876 \n",
      "Epoch [35/100]    2 time: 0.9324s, mse: 0.00123659 \n",
      "Epoch [35/100]    3 time: 0.8901s, mse: 0.04653104 \n",
      "[*] Epoch: [35/100] time: 3.7019s, mse: 0.01349535\n",
      "Epoch [36/100]    0 time: 0.9570s, mse: 0.00101072 \n",
      "Epoch [36/100]    1 time: 0.9416s, mse: 0.00129020 \n",
      "Epoch [36/100]    2 time: 0.9336s, mse: 0.00108339 \n",
      "Epoch [36/100]    3 time: 0.8924s, mse: 0.01855789 \n",
      "[*] Epoch: [36/100] time: 3.7250s, mse: 0.00548555\n",
      "Epoch [37/100]    0 time: 0.9443s, mse: 0.00289721 \n",
      "Epoch [37/100]    1 time: 0.9378s, mse: 0.00696407 \n",
      "Epoch [37/100]    2 time: 0.9286s, mse: 0.00248346 \n",
      "Epoch [37/100]    3 time: 0.8783s, mse: 0.00372881 \n",
      "[*] Epoch: [37/100] time: 3.6892s, mse: 0.00401839\n",
      "Epoch [38/100]    0 time: 0.9222s, mse: 0.00140957 \n",
      "Epoch [38/100]    1 time: 0.9360s, mse: 0.00975789 \n",
      "Epoch [38/100]    2 time: 0.9256s, mse: 0.00087676 \n",
      "Epoch [38/100]    3 time: 0.9000s, mse: 0.00382213 \n",
      "[*] Epoch: [38/100] time: 3.6840s, mse: 0.00396659\n",
      "Epoch [39/100]    0 time: 0.9381s, mse: 0.00169451 \n",
      "Epoch [39/100]    1 time: 0.9442s, mse: 0.00067760 \n",
      "Epoch [39/100]    2 time: 0.9225s, mse: 0.00136442 \n",
      "Epoch [39/100]    3 time: 0.8992s, mse: 0.00101846 \n",
      "[*] Epoch: [39/100] time: 3.7042s, mse: 0.00118875\n",
      "Epoch [40/100]    0 time: 0.9429s, mse: 0.00091335 \n",
      "Epoch [40/100]    1 time: 0.9576s, mse: 0.02118698 \n",
      "Epoch [40/100]    2 time: 0.9482s, mse: 0.05086609 \n",
      "Epoch [40/100]    3 time: 0.8989s, mse: 0.01719401 \n",
      "[*] Epoch: [40/100] time: 3.7478s, mse: 0.02254011\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [41/100]    0 time: 0.9626s, mse: 0.00082460 \n",
      "Epoch [41/100]    1 time: 0.9471s, mse: 0.00164269 \n",
      "Epoch [41/100]    2 time: 0.9346s, mse: 0.00298470 \n",
      "Epoch [41/100]    3 time: 0.8968s, mse: 0.00231529 \n",
      "[*] Epoch: [41/100] time: 3.7415s, mse: 0.00194182\n",
      "Epoch [42/100]    0 time: 0.9305s, mse: 0.00359888 \n",
      "Epoch [42/100]    1 time: 0.9454s, mse: 0.00405005 \n",
      "Epoch [42/100]    2 time: 0.9197s, mse: 0.00334281 \n",
      "Epoch [42/100]    3 time: 0.8808s, mse: 0.00220810 \n",
      "[*] Epoch: [42/100] time: 3.6768s, mse: 0.00329996\n",
      "Epoch [43/100]    0 time: 0.9400s, mse: 0.00285343 \n",
      "Epoch [43/100]    1 time: 0.9333s, mse: 0.00481722 \n",
      "Epoch [43/100]    2 time: 0.9241s, mse: 0.00386632 \n",
      "Epoch [43/100]    3 time: 0.8835s, mse: 0.00144671 \n",
      "[*] Epoch: [43/100] time: 3.6812s, mse: 0.00324592\n",
      "Epoch [44/100]    0 time: 0.9215s, mse: 0.00127247 \n",
      "Epoch [44/100]    1 time: 0.9339s, mse: 0.01619430 \n",
      "Epoch [44/100]    2 time: 0.9297s, mse: 0.00065021 \n",
      "Epoch [44/100]    3 time: 0.8880s, mse: 0.00820904 \n",
      "[*] Epoch: [44/100] time: 3.6733s, mse: 0.00658151\n",
      "Epoch [45/100]    0 time: 0.9268s, mse: 0.00306600 \n",
      "Epoch [45/100]    1 time: 0.9368s, mse: 0.00071803 \n",
      "Epoch [45/100]    2 time: 0.9758s, mse: 0.00084666 \n",
      "Epoch [45/100]    3 time: 0.9542s, mse: 0.00105888 \n",
      "[*] Epoch: [45/100] time: 3.7938s, mse: 0.00142239\n",
      "Epoch [46/100]    0 time: 1.0214s, mse: 0.00112713 \n",
      "Epoch [46/100]    1 time: 1.1084s, mse: 0.00229599 \n",
      "Epoch [46/100]    2 time: 1.1054s, mse: 0.00082448 \n",
      "Epoch [46/100]    3 time: 0.9916s, mse: 0.00585164 \n",
      "[*] Epoch: [46/100] time: 4.2272s, mse: 0.00252481\n",
      "Epoch [47/100]    0 time: 0.9848s, mse: 0.00085271 \n",
      "Epoch [47/100]    1 time: 0.9441s, mse: 0.00464196 \n",
      "Epoch [47/100]    2 time: 0.9309s, mse: 0.00099370 \n",
      "Epoch [47/100]    3 time: 0.8998s, mse: 0.00139316 \n",
      "[*] Epoch: [47/100] time: 3.7599s, mse: 0.00197038\n",
      "Epoch [48/100]    0 time: 0.9415s, mse: 0.00064830 \n",
      "Epoch [48/100]    1 time: 0.9398s, mse: 0.00060622 \n",
      "Epoch [48/100]    2 time: 0.9445s, mse: 0.00051086 \n",
      "Epoch [48/100]    3 time: 0.8852s, mse: 0.00915301 \n",
      "[*] Epoch: [48/100] time: 3.7112s, mse: 0.00272960\n",
      "Epoch [49/100]    0 time: 0.9374s, mse: 0.00039083 \n",
      "Epoch [49/100]    1 time: 0.9422s, mse: 0.00082915 \n",
      "Epoch [49/100]    2 time: 0.9320s, mse: 0.04114748 \n",
      "Epoch [49/100]    3 time: 0.8811s, mse: 0.00089708 \n",
      "[*] Epoch: [49/100] time: 3.6929s, mse: 0.01081613\n",
      "Epoch [50/100]    0 time: 0.9387s, mse: 0.00243519 \n",
      "Epoch [50/100]    1 time: 0.9348s, mse: 0.00396689 \n",
      "Epoch [50/100]    2 time: 0.9341s, mse: 0.00140489 \n",
      "Epoch [50/100]    3 time: 0.8948s, mse: 0.00043812 \n",
      "[*] Epoch: [50/100] time: 3.7027s, mse: 0.00206128\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [51/100]    0 time: 0.9241s, mse: 0.00036848 \n",
      "Epoch [51/100]    1 time: 0.9401s, mse: 0.00786337 \n",
      "Epoch [51/100]    2 time: 0.9356s, mse: 0.00493889 \n",
      "Epoch [51/100]    3 time: 0.8852s, mse: 0.00073073 \n",
      "[*] Epoch: [51/100] time: 3.6852s, mse: 0.00347537\n",
      "Epoch [52/100]    0 time: 0.9260s, mse: 0.00059784 \n",
      "Epoch [52/100]    1 time: 0.9481s, mse: 0.00057314 \n",
      "Epoch [52/100]    2 time: 0.9509s, mse: 0.00082237 \n",
      "Epoch [52/100]    3 time: 0.9001s, mse: 0.00040329 \n",
      "[*] Epoch: [52/100] time: 3.7254s, mse: 0.00059916\n",
      "Epoch [53/100]    0 time: 0.9370s, mse: 0.00055041 \n",
      "Epoch [53/100]    1 time: 0.9356s, mse: 0.00195124 \n",
      "Epoch [53/100]    2 time: 0.9403s, mse: 0.00053752 \n",
      "Epoch [53/100]    3 time: 0.8881s, mse: 0.00133920 \n",
      "[*] Epoch: [53/100] time: 3.7013s, mse: 0.00109459\n",
      "Epoch [54/100]    0 time: 0.9431s, mse: 0.00096411 \n",
      "Epoch [54/100]    1 time: 0.9403s, mse: 0.00045628 \n",
      "Epoch [54/100]    2 time: 0.9271s, mse: 0.00066058 \n",
      "Epoch [54/100]    3 time: 0.8882s, mse: 0.00031101 \n",
      "[*] Epoch: [54/100] time: 3.6989s, mse: 0.00059799\n",
      "Epoch [55/100]    0 time: 0.9252s, mse: 0.00030003 \n",
      "Epoch [55/100]    1 time: 0.9352s, mse: 0.00218383 \n",
      "Epoch [55/100]    2 time: 0.9309s, mse: 0.01388507 \n",
      "Epoch [55/100]    3 time: 0.8759s, mse: 0.00055151 \n",
      "[*] Epoch: [55/100] time: 3.6674s, mse: 0.00423011\n",
      "Epoch [56/100]    0 time: 0.9219s, mse: 0.00027467 \n",
      "Epoch [56/100]    1 time: 0.9449s, mse: 0.00089218 \n",
      "Epoch [56/100]    2 time: 0.9363s, mse: 0.00086600 \n",
      "Epoch [56/100]    3 time: 0.8893s, mse: 0.00058665 \n",
      "[*] Epoch: [56/100] time: 3.6927s, mse: 0.00065488\n",
      "Epoch [57/100]    0 time: 0.9182s, mse: 0.00045755 \n",
      "Epoch [57/100]    1 time: 0.9445s, mse: 0.00114252 \n",
      "Epoch [57/100]    2 time: 0.9255s, mse: 0.00258334 \n",
      "Epoch [57/100]    3 time: 0.8806s, mse: 0.00063751 \n",
      "[*] Epoch: [57/100] time: 3.6690s, mse: 0.00120523\n",
      "Epoch [58/100]    0 time: 0.9218s, mse: 0.00029586 \n",
      "Epoch [58/100]    1 time: 0.9383s, mse: 0.00052165 \n",
      "Epoch [58/100]    2 time: 0.9417s, mse: 0.00054631 \n",
      "Epoch [58/100]    3 time: 0.8887s, mse: 0.00860543 \n",
      "[*] Epoch: [58/100] time: 3.6908s, mse: 0.00249231\n",
      "Epoch [59/100]    0 time: 0.9286s, mse: 0.00037252 \n",
      "Epoch [59/100]    1 time: 0.9433s, mse: 0.00046398 \n",
      "Epoch [59/100]    2 time: 0.9303s, mse: 0.00050468 \n",
      "Epoch [59/100]    3 time: 0.8985s, mse: 0.00053533 \n",
      "[*] Epoch: [59/100] time: 3.7009s, mse: 0.00046913\n",
      "Epoch [60/100]    0 time: 0.9351s, mse: 0.00360882 \n",
      "Epoch [60/100]    1 time: 0.9351s, mse: 0.00350928 \n",
      "Epoch [60/100]    2 time: 0.9284s, mse: 0.00507135 \n",
      "Epoch [60/100]    3 time: 0.8911s, mse: 0.00020537 \n",
      "[*] Epoch: [60/100] time: 3.6899s, mse: 0.00309871\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [61/100]    0 time: 0.9664s, mse: 0.00042951 \n",
      "Epoch [61/100]    1 time: 0.9488s, mse: 0.00250550 \n",
      "Epoch [61/100]    2 time: 0.9353s, mse: 0.00136835 \n",
      "Epoch [61/100]    3 time: 0.8926s, mse: 0.00039327 \n",
      "[*] Epoch: [61/100] time: 3.7434s, mse: 0.00117416\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [62/100]    0 time: 0.9336s, mse: 0.00154106 \n",
      "Epoch [62/100]    1 time: 0.9350s, mse: 0.00064188 \n",
      "Epoch [62/100]    2 time: 0.9369s, mse: 0.00055710 \n",
      "Epoch [62/100]    3 time: 0.8867s, mse: 0.00037875 \n",
      "[*] Epoch: [62/100] time: 3.6925s, mse: 0.00077970\n",
      "Epoch [63/100]    0 time: 0.9348s, mse: 0.00078128 \n",
      "Epoch [63/100]    1 time: 0.9303s, mse: 0.00170283 \n",
      "Epoch [63/100]    2 time: 0.9264s, mse: 0.00107444 \n",
      "Epoch [63/100]    3 time: 0.8866s, mse: 0.00024128 \n",
      "[*] Epoch: [63/100] time: 3.6784s, mse: 0.00094996\n",
      "Epoch [64/100]    0 time: 0.9275s, mse: 0.00100131 \n",
      "Epoch [64/100]    1 time: 0.9350s, mse: 0.00865388 \n",
      "Epoch [64/100]    2 time: 0.9347s, mse: 0.00071146 \n",
      "Epoch [64/100]    3 time: 0.8953s, mse: 0.00037998 \n",
      "[*] Epoch: [64/100] time: 3.6928s, mse: 0.00268666\n",
      "Epoch [65/100]    0 time: 0.9450s, mse: 0.00038980 \n",
      "Epoch [65/100]    1 time: 0.9482s, mse: 0.00160033 \n",
      "Epoch [65/100]    2 time: 0.9280s, mse: 0.00060800 \n",
      "Epoch [65/100]    3 time: 0.8814s, mse: 0.00376350 \n",
      "[*] Epoch: [65/100] time: 3.7029s, mse: 0.00159041\n",
      "Epoch [66/100]    0 time: 0.9307s, mse: 0.00023335 \n",
      "Epoch [66/100]    1 time: 0.9394s, mse: 0.00205025 \n",
      "Epoch [66/100]    2 time: 0.9261s, mse: 0.00121237 \n",
      "Epoch [66/100]    3 time: 0.8907s, mse: 0.00047142 \n",
      "[*] Epoch: [66/100] time: 3.6872s, mse: 0.00099185\n",
      "Epoch [67/100]    0 time: 0.9329s, mse: 0.00040907 \n",
      "Epoch [67/100]    1 time: 0.9280s, mse: 0.00408547 \n",
      "Epoch [67/100]    2 time: 0.9246s, mse: 0.00118242 \n",
      "Epoch [67/100]    3 time: 0.8930s, mse: 0.02165963 \n",
      "[*] Epoch: [67/100] time: 3.6788s, mse: 0.00683415\n",
      "Epoch [68/100]    0 time: 0.9327s, mse: 0.00018128 \n",
      "Epoch [68/100]    1 time: 0.9403s, mse: 0.00042644 \n",
      "Epoch [68/100]    2 time: 0.9275s, mse: 0.00062446 \n",
      "Epoch [68/100]    3 time: 0.8835s, mse: 0.00103407 \n",
      "[*] Epoch: [68/100] time: 3.6842s, mse: 0.00056656\n",
      "Epoch [69/100]    0 time: 0.9226s, mse: 0.00031499 \n",
      "Epoch [69/100]    1 time: 0.9319s, mse: 0.00037276 \n",
      "Epoch [69/100]    2 time: 0.9288s, mse: 0.00049185 \n",
      "Epoch [69/100]    3 time: 0.9015s, mse: 0.00650347 \n",
      "[*] Epoch: [69/100] time: 3.6851s, mse: 0.00192077\n",
      "Epoch [70/100]    0 time: 0.9291s, mse: 0.00026764 \n",
      "Epoch [70/100]    1 time: 0.9399s, mse: 0.00198823 \n",
      "Epoch [70/100]    2 time: 0.9301s, mse: 0.00062775 \n",
      "Epoch [70/100]    3 time: 0.8899s, mse: 0.01113100 \n",
      "[*] Epoch: [70/100] time: 3.6892s, mse: 0.00350365\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [71/100]    0 time: 0.9653s, mse: 0.00031549 \n",
      "Epoch [71/100]    1 time: 0.9458s, mse: 0.00082325 \n",
      "Epoch [71/100]    2 time: 0.9463s, mse: 0.01579883 \n",
      "Epoch [71/100]    3 time: 0.8950s, mse: 0.00028641 \n",
      "[*] Epoch: [71/100] time: 3.7526s, mse: 0.00430600\n",
      "Epoch [72/100]    0 time: 0.9528s, mse: 0.00026677 \n",
      "Epoch [72/100]    1 time: 0.9320s, mse: 0.00138924 \n",
      "Epoch [72/100]    2 time: 0.9225s, mse: 0.00110779 \n",
      "Epoch [72/100]    3 time: 0.8846s, mse: 0.00889025 \n",
      "[*] Epoch: [72/100] time: 3.6922s, mse: 0.00291351\n",
      "Epoch [73/100]    0 time: 0.9351s, mse: 0.00026025 \n",
      "Epoch [73/100]    1 time: 0.9353s, mse: 0.00052055 \n",
      "Epoch [73/100]    2 time: 0.9257s, mse: 0.00159850 \n",
      "Epoch [73/100]    3 time: 0.8893s, mse: 0.00030823 \n",
      "[*] Epoch: [73/100] time: 3.6856s, mse: 0.00067188\n",
      "Epoch [74/100]    0 time: 0.9276s, mse: 0.00018037 \n",
      "Epoch [74/100]    1 time: 0.9428s, mse: 0.00025558 \n",
      "Epoch [74/100]    2 time: 0.9268s, mse: 0.00284995 \n",
      "Epoch [74/100]    3 time: 0.8893s, mse: 0.00035525 \n",
      "[*] Epoch: [74/100] time: 3.6867s, mse: 0.00091029\n",
      "Epoch [75/100]    0 time: 0.9252s, mse: 0.00013018 \n",
      "Epoch [75/100]    1 time: 0.9329s, mse: 0.00828815 \n",
      "Epoch [75/100]    2 time: 0.9381s, mse: 0.00113628 \n",
      "Epoch [75/100]    3 time: 0.8773s, mse: 0.00032245 \n",
      "[*] Epoch: [75/100] time: 3.6738s, mse: 0.00246927\n",
      "Epoch [76/100]    0 time: 0.9333s, mse: 0.00017523 \n",
      "Epoch [76/100]    1 time: 0.9450s, mse: 0.00260716 \n",
      "Epoch [76/100]    2 time: 0.9302s, mse: 0.00115341 \n",
      "Epoch [76/100]    3 time: 0.8925s, mse: 0.00034112 \n",
      "[*] Epoch: [76/100] time: 3.7013s, mse: 0.00106923\n",
      "Epoch [77/100]    0 time: 0.9329s, mse: 0.00030632 \n",
      "Epoch [77/100]    1 time: 0.9436s, mse: 0.00184124 \n",
      "Epoch [77/100]    2 time: 0.9293s, mse: 0.02514801 \n",
      "Epoch [77/100]    3 time: 0.8925s, mse: 0.00583876 \n",
      "[*] Epoch: [77/100] time: 3.6985s, mse: 0.00828358\n",
      "Epoch [78/100]    0 time: 0.9256s, mse: 0.00028172 \n",
      "Epoch [78/100]    1 time: 0.9345s, mse: 0.00045332 \n",
      "Epoch [78/100]    2 time: 0.9284s, mse: 0.00261129 \n",
      "Epoch [78/100]    3 time: 0.8946s, mse: 0.00493695 \n",
      "[*] Epoch: [78/100] time: 3.6833s, mse: 0.00207082\n",
      "Epoch [79/100]    0 time: 0.9509s, mse: 0.00346549 \n",
      "Epoch [79/100]    1 time: 0.9403s, mse: 0.00067659 \n",
      "Epoch [79/100]    2 time: 0.9381s, mse: 0.00606710 \n",
      "Epoch [79/100]    3 time: 0.8884s, mse: 0.00015960 \n",
      "[*] Epoch: [79/100] time: 3.7180s, mse: 0.00259220\n",
      "Epoch [80/100]    0 time: 0.9281s, mse: 0.00010429 \n",
      "Epoch [80/100]    1 time: 0.9512s, mse: 0.00201104 \n",
      "Epoch [80/100]    2 time: 0.9275s, mse: 0.00122063 \n",
      "Epoch [80/100]    3 time: 0.8878s, mse: 0.00013504 \n",
      "[*] Epoch: [80/100] time: 3.6949s, mse: 0.00086775\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [81/100]    0 time: 0.9616s, mse: 0.00055120 \n",
      "Epoch [81/100]    1 time: 0.9382s, mse: 0.00086857 \n",
      "Epoch [81/100]    2 time: 0.9405s, mse: 0.00190825 \n",
      "Epoch [81/100]    3 time: 0.8872s, mse: 0.00018068 \n",
      "[*] Epoch: [81/100] time: 3.7279s, mse: 0.00087718\n",
      "Epoch [82/100]    0 time: 0.9227s, mse: 0.00014118 \n",
      "Epoch [82/100]    1 time: 0.9279s, mse: 0.00179144 \n",
      "Epoch [82/100]    2 time: 0.9291s, mse: 0.00191914 \n",
      "Epoch [82/100]    3 time: 0.8909s, mse: 0.00013907 \n",
      "[*] Epoch: [82/100] time: 3.6708s, mse: 0.00099771\n",
      "Epoch [83/100]    0 time: 0.9317s, mse: 0.00024585 \n",
      "Epoch [83/100]    1 time: 0.9454s, mse: 0.00216865 \n",
      "Epoch [83/100]    2 time: 0.9237s, mse: 0.00078405 \n",
      "Epoch [83/100]    3 time: 0.8889s, mse: 0.00022584 \n",
      "[*] Epoch: [83/100] time: 3.6900s, mse: 0.00085610\n",
      "Epoch [84/100]    0 time: 0.9241s, mse: 0.00010591 \n",
      "Epoch [84/100]    1 time: 0.9379s, mse: 0.00363772 \n",
      "Epoch [84/100]    2 time: 0.9281s, mse: 0.00051308 \n",
      "Epoch [84/100]    3 time: 0.8890s, mse: 0.00028042 \n",
      "[*] Epoch: [84/100] time: 3.6794s, mse: 0.00113428\n",
      "Epoch [85/100]    0 time: 0.9371s, mse: 0.00277596 \n",
      "Epoch [85/100]    1 time: 0.9453s, mse: 0.00205774 \n",
      "Epoch [85/100]    2 time: 0.9217s, mse: 0.00061591 \n",
      "Epoch [85/100]    3 time: 0.8832s, mse: 0.00024222 \n",
      "[*] Epoch: [85/100] time: 3.6876s, mse: 0.00142296\n",
      "Epoch [86/100]    0 time: 0.9651s, mse: 0.00011417 \n",
      "Epoch [86/100]    1 time: 0.9332s, mse: 0.00362592 \n",
      "Epoch [86/100]    2 time: 0.9330s, mse: 0.00063252 \n",
      "Epoch [86/100]    3 time: 0.8878s, mse: 0.00365020 \n",
      "[*] Epoch: [86/100] time: 3.7194s, mse: 0.00200570\n",
      "Epoch [87/100]    0 time: 0.9390s, mse: 0.00010594 \n",
      "Epoch [87/100]    1 time: 0.9270s, mse: 0.00087675 \n",
      "Epoch [87/100]    2 time: 0.9343s, mse: 0.00390192 \n",
      "Epoch [87/100]    3 time: 0.8914s, mse: 0.00023531 \n",
      "[*] Epoch: [87/100] time: 3.6919s, mse: 0.00127998\n",
      "Epoch [88/100]    0 time: 0.9325s, mse: 0.00028277 \n",
      "Epoch [88/100]    1 time: 0.9410s, mse: 0.00063288 \n",
      "Epoch [88/100]    2 time: 0.9291s, mse: 0.00043588 \n",
      "Epoch [88/100]    3 time: 0.8913s, mse: 0.00057362 \n",
      "[*] Epoch: [88/100] time: 3.6942s, mse: 0.00048129\n",
      "Epoch [89/100]    0 time: 0.9299s, mse: 0.00014843 \n",
      "Epoch [89/100]    1 time: 0.9458s, mse: 0.00111869 \n",
      "Epoch [89/100]    2 time: 0.9483s, mse: 0.00137171 \n",
      "Epoch [89/100]    3 time: 0.8924s, mse: 0.00030314 \n",
      "[*] Epoch: [89/100] time: 3.7167s, mse: 0.00073549\n",
      "Epoch [90/100]    0 time: 0.9208s, mse: 0.00010248 \n",
      "Epoch [90/100]    1 time: 0.9395s, mse: 0.00236426 \n",
      "Epoch [90/100]    2 time: 0.9316s, mse: 0.00327166 \n",
      "Epoch [90/100]    3 time: 0.8859s, mse: 0.02321876 \n",
      "[*] Epoch: [90/100] time: 3.6780s, mse: 0.00723929\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n",
      "Epoch [91/100]    0 time: 0.9322s, mse: 0.00023367 \n",
      "Epoch [91/100]    1 time: 0.9318s, mse: 0.00155381 \n",
      "Epoch [91/100]    2 time: 0.9281s, mse: 0.00057258 \n",
      "Epoch [91/100]    3 time: 0.8862s, mse: 0.00109277 \n",
      "[*] Epoch: [91/100] time: 3.6784s, mse: 0.00086321\n",
      "Epoch [92/100]    0 time: 0.9373s, mse: 0.00039391 \n",
      "Epoch [92/100]    1 time: 0.9346s, mse: 0.00262042 \n",
      "Epoch [92/100]    2 time: 0.9259s, mse: 0.00084665 \n",
      "Epoch [92/100]    3 time: 0.8904s, mse: 0.00083314 \n",
      "[*] Epoch: [92/100] time: 3.6884s, mse: 0.00117353\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [93/100]    0 time: 0.9262s, mse: 0.00015552 \n",
      "Epoch [93/100]    1 time: 0.9434s, mse: 0.00025694 \n",
      "Epoch [93/100]    2 time: 0.9356s, mse: 0.00100932 \n",
      "Epoch [93/100]    3 time: 0.8882s, mse: 0.00017151 \n",
      "[*] Epoch: [93/100] time: 3.6937s, mse: 0.00039832\n",
      "Epoch [94/100]    0 time: 0.9224s, mse: 0.00033664 \n",
      "Epoch [94/100]    1 time: 0.9351s, mse: 0.00045634 \n",
      "Epoch [94/100]    2 time: 0.9382s, mse: 0.00527926 \n",
      "Epoch [94/100]    3 time: 0.8883s, mse: 0.00016641 \n",
      "[*] Epoch: [94/100] time: 3.6842s, mse: 0.00155966\n",
      "Epoch [95/100]    0 time: 0.9358s, mse: 0.00056490 \n",
      "Epoch [95/100]    1 time: 0.9473s, mse: 0.00056219 \n",
      "Epoch [95/100]    2 time: 0.9365s, mse: 0.00173014 \n",
      "Epoch [95/100]    3 time: 0.8949s, mse: 0.00015542 \n",
      "[*] Epoch: [95/100] time: 3.7148s, mse: 0.00075316\n",
      "Epoch [96/100]    0 time: 0.9359s, mse: 0.00012418 \n",
      "Epoch [96/100]    1 time: 0.9308s, mse: 0.00032499 \n",
      "Epoch [96/100]    2 time: 0.9267s, mse: 0.00053001 \n",
      "Epoch [96/100]    3 time: 0.8850s, mse: 0.00011032 \n",
      "[*] Epoch: [96/100] time: 3.6787s, mse: 0.00027238\n",
      "Epoch [97/100]    0 time: 0.9294s, mse: 0.00016589 \n",
      "Epoch [97/100]    1 time: 0.9411s, mse: 0.00025838 \n",
      "Epoch [97/100]    2 time: 0.9280s, mse: 0.01759634 \n",
      "Epoch [97/100]    3 time: 0.9021s, mse: 0.00012252 \n",
      "[*] Epoch: [97/100] time: 3.7008s, mse: 0.00453578\n",
      "Epoch [98/100]    0 time: 0.9341s, mse: 0.00014910 \n",
      "Epoch [98/100]    1 time: 0.9349s, mse: 0.00028587 \n",
      "Epoch [98/100]    2 time: 0.9482s, mse: 0.00174095 \n",
      "Epoch [98/100]    3 time: 0.8855s, mse: 0.01484464 \n",
      "[*] Epoch: [98/100] time: 3.7029s, mse: 0.00425514\n",
      "Epoch [99/100]    0 time: 0.9270s, mse: 0.00015229 \n",
      "Epoch [99/100]    1 time: 0.9441s, mse: 0.00398940 \n",
      "Epoch [99/100]    2 time: 0.9288s, mse: 0.00062760 \n",
      "Epoch [99/100]    3 time: 0.8947s, mse: 0.01343486 \n",
      "[*] Epoch: [99/100] time: 3.6948s, mse: 0.00455104\n",
      "Epoch [100/100]    0 time: 0.9331s, mse: 0.00050674 \n",
      "Epoch [100/100]    1 time: 0.9334s, mse: 0.00300861 \n",
      "Epoch [100/100]    2 time: 0.9335s, mse: 0.00057232 \n",
      "Epoch [100/100]    3 time: 0.8960s, mse: 0.00008205 \n",
      "[*] Epoch: [100/100] time: 3.6962s, mse: 0.00104243\n",
      "[TL] [*] Saving TL params into training_dir/g_srgan_init.npz\n",
      "[TL] [*] Saved\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(0, n_epoch_init + 1):\n",
    "    epoch_time = time.time()\n",
    "    total_mse_loss, n_iter = 0, 0\n",
    "\n",
    "    for idx in range(0, len(train_data_list), batch_size):\n",
    "        step_time = time.time()\n",
    "        \n",
    "        batch_file_name = train_data_list[idx: idx+batch_size]\n",
    "        inputs_rgbs, label_raws = get_inputs_labels(train_data_dir, batch_file_name, 2)\n",
    "        \n",
    "        ## update G\n",
    "        errM, _ = sess.run([mse_loss, g_optim_init], {t_image: inputs_rgbs, t_target_image: label_raws})\n",
    "        print(\"Epoch [%2d/%2d] %4d time: %4.4fs, mse: %.8f \" % (epoch, n_epoch_init, n_iter, time.time() - step_time, errM))\n",
    "        total_mse_loss += errM\n",
    "        n_iter += 1\n",
    "    log = \"[*] Epoch: [%2d/%2d] time: %4.4fs, mse: %.8f\" % (epoch, n_epoch_init, time.time() - epoch_time, total_mse_loss / n_iter)\n",
    "    print(log)\n",
    "\n",
    "    ## quick evaluation on train set\n",
    "#     if (epoch != 0) and (epoch % 10 == 0):\n",
    "#         out = sess.run(net_g_test.outputs, {t_image: sample_imgs_96})  #; print('gen sub-image:', out.shape, out.min(), out.max())\n",
    "#         print(\"[*] save images\")\n",
    "#         tl.vis.save_images(out, [ni, ni], save_dir_ginit + '/train_%d.png' % epoch)\n",
    "\n",
    "    ## save model\n",
    "    if (epoch != 0) and (epoch % 10 == 0):\n",
    "        tl.files.save_npz(net_g.all_params, name=training_dir + '/g_srgan_init.npz', sess=sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ** init lr: 0.000100  decay_every_init: 1000, lr_decay: 0.100000 (for GAN)\n",
      "Epoch [ 0/2000]    0 time: 1.5955s, d_loss: 2.13979721 g_loss: 0.03410675 (mse: 0.000407 adv: 0.033700)\n",
      "Epoch [ 0/2000]    1 time: 1.5941s, d_loss: 2.55675888 g_loss: 0.00868321 (mse: 0.000189 adv: 0.008494)\n",
      "Epoch [ 0/2000]    2 time: 1.5318s, d_loss: 2.59092546 g_loss: 0.02898122 (mse: 0.006354 adv: 0.022627)\n",
      "Epoch [ 0/2000]    3 time: 1.4577s, d_loss: 2.33331823 g_loss: 0.00952866 (mse: 0.000309 adv: 0.009220)\n",
      "[*] Epoch: [ 0/2000] time: 6.1793s, d_loss: 2.40519994 g_loss: 0.02032496\n",
      "Epoch [ 1/2000]    0 time: 1.5098s, d_loss: 1.96535134 g_loss: 0.01540101 (mse: 0.000368 adv: 0.015033)\n",
      "Epoch [ 1/2000]    1 time: 1.5604s, d_loss: 0.69386184 g_loss: 0.01694207 (mse: 0.003474 adv: 0.013468)\n",
      "Epoch [ 1/2000]    2 time: 1.5769s, d_loss: 1.97620749 g_loss: 0.03274266 (mse: 0.018687 adv: 0.014056)\n",
      "Epoch [ 1/2000]    3 time: 1.4568s, d_loss: 7.11636019 g_loss: 0.00418333 (mse: 0.000659 adv: 0.003524)\n",
      "[*] Epoch: [ 1/2000] time: 6.1041s, d_loss: 2.93794522 g_loss: 0.01731727\n",
      "Epoch [ 2/2000]    0 time: 1.5182s, d_loss: 1.58695316 g_loss: 0.00145444 (mse: 0.000741 adv: 0.000714)\n",
      "Epoch [ 2/2000]    1 time: 1.5299s, d_loss: 6.28388119 g_loss: 0.00141486 (mse: 0.001023 adv: 0.000392)\n",
      "Epoch [ 2/2000]    2 time: 1.5504s, d_loss: 7.86682701 g_loss: 0.00372596 (mse: 0.003156 adv: 0.000570)\n",
      "Epoch [ 2/2000]    3 time: 1.4839s, d_loss: 2.37826681 g_loss: 0.01314932 (mse: 0.002423 adv: 0.010726)\n",
      "[*] Epoch: [ 2/2000] time: 6.0826s, d_loss: 4.52898204 g_loss: 0.00493614\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(0, 2 + 1):\n",
    "        ## update learning rate\n",
    "        if epoch != 0 and (epoch % decay_every == 0):\n",
    "            new_lr_decay = lr_decay**(epoch // decay_every)\n",
    "            sess.run(tf.assign(lr_v, lr_init * new_lr_decay))\n",
    "            log = \" ** new learning rate: %f (for GAN)\" % (lr_init * new_lr_decay)\n",
    "            print(log)\n",
    "        elif epoch == 0:\n",
    "            sess.run(tf.assign(lr_v, lr_init))\n",
    "            log = \" ** init lr: %f  decay_every_init: %d, lr_decay: %f (for GAN)\" % (lr_init, decay_every, lr_decay)\n",
    "            print(log)\n",
    "\n",
    "        epoch_time = time.time()\n",
    "        total_d_loss, total_g_loss, n_iter = 0, 0, 0\n",
    "\n",
    "        for idx in range(0, len(train_data_list), batch_size):\n",
    "            step_time = time.time()\n",
    "            batch_file_name = train_data_list[idx: idx+batch_size]\n",
    "            inputs_rgbs, label_raws = get_inputs_labels(train_data_dir, batch_file_name, 2)\n",
    "            ## update D\n",
    "            errD, _ = sess.run([d_loss, d_optim], {t_image: inputs_rgbs, t_target_image: label_raws})\n",
    "            ## update G\n",
    "            errG, errM, errA, _ = sess.run([g_loss, mse_loss, g_gan_loss, g_optim],\n",
    "                                                 {t_image: inputs_rgbs, t_target_image: label_raws})\n",
    "            print(\"Epoch [%2d/%2d] %4d time: %4.4fs, d_loss: %.8f g_loss: %.8f (mse: %.6f adv: %.6f)\" %\n",
    "                  (epoch, n_epoch, n_iter, time.time() - step_time, errD, errG, errM, errA))\n",
    "            total_d_loss += errD\n",
    "            total_g_loss += errG\n",
    "            n_iter += 1\n",
    "\n",
    "        log = \"[*] Epoch: [%2d/%2d] time: %4.4fs, d_loss: %.8f g_loss: %.8f\" % (epoch, n_epoch, time.time() - epoch_time, total_d_loss / n_iter,\n",
    "                                                                                total_g_loss / n_iter)\n",
    "        print(log)\n",
    "\n",
    "        ## quick evaluation on train set\n",
    "#         if (epoch != 0) and (epoch % 10 == 0):\n",
    "#             out = sess.run(net_g_test.outputs, {t_image: sample_imgs_96})  #; print('gen sub-image:', out.shape, out.min(), out.max())\n",
    "#             print(\"[*] save images\")\n",
    "#             tl.vis.save_images(out, [ni, ni], save_dir_gan + '/train_%d.png' % epoch)\n",
    "\n",
    "        ## save model\n",
    "        if (epoch != 0) and (epoch % 10 == 0):\n",
    "            tl.files.save_npz(net_g.all_params, name=training_dir + '/g_srgan.npz'.format(tl.global_flag['mode']), sess=sess)\n",
    "            tl.files.save_npz(net_d.all_params, name=training_dir + '/d_srgan.npz'.format(tl.global_flag['mode']), sess=sess)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}