restnet-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "import numpy as np\n",
    "import os\n",
    "import os.path\n",
    "import xlrd\n",
    "from sklearn import svm\n",
    "import numpy as np\n",
    "import random as rd\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000L, 3L) (10000L,)\n",
      "[[  4.75730154   4.29918937   6.41209383]\n",
      " [  9.71655231   5.51019235  11.17021077]] [ 1.  1.]\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "csvfile = open('angle.csv')\n",
    "reader = csv.reader(csvfile)\n",
    "\n",
    "y = []\n",
    "X = []\n",
    "\n",
    "data = []\n",
    "for line in reader:\n",
    "    data.append([float(line[0]),float(line[1]),float(line[2]),int(line[3])])\n",
    "\n",
    "csvfile.close() \n",
    "\n",
    "for i in range(10):\n",
    "    rd.shuffle(data)\n",
    "    \n",
    "data = np.array(data)\n",
    "X = data[:,:3]\n",
    "y = data[:,3]\n",
    "print X.shape,y.shape\n",
    "print X[:2],y[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 导入训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6000L, 3L) (6000L, 2L)\n",
      "(4000L, 3L) (4000L, 2L)\n"
     ]
    }
   ],
   "source": [
    "sc = preprocessing.StandardScaler()\n",
    "Xstd = sc.fit_transform(X)\n",
    "y_onehot = keras.utils.to_categorical(y, num_classes=2)\n",
    "\n",
    "Xtrain = Xstd[:6000]\n",
    "ytrain = y_onehot[:6000]\n",
    "\n",
    "Xtest = Xstd[6000:]\n",
    "ytest = y_onehot[6000:]\n",
    "print Xtrain.shape,ytrain.shape\n",
    "print Xtest.shape,ytest.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sigmoid 20层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6000 samples, validate on 4000 samples\n",
      "Epoch 1/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.8763 - acc: 0.5110 - val_loss: 0.8897 - val_acc: 0.4835\n",
      "Epoch 2/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.8388 - acc: 0.5110 - val_loss: 0.8515 - val_acc: 0.4835\n",
      "Epoch 3/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.8071 - acc: 0.5110 - val_loss: 0.8191 - val_acc: 0.4835\n",
      "Epoch 4/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7809 - acc: 0.5110 - val_loss: 0.7921 - val_acc: 0.4835\n",
      "Epoch 5/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7594 - acc: 0.5110 - val_loss: 0.7699 - val_acc: 0.4835\n",
      "Epoch 6/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7420 - acc: 0.5110 - val_loss: 0.7516 - val_acc: 0.4835\n",
      "Epoch 7/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7283 - acc: 0.5110 - val_loss: 0.7369 - val_acc: 0.4835\n",
      "Epoch 8/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7175 - acc: 0.5110 - val_loss: 0.7253 - val_acc: 0.4835\n",
      "Epoch 9/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7095 - acc: 0.5110 - val_loss: 0.7163 - val_acc: 0.4835\n",
      "Epoch 10/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7035 - acc: 0.5110 - val_loss: 0.7094 - val_acc: 0.4835\n",
      "Epoch 11/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6993 - acc: 0.5110 - val_loss: 0.7041 - val_acc: 0.4835\n",
      "Epoch 12/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6965 - acc: 0.5110 - val_loss: 0.7006 - val_acc: 0.4835\n",
      "Epoch 13/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6948 - acc: 0.5110 - val_loss: 0.6982 - val_acc: 0.4835\n",
      "Epoch 14/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6939 - acc: 0.5110 - val_loss: 0.6967 - val_acc: 0.4835\n",
      "Epoch 15/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6934 - acc: 0.5110 - val_loss: 0.6957 - val_acc: 0.4835\n",
      "Epoch 16/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6931 - acc: 0.5110 - val_loss: 0.6951 - val_acc: 0.4835\n",
      "Epoch 17/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6947 - val_acc: 0.4835\n",
      "Epoch 18/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6945 - val_acc: 0.4835\n",
      "Epoch 19/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6943 - val_acc: 0.4835\n",
      "Epoch 20/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 21/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 22/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 23/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 24/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 25/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 26/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 27/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 28/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 29/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 30/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 31/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 32/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 33/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 34/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 35/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 36/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 37/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 38/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 39/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 40/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 41/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 42/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 43/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 44/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 45/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 46/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 47/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 48/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 49/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 50/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 51/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 52/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 53/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 54/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 55/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 56/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 57/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 58/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 59/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 60/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 61/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 62/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 63/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 64/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 65/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 66/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6943 - val_acc: 0.4835\n",
      "Epoch 67/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6943 - val_acc: 0.4835\n",
      "Epoch 68/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 69/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 70/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 71/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 72/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 73/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 74/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 75/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 76/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 77/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 78/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 79/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 80/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 81/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 82/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 83/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 84/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 85/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 86/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 87/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 88/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 89/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 90/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 91/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 92/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 93/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 94/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 95/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 96/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 97/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 98/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 99/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 100/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 101/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 102/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 103/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 104/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 105/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 106/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 107/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 108/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 109/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 110/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 111/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 112/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 113/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 114/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 115/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 116/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 117/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 118/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 119/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 120/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 121/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 122/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 123/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 124/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 125/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 126/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 127/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 128/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 129/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 130/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 131/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 132/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 133/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 134/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 135/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 136/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 137/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 138/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 139/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 140/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 141/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 142/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 143/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 144/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 145/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 146/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6939 - val_acc: 0.4835\n",
      "Epoch 147/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 148/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 149/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6939 - val_acc: 0.4835\n",
      "Epoch 150/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 151/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 152/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 153/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 154/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 155/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 156/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 157/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 158/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 159/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 160/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 161/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 162/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 163/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 164/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 165/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 166/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 167/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 168/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6939 - val_acc: 0.4835\n",
      "Epoch 169/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 170/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 171/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 172/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 173/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 174/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 175/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 176/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 177/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 178/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 179/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 180/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 181/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 182/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 183/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 184/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 185/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 186/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 187/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 188/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 189/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 190/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 191/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 192/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 193/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 194/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 195/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6943 - val_acc: 0.4835\n",
      "Epoch 196/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 197/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 198/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 199/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 200/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x3adc9080>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input0 = keras.layers.Input(shape=(3,))\n",
    "out0 = keras.layers.Dense(units=3, activation='sigmoid')(input0)\n",
    "out1 = keras.layers.Dense(units=3, activation='sigmoid')(out0)\n",
    "# out1 = keras.layers.add([out1, input0])\n",
    "\n",
    "out2 = keras.layers.Dense(units=3, activation='sigmoid')(out1)\n",
    "out3 = keras.layers.Dense(units=3, activation='sigmoid')(out2)\n",
    "# out3 = keras.layers.add([out3, out1])\n",
    "\n",
    "out4 = keras.layers.Dense(units=3, activation='sigmoid')(out3)\n",
    "out5 = keras.layers.Dense(units=3, activation='sigmoid')(out4)\n",
    "# out5 = keras.layers.add([out5, out3])\n",
    "\n",
    "out6 = keras.layers.Dense(units=3, activation='sigmoid')(out5)\n",
    "out7 = keras.layers.Dense(units=3, activation='sigmoid')(out6)\n",
    "# out7 = keras.layers.add([out7, out5])\n",
    "\n",
    "out8 = keras.layers.Dense(units=3, activation='sigmoid')(out7)\n",
    "out9 = keras.layers.Dense(units=3, activation='sigmoid')(out8)\n",
    "# out9 = keras.layers.add([out9, out7])\n",
    "\n",
    "out10 = keras.layers.Dense(units=3, activation='sigmoid')(out9)\n",
    "out11 = keras.layers.Dense(units=3, activation='sigmoid')(out10)\n",
    "# out11 = keras.layers.add([out11, out9])\n",
    "\n",
    "out12 = keras.layers.Dense(units=3, activation='sigmoid')(out11)\n",
    "out13 = keras.layers.Dense(units=3, activation='sigmoid')(out12)\n",
    "# out13 = keras.layers.add([out13, out11])\n",
    "\n",
    "out14 = keras.layers.Dense(units=3, activation='sigmoid')(out13)\n",
    "out15 = keras.layers.Dense(units=3, activation='sigmoid')(out14)\n",
    "# out15 = keras.layers.add([out15, out13])\n",
    "\n",
    "out16 = keras.layers.Dense(units=3, activation='sigmoid')(out15)\n",
    "out17 = keras.layers.Dense(units=3, activation='sigmoid')(out16)\n",
    "# out17 = keras.layers.add([out17, out15])\n",
    "\n",
    "out18 = keras.layers.Dense(units=3, activation='sigmoid')(out17)\n",
    "out19 = keras.layers.Dense(units=3, activation='sigmoid')(out18)\n",
    "# out19 = keras.layers.add([out19, out17])\n",
    "\n",
    "out20 = keras.layers.Dense(units=3, activation='sigmoid')(out19)\n",
    "out21 = keras.layers.Dense(units=3, activation='sigmoid')(out20)\n",
    "# out21 = keras.layers.add([out21, out19])\n",
    "\n",
    "out = keras.layers.Dense(units=2, activation='softmax')(out21)\n",
    "model = keras.models.Model(inputs=input0, outputs=out)\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model, to_file='mode1.png',show_shapes=True)\n",
    "\n",
    "model.fit(Xtrain, ytrain,\n",
    "          batch_size=128,\n",
    "          epochs=200,\n",
    "          verbose=1,\n",
    "          validation_data=(Xtest, ytest))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sigmoid 20层 + restnet 直连结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6000 samples, validate on 4000 samples\n",
      "Epoch 1/200\n",
      "6000/6000 [==============================] - 0s - loss: 7.5484 - acc: 0.5110 - val_loss: 7.6239 - val_acc: 0.4835\n",
      "Epoch 2/200\n",
      "6000/6000 [==============================] - 0s - loss: 6.5636 - acc: 0.5110 - val_loss: 6.0289 - val_acc: 0.4835\n",
      "Epoch 3/200\n",
      "6000/6000 [==============================] - 0s - loss: 4.9611 - acc: 0.5110 - val_loss: 4.4310 - val_acc: 0.4835\n",
      "Epoch 4/200\n",
      "6000/6000 [==============================] - 0s - loss: 3.6190 - acc: 0.5110 - val_loss: 3.2119 - val_acc: 0.4835\n",
      "Epoch 5/200\n",
      "6000/6000 [==============================] - 0s - loss: 2.6100 - acc: 0.5110 - val_loss: 2.2880 - val_acc: 0.4835\n",
      "Epoch 6/200\n",
      "6000/6000 [==============================] - 0s - loss: 1.8365 - acc: 0.5110 - val_loss: 1.5823 - val_acc: 0.4835\n",
      "Epoch 7/200\n",
      "6000/6000 [==============================] - 0s - loss: 1.2803 - acc: 0.5110 - val_loss: 1.1164 - val_acc: 0.4835\n",
      "Epoch 8/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.9627 - acc: 0.5038 - val_loss: 0.8891 - val_acc: 0.4482\n",
      "Epoch 9/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.8336 - acc: 0.4338 - val_loss: 0.8069 - val_acc: 0.4238\n",
      "Epoch 10/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7913 - acc: 0.4257 - val_loss: 0.7775 - val_acc: 0.4355\n",
      "Epoch 11/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7753 - acc: 0.4372 - val_loss: 0.7630 - val_acc: 0.4495\n",
      "Epoch 12/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7656 - acc: 0.4482 - val_loss: 0.7537 - val_acc: 0.4572\n",
      "Epoch 13/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7571 - acc: 0.4560 - val_loss: 0.7453 - val_acc: 0.4698\n",
      "Epoch 14/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7488 - acc: 0.4657 - val_loss: 0.7370 - val_acc: 0.4793\n",
      "Epoch 15/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7404 - acc: 0.4757 - val_loss: 0.7292 - val_acc: 0.4860\n",
      "Epoch 16/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7320 - acc: 0.4860 - val_loss: 0.7215 - val_acc: 0.4963\n",
      "Epoch 17/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7235 - acc: 0.4935 - val_loss: 0.7149 - val_acc: 0.5098\n",
      "Epoch 18/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7153 - acc: 0.5052 - val_loss: 0.7065 - val_acc: 0.5210\n",
      "Epoch 19/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7071 - acc: 0.5225 - val_loss: 0.6994 - val_acc: 0.5333\n",
      "Epoch 20/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6991 - acc: 0.5390 - val_loss: 0.6922 - val_acc: 0.5497\n",
      "Epoch 21/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6915 - acc: 0.5565 - val_loss: 0.6852 - val_acc: 0.5647\n",
      "Epoch 22/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6841 - acc: 0.5727 - val_loss: 0.6788 - val_acc: 0.5805\n",
      "Epoch 23/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6770 - acc: 0.5855 - val_loss: 0.6718 - val_acc: 0.5945\n",
      "Epoch 24/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6704 - acc: 0.5997 - val_loss: 0.6657 - val_acc: 0.6090\n",
      "Epoch 25/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6639 - acc: 0.6072 - val_loss: 0.6604 - val_acc: 0.6195\n",
      "Epoch 26/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6576 - acc: 0.6197 - val_loss: 0.6548 - val_acc: 0.6275\n",
      "Epoch 27/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6514 - acc: 0.6270 - val_loss: 0.6493 - val_acc: 0.6362\n",
      "Epoch 28/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6456 - acc: 0.6350 - val_loss: 0.6433 - val_acc: 0.6430\n",
      "Epoch 29/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6399 - acc: 0.6405 - val_loss: 0.6390 - val_acc: 0.6472\n",
      "Epoch 30/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6344 - acc: 0.6453 - val_loss: 0.6337 - val_acc: 0.6518\n",
      "Epoch 31/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6294 - acc: 0.6507 - val_loss: 0.6298 - val_acc: 0.6580\n",
      "Epoch 32/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6246 - acc: 0.6572 - val_loss: 0.6257 - val_acc: 0.6633\n",
      "Epoch 33/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6202 - acc: 0.6553 - val_loss: 0.6220 - val_acc: 0.6627\n",
      "Epoch 34/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6160 - acc: 0.6592 - val_loss: 0.6184 - val_acc: 0.6663\n",
      "Epoch 35/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6121 - acc: 0.6625 - val_loss: 0.6153 - val_acc: 0.6685\n",
      "Epoch 36/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6085 - acc: 0.6642 - val_loss: 0.6116 - val_acc: 0.6665\n",
      "Epoch 37/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6052 - acc: 0.6647 - val_loss: 0.6083 - val_acc: 0.6680\n",
      "Epoch 38/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6020 - acc: 0.6670 - val_loss: 0.6061 - val_acc: 0.6715\n",
      "Epoch 39/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5992 - acc: 0.6653 - val_loss: 0.6036 - val_acc: 0.6723\n",
      "Epoch 40/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5965 - acc: 0.6705 - val_loss: 0.6013 - val_acc: 0.6743\n",
      "Epoch 41/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5939 - acc: 0.6712 - val_loss: 0.5984 - val_acc: 0.6725\n",
      "Epoch 42/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5913 - acc: 0.6705 - val_loss: 0.5975 - val_acc: 0.6793\n",
      "Epoch 43/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5889 - acc: 0.6728 - val_loss: 0.5944 - val_acc: 0.6773\n",
      "Epoch 44/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5866 - acc: 0.6785 - val_loss: 0.5920 - val_acc: 0.6770\n",
      "Epoch 45/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5843 - acc: 0.6782 - val_loss: 0.5901 - val_acc: 0.6770\n",
      "Epoch 46/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5820 - acc: 0.6827 - val_loss: 0.5882 - val_acc: 0.6830\n",
      "Epoch 47/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5799 - acc: 0.6835 - val_loss: 0.5864 - val_acc: 0.6850\n",
      "Epoch 48/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5777 - acc: 0.6843 - val_loss: 0.5850 - val_acc: 0.6890\n",
      "Epoch 49/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5758 - acc: 0.6875 - val_loss: 0.5828 - val_acc: 0.6893\n",
      "Epoch 50/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5738 - acc: 0.6880 - val_loss: 0.5815 - val_acc: 0.6915\n",
      "Epoch 51/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5719 - acc: 0.6940 - val_loss: 0.5786 - val_acc: 0.6885\n",
      "Epoch 52/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5700 - acc: 0.6915 - val_loss: 0.5777 - val_acc: 0.6937\n",
      "Epoch 53/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5682 - acc: 0.6982 - val_loss: 0.5755 - val_acc: 0.6947\n",
      "Epoch 54/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5665 - acc: 0.6967 - val_loss: 0.5746 - val_acc: 0.6990\n",
      "Epoch 55/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5648 - acc: 0.7027 - val_loss: 0.5729 - val_acc: 0.7010\n",
      "Epoch 56/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5631 - acc: 0.7043 - val_loss: 0.5712 - val_acc: 0.7023\n",
      "Epoch 57/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5614 - acc: 0.7028 - val_loss: 0.5708 - val_acc: 0.7115\n",
      "Epoch 58/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5599 - acc: 0.7078 - val_loss: 0.5677 - val_acc: 0.7017\n",
      "Epoch 59/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5581 - acc: 0.7082 - val_loss: 0.5663 - val_acc: 0.7047\n",
      "Epoch 60/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5565 - acc: 0.7100 - val_loss: 0.5645 - val_acc: 0.7045\n",
      "Epoch 61/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5551 - acc: 0.7130 - val_loss: 0.5624 - val_acc: 0.7000\n",
      "Epoch 62/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5535 - acc: 0.7103 - val_loss: 0.5619 - val_acc: 0.7123\n",
      "Epoch 63/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5519 - acc: 0.7142 - val_loss: 0.5599 - val_acc: 0.7073\n",
      "Epoch 64/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5504 - acc: 0.7168 - val_loss: 0.5584 - val_acc: 0.7077\n",
      "Epoch 65/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5490 - acc: 0.7143 - val_loss: 0.5578 - val_acc: 0.7205\n",
      "Epoch 66/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5473 - acc: 0.7165 - val_loss: 0.5568 - val_acc: 0.7230\n",
      "Epoch 67/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5460 - acc: 0.7225 - val_loss: 0.5544 - val_acc: 0.7177\n",
      "Epoch 68/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5445 - acc: 0.7190 - val_loss: 0.5535 - val_acc: 0.7238\n",
      "Epoch 69/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5430 - acc: 0.7235 - val_loss: 0.5515 - val_acc: 0.7228\n",
      "Epoch 70/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5416 - acc: 0.7212 - val_loss: 0.5505 - val_acc: 0.7248\n",
      "Epoch 71/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5400 - acc: 0.7223 - val_loss: 0.5495 - val_acc: 0.7280\n",
      "Epoch 72/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5388 - acc: 0.7275 - val_loss: 0.5486 - val_acc: 0.7300\n",
      "Epoch 73/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5372 - acc: 0.7305 - val_loss: 0.5466 - val_acc: 0.7288\n",
      "Epoch 74/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5359 - acc: 0.7307 - val_loss: 0.5451 - val_acc: 0.7288\n",
      "Epoch 75/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5343 - acc: 0.7335 - val_loss: 0.5434 - val_acc: 0.7272\n",
      "Epoch 76/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5330 - acc: 0.7358 - val_loss: 0.5421 - val_acc: 0.7282\n",
      "Epoch 77/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5316 - acc: 0.7360 - val_loss: 0.5414 - val_acc: 0.7335\n",
      "Epoch 78/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5302 - acc: 0.7375 - val_loss: 0.5411 - val_acc: 0.7408\n",
      "Epoch 79/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5290 - acc: 0.7418 - val_loss: 0.5392 - val_acc: 0.7378\n",
      "Epoch 80/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5276 - acc: 0.7432 - val_loss: 0.5381 - val_acc: 0.7400\n",
      "Epoch 81/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5263 - acc: 0.7458 - val_loss: 0.5360 - val_acc: 0.7352\n",
      "Epoch 82/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5250 - acc: 0.7447 - val_loss: 0.5350 - val_acc: 0.7382\n",
      "Epoch 83/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5237 - acc: 0.7482 - val_loss: 0.5334 - val_acc: 0.7368\n",
      "Epoch 84/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5225 - acc: 0.7465 - val_loss: 0.5321 - val_acc: 0.7373\n",
      "Epoch 85/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5211 - acc: 0.7460 - val_loss: 0.5314 - val_acc: 0.7438\n",
      "Epoch 86/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5199 - acc: 0.7530 - val_loss: 0.5298 - val_acc: 0.7428\n",
      "Epoch 87/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5186 - acc: 0.7538 - val_loss: 0.5283 - val_acc: 0.7418\n",
      "Epoch 88/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5173 - acc: 0.7550 - val_loss: 0.5270 - val_acc: 0.7422\n",
      "Epoch 89/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5158 - acc: 0.7545 - val_loss: 0.5270 - val_acc: 0.7505\n",
      "Epoch 90/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5148 - acc: 0.7572 - val_loss: 0.5252 - val_acc: 0.7465\n",
      "Epoch 91/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5135 - acc: 0.7577 - val_loss: 0.5238 - val_acc: 0.7458\n",
      "Epoch 92/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5122 - acc: 0.7572 - val_loss: 0.5234 - val_acc: 0.7533\n",
      "Epoch 93/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5110 - acc: 0.7597 - val_loss: 0.5224 - val_acc: 0.7550\n",
      "Epoch 94/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5097 - acc: 0.7635 - val_loss: 0.5200 - val_acc: 0.7475\n",
      "Epoch 95/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5084 - acc: 0.7628 - val_loss: 0.5183 - val_acc: 0.7438\n",
      "Epoch 96/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5073 - acc: 0.7585 - val_loss: 0.5178 - val_acc: 0.7490\n",
      "Epoch 97/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5060 - acc: 0.7653 - val_loss: 0.5163 - val_acc: 0.7475\n",
      "Epoch 98/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5046 - acc: 0.7667 - val_loss: 0.5143 - val_acc: 0.7420\n",
      "Epoch 99/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5037 - acc: 0.7612 - val_loss: 0.5140 - val_acc: 0.7500\n",
      "Epoch 100/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5022 - acc: 0.7637 - val_loss: 0.5143 - val_acc: 0.7580\n",
      "Epoch 101/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5012 - acc: 0.7680 - val_loss: 0.5118 - val_acc: 0.7535\n",
      "Epoch 102/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5000 - acc: 0.7665 - val_loss: 0.5105 - val_acc: 0.7535\n",
      "Epoch 103/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4987 - acc: 0.7673 - val_loss: 0.5093 - val_acc: 0.7530\n",
      "Epoch 104/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4974 - acc: 0.7663 - val_loss: 0.5079 - val_acc: 0.7515\n",
      "Epoch 105/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4963 - acc: 0.7662 - val_loss: 0.5075 - val_acc: 0.7572\n",
      "Epoch 106/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4950 - acc: 0.7698 - val_loss: 0.5054 - val_acc: 0.7522\n",
      "Epoch 107/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4938 - acc: 0.7637 - val_loss: 0.5062 - val_acc: 0.7610\n",
      "Epoch 108/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4927 - acc: 0.7717 - val_loss: 0.5035 - val_acc: 0.7570\n",
      "Epoch 109/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4914 - acc: 0.7697 - val_loss: 0.5021 - val_acc: 0.7562\n",
      "Epoch 110/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4901 - acc: 0.7738 - val_loss: 0.5004 - val_acc: 0.7522\n",
      "Epoch 111/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4890 - acc: 0.7683 - val_loss: 0.5003 - val_acc: 0.7600\n",
      "Epoch 112/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4878 - acc: 0.7712 - val_loss: 0.4991 - val_acc: 0.7600\n",
      "Epoch 113/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4864 - acc: 0.7723 - val_loss: 0.4992 - val_acc: 0.7650\n",
      "Epoch 114/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4853 - acc: 0.7752 - val_loss: 0.4973 - val_acc: 0.7625\n",
      "Epoch 115/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4841 - acc: 0.7755 - val_loss: 0.4953 - val_acc: 0.7600\n",
      "Epoch 116/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4828 - acc: 0.7743 - val_loss: 0.4938 - val_acc: 0.7592\n",
      "Epoch 117/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4817 - acc: 0.7747 - val_loss: 0.4927 - val_acc: 0.7600\n",
      "Epoch 118/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4804 - acc: 0.7760 - val_loss: 0.4914 - val_acc: 0.7602\n",
      "Epoch 119/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4792 - acc: 0.7748 - val_loss: 0.4906 - val_acc: 0.7622\n",
      "Epoch 120/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4779 - acc: 0.7800 - val_loss: 0.4885 - val_acc: 0.7590\n",
      "Epoch 121/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4768 - acc: 0.7763 - val_loss: 0.4880 - val_acc: 0.7635\n",
      "Epoch 122/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4753 - acc: 0.7823 - val_loss: 0.4857 - val_acc: 0.7578\n",
      "Epoch 123/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4744 - acc: 0.7768 - val_loss: 0.4850 - val_acc: 0.7612\n",
      "Epoch 124/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4731 - acc: 0.7778 - val_loss: 0.4850 - val_acc: 0.7675\n",
      "Epoch 125/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4718 - acc: 0.7838 - val_loss: 0.4824 - val_acc: 0.7618\n",
      "Epoch 126/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4706 - acc: 0.7810 - val_loss: 0.4814 - val_acc: 0.7645\n",
      "Epoch 127/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4694 - acc: 0.7793 - val_loss: 0.4808 - val_acc: 0.7672\n",
      "Epoch 128/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4680 - acc: 0.7823 - val_loss: 0.4796 - val_acc: 0.7708\n",
      "Epoch 129/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4668 - acc: 0.7850 - val_loss: 0.4779 - val_acc: 0.7693\n",
      "Epoch 130/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4654 - acc: 0.7863 - val_loss: 0.4759 - val_acc: 0.7642\n",
      "Epoch 131/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4641 - acc: 0.7830 - val_loss: 0.4764 - val_acc: 0.7752\n",
      "Epoch 132/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4628 - acc: 0.7882 - val_loss: 0.4734 - val_acc: 0.7660\n",
      "Epoch 133/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4616 - acc: 0.7865 - val_loss: 0.4726 - val_acc: 0.7715\n",
      "Epoch 134/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4601 - acc: 0.7837 - val_loss: 0.4719 - val_acc: 0.7758\n",
      "Epoch 135/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4588 - acc: 0.7900 - val_loss: 0.4703 - val_acc: 0.7758\n",
      "Epoch 136/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4575 - acc: 0.7893 - val_loss: 0.4685 - val_acc: 0.7758\n",
      "Epoch 137/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4561 - acc: 0.7888 - val_loss: 0.4674 - val_acc: 0.7772\n",
      "Epoch 138/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4547 - acc: 0.7908 - val_loss: 0.4654 - val_acc: 0.7750\n",
      "Epoch 139/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4533 - acc: 0.7908 - val_loss: 0.4639 - val_acc: 0.7755\n",
      "Epoch 140/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4517 - acc: 0.7868 - val_loss: 0.4644 - val_acc: 0.7775\n",
      "Epoch 141/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4505 - acc: 0.7910 - val_loss: 0.4616 - val_acc: 0.7780\n",
      "Epoch 142/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4490 - acc: 0.7917 - val_loss: 0.4592 - val_acc: 0.7752\n",
      "Epoch 143/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4476 - acc: 0.7925 - val_loss: 0.4582 - val_acc: 0.7798\n",
      "Epoch 144/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4461 - acc: 0.7927 - val_loss: 0.4572 - val_acc: 0.7802\n",
      "Epoch 145/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4447 - acc: 0.7927 - val_loss: 0.4559 - val_acc: 0.7800\n",
      "Epoch 146/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4432 - acc: 0.7942 - val_loss: 0.4532 - val_acc: 0.7805\n",
      "Epoch 147/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4415 - acc: 0.7932 - val_loss: 0.4536 - val_acc: 0.7808\n",
      "Epoch 148/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4402 - acc: 0.7943 - val_loss: 0.4513 - val_acc: 0.7810\n",
      "Epoch 149/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4387 - acc: 0.7952 - val_loss: 0.4495 - val_acc: 0.7810\n",
      "Epoch 150/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4372 - acc: 0.7948 - val_loss: 0.4476 - val_acc: 0.7827\n",
      "Epoch 151/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4356 - acc: 0.7958 - val_loss: 0.4460 - val_acc: 0.7827\n",
      "Epoch 152/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4340 - acc: 0.7955 - val_loss: 0.4438 - val_acc: 0.7843\n",
      "Epoch 153/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4325 - acc: 0.7945 - val_loss: 0.4427 - val_acc: 0.7843\n",
      "Epoch 154/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4309 - acc: 0.7963 - val_loss: 0.4415 - val_acc: 0.7845\n",
      "Epoch 155/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4292 - acc: 0.7965 - val_loss: 0.4391 - val_acc: 0.7850\n",
      "Epoch 156/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4276 - acc: 0.7970 - val_loss: 0.4375 - val_acc: 0.7853\n",
      "Epoch 157/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4259 - acc: 0.7980 - val_loss: 0.4356 - val_acc: 0.7850\n",
      "Epoch 158/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4241 - acc: 0.7973 - val_loss: 0.4347 - val_acc: 0.7860\n",
      "Epoch 159/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4224 - acc: 0.7980 - val_loss: 0.4330 - val_acc: 0.7863\n",
      "Epoch 160/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4206 - acc: 0.7978 - val_loss: 0.4321 - val_acc: 0.7867\n",
      "Epoch 161/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4190 - acc: 0.7983 - val_loss: 0.4288 - val_acc: 0.7865\n",
      "Epoch 162/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4171 - acc: 0.7985 - val_loss: 0.4260 - val_acc: 0.7890\n",
      "Epoch 163/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4152 - acc: 0.7987 - val_loss: 0.4249 - val_acc: 0.7873\n",
      "Epoch 164/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4133 - acc: 0.7992 - val_loss: 0.4228 - val_acc: 0.7885\n",
      "Epoch 165/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4112 - acc: 0.7997 - val_loss: 0.4222 - val_acc: 0.7905\n",
      "Epoch 166/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4096 - acc: 0.8005 - val_loss: 0.4192 - val_acc: 0.7903\n",
      "Epoch 167/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4075 - acc: 0.8008 - val_loss: 0.4167 - val_acc: 0.7932\n",
      "Epoch 168/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4054 - acc: 0.8012 - val_loss: 0.4156 - val_acc: 0.7930\n",
      "Epoch 169/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4035 - acc: 0.8017 - val_loss: 0.4125 - val_acc: 0.7930\n",
      "Epoch 170/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4014 - acc: 0.8025 - val_loss: 0.4104 - val_acc: 0.7950\n",
      "Epoch 171/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3994 - acc: 0.8043 - val_loss: 0.4082 - val_acc: 0.7947\n",
      "Epoch 172/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3973 - acc: 0.8055 - val_loss: 0.4061 - val_acc: 0.7965\n",
      "Epoch 173/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3952 - acc: 0.8065 - val_loss: 0.4034 - val_acc: 0.8010\n",
      "Epoch 174/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3929 - acc: 0.8102 - val_loss: 0.4011 - val_acc: 0.8040\n",
      "Epoch 175/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3908 - acc: 0.8122 - val_loss: 0.3989 - val_acc: 0.8065\n",
      "Epoch 176/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3886 - acc: 0.8138 - val_loss: 0.3968 - val_acc: 0.8083\n",
      "Epoch 177/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3862 - acc: 0.8172 - val_loss: 0.3938 - val_acc: 0.8107\n",
      "Epoch 178/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3839 - acc: 0.8225 - val_loss: 0.3912 - val_acc: 0.8133\n",
      "Epoch 179/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3816 - acc: 0.8243 - val_loss: 0.3888 - val_acc: 0.8165\n",
      "Epoch 180/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3792 - acc: 0.8297 - val_loss: 0.3865 - val_acc: 0.8197\n",
      "Epoch 181/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3767 - acc: 0.8317 - val_loss: 0.3838 - val_acc: 0.8233\n",
      "Epoch 182/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3741 - acc: 0.8342 - val_loss: 0.3816 - val_acc: 0.8265\n",
      "Epoch 183/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3715 - acc: 0.8387 - val_loss: 0.3796 - val_acc: 0.8260\n",
      "Epoch 184/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3690 - acc: 0.8423 - val_loss: 0.3767 - val_acc: 0.8277\n",
      "Epoch 185/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3664 - acc: 0.8443 - val_loss: 0.3734 - val_acc: 0.8333\n",
      "Epoch 186/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3636 - acc: 0.8465 - val_loss: 0.3713 - val_acc: 0.8340\n",
      "Epoch 187/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3609 - acc: 0.8487 - val_loss: 0.3676 - val_acc: 0.8377\n",
      "Epoch 188/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3582 - acc: 0.8525 - val_loss: 0.3647 - val_acc: 0.8410\n",
      "Epoch 189/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3553 - acc: 0.8537 - val_loss: 0.3631 - val_acc: 0.8393\n",
      "Epoch 190/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3526 - acc: 0.8553 - val_loss: 0.3596 - val_acc: 0.8440\n",
      "Epoch 191/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3496 - acc: 0.8568 - val_loss: 0.3565 - val_acc: 0.8472\n",
      "Epoch 192/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3467 - acc: 0.8607 - val_loss: 0.3532 - val_acc: 0.8530\n",
      "Epoch 193/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3436 - acc: 0.8632 - val_loss: 0.3507 - val_acc: 0.8538\n",
      "Epoch 194/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3406 - acc: 0.8647 - val_loss: 0.3473 - val_acc: 0.8570\n",
      "Epoch 195/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3376 - acc: 0.8665 - val_loss: 0.3443 - val_acc: 0.8582\n",
      "Epoch 196/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3345 - acc: 0.8688 - val_loss: 0.3411 - val_acc: 0.8605\n",
      "Epoch 197/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3314 - acc: 0.8705 - val_loss: 0.3381 - val_acc: 0.8625\n",
      "Epoch 198/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3284 - acc: 0.8737 - val_loss: 0.3356 - val_acc: 0.8630\n",
      "Epoch 199/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3251 - acc: 0.8742 - val_loss: 0.3314 - val_acc: 0.8665\n",
      "Epoch 200/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3224 - acc: 0.8767 - val_loss: 0.3285 - val_acc: 0.8680\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x3f621710>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input0 = keras.layers.Input(shape=(3,))\n",
    "out0 = keras.layers.Dense(units=3, activation='sigmoid')(input0)\n",
    "out1 = keras.layers.Dense(units=3, activation='sigmoid')(out0)\n",
    "out1 = keras.layers.add([out1, input0])\n",
    "\n",
    "out2 = keras.layers.Dense(units=3, activation='sigmoid')(out1)\n",
    "out3 = keras.layers.Dense(units=3, activation='sigmoid')(out2)\n",
    "out3 = keras.layers.add([out3, out1])\n",
    "\n",
    "out4 = keras.layers.Dense(units=3, activation='sigmoid')(out3)\n",
    "out5 = keras.layers.Dense(units=3, activation='sigmoid')(out4)\n",
    "out5 = keras.layers.add([out5, out3])\n",
    "\n",
    "out6 = keras.layers.Dense(units=3, activation='sigmoid')(out5)\n",
    "out7 = keras.layers.Dense(units=3, activation='sigmoid')(out6)\n",
    "out7 = keras.layers.add([out7, out5])\n",
    "\n",
    "out8 = keras.layers.Dense(units=3, activation='sigmoid')(out7)\n",
    "out9 = keras.layers.Dense(units=3, activation='sigmoid')(out8)\n",
    "out9 = keras.layers.add([out9, out7])\n",
    "\n",
    "out10 = keras.layers.Dense(units=3, activation='sigmoid')(out9)\n",
    "out11 = keras.layers.Dense(units=3, activation='sigmoid')(out10)\n",
    "out11 = keras.layers.add([out11, out9])\n",
    "\n",
    "out12 = keras.layers.Dense(units=3, activation='sigmoid')(out11)\n",
    "out13 = keras.layers.Dense(units=3, activation='sigmoid')(out12)\n",
    "out13 = keras.layers.add([out13, out11])\n",
    "\n",
    "out14 = keras.layers.Dense(units=3, activation='sigmoid')(out13)\n",
    "out15 = keras.layers.Dense(units=3, activation='sigmoid')(out14)\n",
    "out15 = keras.layers.add([out15, out13])\n",
    "\n",
    "out16 = keras.layers.Dense(units=3, activation='sigmoid')(out15)\n",
    "out17 = keras.layers.Dense(units=3, activation='sigmoid')(out16)\n",
    "out17 = keras.layers.add([out17, out15])\n",
    "\n",
    "out18 = keras.layers.Dense(units=3, activation='sigmoid')(out17)\n",
    "out19 = keras.layers.Dense(units=3, activation='sigmoid')(out18)\n",
    "out19 = keras.layers.add([out19, out17])\n",
    "\n",
    "out20 = keras.layers.Dense(units=3, activation='sigmoid')(out19)\n",
    "out21 = keras.layers.Dense(units=3, activation='sigmoid')(out20)\n",
    "out21 = keras.layers.add([out21, out19])\n",
    "\n",
    "out = keras.layers.Dense(units=2, activation='softmax')(out21)\n",
    "model = keras.models.Model(inputs=input0, outputs=out)\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model, to_file='mode2.png',show_shapes=True)\n",
    "\n",
    "model.fit(Xtrain, ytrain,\n",
    "          batch_size=128,\n",
    "          epochs=200,\n",
    "          verbose=1,\n",
    "          validation_data=(Xtest, ytest))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# relu 20层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6000 samples, validate on 4000 samples\n",
      "Epoch 1/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6931 - acc: 0.5077 - val_loss: 0.6933 - val_acc: 0.4835\n",
      "Epoch 2/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6935 - val_acc: 0.4835\n",
      "Epoch 3/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6936 - val_acc: 0.4835\n",
      "Epoch 4/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6937 - val_acc: 0.4835\n",
      "Epoch 5/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6938 - val_acc: 0.4835\n",
      "Epoch 6/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6939 - val_acc: 0.4835\n",
      "Epoch 7/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6939 - val_acc: 0.4835\n",
      "Epoch 8/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6939 - val_acc: 0.4835\n",
      "Epoch 9/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 10/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 11/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 12/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 13/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 14/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 15/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 16/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 17/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 18/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 19/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 20/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 21/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 22/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 23/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 24/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 25/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 26/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 27/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 28/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6940 - val_acc: 0.4835\n",
      "Epoch 29/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 30/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 31/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 32/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 33/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 34/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6930 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 35/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 36/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 37/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 38/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 39/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 40/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 41/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 42/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 43/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 44/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 45/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 46/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 47/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 48/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 49/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 50/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 51/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 52/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 53/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 54/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 55/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 56/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 57/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 58/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 59/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 60/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 61/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 62/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 63/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 64/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 65/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 66/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 67/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 68/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 69/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 70/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 71/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 72/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 73/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 74/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 75/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 76/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 77/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 78/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 79/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 80/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 81/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 82/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 83/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 84/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 85/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 86/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 87/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 88/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 89/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 90/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 91/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 92/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 93/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 94/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 95/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 96/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 97/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 98/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 99/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 100/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 101/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 102/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 103/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 104/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 105/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 106/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 107/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 108/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 109/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 110/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 111/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 112/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 113/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 114/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 115/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 116/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 117/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 118/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 119/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 120/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 121/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 122/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 123/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 124/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 125/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 126/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 127/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 128/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 129/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 130/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 131/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 132/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 133/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 134/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 135/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 136/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 137/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 138/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 139/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 140/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 141/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 142/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 143/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 144/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 145/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 146/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 147/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 148/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 149/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 150/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 151/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 152/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 153/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 154/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 155/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 156/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 157/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 158/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 159/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 160/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 161/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 162/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 163/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 164/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 165/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 166/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 167/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 168/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 169/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 170/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 171/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 172/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 173/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 174/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 175/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 176/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 177/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 178/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 179/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 180/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 181/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 182/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 183/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 184/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 185/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 186/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 187/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 188/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 189/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 190/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 191/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 192/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 193/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 194/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6942 - val_acc: 0.4835\n",
      "Epoch 195/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 196/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 197/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 198/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 199/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n",
      "Epoch 200/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6929 - acc: 0.5110 - val_loss: 0.6941 - val_acc: 0.4835\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x4cae4828>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input0 = keras.layers.Input(shape=(3,))\n",
    "out0 = keras.layers.Dense(units=3, activation='relu')(input0)\n",
    "out1 = keras.layers.Dense(units=3, activation='relu')(out0)\n",
    "# out1 = keras.layers.add([out1, input0])\n",
    "\n",
    "out2 = keras.layers.Dense(units=3, activation='relu')(out1)\n",
    "out3 = keras.layers.Dense(units=3, activation='relu')(out2)\n",
    "# out3 = keras.layers.add([out3, out1])\n",
    "\n",
    "out4 = keras.layers.Dense(units=3, activation='relu')(out3)\n",
    "out5 = keras.layers.Dense(units=3, activation='relu')(out4)\n",
    "# out5 = keras.layers.add([out5, out3])\n",
    "\n",
    "out6 = keras.layers.Dense(units=3, activation='relu')(out5)\n",
    "out7 = keras.layers.Dense(units=3, activation='relu')(out6)\n",
    "# out7 = keras.layers.add([out7, out5])\n",
    "\n",
    "out8 = keras.layers.Dense(units=3, activation='relu')(out7)\n",
    "out9 = keras.layers.Dense(units=3, activation='relu')(out8)\n",
    "# out9 = keras.layers.add([out9, out7])\n",
    "\n",
    "out10 = keras.layers.Dense(units=3, activation='relu')(out9)\n",
    "out11 = keras.layers.Dense(units=3, activation='relu')(out10)\n",
    "# out11 = keras.layers.add([out11, out9])\n",
    "\n",
    "out12 = keras.layers.Dense(units=3, activation='relu')(out11)\n",
    "out13 = keras.layers.Dense(units=3, activation='relu')(out12)\n",
    "# out13 = keras.layers.add([out13, out11])\n",
    "\n",
    "out14 = keras.layers.Dense(units=3, activation='relu')(out13)\n",
    "out15 = keras.layers.Dense(units=3, activation='relu')(out14)\n",
    "# out15 = keras.layers.add([out15, out13])\n",
    "\n",
    "out16 = keras.layers.Dense(units=3, activation='relu')(out15)\n",
    "out17 = keras.layers.Dense(units=3, activation='relu')(out16)\n",
    "# out17 = keras.layers.add([out17, out15])\n",
    "\n",
    "out18 = keras.layers.Dense(units=3, activation='relu')(out17)\n",
    "out19 = keras.layers.Dense(units=3, activation='relu')(out18)\n",
    "# out19 = keras.layers.add([out19, out17])\n",
    "\n",
    "out20 = keras.layers.Dense(units=3, activation='relu')(out19)\n",
    "out21 = keras.layers.Dense(units=3, activation='relu')(out20)\n",
    "# out21 = keras.layers.add([out21, out19])\n",
    "\n",
    "out = keras.layers.Dense(units=2, activation='softmax')(out21)\n",
    "model = keras.models.Model(inputs=input0, outputs=out)\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model, to_file='mode3.png',show_shapes=True)\n",
    "\n",
    "model.fit(Xtrain, ytrain,\n",
    "          batch_size=128,\n",
    "          epochs=200,\n",
    "          verbose=1,\n",
    "          validation_data=(Xtest, ytest))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# relu 20层 + restnet 直连结构"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 6000 samples, validate on 4000 samples\n",
      "Epoch 1/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7667 - acc: 0.3543 - val_loss: 0.7280 - val_acc: 0.4103\n",
      "Epoch 2/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.7093 - acc: 0.4697 - val_loss: 0.6865 - val_acc: 0.5453\n",
      "Epoch 3/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6725 - acc: 0.5918 - val_loss: 0.6518 - val_acc: 0.6255\n",
      "Epoch 4/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6369 - acc: 0.6323 - val_loss: 0.6183 - val_acc: 0.6593\n",
      "Epoch 5/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.6012 - acc: 0.6708 - val_loss: 0.5825 - val_acc: 0.6925\n",
      "Epoch 6/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5638 - acc: 0.6983 - val_loss: 0.5474 - val_acc: 0.7117\n",
      "Epoch 7/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.5264 - acc: 0.7305 - val_loss: 0.5109 - val_acc: 0.7382\n",
      "Epoch 8/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4818 - acc: 0.7718 - val_loss: 0.4659 - val_acc: 0.7758\n",
      "Epoch 9/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.4345 - acc: 0.8107 - val_loss: 0.4217 - val_acc: 0.8127\n",
      "Epoch 10/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3816 - acc: 0.8445 - val_loss: 0.3674 - val_acc: 0.8472\n",
      "Epoch 11/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3366 - acc: 0.8728 - val_loss: 0.3331 - val_acc: 0.8668\n",
      "Epoch 12/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.3018 - acc: 0.8895 - val_loss: 0.3014 - val_acc: 0.8805\n",
      "Epoch 13/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.2752 - acc: 0.8995 - val_loss: 0.2771 - val_acc: 0.8910\n",
      "Epoch 14/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.2550 - acc: 0.9082 - val_loss: 0.2608 - val_acc: 0.9000\n",
      "Epoch 15/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.2394 - acc: 0.9143 - val_loss: 0.2458 - val_acc: 0.9095\n",
      "Epoch 16/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.2251 - acc: 0.9207 - val_loss: 0.2340 - val_acc: 0.9135\n",
      "Epoch 17/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.2134 - acc: 0.9272 - val_loss: 0.2239 - val_acc: 0.9177\n",
      "Epoch 18/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.2046 - acc: 0.9308 - val_loss: 0.2164 - val_acc: 0.9203\n",
      "Epoch 19/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1973 - acc: 0.9332 - val_loss: 0.2111 - val_acc: 0.9215\n",
      "Epoch 20/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1920 - acc: 0.9350 - val_loss: 0.2063 - val_acc: 0.9270\n",
      "Epoch 21/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1870 - acc: 0.9360 - val_loss: 0.2014 - val_acc: 0.9270\n",
      "Epoch 22/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1822 - acc: 0.9397 - val_loss: 0.1975 - val_acc: 0.9283\n",
      "Epoch 23/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1780 - acc: 0.9417 - val_loss: 0.1947 - val_acc: 0.9295\n",
      "Epoch 24/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1739 - acc: 0.9447 - val_loss: 0.1929 - val_acc: 0.9300\n",
      "Epoch 25/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1709 - acc: 0.9460 - val_loss: 0.1886 - val_acc: 0.9330\n",
      "Epoch 26/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1681 - acc: 0.9470 - val_loss: 0.1859 - val_acc: 0.9333\n",
      "Epoch 27/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1658 - acc: 0.9473 - val_loss: 0.1831 - val_acc: 0.9360\n",
      "Epoch 28/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1635 - acc: 0.9485 - val_loss: 0.1817 - val_acc: 0.9357\n",
      "Epoch 29/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1616 - acc: 0.9493 - val_loss: 0.1780 - val_acc: 0.9383\n",
      "Epoch 30/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1589 - acc: 0.9495 - val_loss: 0.1752 - val_acc: 0.9403\n",
      "Epoch 31/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1574 - acc: 0.9503 - val_loss: 0.1746 - val_acc: 0.9413\n",
      "Epoch 32/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1557 - acc: 0.9505 - val_loss: 0.1722 - val_acc: 0.9413\n",
      "Epoch 33/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1537 - acc: 0.9515 - val_loss: 0.1691 - val_acc: 0.9437\n",
      "Epoch 34/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1522 - acc: 0.9533 - val_loss: 0.1691 - val_acc: 0.9423\n",
      "Epoch 35/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1501 - acc: 0.9530 - val_loss: 0.1648 - val_acc: 0.9460\n",
      "Epoch 36/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1484 - acc: 0.9538 - val_loss: 0.1637 - val_acc: 0.9450\n",
      "Epoch 37/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1469 - acc: 0.9545 - val_loss: 0.1612 - val_acc: 0.9473\n",
      "Epoch 38/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1450 - acc: 0.9555 - val_loss: 0.1617 - val_acc: 0.9453\n",
      "Epoch 39/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1437 - acc: 0.9555 - val_loss: 0.1583 - val_acc: 0.9490\n",
      "Epoch 40/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1418 - acc: 0.9570 - val_loss: 0.1574 - val_acc: 0.9475\n",
      "Epoch 41/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1405 - acc: 0.9575 - val_loss: 0.1559 - val_acc: 0.9483\n",
      "Epoch 42/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1392 - acc: 0.9573 - val_loss: 0.1555 - val_acc: 0.9497\n",
      "Epoch 43/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1378 - acc: 0.9587 - val_loss: 0.1537 - val_acc: 0.9505\n",
      "Epoch 44/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1367 - acc: 0.9597 - val_loss: 0.1530 - val_acc: 0.9497\n",
      "Epoch 45/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1358 - acc: 0.9600 - val_loss: 0.1512 - val_acc: 0.9502\n",
      "Epoch 46/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1342 - acc: 0.9607 - val_loss: 0.1498 - val_acc: 0.9525\n",
      "Epoch 47/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1333 - acc: 0.9612 - val_loss: 0.1491 - val_acc: 0.9517\n",
      "Epoch 48/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1322 - acc: 0.9610 - val_loss: 0.1481 - val_acc: 0.9535\n",
      "Epoch 49/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1311 - acc: 0.9613 - val_loss: 0.1480 - val_acc: 0.9533\n",
      "Epoch 50/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1301 - acc: 0.9623 - val_loss: 0.1470 - val_acc: 0.9537\n",
      "Epoch 51/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1293 - acc: 0.9622 - val_loss: 0.1455 - val_acc: 0.9540\n",
      "Epoch 52/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1279 - acc: 0.9630 - val_loss: 0.1444 - val_acc: 0.9555\n",
      "Epoch 53/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1270 - acc: 0.9628 - val_loss: 0.1438 - val_acc: 0.9543\n",
      "Epoch 54/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1256 - acc: 0.9637 - val_loss: 0.1428 - val_acc: 0.9560\n",
      "Epoch 55/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1249 - acc: 0.9633 - val_loss: 0.1418 - val_acc: 0.9565\n",
      "Epoch 56/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1236 - acc: 0.9643 - val_loss: 0.1412 - val_acc: 0.9563\n",
      "Epoch 57/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1229 - acc: 0.9642 - val_loss: 0.1403 - val_acc: 0.9583\n",
      "Epoch 58/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1219 - acc: 0.9650 - val_loss: 0.1400 - val_acc: 0.9577\n",
      "Epoch 59/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1210 - acc: 0.9648 - val_loss: 0.1392 - val_acc: 0.9593\n",
      "Epoch 60/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1201 - acc: 0.9650 - val_loss: 0.1379 - val_acc: 0.9593\n",
      "Epoch 61/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1194 - acc: 0.9650 - val_loss: 0.1375 - val_acc: 0.9600\n",
      "Epoch 62/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1184 - acc: 0.9657 - val_loss: 0.1377 - val_acc: 0.9590\n",
      "Epoch 63/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1177 - acc: 0.9657 - val_loss: 0.1373 - val_acc: 0.9587\n",
      "Epoch 64/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1171 - acc: 0.9650 - val_loss: 0.1362 - val_acc: 0.9587\n",
      "Epoch 65/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1162 - acc: 0.9662 - val_loss: 0.1350 - val_acc: 0.9595\n",
      "Epoch 66/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1152 - acc: 0.9672 - val_loss: 0.1336 - val_acc: 0.9600\n",
      "Epoch 67/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1140 - acc: 0.9672 - val_loss: 0.1343 - val_acc: 0.9613\n",
      "Epoch 68/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1133 - acc: 0.9675 - val_loss: 0.1327 - val_acc: 0.9613\n",
      "Epoch 69/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1128 - acc: 0.9673 - val_loss: 0.1315 - val_acc: 0.9600\n",
      "Epoch 70/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1120 - acc: 0.9670 - val_loss: 0.1304 - val_acc: 0.9615\n",
      "Epoch 71/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1111 - acc: 0.9687 - val_loss: 0.1304 - val_acc: 0.9615\n",
      "Epoch 72/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1100 - acc: 0.9685 - val_loss: 0.1296 - val_acc: 0.9633\n",
      "Epoch 73/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1096 - acc: 0.9692 - val_loss: 0.1287 - val_acc: 0.9627\n",
      "Epoch 74/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1086 - acc: 0.9695 - val_loss: 0.1280 - val_acc: 0.9625\n",
      "Epoch 75/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1080 - acc: 0.9692 - val_loss: 0.1267 - val_acc: 0.9633\n",
      "Epoch 76/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1074 - acc: 0.9692 - val_loss: 0.1262 - val_acc: 0.9635\n",
      "Epoch 77/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1064 - acc: 0.9698 - val_loss: 0.1263 - val_acc: 0.9630\n",
      "Epoch 78/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1057 - acc: 0.9687 - val_loss: 0.1251 - val_acc: 0.9637\n",
      "Epoch 79/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1049 - acc: 0.9697 - val_loss: 0.1252 - val_acc: 0.9630\n",
      "Epoch 80/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1039 - acc: 0.9695 - val_loss: 0.1222 - val_acc: 0.9645\n",
      "Epoch 81/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1034 - acc: 0.9700 - val_loss: 0.1223 - val_acc: 0.9637\n",
      "Epoch 82/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1022 - acc: 0.9705 - val_loss: 0.1213 - val_acc: 0.9643\n",
      "Epoch 83/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.1012 - acc: 0.9712 - val_loss: 0.1207 - val_acc: 0.9645\n",
      "Epoch 84/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0997 - acc: 0.9712 - val_loss: 0.1210 - val_acc: 0.9643\n",
      "Epoch 85/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0990 - acc: 0.9722 - val_loss: 0.1182 - val_acc: 0.9657\n",
      "Epoch 86/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0977 - acc: 0.9728 - val_loss: 0.1179 - val_acc: 0.9660\n",
      "Epoch 87/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0968 - acc: 0.9723 - val_loss: 0.1167 - val_acc: 0.9660\n",
      "Epoch 88/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0960 - acc: 0.9733 - val_loss: 0.1163 - val_acc: 0.9660\n",
      "Epoch 89/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0951 - acc: 0.9733 - val_loss: 0.1156 - val_acc: 0.9670\n",
      "Epoch 90/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0942 - acc: 0.9740 - val_loss: 0.1135 - val_acc: 0.9662\n",
      "Epoch 91/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0932 - acc: 0.9747 - val_loss: 0.1126 - val_acc: 0.9670\n",
      "Epoch 92/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0928 - acc: 0.9740 - val_loss: 0.1125 - val_acc: 0.9677\n",
      "Epoch 93/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0920 - acc: 0.9747 - val_loss: 0.1141 - val_acc: 0.9662\n",
      "Epoch 94/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0913 - acc: 0.9750 - val_loss: 0.1123 - val_acc: 0.9670\n",
      "Epoch 95/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0906 - acc: 0.9747 - val_loss: 0.1104 - val_acc: 0.9688\n",
      "Epoch 96/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0900 - acc: 0.9748 - val_loss: 0.1116 - val_acc: 0.9680\n",
      "Epoch 97/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0891 - acc: 0.9753 - val_loss: 0.1108 - val_acc: 0.9675\n",
      "Epoch 98/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0884 - acc: 0.9755 - val_loss: 0.1093 - val_acc: 0.9690\n",
      "Epoch 99/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0881 - acc: 0.9763 - val_loss: 0.1088 - val_acc: 0.9692\n",
      "Epoch 100/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0872 - acc: 0.9767 - val_loss: 0.1109 - val_acc: 0.9692\n",
      "Epoch 101/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0870 - acc: 0.9760 - val_loss: 0.1088 - val_acc: 0.9685\n",
      "Epoch 102/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0858 - acc: 0.9762 - val_loss: 0.1078 - val_acc: 0.9700\n",
      "Epoch 103/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0861 - acc: 0.9765 - val_loss: 0.1065 - val_acc: 0.9713\n",
      "Epoch 104/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0853 - acc: 0.9768 - val_loss: 0.1063 - val_acc: 0.9700\n",
      "Epoch 105/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0846 - acc: 0.9770 - val_loss: 0.1083 - val_acc: 0.9685\n",
      "Epoch 106/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0842 - acc: 0.9783 - val_loss: 0.1065 - val_acc: 0.9698\n",
      "Epoch 107/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0839 - acc: 0.9777 - val_loss: 0.1059 - val_acc: 0.9710\n",
      "Epoch 108/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0837 - acc: 0.9772 - val_loss: 0.1052 - val_acc: 0.9708\n",
      "Epoch 109/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0829 - acc: 0.9780 - val_loss: 0.1055 - val_acc: 0.9708\n",
      "Epoch 110/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0826 - acc: 0.9778 - val_loss: 0.1047 - val_acc: 0.9720\n",
      "Epoch 111/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0824 - acc: 0.9782 - val_loss: 0.1041 - val_acc: 0.9708\n",
      "Epoch 112/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0817 - acc: 0.9788 - val_loss: 0.1050 - val_acc: 0.9718\n",
      "Epoch 113/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0814 - acc: 0.9782 - val_loss: 0.1027 - val_acc: 0.9722\n",
      "Epoch 114/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0809 - acc: 0.9777 - val_loss: 0.1035 - val_acc: 0.9713\n",
      "Epoch 115/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0807 - acc: 0.9778 - val_loss: 0.1032 - val_acc: 0.9710\n",
      "Epoch 116/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0805 - acc: 0.9785 - val_loss: 0.1021 - val_acc: 0.9728\n",
      "Epoch 117/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0799 - acc: 0.9785 - val_loss: 0.1016 - val_acc: 0.9730\n",
      "Epoch 118/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0794 - acc: 0.9788 - val_loss: 0.1026 - val_acc: 0.9720\n",
      "Epoch 119/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0792 - acc: 0.9783 - val_loss: 0.1026 - val_acc: 0.9722\n",
      "Epoch 120/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0786 - acc: 0.9790 - val_loss: 0.1034 - val_acc: 0.9720\n",
      "Epoch 121/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0785 - acc: 0.9785 - val_loss: 0.1027 - val_acc: 0.9715\n",
      "Epoch 122/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0783 - acc: 0.9787 - val_loss: 0.1019 - val_acc: 0.9722\n",
      "Epoch 123/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0779 - acc: 0.9792 - val_loss: 0.1026 - val_acc: 0.9718\n",
      "Epoch 124/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0773 - acc: 0.9785 - val_loss: 0.1006 - val_acc: 0.9728\n",
      "Epoch 125/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0771 - acc: 0.9795 - val_loss: 0.1006 - val_acc: 0.9730\n",
      "Epoch 126/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0770 - acc: 0.9785 - val_loss: 0.1017 - val_acc: 0.9728\n",
      "Epoch 127/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0769 - acc: 0.9792 - val_loss: 0.1004 - val_acc: 0.9732\n",
      "Epoch 128/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0762 - acc: 0.9795 - val_loss: 0.0976 - val_acc: 0.9738\n",
      "Epoch 129/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0760 - acc: 0.9797 - val_loss: 0.0990 - val_acc: 0.9732\n",
      "Epoch 130/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0759 - acc: 0.9797 - val_loss: 0.0983 - val_acc: 0.9732\n",
      "Epoch 131/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0747 - acc: 0.9800 - val_loss: 0.0978 - val_acc: 0.9740\n",
      "Epoch 132/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0745 - acc: 0.9797 - val_loss: 0.0972 - val_acc: 0.9730\n",
      "Epoch 133/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0748 - acc: 0.9795 - val_loss: 0.0968 - val_acc: 0.9738\n",
      "Epoch 134/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0737 - acc: 0.9798 - val_loss: 0.0974 - val_acc: 0.9742\n",
      "Epoch 135/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0732 - acc: 0.9803 - val_loss: 0.0953 - val_acc: 0.9750\n",
      "Epoch 136/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0732 - acc: 0.9802 - val_loss: 0.0956 - val_acc: 0.9740\n",
      "Epoch 137/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0729 - acc: 0.9803 - val_loss: 0.0952 - val_acc: 0.9740\n",
      "Epoch 138/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0728 - acc: 0.9798 - val_loss: 0.0947 - val_acc: 0.9742\n",
      "Epoch 139/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0721 - acc: 0.9805 - val_loss: 0.0953 - val_acc: 0.9738\n",
      "Epoch 140/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0719 - acc: 0.9808 - val_loss: 0.0942 - val_acc: 0.9748\n",
      "Epoch 141/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0719 - acc: 0.9803 - val_loss: 0.0926 - val_acc: 0.9755\n",
      "Epoch 142/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0713 - acc: 0.9810 - val_loss: 0.0932 - val_acc: 0.9755\n",
      "Epoch 143/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0710 - acc: 0.9810 - val_loss: 0.0920 - val_acc: 0.9758\n",
      "Epoch 144/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0703 - acc: 0.9815 - val_loss: 0.0936 - val_acc: 0.9745\n",
      "Epoch 145/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0706 - acc: 0.9813 - val_loss: 0.0933 - val_acc: 0.9740\n",
      "Epoch 146/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0700 - acc: 0.9807 - val_loss: 0.0921 - val_acc: 0.9742\n",
      "Epoch 147/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0700 - acc: 0.9810 - val_loss: 0.0925 - val_acc: 0.9755\n",
      "Epoch 148/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0695 - acc: 0.9822 - val_loss: 0.0918 - val_acc: 0.9748\n",
      "Epoch 149/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0700 - acc: 0.9808 - val_loss: 0.0900 - val_acc: 0.9762\n",
      "Epoch 150/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0692 - acc: 0.9818 - val_loss: 0.0907 - val_acc: 0.9758\n",
      "Epoch 151/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0692 - acc: 0.9817 - val_loss: 0.0905 - val_acc: 0.9758\n",
      "Epoch 152/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0689 - acc: 0.9813 - val_loss: 0.0901 - val_acc: 0.9762\n",
      "Epoch 153/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0687 - acc: 0.9818 - val_loss: 0.0908 - val_acc: 0.9750\n",
      "Epoch 154/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0684 - acc: 0.9823 - val_loss: 0.0905 - val_acc: 0.9758\n",
      "Epoch 155/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0685 - acc: 0.9813 - val_loss: 0.0895 - val_acc: 0.9760\n",
      "Epoch 156/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0679 - acc: 0.9820 - val_loss: 0.0879 - val_acc: 0.9770\n",
      "Epoch 157/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0679 - acc: 0.9822 - val_loss: 0.0895 - val_acc: 0.9755\n",
      "Epoch 158/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0675 - acc: 0.9818 - val_loss: 0.0960 - val_acc: 0.9725\n",
      "Epoch 159/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0680 - acc: 0.9822 - val_loss: 0.0892 - val_acc: 0.9765\n",
      "Epoch 160/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0669 - acc: 0.9825 - val_loss: 0.0878 - val_acc: 0.9770\n",
      "Epoch 161/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0675 - acc: 0.9823 - val_loss: 0.0878 - val_acc: 0.9762\n",
      "Epoch 162/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0672 - acc: 0.9818 - val_loss: 0.0884 - val_acc: 0.9772\n",
      "Epoch 163/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0660 - acc: 0.9825 - val_loss: 0.0886 - val_acc: 0.9768\n",
      "Epoch 164/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0668 - acc: 0.9822 - val_loss: 0.0896 - val_acc: 0.9752\n",
      "Epoch 165/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0664 - acc: 0.9822 - val_loss: 0.0886 - val_acc: 0.9760\n",
      "Epoch 166/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0661 - acc: 0.9825 - val_loss: 0.0887 - val_acc: 0.9762\n",
      "Epoch 167/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0648 - acc: 0.9823 - val_loss: 0.0963 - val_acc: 0.9735\n",
      "Epoch 168/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0659 - acc: 0.9825 - val_loss: 0.0872 - val_acc: 0.9775\n",
      "Epoch 169/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0659 - acc: 0.9827 - val_loss: 0.0865 - val_acc: 0.9770\n",
      "Epoch 170/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0650 - acc: 0.9832 - val_loss: 0.0882 - val_acc: 0.9760\n",
      "Epoch 171/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0654 - acc: 0.9828 - val_loss: 0.0874 - val_acc: 0.9770\n",
      "Epoch 172/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0652 - acc: 0.9828 - val_loss: 0.0899 - val_acc: 0.9758\n",
      "Epoch 173/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0652 - acc: 0.9827 - val_loss: 0.0870 - val_acc: 0.9772\n",
      "Epoch 174/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0643 - acc: 0.9835 - val_loss: 0.0861 - val_acc: 0.9762\n",
      "Epoch 175/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0643 - acc: 0.9837 - val_loss: 0.0880 - val_acc: 0.9760\n",
      "Epoch 176/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0642 - acc: 0.9825 - val_loss: 0.0854 - val_acc: 0.9768\n",
      "Epoch 177/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0642 - acc: 0.9833 - val_loss: 0.0866 - val_acc: 0.9770\n",
      "Epoch 178/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0636 - acc: 0.9832 - val_loss: 0.0862 - val_acc: 0.9772\n",
      "Epoch 179/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0639 - acc: 0.9835 - val_loss: 0.0871 - val_acc: 0.9768\n",
      "Epoch 180/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0637 - acc: 0.9838 - val_loss: 0.0870 - val_acc: 0.9765\n",
      "Epoch 181/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0639 - acc: 0.9833 - val_loss: 0.0856 - val_acc: 0.9772\n",
      "Epoch 182/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0638 - acc: 0.9833 - val_loss: 0.0849 - val_acc: 0.9775\n",
      "Epoch 183/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0633 - acc: 0.9832 - val_loss: 0.0857 - val_acc: 0.9765\n",
      "Epoch 184/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0628 - acc: 0.9837 - val_loss: 0.0874 - val_acc: 0.9760\n",
      "Epoch 185/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0621 - acc: 0.9842 - val_loss: 0.0848 - val_acc: 0.9772\n",
      "Epoch 186/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0624 - acc: 0.9843 - val_loss: 0.0846 - val_acc: 0.9770\n",
      "Epoch 187/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0617 - acc: 0.9843 - val_loss: 0.0843 - val_acc: 0.9772\n",
      "Epoch 188/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0617 - acc: 0.9838 - val_loss: 0.0838 - val_acc: 0.9780\n",
      "Epoch 189/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0622 - acc: 0.9833 - val_loss: 0.0842 - val_acc: 0.9775\n",
      "Epoch 190/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0623 - acc: 0.9838 - val_loss: 0.0839 - val_acc: 0.9770\n",
      "Epoch 191/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0616 - acc: 0.9840 - val_loss: 0.0831 - val_acc: 0.9775\n",
      "Epoch 192/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0608 - acc: 0.9843 - val_loss: 0.0847 - val_acc: 0.9775\n",
      "Epoch 193/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0620 - acc: 0.9837 - val_loss: 0.0820 - val_acc: 0.9780\n",
      "Epoch 194/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0606 - acc: 0.9845 - val_loss: 0.0835 - val_acc: 0.9778\n",
      "Epoch 195/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0607 - acc: 0.9842 - val_loss: 0.0851 - val_acc: 0.9775\n",
      "Epoch 196/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0607 - acc: 0.9842 - val_loss: 0.0836 - val_acc: 0.9780\n",
      "Epoch 197/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0605 - acc: 0.9843 - val_loss: 0.0837 - val_acc: 0.9775\n",
      "Epoch 198/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0608 - acc: 0.9840 - val_loss: 0.0829 - val_acc: 0.9780\n",
      "Epoch 199/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0603 - acc: 0.9842 - val_loss: 0.0838 - val_acc: 0.9775\n",
      "Epoch 200/200\n",
      "6000/6000 [==============================] - 0s - loss: 0.0607 - acc: 0.9843 - val_loss: 0.0830 - val_acc: 0.9772\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x51b62ba8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input0 = keras.layers.Input(shape=(3,))\n",
    "out0 = keras.layers.Dense(units=3, activation='relu')(input0)\n",
    "out1 = keras.layers.Dense(units=3, activation='relu')(out0)\n",
    "out1 = keras.layers.add([out1, input0])\n",
    "\n",
    "out2 = keras.layers.Dense(units=3, activation='relu')(out1)\n",
    "out3 = keras.layers.Dense(units=3, activation='relu')(out2)\n",
    "out3 = keras.layers.add([out3, out1])\n",
    "\n",
    "out4 = keras.layers.Dense(units=3, activation='relu')(out3)\n",
    "out5 = keras.layers.Dense(units=3, activation='relu')(out4)\n",
    "out5 = keras.layers.add([out5, out3])\n",
    "\n",
    "out6 = keras.layers.Dense(units=3, activation='relu')(out5)\n",
    "out7 = keras.layers.Dense(units=3, activation='relu')(out6)\n",
    "out7 = keras.layers.add([out7, out5])\n",
    "\n",
    "out8 = keras.layers.Dense(units=3, activation='relu')(out7)\n",
    "out9 = keras.layers.Dense(units=3, activation='relu')(out8)\n",
    "out9 = keras.layers.add([out9, out7])\n",
    "\n",
    "out10 = keras.layers.Dense(units=3, activation='relu')(out9)\n",
    "out11 = keras.layers.Dense(units=3, activation='relu')(out10)\n",
    "out11 = keras.layers.add([out11, out9])\n",
    "\n",
    "out12 = keras.layers.Dense(units=3, activation='relu')(out11)\n",
    "out13 = keras.layers.Dense(units=3, activation='relu')(out12)\n",
    "out13 = keras.layers.add([out13, out11])\n",
    "\n",
    "out14 = keras.layers.Dense(units=3, activation='relu')(out13)\n",
    "out15 = keras.layers.Dense(units=3, activation='relu')(out14)\n",
    "out15 = keras.layers.add([out15, out13])\n",
    "\n",
    "out16 = keras.layers.Dense(units=3, activation='relu')(out15)\n",
    "out17 = keras.layers.Dense(units=3, activation='relu')(out16)\n",
    "out17 = keras.layers.add([out17, out15])\n",
    "\n",
    "out18 = keras.layers.Dense(units=3, activation='relu')(out17)\n",
    "out19 = keras.layers.Dense(units=3, activation='relu')(out18)\n",
    "out19 = keras.layers.add([out19, out17])\n",
    "\n",
    "out20 = keras.layers.Dense(units=3, activation='relu')(out19)\n",
    "out21 = keras.layers.Dense(units=3, activation='relu')(out20)\n",
    "out21 = keras.layers.add([out21, out19])\n",
    "\n",
    "out = keras.layers.Dense(units=2, activation='softmax')(out21)\n",
    "model = keras.models.Model(inputs=input0, outputs=out)\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "from keras.utils.vis_utils import plot_model\n",
    "plot_model(model, to_file='mode4.png',show_shapes=True)\n",
    "\n",
    "model.fit(Xtrain, ytrain,\n",
    "          batch_size=128,\n",
    "          epochs=200,\n",
    "          verbose=1,\n",
    "          validation_data=(Xtest, ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}