LeNet_Dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import mnist\n",
    "from keras.layers import Flatten, Conv2D, MaxPool2D, Dense, Input,AveragePooling2D,Reshape,Activation,Dropout\n",
    "from keras.models import Sequential, Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import plot_model, to_categorical\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "(10000, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "#数据变成TensorFow为Backend的形式\n",
    "print (x_train.shape)\n",
    "print (x_test.shape)\n",
    "x_train=x_train.reshape(60000,28,28,1)\n",
    "x_test=x_test.reshape(10000,28,28,1)\n",
    "#把标签变成one-hot编码的形式\n",
    "y_train=to_categorical(y_train,num_classes=10)\n",
    "y_test=to_categorical(y_test,num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 24, 24, 6)         156       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 12, 12, 6)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 8, 8, 16)          2416      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 4, 4, 16)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 120)               30840     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 84)                10164     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                850       \n",
      "=================================================================\n",
      "Total params: 44,426\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 构建LeNet-5网络\n",
    "lenet = Sequential()\n",
    "lenet.add(Conv2D(input_shape = (28,28,1), filters=6, kernel_size=(5,5), padding='valid', activation='relu'))\n",
    "lenet.add(MaxPool2D(pool_size=(2,2), strides=2))\n",
    "lenet.add(Conv2D(input_shape=(16,16,8), filters=16, kernel_size=(5,5), padding='valid', activation='relu'))\n",
    "lenet.add(MaxPool2D(pool_size=(2,2), strides=2))\n",
    "lenet.add(Flatten())\n",
    "lenet.add(Dense(120, activation='relu'))\n",
    "lenet.add(Dense(84, activation='relu'))\n",
    "lenet.add(Dense(10, activation='softmax'))\n",
    "lenet.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "60000/60000 [==============================] - 11s 181us/step - loss: 1.1482 - acc: 0.8318\n",
      "Epoch 2/20\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 0.1317 - acc: 0.9598\n",
      "Epoch 3/20\n",
      "60000/60000 [==============================] - 11s 178us/step - loss: 0.0980 - acc: 0.9701\n",
      "Epoch 4/20\n",
      "60000/60000 [==============================] - 11s 175us/step - loss: 0.0807 - acc: 0.9753\n",
      "Epoch 5/20\n",
      "60000/60000 [==============================] - 10s 175us/step - loss: 0.0703 - acc: 0.9787\n",
      "Epoch 6/20\n",
      "60000/60000 [==============================] - 11s 179us/step - loss: 0.0626 - acc: 0.9807\n",
      "Epoch 7/20\n",
      "60000/60000 [==============================] - 11s 186us/step - loss: 0.0547 - acc: 0.9825\n",
      "Epoch 8/20\n",
      "60000/60000 [==============================] - 12s 202us/step - loss: 0.0486 - acc: 0.9848\n",
      "Epoch 9/20\n",
      "60000/60000 [==============================] - 12s 193us/step - loss: 0.0445 - acc: 0.9868\n",
      "Epoch 10/20\n",
      "60000/60000 [==============================] - 12s 203us/step - loss: 0.0396 - acc: 0.9878\n",
      "Epoch 11/20\n",
      "60000/60000 [==============================] - 12s 201us/step - loss: 0.0368 - acc: 0.9882\n",
      "Epoch 12/20\n",
      "60000/60000 [==============================] - 13s 216us/step - loss: 0.0336 - acc: 0.9891\n",
      "Epoch 13/20\n",
      "60000/60000 [==============================] - 12s 200us/step - loss: 0.0297 - acc: 0.9907\n",
      "Epoch 14/20\n",
      "60000/60000 [==============================] - 11s 186us/step - loss: 0.0282 - acc: 0.9913\n",
      "Epoch 15/20\n",
      "60000/60000 [==============================] - 11s 184us/step - loss: 0.0251 - acc: 0.9919\n",
      "Epoch 16/20\n",
      "60000/60000 [==============================] - 11s 184us/step - loss: 0.0230 - acc: 0.9924\n",
      "Epoch 17/20\n",
      "60000/60000 [==============================] - 11s 184us/step - loss: 0.0211 - acc: 0.9935\n",
      "Epoch 18/20\n",
      "60000/60000 [==============================] - 11s 184us/step - loss: 0.0192 - acc: 0.9940\n",
      "Epoch 19/20\n",
      "60000/60000 [==============================] - 11s 185us/step - loss: 0.0172 - acc: 0.9948\n",
      "Epoch 20/20\n",
      "60000/60000 [==============================] - 11s 185us/step - loss: 0.0155 - acc: 0.9953\n"
     ]
    }
   ],
   "source": [
    "lenet.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.01), metrics=['accuracy'])\n",
    "lenet_history = lenet.fit(x_train, y_train, batch_size=128, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 126us/step\n",
      "Testing Loss: 0.06592238205773292\n",
      "Testing Accuracy of: 0.9826\n"
     ]
    }
   ],
   "source": [
    "lenet_score=lenet.evaluate(x_test,y_test)\n",
    "print(\"Testing Loss:\", lenet_score[0])\n",
    "print(\"Testing Accuracy of:\", lenet_score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_3 (Conv2D)            (None, 24, 24, 6)         156       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 12, 12, 6)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 8, 8, 16)          2416      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 4, 4, 16)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 120)               30840     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 120)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 84)                10164     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 84)                0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                850       \n",
      "=================================================================\n",
      "Total params: 44,426\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 构建LeNet-5网络\n",
    "lenet_drop = Sequential()\n",
    "lenet_drop.add(Conv2D(input_shape = (28,28,1), filters=6, kernel_size=(5,5), padding='valid', activation='relu'))\n",
    "lenet_drop.add(MaxPool2D(pool_size=(2,2), strides=2))\n",
    "lenet_drop.add(Conv2D(input_shape=(16,16,8), filters=16, kernel_size=(5,5), padding='valid', activation='relu'))\n",
    "lenet_drop.add(MaxPool2D(pool_size=(2,2), strides=2))\n",
    "lenet_drop.add(Flatten())\n",
    "lenet_drop.add(Dense(120, activation='relu'))\n",
    "lenet_drop.add(Dropout(0.2))\n",
    "lenet_drop.add(Dense(84, activation='relu'))\n",
    "lenet_drop.add(Dropout(0.2))\n",
    "lenet_drop.add(Dense(10, activation='softmax'))\n",
    "lenet_drop.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "60000/60000 [==============================] - 11s 186us/step - loss: 14.4106 - acc: 0.1054\n",
      "Epoch 2/20\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 1.6244 - acc: 0.6266\n",
      "Epoch 3/20\n",
      "60000/60000 [==============================] - 11s 184us/step - loss: 0.3582 - acc: 0.8957\n",
      "Epoch 4/20\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 0.2443 - acc: 0.9284\n",
      "Epoch 5/20\n",
      "60000/60000 [==============================] - 11s 181us/step - loss: 0.1958 - acc: 0.9431\n",
      "Epoch 6/20\n",
      "60000/60000 [==============================] - 11s 181us/step - loss: 0.1689 - acc: 0.9507\n",
      "Epoch 7/20\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 0.1504 - acc: 0.9555\n",
      "Epoch 8/20\n",
      "60000/60000 [==============================] - 11s 185us/step - loss: 0.1354 - acc: 0.9602\n",
      "Epoch 9/20\n",
      "60000/60000 [==============================] - 11s 179us/step - loss: 0.1239 - acc: 0.9634\n",
      "Epoch 10/20\n",
      "60000/60000 [==============================] - 11s 182us/step - loss: 0.1177 - acc: 0.9654\n",
      "Epoch 11/20\n",
      "60000/60000 [==============================] - 11s 177us/step - loss: 0.1081 - acc: 0.9678\n",
      "Epoch 12/20\n",
      "60000/60000 [==============================] - 11s 176us/step - loss: 0.1034 - acc: 0.9694\n",
      "Epoch 13/20\n",
      "60000/60000 [==============================] - 11s 176us/step - loss: 0.0975 - acc: 0.9711\n",
      "Epoch 14/20\n",
      "60000/60000 [==============================] - 11s 183us/step - loss: 0.0942 - acc: 0.9719\n",
      "Epoch 15/20\n",
      "60000/60000 [==============================] - 11s 179us/step - loss: 0.0884 - acc: 0.9740\n",
      "Epoch 16/20\n",
      "60000/60000 [==============================] - 11s 186us/step - loss: 0.0828 - acc: 0.9753\n",
      "Epoch 17/20\n",
      "60000/60000 [==============================] - 11s 191us/step - loss: 0.0824 - acc: 0.9757\n",
      "Epoch 18/20\n",
      "60000/60000 [==============================] - 11s 183us/step - loss: 0.0779 - acc: 0.9763\n",
      "Epoch 19/20\n",
      "60000/60000 [==============================] - 11s 181us/step - loss: 0.0757 - acc: 0.9765\n",
      "Epoch 20/20\n",
      "60000/60000 [==============================] - 11s 181us/step - loss: 0.0735 - acc: 0.9776\n"
     ]
    }
   ],
   "source": [
    "lenet_drop.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.01), metrics=['accuracy'])\n",
    "lenet_drop_history = lenet_drop.fit(x_train, y_train, batch_size=128, epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 131us/step\n",
      "Testing Loss: 0.05491561537194575\n",
      "Testing Accuracy of: 0.9841\n"
     ]
    }
   ],
   "source": [
    "lenet_drop_score=lenet_drop.evaluate(x_test,y_test)\n",
    "print(\"Testing Loss:\", lenet_drop_score[0])\n",
    "print(\"Testing Accuracy of:\", lenet_drop_score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['loss', 'acc'])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt8FPW9//HXZ5NAIEFBDFRFC/Z4QSEGjaBSq5XWWqtgLV5BoPYUL8eKp9VW2x5tfZz21J/WSy1HqlW01mq9FGqrHlGsVYpFgVJFRRGLAspFFCTck/38/pjZsFl2k5Vkd5LM+/l47GNn5/b97OxmP5nvzHzG3B0REYmvRNQBiIhItJQIRERiTolARCTmlAhERGJOiUBEJOaUCEREYk6JQDo9M7vbzP47z3mXmtkXCh2TSHuiRCAiEnNKBCIdhJmVRh2DdE5KBNIuhF0yV5jZy2a20czuNLO+ZvaEmW0ws6fNrFfa/CPN7FUzW2dmz5rZwLRpQ8xsfrjc74HyjLZOMbMF4bKzzaw6zxi/Ymb/MLOPzWyZmf0oY/pnw/WtC6dPCMd3M7Ofm9k7ZrbezGaF4443s+VZtsMXwuEfmdnDZvZbM/sYmGBmQ83shbCN983sl2bWJW35Q83sKTP70MxWmdn3zexTZrbJzHqnzXe4ma0xs7J83rt0bkoE0p58DfgicCBwKvAE8H2giuC7eimAmR0I3A9cFk57HPiTmXUJfxSnA/cCewAPheslXHYIcBdwAdAb+BXwqJl1zSO+jcA4oCfwFeAiMzstXO+nw3hvDWOqARaEy90AHAEcE8b0XSCZ5zYZBTwctnkf0AD8J7AncDQwArg4jKEH8DTwf8DewL8BM919JfAscGbaes8DHnD37XnGIZ2YEoG0J7e6+yp3XwE8D8xx93+4+xZgGjAknO8s4DF3fyr8IbsB6EbwQ3sUUAbc7O7b3f1h4KW0NiYCv3L3Oe7e4O73AFvD5Zrl7s+6+yvunnT3lwmS0XHh5HOBp939/rDdte6+wMwSwPnAJHdfEbY529235rlNXnD36WGbm919nrv/3d3r3X0pQSJLxXAKsNLdf+7uW9x9g7vPCafdA4wFMLMS4ByCZCmiRCDtyqq04c1ZXleGw3sD76QmuHsSWAbsE05b4U2rKb6TNvxp4Dth18o6M1sH7Bsu1ywzG2Zmfwm7VNYDFxL8Z064jiVZFtuToGsq27R8LMuI4UAz+7OZrQy7i36aRwwAfwQOMbMBBHtd6939xV2MSToZJQLpiN4j+EEHwMyM4EdwBfA+sE84LmW/tOFlwE/cvWfao7u7359Hu78DHgX2dffdgSlAqp1lwGeyLPMBsCXHtI1A97T3UULQrZQuszzwbcAi4AB3342g6yw9hv2zBR7uVT1IsFdwHtobkDRKBNIRPQh8xcxGhAc7v0PQvTMbeAGoBy41szIzOx0YmrbsHcCF4X/3ZmYV4UHgHnm02wP40N23mNlQgu6glPuAL5jZmWZWama9zawm3Fu5C7jRzPY2sxIzOzo8JvEmUB62Xwb8EGjpWEUP4GOgzswOBi5Km/ZnYC8zu8zMuppZDzMbljb9N8AEYCRKBJJGiUA6HHd/g+A/21sJ/uM+FTjV3be5+zbgdIIfvA8Jjif8IW3ZucA3gV8CHwFvhfPm42LgWjPbAFxNkJBS630XOJkgKX1IcKD4sHDy5cArBMcqPgSuAxLuvj5c568J9mY2Ak3OIsricoIEtIEgqf0+LYYNBN0+pwIrgcXA59Om/43gIPV8d0/vLpOYM92YRiQ+zOwZ4Hfu/uuoY5H2Q4lAJCbM7EjgKYJjHBuijkfaD3UNicSAmd1DcI3BZUoCkkl7BCIiMac9AhGRmOsQRaz23HNP79+/f9RhiIh0KPPmzfvA3TOvTdlJh0gE/fv3Z+7cuVGHISLSoZhZXqcJq2tIRCTmlAhERGJOiUBEJOY6xDECEcnf9u3bWb58OVu2bIk6FCmS8vJy+vXrR1nZrt1nSIlApJNZvnw5PXr0oH///jQtwiqdkbuzdu1ali9fzoABA3ZpHeoaEulktmzZQu/evZUEYsLM6N27d6v2AJUIRDohJYF4ae3n3bkTwZtPwvM3Rh2FiEi71rkTwZJnlAhEIlBZWdnyTJ/Q9OnTee2117JO+9GPfsQNN9zQ5m3GRedOBBVVsG0DbNsUdSQi0krNJQJpnc6dCCr7Bs8bV0cbh0iMXX/99Rx55JFUV1dzzTXXALB06VIGDhzIN7/5TQ499FBOPPFENm/eDMCSJUs46aSTOOKIIzj22GNZtGgRs2fP5tFHH+WKK66gpqaGJUuW5GxvwYIFHHXUUVRXV/PVr36Vjz76CIBf/OIXHHLIIVRXV3P22WcD8Ne//pWamhpqamoYMmQIGzbEs0J3wU4fNbO7gFOA1e4+KGPad4AbgCp3/6BQMVDZJ3iuWwO9+hesGZH26sd/epXX3vu4Tdd5yN67cc2ph+Y174wZM1i8eDEvvvgi7s7IkSN57rnn2G+//Vi8eDH3338/d9xxB2eeeSaPPPIIY8eOZeLEiUyZMoUDDjiAOXPmcPHFF/PMM88wcuRITjnlFEaPHt1sm+PGjePWW2/luOOO4+qrr+bHP/4xN998Mz/72c/417/+RdeuXVm3bh0AN9xwA5MnT2b48OHU1dVRXl7e6u3TERXyOoK7Ce4L+5v0kWa2L3Ai8G4B2w5UhEX3tEcgEokZM2YwY8YMhgwZAkBdXR2LFy9mv/32Y8CAAdTU1ABwxBFHsHTpUurq6pg9ezZnnHFG4zq2bt2ad3vr169n3bp1HHfccQCMHz++cV3V1dWMGTOG0047jdNOOw2A4cOH8+1vf5sxY8Zw+umn069fvzZ53x1NwRKBuz9nZv2zTLoJ+C7wx0K13SjVNVS3quBNibRH+f7nXijuzlVXXcUFF1zQZPzSpUvp2rVr4+uSkhI2b95MMpmkZ8+eLFiwoM1jeeyxx3juuef405/+xE9+8hNeeeUVrrzySr7yla/w+OOPM3z4cJ588kkOPvjgNm+7vSvqMQIzGwWscPd/5jHvRDOba2Zz16xZs2sNpvYI6nZxeRFplS996Uvcdddd1NXVAbBixQpWr869h77bbrsxYMAAHnroISBIJP/8Z/Bz0aNHjxb78HfffXd69erF888/D8C9997LcccdRzKZZNmyZXz+85/nuuuuY/369dTV1bFkyRIGDx7M9773PY488kgWLVrUFm+7wylaIjCz7sD3gavzmd/db3f3Wnevrapq8b4K2ZV2gW691DUkEpETTzyRc889l6OPPprBgwczevToFn/M77vvPu68804OO+wwDj30UP74x6Dz4Oyzz+b6669nyJAhzR4svueee7jiiiuorq5mwYIFXH311TQ0NDB27FgGDx7MkCFDuPTSS+nZsyc333wzgwYNorq6mrKyMr785S+36fvvKAp6z+Kwa+jP7j7IzAYDM4HUuZz9gPeAoe6+srn11NbW+i7fmOaXQ6HqQDjrt7u2vEgH8/rrrzNw4MCow5Aiy/a5m9k8d69tadmiFZ1z91eAPqnXZrYUqC3oWUMQnDmkriERkZwK1jVkZvcDLwAHmdlyM/tGodpqVmUfdQ2JiDSjkGcNndPC9P6FaruJij5Qp0QgIpJL576yGKCyCrbVqcyEiEgOMUgEKjMhItKczp8IKlJlJpQIRESy6fyJoDJ1UZkSgUixFLsMdTHaz9ezzz7L7Nmzs067++67ueSSS4ocUctikAjUNSTSGbS2DHV9fX0bRpNbc4mgver8iaBCewQiUSpGGep//etfjVcv//CHP2wc/+yzz3LssccycuRIDjnkEABuvPFGBg0axKBBg7j55psb4zn44IMZM2YMAwcOZPTo0WzaFJxgMnPmTIYMGcLgwYM5//zzG4vg9e/fnw8+CC6Dmjt3LscffzxLly5lypQp3HTTTdTU1DSWushm6dKlnHDCCVRXVzNixAjefTeow/nQQw8xaNAgDjvsMD73uc8B8OqrrzJ06FBqamqorq5m8eLFrftQMhTtgrLIlJQFZSaUCCSOnrgSVr7Stuv81GD48s/ymrVYZagnTZrERRddxLhx45g8eXKTafPnz2fhwoUMGDCAefPmMXXqVObMmYO7M2zYMI477jh69erFG2+8wZ133snw4cM5//zz+d///V8uueQSJkyYwMyZMznwwAMZN24ct912G5dddlnW99u/f38uvPBCKisrufzyy5vdNt/61rcYP34848eP56677uLSSy9l+vTpXHvttTz55JPss88+jeWyp0yZwqRJkxgzZgzbtm2joaEhr+2fr86/RwBB95AqkIoUXXoZ6sMPP5xFixY1/jfbUhnqmpoaLrjgAt5///0W2/nb3/7GOecEly6dd955TaYNHTqUAQMGADBr1iy++tWvUlFRQWVlJaeffnrjf+377rsvw4cPB2Ds2LHMmjWLN954gwEDBnDggQcCQVnr5557rg22DLzwwguce+65jTHPmjULCEpjT5gwgTvuuKPxB//oo4/mpz/9Kddddx3vvPMO3bp1a5MYUjr/HgEE3UMbVWZCYijP/9wLpZhlqM0s6/iKiopdWj7X+lJKS0tJJpMAbNmyJa828jFlyhTmzJnDY489xhFHHMG8efM499xzGTZsGI899hgnn3wyv/rVrzjhhBParM0Y7RGoa0ik2IpVhnr48OE88MADQFC9NJdjjz2W6dOns2nTJjZu3Mi0adM49thjAXj33Xd54YUXAPjd737HZz/7WQ466CCWLl3KW2+9Bewoaw1BN9C8efMAeOSRRxrbyKdcNsAxxxzTJOZUHEuWLGHYsGFce+21VFVVsWzZMt5++232339/Lr30UkaNGsXLL7/c4vo/iZgkApWZEIlCscpQ33LLLUyePJnBgwezYsWKnOs+/PDDmTBhAkOHDmXYsGH8+7//e+Pd0w466CAmT57MwIED+eijj7jooosoLy9n6tSpnHHGGQwePJhEIsGFF14IwDXXXMOkSZOora2lpKSksY1TTz2VadOmtXiw+NZbb2Xq1KlUV1dz7733cssttwBwxRVXMHjwYAYNGsQxxxzDYYcdxoMPPsigQYOoqalh4cKFjBs3rtlt+EkVtAx1W2lVGWqA52+EmT+G778HXfLbTRTpqFSG+pNbunQpp5xyCgsXLow6lF3WmjLUMdkjSN2yUnsFIiKZYpIIVGZCRHLr379/h94baK14JILURWW6ulhioiN0+Urbae3nHY9EoK4hiZHy8nLWrl2rZBAT7s7atWspLy/f5XXE5DqCPYNnJQKJgX79+rF8+XLWrNG1M3FRXl5Ov379dnn5eCSCkjLotoe6hiQWysrKGq+kFclHIe9ZfJeZrTazhWnjrjezRWb2splNM7OehWp/J7qoTEQkq0IeI7gbOClj3FPAIHevBt4Eripg+01VVikRiIhkUbBE4O7PAR9mjJvh7qmi4H8Hdr1T65Oq7KuuIRGRLKI8a+h84IlcE81sopnNNbO5bXLQq0JlJkREsokkEZjZD4B6IGd1KHe/3d1r3b22qqqq9Y1WVsH2TbC1rvXrEhHpRIp+1pCZTQBOAUZ4MU90Tr9lZdfo7mcqItLeFHWPwMxOAr4LjHT3TcVsmwqVmRARyaaQp4/eD7wAHGRmy83sG8AvgR7AU2a2wMymFKr9nVTq3sUiItkUrGvI3c/JMvrOQrXXovSuIRERaRSPWkMA3fcETHsEIiIZ4pMISkqh+x5KBCIiGeKTCCC8qEyFuERE0sUrEVRUQd2qqKMQEWlX4pUIdBN7EZGdxCwRhF1DumGHiEijeCWCirDMxDaVmRARSYlXItAtK0VEdhKzRJC6ib3OHBIRSYlXImisN6Qzh0REUuKVCNQ1JCKyk3glgu69UZkJEZGm4pUISkqDZKDCcyIijeKVCCDoHqrTwWIRkZQYJgKVmRARSRe/RFDRR11DIiJp4pcIKvsEXUMqMyEiAsQ1EdRvhq0boo5ERKRdiGEiSN2yUgeMRUSgsDevv8vMVpvZwrRxe5jZU2a2OHzuVaj2c6rQTexFRNIVco/gbuCkjHFXAjPd/QBgZvi6uCpVZkJEJF3BEoG7Pwd8mDF6FHBPOHwPcFqh2s9JXUMiIk0U+xhBX3d/PxxeCfTNNaOZTTSzuWY2d82aNvzR7t4bLKGuIRGRUGQHi93dgZzncLr77e5e6+61VVVVbddwoiRIBuoaEhEBip8IVpnZXgDhczT/lqduWSkiIkVPBI8C48Ph8cAfi9x+oKJKXUMiIqFCnj56P/ACcJCZLTezbwA/A75oZouBL4Svi6+yjxKBiEiotFArdvdzckwaUag281YZ1htyB7OooxERiVT8riyGoPBc/RbY+nHUkYiIRC6eiaDxojIdMBYRiXciUDlqEZGYJoIKlZkQEUmJZyJIlZlQ15CISEwTQfc9gjIT6hoSEYlpIkiUQPc91TUkIkJcEwEE3UPqGhIRiXMiqFLXkIgIcU4EFSozISICcU4EqXpDnrMStohILMQ7ETRsVZkJEYm9+CaCxovK1D0kIvEW30RQqUQgIgJKBLqWQERiL8aJICwzoVtWikjMxTcRdNsDrERdQyISe/FNBIkEVKjMhIhIJInAzP7TzF41s4Vmdr+ZlUcRR3DLSnUNiUi8FT0RmNk+wKVArbsPAkqAs4sdB6Cri0VEiK5rqBToZmalQHfgvUiiqFQiEBEpeiJw9xXADcC7wPvAenefkTmfmU00s7lmNnfNmgJ131T2CQrPqcyEiMRYFF1DvYBRwABgb6DCzMZmzufut7t7rbvXVlVVFSaYij7QsA22rC/M+kVEOoAouoa+APzL3de4+3bgD8AxEcShq4tFRIgmEbwLHGVm3c3MgBHA6xHEsSMR6L4EIhJjURwjmAM8DMwHXgljuL3YcQAqPCciQp6JwMwmmdluFrjTzOab2Ym72qi7X+PuB7v7IHc/z9237uq6WiVVZkKJQERiLN89gvPd/WPgRKAXcB7ws4JFVSzdegVlJtQ1JCIxlm8isPD5ZOBed381bVzHlUhARZX2CEQk1vJNBPPMbAZBInjSzHoAycKFVUS6qExEYq40z/m+AdQAb7v7JjPbA/h64cIqotRFZSIiMZXvHsHRwBvuvi68+OuHQOe4Ckv1hkQk5vJNBLcBm8zsMOA7wBLgNwWLqphSFUhVZkJEYirfRFDv7k5QGuKX7j4Z6FG4sIqoMlVmYl3UkYiIRCLfRLDBzK4iOG30MTNLAGWFC6uIdFGZiMRcvongLGArwfUEK4F+wPUFi6qYVG9IRGIur0QQ/vjfB+xuZqcAW9y98xwjAJ05JCKxlW+JiTOBF4EzgDOBOWY2upCBFY26hkQk5vK9juAHwJHuvhrAzKqApwmKx3Vs3XpBolSJQERiK99jBIlUEgit/QTLtm+pMhPqGhKRmMp3j+D/zOxJ4P7w9VnA44UJKQIqMyEiMZZXInD3K8zsa8DwcNTt7j6tcGEVma4uFpEYy3ePAHd/BHikgLFEp7IPrH4t6ihERCLRbCIwsw1AttoLBri771aQqIot1TXkDtbxq2uLiHwSzSYCd+8cZSRaUtEHktth80fQfY+ooxERKarOceZPa+nqYhGJsUgSgZn1NLOHzWyRmb1uZkdHEUcjXV0sIjGW98HiNnYL8H/uPtrMugDdI4ojoKuLRSTGip4IzGx34HPABAB33wZsK3YcTahrSERiLIquoQHAGmCqmf3DzH5tZhWZM5nZRDOba2Zz16xZU9iIyntCokxdQyISS1EkglLgcOA2dx8CbASuzJzJ3W9391p3r62qqipsRKkyE3UFTjgiIu1QFIlgObDc3eeErx8mSAzRquwDdauijkJEpOiKngjCexssM7ODwlEjgOgv663so64hEYmlqM4a+hZwX3jG0NvA1yOKY4eKPrByYdRRiIgUXSSJwN0XALVRtJ1Tao8gmQyOGYiIxIR+8VIq+0CyHrasizoSEZGiUiJIqQjPTNK1BCISM0oEKZV9g2edOSQiMaNEkNJYb0jXEohIvCgRpKhrSERiSokgpVuvoMyEuoZEJGaUCFLMwlNI1TUkIvGiRJCuokp7BCISO0oE6Sr76hiBiMSOEkG6yip1DYlI7CgRpEvtESSTUUciIlI0SgTpKvqAN8Dmj6KORESkaJQI0lWG1xKoHLWIxIgSQTqVmRCRGFIiSFeRuom9DhiLSHwoEaRT15CIxJASQbrynlDSRV1DIhIrSgTpzILuIXUNiUiMRJYIzKzEzP5hZn+OKoasKqvUNSQisRLlHsEk4PUI28+usq+6hkQkViJJBGbWD/gK8Oso2m9WRZW6hkQkVqLaI7gZ+C6Qs5aDmU00s7lmNnfNmiL+MFf2DeoNqcyEiMRE0ROBmZ0CrHb3ec3N5+63u3utu9dWVVUVKTqCexJ4A2z+sHhtiohEKIo9guHASDNbCjwAnGBmv40gjux0y0oRiZmiJwJ3v8rd+7l7f+Bs4Bl3H1vsOHJSmQkRiRldR5CpMiwzofsSiEhMlEbZuLs/CzwbZQw7UdeQiMSM9ggyle8OJV3VNSQisaFEkMks6B5S15CIxIQSQTYVVeoaEpHYUCLIJnXvYhGRGFAiyEaF50QkRpQIsqnoAxs/gGRD1JGIiBScEkE2lX2DMhObVGZCRDo/JYJsdMtKEYkRJYJsGstMKBGISOenRJBNRVhmQolARGJAiSAbdQ2JSIwoEWTTdTcoLVeZCRGJBSWCbMyC7iHdslJEYkCJIBddVCYiMaFEkIvKTIhITCgR5KLCcyISE0oEuVT2gU0qMyEinZ8SQS6VfcGTsGlt1JGIiBSUEkEuumWliMRE0ROBme1rZn8xs9fM7FUzm1TsGPKSKjOhM4dEpJOL4ub19cB33H2+mfUA5pnZU+7+WgSx5FapMhMiEg9F3yNw9/fdfX44vAF4Hdin2HG0SF1DIhITkR4jMLP+wBBgTpZpE81srpnNXbMmgit8u/aA0m7qGhKRTi+yRGBmlcAjwGXu/nHmdHe/3d1r3b22qqoqigCDq4u1RyAinVwkicDMygiSwH3u/ocoYshLRR8lAhHp9KI4a8iAO4HX3f3GYrf/iVT2hY0qPCcinVsUewTDgfOAE8xsQfg4OYI4WlbZB9a9q3sXi0inFsVZQ7Pc3dy92t1rwsfjxY4jLzVjoH4r3DcattZFHY2ISEHoyuLm7HsknHE3vLcAfj82SAoiIp2MEkFLDj4ZRv0S3v4L/GGiitCJSKcTxZXFHU/NucFxghk/gMd6wSk3BaeXioh0AkoE+TrmkqAS6awboXtvGPFfUUckItImOnUiWLKmjtfe+5hTD9u7bVY44uogGTx/Q5AMjr64bdYrIhKhTp0IJv/lLf4wfwXvfriJi4//DNba7hyzoFto84fw5FXQfQ847Oy2CVZEJCKd+mDx/5w+mFE1e3P9k2/w/WmvUN+QbP1KEyXwtTthwHEw/WJ444nWr1NEJEKdOhF0LS3h5rNquOTz/8b9Ly7jG/fMpW5rfetXXNoVzr4P9qqGhybAO7Nbv04RkYh06kQAYGZc/qWD+J/TBzPrrQ84c8oLrPp4S+tX3LUHjHkEdt8XfncWvP9y69cpIhKBTp8IUs4Zuh93jq/lnbUbOW3y31i0cqeCp59cRW8YNx267ga//Rp8+Hbr1ykiUmSxSQQAxx/UhwcvPJqkO2fc9gKzFn/Q+pXu3g/OmwbJevjNabBhZevXKSJSRLFKBACH7r070y4ezj69ujFh6os8NHdZ61dadSCMfRg2fgD3ng6bP2r9OkVEiiR2iQBg757dePDCozlq/95c8fDL3PTUm7h761a6zxHBAeS1i4NjBts2tU2wIiIFFstEALBbeRlTv34ko4/oxy0zF3P5Qy+zrb6Vp5d+5vPwtV/D8pfgwXHQsL1tghURKaDYJgKAspIE14+u5ttfPJBH5i9nwtQXWb+5lT/eh4wKLjp76ymYfhEk2+DaBRGRAurUVxbnw8y4dMQB9OvVje898jJnTJnN1K8PZZ+e3XZ9pUdMCIrUzfxxsFfw6eHQoy/02Cu461llXygrb7P3ICLSGtbqvvEiqK2t9blz5xa8ndlvfcAFv51HeVkJUyccyaB9dt/1lbnDM/8Ns24Cz1K6urwn9PhUkBSyPn8qSB5de+x6DCISa2Y2z91rW5xPiaCpN1dt4OtTX+KjTduYfO7hfP7gPq1bYbIhKFS3YSXUrQqfV8KGVTs/N2S58U1JV+jSHcoqoKxbM8Phc1n3psNl3aGkC5SUhY8uac8Zw4nStOFY9xqKdArtOhGY2UnALUAJ8Gt3/1lz8xczEQCs/ngL59/zEq+99zEXHf8Z+vXqTpeSBF3LEuFzCV1KEnQpTdC18VHS+LpL+ChNWP6F7txhy7qdE8SmtcEZSNvDx07DG2H75h3jaaPP00p2JIpESZAkEmXhc+p1aZ6vE8H6UtOsJJynJGN8Isc84fKW2DHNEuF6M6el2kpNCx9YUDQw6zDhcyK8z0TasGWsp3FcSZZpGcskSna01eIzzUzPElf6e9C9MSSHdpsIzKwEeBP4IrAceAk4x91fy7VMsRMBwMat9Ux64B88/frqXV5HwoID0qUJoyRhlJYkSJilvTZKLBjOfF2aSDSON4OShJEwI2GEzxb81lmwTMKCI/9dbDvlvply30pXttLVt1DiDZRRT6nXU0o9pb6dEuop83oS4etSD6aX+HZKvJ6S1HxeT8IbKPEGEjQEwzRgyXoSJIPpNGDJHdNTD/N6Esl6jCTmSRJej3kS84a054bGaaSNS2TrTpNmea6El55s0seRempuetMEZM2tPzNB5UxmLSW7LMu3Rt4Jt4VE3WSbZK4/2/RdfJ057phvwacGfZJ3nLaa/BJBFAeLhwJvufvbAGb2ADAKyJkIolDRtZQ7xtXy0abtbK1vYFt9km31SbaGj2C4oXHctvok2xqSbN3ewLaGZJPxDUmnwZ2GpFOfdBoaguekh6+TSeobvOl84eut9Q0kHdydpENDuJw7JD2Yv3E4mT6cIOndSHo57o4DybTpTvCcdMBTw8H49tJbaCQpJUmCJIZTQpKStOEETiKcXkISs+xiortlAAAIjUlEQVTzGE4ieGeNw4ZjQIJk2FbT8WapeZON86Wml6TFlGh87HhdQpKEJXdeZzhM2rBlGSbL+ETjsjuvM2G53tuO8UGbmTE0fU0z0y38PHaMI3yPqW23Y1smrOm2TcWRGp++7Xe8Tm3f9DbDWNJzVovfGct4nbnNw+cwFsj9OaRvB9LWkR6LtTg+xTPWk335bNPW7TWKgbuYCPIVRSLYB0i/nHc5MCxzJjObCEwE2G+//YoT2c4xsEdFl0jajpKHycUJEo8Tvk5LGJkJJOk0ztf42psu5+xIaDTO23S5VBLKXGZHgtoRm3vmtGBdpE9vMn8wL2lttbj+8I8xNS61fZpurx1tp8cfNtU4f6rtzHVmLu9pK05/nzvW13S9O9blGXHuaCs9pszPeuf30fS9ZdsGqdhyzdPk88hYaeZ2zNVu+vLNxe4Zy+RaLtt8ZNk+n3Q9eb3HJnHvNPvO86VNvPjTn6HQ2u3po+5+O3A7BF1DEYcTK2bWuGdaklD/s0hnF8WpISuAfdNe9wvHiYhIBKJIBC8BB5jZADPrApwNPBpBHCIiQgRdQ+5eb2aXAE8SnD56l7u/Wuw4REQkEMkxAnd/HHg8irZFRKQpXT4qIhJzSgQiIjGnRCAiEnNKBCIiMdchqo+a2RrgnV1cfE+gDe5SXzCKr3UUX+sovtZrzzF+2t2rWpqpQySC1jCzufkUXYqK4msdxdc6iq/1OkKMLVHXkIhIzCkRiIjEXBwSwe1RB9ACxdc6iq91FF/rdYQYm9XpjxGIiEjz4rBHICIizVAiEBGJuU6TCMzsJDN7w8zeMrMrs0zvama/D6fPMbP+RYxtXzP7i5m9ZmavmtmkLPMcb2brzWxB+Li6WPGF7S81s1fCtne6QbQFfhFuv5fN7PAixnZQ2nZZYGYfm9llGfMUdfuZ2V1mttrMFqaN28PMnjKzxeFzrxzLjg/nWWxm44sY3/Vmtij8/KaZWc8cyzb7XShgfD8ysxVpn+HJOZZt9m+9gPH9Pi22pWa2IMeyBd9+bS64nWDHfhCUs14C7A90Af4JHJIxz8XAlHD4bOD3RYxvL+DwcLgH8GaW+I4H/hzhNlwK7NnM9JOBJwhuqXoUMCfCz3olwYUykW0/4HPA4cDCtHH/D7gyHL4SuC7LcnsAb4fPvcLhXkWK70SgNBy+Llt8+XwXChjfj4DL8/j8m/1bL1R8GdN/Dlwd1fZr60dn2SMYCrzl7m+7+zbgAWBUxjyjgHvC4YeBEWZWlPswuvv77j4/HN4AvE5w7+aOZBTwGw/8HehpZntFEMcIYIm77+qV5m3C3Z8DPswYnf4duwc4LcuiXwKecvcP3f0j4CngpGLE5+4z3L0+fPl3grsDRiLH9stHPn/rrdZcfOHvxpnA/W3dblQ6SyLYB1iW9no5O//QNs4T/jGsB3oXJbo0YZfUEGBOlslHm9k/zewJMzu0qIEFt8ueYWbzzGxilun5bONiOJvcf4BRbj+Avu7+fji8EuibZZ72sh3PJ9jDy6al70IhXRJ2Xd2Vo2utPWy/Y4FV7r44x/Qot98u6SyJoEMws0rgEeAyd/84Y/J8gu6Ow4BbgelFDu+z7n448GXgP8zsc0Vuv0XhrU1HAg9lmRz19mvCgz6Cdnlutpn9AKgH7ssxS1TfhduAzwA1wPsE3S/t0Tk0vzfQ7v+WMnWWRLAC2Dftdb9wXNZ5zKwU2B1YW5TogjbLCJLAfe7+h8zp7v6xu9eFw48DZWa2Z7Hic/cV4fNqYBrBLni6fLZxoX0ZmO/uqzInRL39QqtS3WXh8+os80S6Hc1sAnAKMCZMVjvJ47tQEO6+yt0b3D0J3JGj3ai3XylwOvD7XPNEtf1ao7MkgpeAA8xsQPhf49nAoxnzPAqkztAYDTyT6w+hrYV9incCr7v7jTnm+VTqmIWZDSX4bIqSqMyswsx6pIYJDiouzJjtUWBcePbQUcD6tG6QYsn5n1iU2y9N+ndsPPDHLPM8CZxoZr3Cro8Tw3EFZ2YnAd8FRrr7phzz5PNdKFR86cecvpqj3Xz+1gvpC8Aid1+ebWKU269Voj5a3VYPgrNa3iQ4o+AH4bhrCb70AOUEXQpvAS8C+xcxts8SdBO8DCwIHycDFwIXhvNcArxKcBbE34Fjihjf/mG7/wxjSG2/9PgMmBxu31eA2iJ/vhUEP+y7p42LbPsRJKT3ge0E/dTfIDjmNBNYDDwN7BHOWwv8Om3Z88Pv4VvA14sY31sE/eup72DqLLq9gceb+y4UKb57w+/WywQ/7ntlxhe+3ulvvRjxhePvTn3n0uYt+vZr64dKTIiIxFxn6RoSEZFdpEQgIhJzSgQiIjGnRCAiEnNKBCIiMadEIFJgYWXUP0cdh0guSgQiIjGnRCASMrOxZvZiWEf+V2ZWYmZ1ZnaTBfeRmGlmVeG8NWb297Ta/r3C8f9mZk+Hxe/mm9lnwtVXmtnD4f0A7itW5VuRfCgRiABmNhA4Cxju7jVAAzCG4Irmue5+KPBX4Jpwkd8A33P3aoKrYVPj7wMme1D87hiCq1MhqDh7GXAIwdWnwwv+pkTyVBp1ACLtxAjgCOCl8J/1bgRF45LsKDD2W+APZrY70NPd/xqOvwd4KKwxs4+7TwNw9y0A4fpe9LA+TXhnq/7ArMK/LZGWKRGIBAy4x92vajLS7L8y5tvVmixb04Yb0N+etCPqGhIJzARGm1kfaLz/8KcJ/kZGh/OcC8xy9/XAR2Z2bDj+POCvHtx9brmZnRauo6uZdS/quxDZBfqvRARw99fM7IcEd5ZKEFSd/A9gIzA0nLaa4DgCBGWmp4Q/9G8DXw/Hnwf8ysyuDddxRhHfhsguUfVRkWaYWZ27V0Ydh0ghqWtIRCTmtEcgIhJz2iMQEYk5JQIRkZhTIhARiTklAhGRmFMiEBGJuf8PLvk5Wzhc3KkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(lenet_history.history.keys())\n",
    "# summarize history for accuracy\n",
    "plt.plot(lenet_history.history['loss'])\n",
    "plt.plot(lenet_drop_history.history['loss'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['lenet loss','lenet dropout loss'], loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}