rough_draft_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[False False False False False  True False False False False False False\n",
      " False False False  True  True False False False]\n"
     ]
    }
   ],
   "source": [
    "# We'll start off by importing keras as well as the MNIST dataset which conveniently comes included in the keras library.\n",
    "# Keras comes with six different out of the box datasets which can be viewed here: https://keras.io/datasets/\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "\n",
    "# Next we'll load the MNIST data as our test and training datasets \n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "# Now that we have the MNIST data we want to select from the digits 0-9 only 3 and 8 to work with to keep our training time manageable. \n",
    "# An easy way to do this is to use numpy's logical_or function which will return to us an array with the \n",
    "# boolean value at each position based on whether or not that value meets one of the conditions passed to the function. \n",
    "# items that meet the conditions that we pass to it. In this case the number 3s are at index 2 and 8s at index 7.\n",
    "\n",
    "import numpy\n",
    "\n",
    "train_picks = numpy.logical_or(y_train==2,y_train==7)\n",
    "test_picks = numpy.logical_or(y_test==2,y_test==7)\n",
    "\n",
    "# Lets take a look at the first 20 elements of one the arrays that logical_or returns to us:\n",
    "print(train_picks[:20])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (12223, 28, 28, 1)\n",
      "12223 train samples\n",
      "2060 test samples\n"
     ]
    }
   ],
   "source": [
    "# We can then select only those elements from our x_train and y_train datasets that match the boolean values of....\n",
    "# I'm going to come back to explaining the data prep if I end up having time for it...\n",
    "\n",
    "from keras import backend as K\n",
    "\n",
    "num_classes=2\n",
    "\n",
    "####### DATA PREP!! #######\n",
    "\n",
    "x_train = x_train[train_picks]\n",
    "x_test = x_test[test_picks]\n",
    "\n",
    "y_train = numpy.array(y_train[train_picks]==7,dtype=int)\n",
    "y_test = numpy.array(y_test[test_picks]==7,dtype=int)\n",
    "\n",
    "# print(\"x_train: \", x_train[:10])\n",
    "# print(\"x_test: \", x_test[:10])\n",
    "\n",
    "# print(\"y_train: \", y_train[:10])\n",
    "# print(\"y_test: \", y_test[:10])\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# Don't really know what this if statements is doing... - Something with setting the input_shape which we need for \n",
    "#Setting up the Conv2D later on.\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Check Shape\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 12223 samples, validate on 2060 samples\n",
      "Epoch 1/12\n",
      "12223/12223 [==============================] - 12s - loss: 0.2381 - acc: 0.9499 - val_loss: 0.1043 - val_acc: 0.9728\n",
      "Epoch 2/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0653 - acc: 0.9813 - val_loss: 0.0718 - val_acc: 0.9757\n",
      "Epoch 3/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0478 - acc: 0.9851 - val_loss: 0.0647 - val_acc: 0.9772\n",
      "Epoch 4/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0424 - acc: 0.9873 - val_loss: 0.0637 - val_acc: 0.9801\n",
      "Epoch 5/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0396 - acc: 0.9872 - val_loss: 0.0623 - val_acc: 0.9816\n",
      "Epoch 6/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0378 - acc: 0.9881 - val_loss: 0.0552 - val_acc: 0.9811\n",
      "Epoch 7/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0365 - acc: 0.9891 - val_loss: 0.0631 - val_acc: 0.9806\n",
      "Epoch 8/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0353 - acc: 0.9894 - val_loss: 0.0579 - val_acc: 0.9825\n",
      "Epoch 9/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0346 - acc: 0.9894 - val_loss: 0.0590 - val_acc: 0.9816\n",
      "Epoch 10/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0337 - acc: 0.9899 - val_loss: 0.0534 - val_acc: 0.9830\n",
      "Epoch 11/12\n",
      "12223/12223 [==============================] - 1s - loss: 0.0331 - acc: 0.9892 - val_loss: 0.0544 - val_acc: 0.9825\n",
      "Epoch 12/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0326 - acc: 0.9903 - val_loss: 0.0551 - val_acc: 0.9825\n",
      "Test loss: 0.055062100917\n",
      "Test accuracy: 0.982524271845\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Flatten, Dense, Conv2D\n",
    "\n",
    "# variables for a bunch of params\n",
    "num_classes = 2\n",
    "filters = 3\n",
    "kernel_size = 3\n",
    "batch_size = 128\n",
    "epochs = 12\n",
    "\n",
    "model = Sequential()\n",
    "# This is an older version of the keras API Convolutional2D\n",
    "# model.add(Conv2D(filters, kernel_size, kernel_size, border_mode='same', input_shape=(1,28,28)))\n",
    "# Should be more like this: input shape was created above in the if-else statement.\n",
    "model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer=\"adadelta\", metrics=['accuracy'])\n",
    "\n",
    "# model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "#               optimizer=keras.optimizers.Adadelta(),\n",
    "#               metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 12223 samples, validate on 2060 samples\n",
      "Epoch 1/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.2844 - acc: 0.8963 - val_loss: 0.0945 - val_acc: 0.9631\n",
      "Epoch 2/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0743 - acc: 0.9741 - val_loss: 0.0655 - val_acc: 0.9777\n",
      "Epoch 3/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0516 - acc: 0.9829 - val_loss: 0.0536 - val_acc: 0.9811\n",
      "Epoch 4/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0426 - acc: 0.9868 - val_loss: 0.0483 - val_acc: 0.9850\n",
      "Epoch 5/12\n",
      "12223/12223 [==============================] - 2s - loss: 0.0358 - acc: 0.9885 - val_loss: 0.0471 - val_acc: 0.9850\n",
      "Epoch 6/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0357 - acc: 0.9879 - val_loss: 0.0416 - val_acc: 0.9859\n",
      "Epoch 7/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0327 - acc: 0.9900 - val_loss: 0.0396 - val_acc: 0.9879\n",
      "Epoch 8/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0323 - acc: 0.9891 - val_loss: 0.0388 - val_acc: 0.9883\n",
      "Epoch 9/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0298 - acc: 0.9905 - val_loss: 0.0360 - val_acc: 0.9888\n",
      "Epoch 10/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0308 - acc: 0.9901 - val_loss: 0.0342 - val_acc: 0.9883\n",
      "Epoch 11/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0273 - acc: 0.9911 - val_loss: 0.0347 - val_acc: 0.9893\n",
      "Epoch 12/12\n",
      "12223/12223 [==============================] - 3s - loss: 0.0305 - acc: 0.9908 - val_loss: 0.0411 - val_acc: 0.9874\n",
      "Test loss: 0.0411417842882\n",
      "Test accuracy: 0.987378640777\n"
     ]
    }
   ],
   "source": [
    "# Adding dropout and MaxPooling2D - I'm going to repeat a lot of code on purpose\n",
    "\n",
    "# Creates model comprised of 2 convolutional layers followed by dense layers\n",
    "# dense_layer_sizes: List of layer sizes.\n",
    "\n",
    "# This list has one number for each layer:\n",
    "# filters: Number of convolutional filters in each convolutional layer\n",
    "# kernel_size: Convolutional kernel size\n",
    "# pool_size: Size of pooling area for max pooling\n",
    "\n",
    "# This doesn't necessarily improve our accuracy a great amount, but it gives us access to more parameters to tune\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dropout, MaxPooling2D\n",
    "from keras.layers import Activation, Flatten, Dense, Conv2D\n",
    "\n",
    "num_classes = 2\n",
    "filters = 3\n",
    "kernel_size = 3\n",
    "batch_size = 128\n",
    "epochs = 12\n",
    "pool_size = 2\n",
    "dense_layer_sizes = [32, 64]\n",
    "\n",
    "model = Sequential()\n",
    "\n",
    "model.add(Conv2D(filters, kernel_size,\n",
    "                     padding='valid',\n",
    "                     input_shape=input_shape))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(filters, kernel_size))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=pool_size))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "for layer_size in dense_layer_sizes:\n",
    "    model.add(Dense(layer_size))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "# Different compile function here. Not sure why they're different.\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Once deleted, variables cannot be recovered. Proceed (y/[n])? y\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.0947 - acc: 0.9665    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0619 - acc: 0.9804    \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0566 - acc: 0.9813    \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0519 - acc: 0.9843    \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0447 - acc: 0.9875    \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0449 - acc: 0.9869    \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.1061 - acc: 0.9637    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0641 - acc: 0.9780     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0560 - acc: 0.9816    \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0504 - acc: 0.9855     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0495 - acc: 0.9845     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0479 - acc: 0.9864     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.1042 - acc: 0.9656    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0634 - acc: 0.9800     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0578 - acc: 0.9828    \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0523 - acc: 0.9854    \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0464 - acc: 0.9886    \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0452 - acc: 0.9872    \n",
      "8105/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1099 - acc: 0.9596     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0649 - acc: 0.9790     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0558 - acc: 0.9818     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0521 - acc: 0.9838     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0522 - acc: 0.9854     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0467 - acc: 0.9861     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1111 - acc: 0.9607     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0582 - acc: 0.9807     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0536 - acc: 0.9826     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0470 - acc: 0.9852     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0424 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0418 - acc: 0.9867     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1234 - acc: 0.9578     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0663 - acc: 0.9778     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0609 - acc: 0.9798     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0555 - acc: 0.9829     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0540 - acc: 0.9847     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0523 - acc: 0.9842     \n",
      "8110/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1284 - acc: 0.9523     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0669 - acc: 0.9778     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0573 - acc: 0.9820     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0538 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0477 - acc: 0.9851     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0503 - acc: 0.9847     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1192 - acc: 0.9590     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0595 - acc: 0.9815     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0513 - acc: 0.9834     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0475 - acc: 0.9849     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0464 - acc: 0.9865     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0403 - acc: 0.9869     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1336 - acc: 0.9518     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0676 - acc: 0.9780     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0601 - acc: 0.9821     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0514 - acc: 0.9836     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0504 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0438 - acc: 0.9864     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1528 - acc: 0.9426     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0620 - acc: 0.9788     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0586 - acc: 0.9811     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0529 - acc: 0.9847     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0512 - acc: 0.9849     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0460 - acc: 0.9854     \n",
      "7920/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1502 - acc: 0.9502     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0623 - acc: 0.9783     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0532 - acc: 0.9828     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0494 - acc: 0.9838     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0464 - acc: 0.9863     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0437 - acc: 0.9853     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1509 - acc: 0.9471     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0669 - acc: 0.9789     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0562 - acc: 0.9818     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0521 - acc: 0.9839     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0489 - acc: 0.9845     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0444 - acc: 0.9869     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.2194 - acc: 0.9243     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0727 - acc: 0.9757     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0655 - acc: 0.9802     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0539 - acc: 0.9822     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0532 - acc: 0.9833     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0511 - acc: 0.9838     \n",
      "4075/4075 [==============================] - 2s     \n",
      "7920/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1904 - acc: 0.9356     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0632 - acc: 0.9793     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0534 - acc: 0.9810     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0527 - acc: 0.9834     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0435 - acc: 0.9860     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0443 - acc: 0.9843     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.2134 - acc: 0.9211     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0779 - acc: 0.9726     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0586 - acc: 0.9825     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0564 - acc: 0.9831     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0539 - acc: 0.9831     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0495 - acc: 0.9845     \n",
      "4074/4074 [==============================] - 2s     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.2097 - acc: 0.9282     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0727 - acc: 0.9763     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0591 - acc: 0.9796     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0529 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0475 - acc: 0.9847     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0447 - acc: 0.9866     \n",
      "7920/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.2448 - acc: 0.9105     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0779 - acc: 0.9740     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0642 - acc: 0.9794     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0549 - acc: 0.9822     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0514 - acc: 0.9842     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0472 - acc: 0.9854     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.2528 - acc: 0.8887     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0741 - acc: 0.9771     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0611 - acc: 0.9805     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0566 - acc: 0.9834     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0489 - acc: 0.9845     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0483 - acc: 0.9856     \n",
      "4074/4074 [==============================] - 2s     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.2646 - acc: 0.9127     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0768 - acc: 0.9740     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0646 - acc: 0.9775     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0559 - acc: 0.9816     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0523 - acc: 0.9839     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0471 - acc: 0.9839     \n",
      "4075/4075 [==============================] - 2s     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.2862 - acc: 0.8927     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0867 - acc: 0.9708     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0696 - acc: 0.9764     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0595 - acc: 0.9807     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0585 - acc: 0.9825     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0518 - acc: 0.9839     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.2774 - acc: 0.9038     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0824 - acc: 0.9745     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0642 - acc: 0.9804     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0595 - acc: 0.9804     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0522 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0518 - acc: 0.9837     \n",
      "7950/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.1253 - acc: 0.9562     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0581 - acc: 0.9816     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0492 - acc: 0.9840     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0452 - acc: 0.9864     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0419 - acc: 0.9876     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0409 - acc: 0.9879     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [32], 'batch_size': 40, 'filters': 8, 'epochs': 6, 'pool_size': 2}\n"
     ]
    }
   ],
   "source": [
    "# We will reset all of our variables and repeat all of our code over again to avoid variable conflicts from previous\n",
    "# examples\n",
    "%reset\n",
    "\n",
    "import numpy\n",
    "from __future__ import print_function\n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dropout, MaxPooling2D\n",
    "from keras.layers import Activation, Flatten, Dense, Conv2D\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "train_picks = numpy.logical_or(y_train==2,y_train==7)\n",
    "test_picks = numpy.logical_or(y_test==2,y_test==7)\n",
    "\n",
    "num_classes=2\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "x_train = x_train[train_picks]\n",
    "x_test = x_test[test_picks]\n",
    "\n",
    "y_train = numpy.array(y_train[train_picks]==7,dtype=int)\n",
    "y_test = numpy.array(y_test[test_picks]==7,dtype=int)\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "\n",
    "# First we'll create our build function that we will return our model to us.\n",
    "def create_model(dense_layer_sizes, filters, kernel_size, pool_size, optimizer):\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    # Create first layer\n",
    "    model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(filters, kernel_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=pool_size))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    # Create second layer\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    # Return the model\n",
    "    return model\n",
    "\n",
    "# This is where we will set all of our variables. param_grid takes a dictionary full of arrays of parameters and \n",
    "# then runs the classifier on all of the different combinations of parameters within those arrays \n",
    "# and then reports back the highest accuracy\n",
    "\n",
    "########## BATCH SIZE TEST ###########\n",
    "\n",
    "# Here we'll create the actual param_grid dictionary using the variables that we set above\n",
    "param_grid={'dense_layer_sizes': [[32]],\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [5, 10, 20, 40, 80, 120, 150]}\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1257 - acc: 0.9545     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0657 - acc: 0.9777     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0572 - acc: 0.9821     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0568 - acc: 0.9821     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0446 - acc: 0.9858     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0471 - acc: 0.9849     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1453 - acc: 0.9488     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0574 - acc: 0.9790     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0552 - acc: 0.9825     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0500 - acc: 0.9843     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0426 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0409 - acc: 0.9877     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1400 - acc: 0.9492     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0599 - acc: 0.9804     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0571 - acc: 0.9823     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0531 - acc: 0.9845     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0495 - acc: 0.9848     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0461 - acc: 0.9861     \n",
      "8130/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1417 - acc: 0.9486     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0614 - acc: 0.9822     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0568 - acc: 0.9827     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0496 - acc: 0.9847     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0488 - acc: 0.9850     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0478 - acc: 0.9849     \n",
      "8050/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1479 - acc: 0.9474     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0533 - acc: 0.9813     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0496 - acc: 0.9829     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0471 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0465 - acc: 0.9847     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0400 - acc: 0.9860     \n",
      "8085/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1514 - acc: 0.9423     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0649 - acc: 0.9798     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0570 - acc: 0.9818     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0534 - acc: 0.9849     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0503 - acc: 0.9853     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0434 - acc: 0.9865     \n",
      "8015/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1604 - acc: 0.9416     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0624 - acc: 0.9796     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0521 - acc: 0.9833     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0463 - acc: 0.9858     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0438 - acc: 0.9858     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0379 - acc: 0.9876     \n",
      "7880/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1594 - acc: 0.9438     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0628 - acc: 0.9805     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0535 - acc: 0.9842     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0474 - acc: 0.9858     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0398 - acc: 0.9874     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0384 - acc: 0.9881     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1512 - acc: 0.9395     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0691 - acc: 0.9789     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0591 - acc: 0.9806     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0526 - acc: 0.9849     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0493 - acc: 0.9843     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0444 - acc: 0.9867     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1550 - acc: 0.9442     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0607 - acc: 0.9805     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0568 - acc: 0.9809     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0501 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0442 - acc: 0.9844     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0436 - acc: 0.9851     \n",
      "8010/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1462 - acc: 0.9502     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0583 - acc: 0.9790     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0501 - acc: 0.9838     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0496 - acc: 0.9843     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0449 - acc: 0.9847     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0383 - acc: 0.9881     \n",
      "8010/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1621 - acc: 0.9396     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0661 - acc: 0.9779     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0601 - acc: 0.9813     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0573 - acc: 0.9821     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0475 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0478 - acc: 0.9859     \n",
      "7965/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1870 - acc: 0.9259     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0727 - acc: 0.9761     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0639 - acc: 0.9797     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0560 - acc: 0.9828     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0531 - acc: 0.9833     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0479 - acc: 0.9842     \n",
      "7950/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1797 - acc: 0.9367     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0601 - acc: 0.9806     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0531 - acc: 0.9832     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0464 - acc: 0.9844     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0423 - acc: 0.9849     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0393 - acc: 0.9874     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1757 - acc: 0.9420     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0622 - acc: 0.9796     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0563 - acc: 0.9827     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0526 - acc: 0.9843     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0504 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0459 - acc: 0.9866     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1642 - acc: 0.9416     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0690 - acc: 0.9777     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0557 - acc: 0.9820     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0580 - acc: 0.9801     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0514 - acc: 0.9831     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0494 - acc: 0.9858     \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1678 - acc: 0.9394     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0672 - acc: 0.9777     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0536 - acc: 0.9828     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0499 - acc: 0.9823     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0493 - acc: 0.9844     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0413 - acc: 0.9869     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1778 - acc: 0.9304     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0689 - acc: 0.9752     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0600 - acc: 0.9809     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0536 - acc: 0.9840     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0507 - acc: 0.9832     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0486 - acc: 0.9858     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "12223/12223 [==============================] - 11s - loss: 0.1222 - acc: 0.9543    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0603 - acc: 0.9798     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0514 - acc: 0.9842     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0463 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0444 - acc: 0.9861     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0396 - acc: 0.9880     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [32], 'batch_size': 40, 'filters': 8, 'epochs': 6, 'pool_size': 2}\n"
     ]
    }
   ],
   "source": [
    "########## BATCH SIZE TEST ###########\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[32]],\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [30, 35, 40, 45, 50, 60]}\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.1474 - acc: 0.9458     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0645 - acc: 0.9779     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0583 - acc: 0.9813     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0519 - acc: 0.9827     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0518 - acc: 0.9836     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0458 - acc: 0.9854     \n",
      "4075/4075 [==============================] - 2s     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1442 - acc: 0.9530     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0610 - acc: 0.9799     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0508 - acc: 0.9827     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0503 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0472 - acc: 0.9848     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0423 - acc: 0.9863     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1719 - acc: 0.9407     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0689 - acc: 0.9768     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0580 - acc: 0.9816     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0524 - acc: 0.9828     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0525 - acc: 0.9833     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0493 - acc: 0.9861     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1365 - acc: 0.9502     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0597 - acc: 0.9793     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0517 - acc: 0.9834     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0452 - acc: 0.9851     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0402 - acc: 0.9885     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0409 - acc: 0.9883     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1386 - acc: 0.9529     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0511 - acc: 0.9826     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0436 - acc: 0.9853     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0433 - acc: 0.9849     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0375 - acc: 0.9874     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0350 - acc: 0.9879     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1315 - acc: 0.9550     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0583 - acc: 0.9795     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0510 - acc: 0.9844     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0439 - acc: 0.9859     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0403 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0405 - acc: 0.9876     \n",
      "4074/4074 [==============================] - 3s     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1512 - acc: 0.9443     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0648 - acc: 0.9791     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0482 - acc: 0.9867     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0486 - acc: 0.9867     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0457 - acc: 0.9865     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0373 - acc: 0.9893     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1303 - acc: 0.9579     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0521 - acc: 0.9832     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0433 - acc: 0.9869     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0389 - acc: 0.9875     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0339 - acc: 0.9907     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0308 - acc: 0.9902     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1539 - acc: 0.9456     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0569 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0485 - acc: 0.9852     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0435 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0406 - acc: 0.9881     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0333 - acc: 0.9896     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1113 - acc: 0.9601     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0569 - acc: 0.9810     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0479 - acc: 0.9848     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0434 - acc: 0.9867     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0401 - acc: 0.9877     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0375 - acc: 0.9886     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1128 - acc: 0.9611     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0485 - acc: 0.9849     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0391 - acc: 0.9870     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0361 - acc: 0.9888     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0313 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0286 - acc: 0.9909     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1249 - acc: 0.9532     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0518 - acc: 0.9848     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0433 - acc: 0.9863     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0396 - acc: 0.9886     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0363 - acc: 0.9902     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0325 - acc: 0.9906     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0987 - acc: 0.9668    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0499 - acc: 0.9835     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0419 - acc: 0.9867     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0355 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0303 - acc: 0.9908     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0269 - acc: 0.9918     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [64, 64], 'batch_size': 40, 'filters': 8, 'epochs': 6, 'pool_size': 2}\n"
     ]
    }
   ],
   "source": [
    "########## Dense Layer Size Test ###########\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[32], [64], [32,32], [64,64]],\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [40]} # Tuned\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.1107 - acc: 0.9610    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0499 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0432 - acc: 0.9865     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0369 - acc: 0.9880     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0365 - acc: 0.9887     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0308 - acc: 0.9897     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1209 - acc: 0.9584     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0499 - acc: 0.9845     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0420 - acc: 0.9867     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0315 - acc: 0.9902     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0289 - acc: 0.9910     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0266 - acc: 0.9923     \n",
      "4074/4074 [==============================] - 3s     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1182 - acc: 0.9577     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0527 - acc: 0.9823     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0463 - acc: 0.9849     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0401 - acc: 0.9885     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0367 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0358 - acc: 0.9881     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1191 - acc: 0.9584     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0530 - acc: 0.9831     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0445 - acc: 0.9870     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0373 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0351 - acc: 0.9901     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0335 - acc: 0.9898     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1175 - acc: 0.9607     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0468 - acc: 0.9843     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0375 - acc: 0.9877     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0330 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0291 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0263 - acc: 0.9914     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1206 - acc: 0.9593     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0522 - acc: 0.9838     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0425 - acc: 0.9866     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0387 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0329 - acc: 0.9897     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0328 - acc: 0.9898     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1134 - acc: 0.9597     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0529 - acc: 0.9821     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0461 - acc: 0.9850     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0391 - acc: 0.9888     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0362 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0339 - acc: 0.9904     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1129 - acc: 0.9599     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0495 - acc: 0.9842     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0404 - acc: 0.9856     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0366 - acc: 0.9871     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0324 - acc: 0.9875     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0309 - acc: 0.9902     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1282 - acc: 0.9558     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0520 - acc: 0.9839     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0443 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0401 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0377 - acc: 0.9890     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0336 - acc: 0.9903     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1107 - acc: 0.9653     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0514 - acc: 0.9816     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0441 - acc: 0.9859     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0388 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0324 - acc: 0.9890     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0324 - acc: 0.9893     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1122 - acc: 0.9605     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0453 - acc: 0.9843     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0393 - acc: 0.9874     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0320 - acc: 0.9883     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0284 - acc: 0.9902     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0260 - acc: 0.9920     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1182 - acc: 0.9594     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0544 - acc: 0.9817     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0491 - acc: 0.9854     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0427 - acc: 0.9865     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0378 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0333 - acc: 0.9896     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.1097 - acc: 0.9617     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0509 - acc: 0.9849     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0438 - acc: 0.9870     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0398 - acc: 0.9869     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0342 - acc: 0.9914     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0310 - acc: 0.9908     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1204 - acc: 0.9594     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0524 - acc: 0.9823     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0456 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0388 - acc: 0.9864     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0375 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0336 - acc: 0.9886     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1214 - acc: 0.9593     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0532 - acc: 0.9832     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0452 - acc: 0.9864     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0432 - acc: 0.9867     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0411 - acc: 0.9881     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0372 - acc: 0.9894     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.1215 - acc: 0.9581     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0584 - acc: 0.9807     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0493 - acc: 0.9839     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0457 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0419 - acc: 0.9880     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0355 - acc: 0.9890     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1211 - acc: 0.9530     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0529 - acc: 0.9820     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0395 - acc: 0.9874     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0374 - acc: 0.9872     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0351 - acc: 0.9890     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0319 - acc: 0.9897     \n",
      "7880/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1072 - acc: 0.9648     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0568 - acc: 0.9832     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0467 - acc: 0.9849     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0415 - acc: 0.9880     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0347 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0340 - acc: 0.9913     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.1241 - acc: 0.9589     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0548 - acc: 0.9818     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0482 - acc: 0.9843     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0445 - acc: 0.9872     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0413 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0332 - acc: 0.9897     \n",
      "7920/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1176 - acc: 0.9584     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0531 - acc: 0.9828     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0398 - acc: 0.9885     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0327 - acc: 0.9901     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0319 - acc: 0.9908     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0285 - acc: 0.9901     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1128 - acc: 0.9629     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0510 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0462 - acc: 0.9869     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0403 - acc: 0.9887     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0362 - acc: 0.9893     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0312 - acc: 0.9910     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0872 - acc: 0.9726    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0400 - acc: 0.9881     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0342 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0304 - acc: 0.9895     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0271 - acc: 0.9919     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0245 - acc: 0.9931     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [64, 64], 'batch_size': 40, 'filters': 8, 'epochs': 6, 'pool_size': 2, 'optimizer': 'SGD'}\n"
     ]
    }
   ],
   "source": [
    "########## Optimizer Test ###########\n",
    "\n",
    "def create_model(dense_layer_sizes, filters, kernel_size, pool_size, optimizer):\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    # Create first layer\n",
    "    model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(filters, kernel_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=pool_size))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    # Create second layer\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    # Return the model\n",
    "    return model\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[64,64]], # Tuned\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [40], # Tuned\n",
    "            'optimizer': ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']}\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "8148/8148 [==============================] - 12s - loss: 0.1107 - acc: 0.9617    \n",
      "Epoch 2/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0507 - acc: 0.9829     \n",
      "Epoch 3/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0439 - acc: 0.9865     \n",
      "Epoch 4/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0366 - acc: 0.9880     \n",
      "Epoch 5/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0315 - acc: 0.9896     \n",
      "Epoch 6/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0303 - acc: 0.9903     \n",
      "Epoch 7/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0288 - acc: 0.9912     \n",
      "Epoch 8/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0265 - acc: 0.9914     \n",
      "Epoch 9/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0247 - acc: 0.9925     \n",
      "Epoch 10/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0217 - acc: 0.9931     \n",
      "Epoch 11/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0217 - acc: 0.9928     \n",
      "Epoch 12/12\n",
      "8148/8148 [==============================] - 4s - loss: 0.0210 - acc: 0.9930     \n",
      "7880/8148 [============================>.] - ETA: 0sEpoch 1/12\n",
      "8149/8149 [==============================] - 8s - loss: 0.1059 - acc: 0.9651     \n",
      "Epoch 2/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0451 - acc: 0.9839     \n",
      "Epoch 3/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0367 - acc: 0.9881     \n",
      "Epoch 4/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0317 - acc: 0.9891     \n",
      "Epoch 5/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0281 - acc: 0.9909     \n",
      "Epoch 6/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0261 - acc: 0.9913     \n",
      "Epoch 7/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0205 - acc: 0.9929     \n",
      "Epoch 8/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0215 - acc: 0.9930     \n",
      "Epoch 9/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0166 - acc: 0.9945     \n",
      "Epoch 10/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0155 - acc: 0.9948     \n",
      "Epoch 11/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0177 - acc: 0.9941     \n",
      "Epoch 12/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0158 - acc: 0.9952     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/12\n",
      "8149/8149 [==============================] - 9s - loss: 0.1306 - acc: 0.9532     \n",
      "Epoch 2/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0454 - acc: 0.9869     \n",
      "Epoch 3/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0364 - acc: 0.9886     \n",
      "Epoch 4/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0340 - acc: 0.9904     \n",
      "Epoch 5/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0285 - acc: 0.9918     \n",
      "Epoch 6/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0253 - acc: 0.9917     \n",
      "Epoch 7/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0207 - acc: 0.9935     \n",
      "Epoch 8/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0222 - acc: 0.9939     \n",
      "Epoch 9/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0194 - acc: 0.9940     \n",
      "Epoch 10/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0163 - acc: 0.9952     \n",
      "Epoch 11/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0182 - acc: 0.9942     \n",
      "Epoch 12/12\n",
      "8149/8149 [==============================] - 4s - loss: 0.0164 - acc: 0.9952     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/12\n",
      "12223/12223 [==============================] - 11s - loss: 0.0919 - acc: 0.9696    \n",
      "Epoch 2/12\n",
      "12223/12223 [==============================] - 6s - loss: 0.0482 - acc: 0.9854     \n",
      "Epoch 3/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0388 - acc: 0.9872     \n",
      "Epoch 4/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0357 - acc: 0.9895     \n",
      "Epoch 5/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0325 - acc: 0.9898     \n",
      "Epoch 6/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0289 - acc: 0.9908     \n",
      "Epoch 7/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0268 - acc: 0.9916     \n",
      "Epoch 8/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0265 - acc: 0.9917     \n",
      "Epoch 9/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0239 - acc: 0.9921     \n",
      "Epoch 10/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0207 - acc: 0.9933     \n",
      "Epoch 11/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0198 - acc: 0.9938     \n",
      "Epoch 12/12\n",
      "12223/12223 [==============================] - 7s - loss: 0.0207 - acc: 0.9929     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [64, 64], 'batch_size': 40, 'filters': 8, 'epochs': 12, 'pool_size': 2, 'optimizer': 'SGD'}\n"
     ]
    }
   ],
   "source": [
    "########## Optimizer Test ###########\n",
    "\n",
    "def create_model(dense_layer_sizes, filters, kernel_size, pool_size, optimizer):\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    # Create first layer\n",
    "    model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(filters, kernel_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=pool_size))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    # Create second layer\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    # Return the model\n",
    "    return model\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[64,64]], # Tuned\n",
    "            'epochs': [12],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [40], # Tuned\n",
    "            'optimizer': ['SGD']} #tuned\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.1123 - acc: 0.9615    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0518 - acc: 0.9815     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0430 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0372 - acc: 0.9871     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0315 - acc: 0.9908     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0318 - acc: 0.9894     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.1279 - acc: 0.9536    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0498 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0407 - acc: 0.9863     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0337 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0304 - acc: 0.9903     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0277 - acc: 0.9914     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.1195 - acc: 0.9564    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0605 - acc: 0.9811     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0507 - acc: 0.9840     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0459 - acc: 0.9860     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0446 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0408 - acc: 0.9879     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.1212 - acc: 0.9594    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0521 - acc: 0.9831     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0471 - acc: 0.9845     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0439 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0372 - acc: 0.9872     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0358 - acc: 0.9898     \n",
      "4075/4075 [==============================] - 6s     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.1136 - acc: 0.9594    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0429 - acc: 0.9854     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0364 - acc: 0.9890     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0322 - acc: 0.9902     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0281 - acc: 0.9913     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0265 - acc: 0.9912     \n",
      "4074/4074 [==============================] - 6s     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.1159 - acc: 0.9600    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0514 - acc: 0.9834     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0424 - acc: 0.9866     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0412 - acc: 0.9859     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0350 - acc: 0.9893     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0343 - acc: 0.9894     \n",
      "4074/4074 [==============================] - 6s     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.1170 - acc: 0.9564    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0485 - acc: 0.9845     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0448 - acc: 0.9859     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0358 - acc: 0.9886     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0329 - acc: 0.9903     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0303 - acc: 0.9908     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.1222 - acc: 0.9569    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0491 - acc: 0.9852     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0420 - acc: 0.9863     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0317 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0291 - acc: 0.9903     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0276 - acc: 0.9907     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.1256 - acc: 0.9553    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0566 - acc: 0.9817     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0480 - acc: 0.9848     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0440 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0393 - acc: 0.9885     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0362 - acc: 0.9896     \n",
      "4074/4074 [==============================] - 6s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.1227 - acc: 0.9526    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0536 - acc: 0.9810     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0466 - acc: 0.9839     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0430 - acc: 0.9859     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0370 - acc: 0.9880     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0320 - acc: 0.9905     \n",
      "4075/4075 [==============================] - 6s     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1196 - acc: 0.9583    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0512 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0431 - acc: 0.9870     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0369 - acc: 0.9887     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0331 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0311 - acc: 0.9909     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1293 - acc: 0.9566    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0533 - acc: 0.9837     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0453 - acc: 0.9860     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0388 - acc: 0.9886     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0395 - acc: 0.9885     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0311 - acc: 0.9896     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 19s - loss: 0.1123 - acc: 0.9605    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0527 - acc: 0.9833     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0421 - acc: 0.9875     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0387 - acc: 0.9864     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0370 - acc: 0.9881     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0304 - acc: 0.9904     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.1157 - acc: 0.9602    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0449 - acc: 0.9855     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0362 - acc: 0.9894     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0308 - acc: 0.9894     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0284 - acc: 0.9924     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0238 - acc: 0.9915     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.1170 - acc: 0.9606    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0505 - acc: 0.9843     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0467 - acc: 0.9859     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0417 - acc: 0.9875     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0365 - acc: 0.9885     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0343 - acc: 0.9887     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 16s - loss: 0.1037 - acc: 0.9602    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0416 - acc: 0.9857     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0330 - acc: 0.9897     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0266 - acc: 0.9917     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0250 - acc: 0.9935     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0218 - acc: 0.9930     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [64, 64], 'batch_size': 40, 'filters': 8, 'optimizer': 'SGD', 'epochs': 6, 'pool_size': 2, 'learn_rate': 0.1}\n"
     ]
    }
   ],
   "source": [
    "########## Learn Rate ###########\n",
    "\n",
    "def create_model(dense_layer_sizes, filters, kernel_size, pool_size, optimizer, learn_rate):\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    # Create first layer\n",
    "    model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(filters, kernel_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=pool_size))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    # Create second layer\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    # Return the model\n",
    "    return model\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[64,64]], # Tuned\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [40], # Tuned\n",
    "            'optimizer': ['SGD'], # Tuned\n",
    "            'learn_rate': [0.001, 0.01, 0.1, 0.2, 0.3]}\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 16s - loss: 0.1219 - acc: 0.9558    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0490 - acc: 0.9836     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0368 - acc: 0.9872     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0343 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0299 - acc: 0.9904     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0270 - acc: 0.9929     \n",
      "4075/4075 [==============================] - 7s     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1127 - acc: 0.9610    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0522 - acc: 0.9832     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0397 - acc: 0.9880     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0309 - acc: 0.9899     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0316 - acc: 0.9893     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0296 - acc: 0.9907     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1212 - acc: 0.9559    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0541 - acc: 0.9812     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0438 - acc: 0.9856     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0426 - acc: 0.9861     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0355 - acc: 0.9897     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0361 - acc: 0.9892     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.1101 - acc: 0.9624    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0524 - acc: 0.9826     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0437 - acc: 0.9875     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0398 - acc: 0.9880     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0374 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0322 - acc: 0.9901     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1230 - acc: 0.9575    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0526 - acc: 0.9802     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0442 - acc: 0.9852     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0374 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0367 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0324 - acc: 0.9893     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1136 - acc: 0.9597    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0497 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0443 - acc: 0.9859     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0418 - acc: 0.9887     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0321 - acc: 0.9898     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0267 - acc: 0.9901     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.1199 - acc: 0.9569    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0505 - acc: 0.9839     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0411 - acc: 0.9872     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0372 - acc: 0.9888     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0334 - acc: 0.9904     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0286 - acc: 0.9912     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1168 - acc: 0.9585    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0500 - acc: 0.9836     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0387 - acc: 0.9876     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0382 - acc: 0.9872     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0321 - acc: 0.9899     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0315 - acc: 0.9901     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1300 - acc: 0.9515    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0498 - acc: 0.9833     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0414 - acc: 0.9861     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0336 - acc: 0.9903     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0306 - acc: 0.9915     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0272 - acc: 0.9909     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.1262 - acc: 0.9566    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0520 - acc: 0.9848     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0404 - acc: 0.9878     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0395 - acc: 0.9882     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0346 - acc: 0.9894     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0312 - acc: 0.9904     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.1176 - acc: 0.9594    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0464 - acc: 0.9845     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0382 - acc: 0.9861     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0336 - acc: 0.9888     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0290 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0267 - acc: 0.9917     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1218 - acc: 0.9613    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0556 - acc: 0.9822     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0466 - acc: 0.9869     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0407 - acc: 0.9881     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0384 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0358 - acc: 0.9892     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.1224 - acc: 0.9550    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0535 - acc: 0.9836     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0453 - acc: 0.9865     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0406 - acc: 0.9883     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0354 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0319 - acc: 0.9907     \n",
      "4075/4075 [==============================] - 7s     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1225 - acc: 0.9564    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0482 - acc: 0.9837     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0383 - acc: 0.9871     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0361 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0317 - acc: 0.9887     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0284 - acc: 0.9912     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1129 - acc: 0.9611    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0545 - acc: 0.9821     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0478 - acc: 0.9848     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0389 - acc: 0.9881     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0384 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0360 - acc: 0.9901     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.1211 - acc: 0.9584    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0526 - acc: 0.9839     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0406 - acc: 0.9882     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0393 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0356 - acc: 0.9887     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0317 - acc: 0.9899     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1247 - acc: 0.9562    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0412 - acc: 0.9858     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0324 - acc: 0.9896     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0291 - acc: 0.9908     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0241 - acc: 0.9921     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0199 - acc: 0.9935     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.1217 - acc: 0.9586    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0599 - acc: 0.9817     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0498 - acc: 0.9840     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0443 - acc: 0.9867     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0421 - acc: 0.9869     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0416 - acc: 0.9872     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 17s - loss: 0.0943 - acc: 0.9677    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0420 - acc: 0.9871     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0318 - acc: 0.9903     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0266 - acc: 0.9922     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0260 - acc: 0.9931     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0243 - acc: 0.9925     \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 3, 'dense_layer_sizes': [64, 64], 'batch_size': 40, 'momentum': 0.4, 'filters': 8, 'optimizer': 'SGD', 'epochs': 6, 'pool_size': 2, 'learn_rate': 0.1}\n"
     ]
    }
   ],
   "source": [
    "########## Learn Rate ###########\n",
    "\n",
    "def create_model(dense_layer_sizes, filters, kernel_size, pool_size, optimizer, learn_rate, momentum):\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    # Create first layer\n",
    "    model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(filters, kernel_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=pool_size))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    # Create second layer\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    # Return the model\n",
    "    return model\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[64,64]], # Tuned\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [40], # Tuned\n",
    "            'optimizer': ['SGD'], # Tuned\n",
    "            'learn_rate': [0.1], # Tuned\n",
    "            'momentum': [0.0, 0.2, 0.4, 0.6, 0.8, 0.9]\n",
    "           }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 18s - loss: 0.1107 - acc: 0.9607    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0549 - acc: 0.9826     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0465 - acc: 0.9848     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0427 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0379 - acc: 0.9886     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0329 - acc: 0.9899     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1227 - acc: 0.9536    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0521 - acc: 0.9828     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0451 - acc: 0.9852     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0404 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0339 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0325 - acc: 0.9903     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.1224 - acc: 0.9559    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0513 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0428 - acc: 0.9856     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0385 - acc: 0.9885     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0325 - acc: 0.9893     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0292 - acc: 0.9904     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.1226 - acc: 0.9550    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0528 - acc: 0.9822     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0433 - acc: 0.9861     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0382 - acc: 0.9880     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0338 - acc: 0.9901     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0270 - acc: 0.9910     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.1280 - acc: 0.9526    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0495 - acc: 0.9842     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0419 - acc: 0.9863     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0341 - acc: 0.9896     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0343 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0302 - acc: 0.9907     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.1177 - acc: 0.9589    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0495 - acc: 0.9839     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0391 - acc: 0.9880     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0365 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0294 - acc: 0.9914     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0281 - acc: 0.9923     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.1231 - acc: 0.9556    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0509 - acc: 0.9853     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0425 - acc: 0.9876     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0365 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0332 - acc: 0.9899     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0287 - acc: 0.9920     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.1354 - acc: 0.9443    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0468 - acc: 0.9860     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0360 - acc: 0.9886     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0284 - acc: 0.9899     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0241 - acc: 0.9928     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0224 - acc: 0.9914     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.1276 - acc: 0.9517    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0516 - acc: 0.9832     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0423 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0392 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0332 - acc: 0.9898     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0302 - acc: 0.9910     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 18s - loss: 0.0925 - acc: 0.9688    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0433 - acc: 0.9861    \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0359 - acc: 0.9894    \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0295 - acc: 0.9906    \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0249 - acc: 0.9917    \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0218 - acc: 0.9935    \n",
      "The parameters of the best model are: \n",
      "{'kernel_size': 5, 'dense_layer_sizes': [64, 64], 'batch_size': 40, 'momentum': 0.4, 'filters': 8, 'optimizer': 'SGD', 'epochs': 6, 'pool_size': 2, 'learn_rate': 0.1}\n"
     ]
    }
   ],
   "source": [
    "########## Learn Rate ###########\n",
    "\n",
    "def create_model(dense_layer_sizes, filters, kernel_size, pool_size, optimizer, learn_rate, momentum):\n",
    "    \n",
    "    model = Sequential()\n",
    "    \n",
    "    # Create first layer\n",
    "    model.add(Conv2D(filters, kernel_size, padding='valid', input_shape=input_shape))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Conv2D(filters, kernel_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(MaxPooling2D(pool_size=pool_size))\n",
    "    model.add(Dropout(0.25))\n",
    "    \n",
    "    # Create second layer\n",
    "    model.add(Flatten())\n",
    "    for layer_size in dense_layer_sizes:\n",
    "        model.add(Dense(layer_size))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes))\n",
    "    model.add(Activation('softmax'))\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "    \n",
    "    # Return the model\n",
    "    return model\n",
    "\n",
    "param_grid={'dense_layer_sizes': [[64,64]], # Tuned\n",
    "            'epochs': [6],\n",
    "            'filters': [8],\n",
    "            'kernel_size': [3,4,5],\n",
    "            'pool_size': [2],\n",
    "            'batch_size': [40], # Tuned\n",
    "            'optimizer': ['SGD'], # Tuned\n",
    "            'learn_rate': [0.1], # Tuned\n",
    "            'momentum': [0.4] # Tuned\n",
    "           }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "grid = GridSearchCV(my_classifier, param_grid, scoring='neg_log_loss', n_jobs=1)\n",
    "grid.fit(x_train, y_train)\n",
    "\n",
    "print('The parameters of the best model are: ')\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}