param_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (12223, 28, 28, 1)\n",
      "12223 train samples\n",
      "2060 test samples\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import numpy \n",
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from keras.optimizers import SGD\n",
    "from keras import backend as K\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "num_classes = 2\n",
    "\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# the data, shuffled and split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "\n",
    "#Only look at 2s and 7s\n",
    "train_picks = numpy.logical_or(y_train==2,y_train==7)\n",
    "test_picks = numpy.logical_or(y_test==2,y_test==7)\n",
    "\n",
    "x_train = x_train[train_picks]\n",
    "x_test = x_test[test_picks]\n",
    "y_train = numpy.array(y_train[train_picks]==7,dtype=int)\n",
    "y_test = numpy.array(y_test[test_picks]==7,dtype=int)\n",
    "\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n",
      "Epoch 1/10\n",
      "8148/8148 [==============================] - 20s - loss: 0.1397 - acc: 0.9500    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 12s - loss: 0.0792 - acc: 0.9757    \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 12s - loss: 0.0672 - acc: 0.9797    \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 12s - loss: 0.0661 - acc: 0.9806    \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 12s - loss: 0.0660 - acc: 0.9812    \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 12s - loss: 0.0600 - acc: 0.9829    \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 11s - loss: 0.0590 - acc: 0.9826    \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 11s - loss: 0.0582 - acc: 0.9856    \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 11s - loss: 0.0566 - acc: 0.9850    \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 11s - loss: 0.0537 - acc: 0.9845    \n",
      "8148/8148 [==============================] - 4s     \n",
      "Epoch 1/10\n",
      "8149/8149 [==============================] - 19s - loss: 0.1424 - acc: 0.9493    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.0781 - acc: 0.9758    \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.0700 - acc: 0.9784    \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.0658 - acc: 0.9815    \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.0594 - acc: 0.9817    \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 12s - loss: 0.0583 - acc: 0.9821    \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0562 - acc: 0.9837     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0568 - acc: 0.9834     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0549 - acc: 0.9848     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0527 - acc: 0.9840     \n",
      "8090/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 20s - loss: 0.1477 - acc: 0.9432    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.0827 - acc: 0.9766    \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.0708 - acc: 0.9800    \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0758 - acc: 0.9775     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0675 - acc: 0.9793     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0638 - acc: 0.9828     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0602 - acc: 0.9840     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0617 - acc: 0.9836     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0560 - acc: 0.9838     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 9s - loss: 0.0590 - acc: 0.9839     \n",
      "8110/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 14s - loss: 0.1965 - acc: 0.9244    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0968 - acc: 0.9747     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0824 - acc: 0.9795     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0801 - acc: 0.9812     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0703 - acc: 0.9817     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0656 - acc: 0.9824     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0616 - acc: 0.9848     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0570 - acc: 0.9842     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0575 - acc: 0.9863     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0501 - acc: 0.9880     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 14s - loss: 0.1646 - acc: 0.9374    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0828 - acc: 0.9730     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0741 - acc: 0.9783     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0658 - acc: 0.9804     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0617 - acc: 0.9800     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0581 - acc: 0.9811     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0496 - acc: 0.9837     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0514 - acc: 0.9840     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0499 - acc: 0.9840     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0480 - acc: 0.9840     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 14s - loss: 0.1861 - acc: 0.9342    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0955 - acc: 0.9736     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0812 - acc: 0.9766     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0729 - acc: 0.9802     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0690 - acc: 0.9804     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0662 - acc: 0.9825     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0644 - acc: 0.9836     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0586 - acc: 0.9834     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0575 - acc: 0.9828     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0520 - acc: 0.9859     \n",
      "4074/4074 [==============================] - 9s     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 10s - loss: 0.1874 - acc: 0.9325    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0828 - acc: 0.9741     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 3s - loss: 0.0730 - acc: 0.9780     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 3s - loss: 0.0674 - acc: 0.9799     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0692 - acc: 0.9796     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0631 - acc: 0.9826     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 3s - loss: 0.0596 - acc: 0.9810     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 3s - loss: 0.0585 - acc: 0.9823     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0579 - acc: 0.9842     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 3s - loss: 0.0544 - acc: 0.9854     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/10\n",
      "8149/8149 [==============================] - 10s - loss: 0.2151 - acc: 0.9172    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0852 - acc: 0.9728     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0709 - acc: 0.9767     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0696 - acc: 0.9775     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0645 - acc: 0.9775     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0601 - acc: 0.9805     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0585 - acc: 0.9804     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0577 - acc: 0.9823     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0531 - acc: 0.9822     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0505 - acc: 0.9815     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/10\n",
      "8149/8149 [==============================] - 10s - loss: 0.2029 - acc: 0.9184    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0797 - acc: 0.9730     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0775 - acc: 0.9756     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0676 - acc: 0.9800     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0616 - acc: 0.9807     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0557 - acc: 0.9828     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0586 - acc: 0.9810     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0550 - acc: 0.9832     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0540 - acc: 0.9838     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0538 - acc: 0.9850     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/10\n",
      "8148/8148 [==============================] - 10s - loss: 0.2150 - acc: 0.9148    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0882 - acc: 0.9710     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0730 - acc: 0.9748     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0702 - acc: 0.9777     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0674 - acc: 0.9780     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0665 - acc: 0.9816     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0682 - acc: 0.9806     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0600 - acc: 0.9818     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0630 - acc: 0.9811     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0591 - acc: 0.9842     \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 10s - loss: 0.2164 - acc: 0.9179    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0793 - acc: 0.9720     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0685 - acc: 0.9758     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0604 - acc: 0.9798     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0601 - acc: 0.9807     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0539 - acc: 0.9816     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0511 - acc: 0.9818     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0496 - acc: 0.9831     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0488 - acc: 0.9831     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0429 - acc: 0.9843     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 10s - loss: 0.2086 - acc: 0.9173    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0859 - acc: 0.9736     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0746 - acc: 0.9751     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0649 - acc: 0.9799     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0659 - acc: 0.9817     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 3s - loss: 0.0600 - acc: 0.9845     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0622 - acc: 0.9829     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0592 - acc: 0.9838     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0532 - acc: 0.9845     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0530 - acc: 0.9850     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 10s - loss: 0.2604 - acc: 0.8925    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0867 - acc: 0.9710     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0752 - acc: 0.9748     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0697 - acc: 0.9789     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0636 - acc: 0.9818     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0686 - acc: 0.9797     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0560 - acc: 0.9833     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0590 - acc: 0.9849     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0564 - acc: 0.9842     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0569 - acc: 0.9834     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 13s - loss: 0.2464 - acc: 0.9140    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0832 - acc: 0.9730     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0707 - acc: 0.9775     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0641 - acc: 0.9791     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0601 - acc: 0.9809     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0562 - acc: 0.9827     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0530 - acc: 0.9831     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0529 - acc: 0.9828     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0526 - acc: 0.9838     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0484 - acc: 0.9837     \n",
      "4074/4074 [==============================] - 9s     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/10\n",
      "8149/8149 [==============================] - 11s - loss: 0.2388 - acc: 0.9108    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.1029 - acc: 0.9671     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0858 - acc: 0.9739     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0785 - acc: 0.9759     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0709 - acc: 0.9807     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0724 - acc: 0.9802     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0701 - acc: 0.9816     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0652 - acc: 0.9813     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0633 - acc: 0.9826     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0623 - acc: 0.9828     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 10s - loss: 0.2796 - acc: 0.8931    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.1051 - acc: 0.9622     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0774 - acc: 0.9704     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0693 - acc: 0.9736     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0692 - acc: 0.9759     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0676 - acc: 0.9759     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0623 - acc: 0.9768     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0613 - acc: 0.9807     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0588 - acc: 0.9813     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 2s - loss: 0.0564 - acc: 0.9811     \n",
      "4075/4075 [==============================] - 7s     \n",
      "7900/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 10s - loss: 0.3128 - acc: 0.8570    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.1191 - acc: 0.9674     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.1066 - acc: 0.9735     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0921 - acc: 0.9800     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0879 - acc: 0.9813     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0862 - acc: 0.9829     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0772 - acc: 0.9855     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0747 - acc: 0.9854     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0750 - acc: 0.9840     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0691 - acc: 0.9883     \n",
      "4074/4074 [==============================] - 7s     \n",
      "7900/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 10s - loss: 0.3265 - acc: 0.8731    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0954 - acc: 0.9663     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0788 - acc: 0.9729     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0713 - acc: 0.9752     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0682 - acc: 0.9780     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0630 - acc: 0.9788     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0590 - acc: 0.9809     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0595 - acc: 0.9809     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0558 - acc: 0.9816     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 2s - loss: 0.0544 - acc: 0.9832     \n",
      "8000/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed: 19.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "12223/12223 [==============================] - 18s - loss: 0.1390 - acc: 0.9477    \n",
      "Epoch 2/10\n",
      "12223/12223 [==============================] - 10s - loss: 0.0692 - acc: 0.9768    \n",
      "Epoch 3/10\n",
      "12223/12223 [==============================] - 10s - loss: 0.0622 - acc: 0.9805    \n",
      "Epoch 4/10\n",
      "12223/12223 [==============================] - 9s - loss: 0.0569 - acc: 0.9801     \n",
      "Epoch 5/10\n",
      "12223/12223 [==============================] - 10s - loss: 0.0561 - acc: 0.9821    \n",
      "Epoch 6/10\n",
      "12223/12223 [==============================] - 10s - loss: 0.0506 - acc: 0.9840    \n",
      "Epoch 7/10\n",
      "12223/12223 [==============================] - 8s - loss: 0.0438 - acc: 0.9855     \n",
      "Epoch 8/10\n",
      "12223/12223 [==============================] - 7s - loss: 0.0478 - acc: 0.9843     \n",
      "Epoch 9/10\n",
      "12223/12223 [==============================] - 8s - loss: 0.0407 - acc: 0.9862     \n",
      "Epoch 10/10\n",
      "12223/12223 [==============================] - 8s - loss: 0.0431 - acc: 0.9871     \n",
      "Best: 0.988546 using {'epochs': 10, 'batch_size': 20}\n",
      "0.986664 (0.001634) with: {'epochs': 10, 'batch_size': 10}\n",
      "0.988546 (0.002018) with: {'epochs': 10, 'batch_size': 20}\n",
      "0.988137 (0.000835) with: {'epochs': 10, 'batch_size': 40}\n",
      "0.988301 (0.000613) with: {'epochs': 10, 'batch_size': 60}\n",
      "0.987810 (0.000418) with: {'epochs': 10, 'batch_size': 80}\n",
      "0.988301 (0.000506) with: {'epochs': 10, 'batch_size': 100}\n",
      "Best: 0.988546 using {'epochs': 10, 'batch_size': 20}\n",
      "1940/2060 [===========================>..] - ETA: 0sTest Accuracy 0.986407764907\n"
     ]
    }
   ],
   "source": [
    "##### BATCH SIZE #####\n",
    "\n",
    "def create_model():\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [10, 20, 40, 60, 80, 100]}\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "Epoch 1/10\n",
      "8148/8148 [==============================] - 19s - loss: 0.1533 - acc: 0.9427    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 7s - loss: 0.0760 - acc: 0.9731     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 7s - loss: 0.0682 - acc: 0.9775     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0686 - acc: 0.9789     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0664 - acc: 0.9795     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0609 - acc: 0.9821     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0585 - acc: 0.9802     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0565 - acc: 0.9829     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0547 - acc: 0.9850     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0552 - acc: 0.9843     \n",
      "4075/4075 [==============================] - 13s    \n",
      "8148/8148 [==============================] - 3s     \n",
      "Epoch 1/10\n",
      "8149/8149 [==============================] - 16s - loss: 0.1516 - acc: 0.9434    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0751 - acc: 0.9764     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0674 - acc: 0.9813     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0657 - acc: 0.9826     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0587 - acc: 0.9823     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0580 - acc: 0.9811     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0515 - acc: 0.9843     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0496 - acc: 0.9858     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0507 - acc: 0.9864     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0506 - acc: 0.9850     \n",
      "4074/4074 [==============================] - 9s     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 16s - loss: 0.1683 - acc: 0.9331    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0803 - acc: 0.9759     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0701 - acc: 0.9801     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0721 - acc: 0.9784     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0649 - acc: 0.9817     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0591 - acc: 0.9832     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0575 - acc: 0.9843     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0549 - acc: 0.9843     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 4s - loss: 0.0556 - acc: 0.9852     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0561 - acc: 0.9856     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 14s - loss: 0.1966 - acc: 0.9242    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0968 - acc: 0.9743     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0826 - acc: 0.9794     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0801 - acc: 0.9816     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0703 - acc: 0.9818     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0656 - acc: 0.9823     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0617 - acc: 0.9845     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 4s - loss: 0.0571 - acc: 0.9843     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0575 - acc: 0.9865     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 4s - loss: 0.0502 - acc: 0.9880     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 14s - loss: 0.1647 - acc: 0.9372    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0829 - acc: 0.9728     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0741 - acc: 0.9779     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0659 - acc: 0.9804     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0619 - acc: 0.9796     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0583 - acc: 0.9815     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0496 - acc: 0.9837     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0514 - acc: 0.9840     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0498 - acc: 0.9842     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0482 - acc: 0.9837     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 15s - loss: 0.1861 - acc: 0.9345    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0955 - acc: 0.9739     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0812 - acc: 0.9767     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0728 - acc: 0.9802     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0691 - acc: 0.9804     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0661 - acc: 0.9823     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0643 - acc: 0.9834     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0585 - acc: 0.9837     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0576 - acc: 0.9831     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0519 - acc: 0.9860     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 14s - loss: 0.1544 - acc: 0.9466    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0828 - acc: 0.9721     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0730 - acc: 0.9766     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0686 - acc: 0.9785     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0672 - acc: 0.9818     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0602 - acc: 0.9818     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0577 - acc: 0.9827     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0580 - acc: 0.9820     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0553 - acc: 0.9845     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 5s - loss: 0.0499 - acc: 0.9853     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 16s - loss: 0.1767 - acc: 0.9302    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0930 - acc: 0.9688     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0752 - acc: 0.9782     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0732 - acc: 0.9767     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0654 - acc: 0.9782     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0622 - acc: 0.9805     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0591 - acc: 0.9821     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0576 - acc: 0.9836     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0536 - acc: 0.9839     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0469 - acc: 0.9858     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 14s - loss: 0.1788 - acc: 0.9320    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0819 - acc: 0.9752     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0785 - acc: 0.9757     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0694 - acc: 0.9791     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0651 - acc: 0.9813     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0622 - acc: 0.9833     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0591 - acc: 0.9818     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0551 - acc: 0.9838     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0543 - acc: 0.9849     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0521 - acc: 0.9861     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 14s - loss: 0.1571 - acc: 0.9429    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0821 - acc: 0.9740     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0736 - acc: 0.9778     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0719 - acc: 0.9780     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0682 - acc: 0.9805     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0653 - acc: 0.9809     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0626 - acc: 0.9813     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0628 - acc: 0.9823     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0584 - acc: 0.9844     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0609 - acc: 0.9832     \n",
      "8148/8148 [==============================] - 2s     \n",
      "Epoch 1/10\n",
      "8149/8149 [==============================] - 14s - loss: 0.1587 - acc: 0.9399    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0769 - acc: 0.9698     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0662 - acc: 0.9757     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0667 - acc: 0.9741     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0605 - acc: 0.9757     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0568 - acc: 0.9766     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0533 - acc: 0.9802     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0509 - acc: 0.9820     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0537 - acc: 0.9839     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0444 - acc: 0.9844     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 14s - loss: 0.1502 - acc: 0.9454    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0782 - acc: 0.9748     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0693 - acc: 0.9774     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0678 - acc: 0.9806     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0634 - acc: 0.9826     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0562 - acc: 0.9852     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0638 - acc: 0.9823     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0544 - acc: 0.9839     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0558 - acc: 0.9854     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0478 - acc: 0.9872     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 14s - loss: 0.1583 - acc: 0.9419    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0778 - acc: 0.9750     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0732 - acc: 0.9772     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0646 - acc: 0.9817     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0618 - acc: 0.9812     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0632 - acc: 0.9832     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0585 - acc: 0.9840     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0538 - acc: 0.9832     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0541 - acc: 0.9833     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0520 - acc: 0.9848     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 15s - loss: 0.1580 - acc: 0.9409    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0763 - acc: 0.9759     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0727 - acc: 0.9762     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0628 - acc: 0.9790     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0594 - acc: 0.9795     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0575 - acc: 0.9837     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0518 - acc: 0.9844     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0509 - acc: 0.9836     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0483 - acc: 0.9848     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0447 - acc: 0.9850     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 15s - loss: 0.1573 - acc: 0.9424    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0920 - acc: 0.9715     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0753 - acc: 0.9757     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0759 - acc: 0.9766     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0691 - acc: 0.9809     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0671 - acc: 0.9795     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0660 - acc: 0.9807     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0627 - acc: 0.9823     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0633 - acc: 0.9827     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0606 - acc: 0.9815     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 196s - loss: 0.1667 - acc: 0.9378   \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0847 - acc: 0.9710     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0709 - acc: 0.9773     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0693 - acc: 0.9789     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0688 - acc: 0.9791     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0637 - acc: 0.9817     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0610 - acc: 0.9811     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0546 - acc: 0.9838     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0512 - acc: 0.9845     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0521 - acc: 0.9860     \n",
      "4075/4075 [==============================] - 11s    \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 17s - loss: 0.1894 - acc: 0.9269    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.1019 - acc: 0.9769     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0864 - acc: 0.9807     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0828 - acc: 0.9817     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0741 - acc: 0.9840     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0721 - acc: 0.9849     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0635 - acc: 0.9861     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0558 - acc: 0.9871     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0541 - acc: 0.9864     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0483 - acc: 0.9890     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 17s - loss: 0.1803 - acc: 0.9356    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0807 - acc: 0.9724     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0786 - acc: 0.9751     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0670 - acc: 0.9783     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0651 - acc: 0.9799     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0602 - acc: 0.9798     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0647 - acc: 0.9810     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0557 - acc: 0.9829     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0542 - acc: 0.9816     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0532 - acc: 0.9832     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8148/8148 [==============================] - 15s - loss: 0.1677 - acc: 0.9410    \n",
      "Epoch 2/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0759 - acc: 0.9726     \n",
      "Epoch 3/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0700 - acc: 0.9761     \n",
      "Epoch 4/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0681 - acc: 0.9766     \n",
      "Epoch 5/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0647 - acc: 0.9777     \n",
      "Epoch 6/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0624 - acc: 0.9766     \n",
      "Epoch 7/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0559 - acc: 0.9815     \n",
      "Epoch 8/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0540 - acc: 0.9818     \n",
      "Epoch 9/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0531 - acc: 0.9837     \n",
      "Epoch 10/10\n",
      "8148/8148 [==============================] - 6s - loss: 0.0517 - acc: 0.9834     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 16s - loss: 0.1838 - acc: 0.9245    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 7s - loss: 0.0874 - acc: 0.9740     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0755 - acc: 0.9788     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0698 - acc: 0.9793     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0608 - acc: 0.9825     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0606 - acc: 0.9825     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0563 - acc: 0.9847     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0567 - acc: 0.9839     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0511 - acc: 0.9850     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0511 - acc: 0.9856     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/10\n",
      "8149/8149 [==============================] - 15s - loss: 0.1759 - acc: 0.9346    \n",
      "Epoch 2/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0879 - acc: 0.9734     \n",
      "Epoch 3/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0781 - acc: 0.9757     \n",
      "Epoch 4/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0775 - acc: 0.9778     \n",
      "Epoch 5/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0733 - acc: 0.9813     \n",
      "Epoch 6/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0745 - acc: 0.9813     \n",
      "Epoch 7/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0701 - acc: 0.9806     \n",
      "Epoch 8/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0636 - acc: 0.9828     \n",
      "Epoch 9/10\n",
      "8149/8149 [==============================] - 6s - loss: 0.0636 - acc: 0.9833     \n",
      "Epoch 10/10\n",
      "8149/8149 [==============================] - 5s - loss: 0.0629 - acc: 0.9816     \n",
      "8040/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed: 33.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "12223/12223 [==============================] - 19s - loss: 0.1337 - acc: 0.9489    \n",
      "Epoch 2/10\n",
      "12223/12223 [==============================] - 10s - loss: 0.0758 - acc: 0.9747    \n",
      "Epoch 3/10\n",
      "12223/12223 [==============================] - 10s - loss: 0.0643 - acc: 0.9799    \n",
      "Epoch 4/10\n",
      "12223/12223 [==============================] - 8s - loss: 0.0586 - acc: 0.9818     \n",
      "Epoch 5/10\n",
      "12223/12223 [==============================] - 8s - loss: 0.0574 - acc: 0.9827     \n",
      "Epoch 6/10\n",
      "12223/12223 [==============================] - 7s - loss: 0.0531 - acc: 0.9841     \n",
      "Epoch 7/10\n",
      "12223/12223 [==============================] - 7s - loss: 0.0512 - acc: 0.9836     \n",
      "Epoch 8/10\n",
      "12223/12223 [==============================] - 7s - loss: 0.0483 - acc: 0.9850     \n",
      "Epoch 9/10\n",
      "12223/12223 [==============================] - 7s - loss: 0.0495 - acc: 0.9854     \n",
      "Epoch 10/10\n",
      "12223/12223 [==============================] - 7s - loss: 0.0477 - acc: 0.9854     \n",
      "Best: 0.988873 using {'optimizer': 'Adamax', 'epochs': 10, 'batch_size': 20}\n",
      "0.987401 (0.001305) with: {'optimizer': 'SGD', 'epochs': 10, 'batch_size': 20}\n",
      "0.988628 (0.001729) with: {'optimizer': 'RMSprop', 'epochs': 10, 'batch_size': 20}\n",
      "0.988301 (0.000759) with: {'optimizer': 'Adagrad', 'epochs': 10, 'batch_size': 20}\n",
      "0.988383 (0.000948) with: {'optimizer': 'Adadelta', 'epochs': 10, 'batch_size': 20}\n",
      "0.987646 (0.000645) with: {'optimizer': 'Adam', 'epochs': 10, 'batch_size': 20}\n",
      "0.988873 (0.000233) with: {'optimizer': 'Adamax', 'epochs': 10, 'batch_size': 20}\n",
      "0.987973 (0.001315) with: {'optimizer': 'Nadam', 'epochs': 10, 'batch_size': 20}\n",
      "Best: 0.988873 using {'optimizer': 'Adamax', 'epochs': 10, 'batch_size': 20}\n",
      "2020/2060 [============================>.] - ETA: 0sTest Accuracy 0.983009706423\n"
     ]
    }
   ],
   "source": [
    "##### OPTIMIZATION ALGORITHM #####\n",
    "\n",
    "def create_model(optimizer):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [10],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.1266 - acc: 0.9548     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0722 - acc: 0.9737     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0656 - acc: 0.9797     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0612 - acc: 0.9784     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0574 - acc: 0.9844     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0453 - acc: 0.9861     \n",
      "7820/8148 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1327 - acc: 0.9525     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0706 - acc: 0.9736     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0562 - acc: 0.9777     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0477 - acc: 0.9804     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0422 - acc: 0.9852     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0385 - acc: 0.9869     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1300 - acc: 0.9555     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0726 - acc: 0.9744     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0601 - acc: 0.9775     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0544 - acc: 0.9811     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0470 - acc: 0.9828     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0442 - acc: 0.9850     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.1313 - acc: 0.9505     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0693 - acc: 0.9769     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0538 - acc: 0.9843     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0532 - acc: 0.9833     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0464 - acc: 0.9861     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0364 - acc: 0.9870     \n",
      "7840/8148 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1101 - acc: 0.9649     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0673 - acc: 0.9786     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0532 - acc: 0.9828     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0471 - acc: 0.9850     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0443 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0379 - acc: 0.9867     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1361 - acc: 0.9534     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0828 - acc: 0.9735     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0596 - acc: 0.9820     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0502 - acc: 0.9833     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0461 - acc: 0.9866     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0455 - acc: 0.9863     \n",
      "7800/8149 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.1209 - acc: 0.9573     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0626 - acc: 0.9805     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0518 - acc: 0.9838     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0466 - acc: 0.9856     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0417 - acc: 0.9867     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0378 - acc: 0.9871     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1440 - acc: 0.9534     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0700 - acc: 0.9794     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0489 - acc: 0.9847     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0502 - acc: 0.9833     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0375 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0388 - acc: 0.9883     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1221 - acc: 0.9589     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0543 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0463 - acc: 0.9845     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0388 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0351 - acc: 0.9867     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0353 - acc: 0.9879     \n",
      "7820/8149 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.1386 - acc: 0.9464     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0802 - acc: 0.9741     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0679 - acc: 0.9779     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0510 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0483 - acc: 0.9828     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0446 - acc: 0.9859     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1321 - acc: 0.9512     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0618 - acc: 0.9789     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0454 - acc: 0.9848     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0372 - acc: 0.9866     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0353 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0302 - acc: 0.9886     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1230 - acc: 0.9545     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0722 - acc: 0.9709     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0687 - acc: 0.9746     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0584 - acc: 0.9812     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0522 - acc: 0.9807     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0472 - acc: 0.9827     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.1646 - acc: 0.9438     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0852 - acc: 0.9709     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0730 - acc: 0.9752     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0718 - acc: 0.9750     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0637 - acc: 0.9786     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0596 - acc: 0.9793     \n",
      "4075/4075 [==============================] - 0s     \n",
      "7820/8148 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1190 - acc: 0.9569     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0828 - acc: 0.9719     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0751 - acc: 0.9767     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0537 - acc: 0.9815     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0562 - acc: 0.9812     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0522 - acc: 0.9828     \n",
      "7800/8149 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1509 - acc: 0.9455     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0772 - acc: 0.9741     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0641 - acc: 0.9816     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0611 - acc: 0.9798     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0543 - acc: 0.9825     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0889 - acc: 0.9807     \n",
      "7840/8149 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.5575 - acc: 0.7982     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 4.1401 - acc: 0.5230     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 4.4864 - acc: 0.4920     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 4.3578 - acc: 0.4950     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 4.3967 - acc: 0.4999     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 4.5254 - acc: 0.4939     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 3.9856 - acc: 0.6667     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.2566 - acc: 0.5333     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.2395 - acc: 0.5345     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 7.2437 - acc: 0.5354     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.2705 - acc: 0.5342     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 7.1924 - acc: 0.5336     \n",
      "7860/8149 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 2.6361 - acc: 0.7547     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 7.1212 - acc: 0.5210     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.2131 - acc: 0.5105     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.2164 - acc: 0.5068     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.3576 - acc: 0.4972     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 7.3084 - acc: 0.5085     \n",
      "7860/8149 [===========================>..] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed:  6.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.1217 - acc: 0.9574     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0576 - acc: 0.9822     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0464 - acc: 0.9860     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 3s - loss: 0.0408 - acc: 0.9872     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0319 - acc: 0.9903     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0351 - acc: 0.9894     \n",
      "Best: 0.992719 using {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.991164 (0.001713) with: {'batch_size': 20, 'momentum': 0.0, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.992310 (0.001408) with: {'batch_size': 20, 'momentum': 0.2, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.992719 (0.000612) with: {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.990592 (0.001223) with: {'batch_size': 20, 'momentum': 0.6, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.983392 (0.004559) with: {'batch_size': 20, 'momentum': 0.8, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.488914 (0.009715) with: {'batch_size': 20, 'momentum': 0.9, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "Best: 0.992719 using {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "1900/2060 [==========================>...] - ETA: 0sTest Accuracy 0.989320386382\n"
     ]
    }
   ],
   "source": [
    "##### MOMENTUM #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [0.0, 0.2, 0.4, 0.6, 0.8, 0.9]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.4815 - acc: 0.7768     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.2418 - acc: 0.9118     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.1697 - acc: 0.9424     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.1425 - acc: 0.9492     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.1251 - acc: 0.9574     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.1142 - acc: 0.9613     \n",
      "7880/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.4420 - acc: 0.8158     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.2230 - acc: 0.9174     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1674 - acc: 0.9357     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1404 - acc: 0.9485     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1341 - acc: 0.9488     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1222 - acc: 0.9548     \n",
      "4074/4074 [==============================] - 0s     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.5970 - acc: 0.7181     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.2919 - acc: 0.8973     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1797 - acc: 0.9369     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1492 - acc: 0.9450     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1333 - acc: 0.9540     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1169 - acc: 0.9586     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.2701 - acc: 0.8744     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.1028 - acc: 0.9642     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0838 - acc: 0.9715     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0789 - acc: 0.9740     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0750 - acc: 0.9755     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0732 - acc: 0.9743     \n",
      "7800/8148 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1895 - acc: 0.9283     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0947 - acc: 0.9728     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0804 - acc: 0.9759     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0677 - acc: 0.9806     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0645 - acc: 0.9817     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0585 - acc: 0.9823     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.2491 - acc: 0.8990     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.1136 - acc: 0.9676     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0960 - acc: 0.9726     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0843 - acc: 0.9758     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0801 - acc: 0.9782     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0719 - acc: 0.9806     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.1217 - acc: 0.9556     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0681 - acc: 0.9772     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0578 - acc: 0.9815     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0485 - acc: 0.9842     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0464 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0418 - acc: 0.9854     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1349 - acc: 0.9550     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0724 - acc: 0.9782     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0543 - acc: 0.9823     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0523 - acc: 0.9833     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0403 - acc: 0.9866     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0375 - acc: 0.9881     \n",
      "4074/4074 [==============================] - 0s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1243 - acc: 0.9575     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0560 - acc: 0.9810     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0459 - acc: 0.9847     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0400 - acc: 0.9865     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0388 - acc: 0.9870     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0335 - acc: 0.9877     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1394 - acc: 0.9491     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0632 - acc: 0.9790     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0510 - acc: 0.9829     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0459 - acc: 0.9850     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0412 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0366 - acc: 0.9891     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1201 - acc: 0.9557     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0586 - acc: 0.9799     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0452 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0346 - acc: 0.9883     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0419 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0336 - acc: 0.9877     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1400 - acc: 0.9545     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0783 - acc: 0.9759     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0736 - acc: 0.9755     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0666 - acc: 0.9799     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0619 - acc: 0.9805     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0524 - acc: 0.9850     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.1403 - acc: 0.9486     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0892 - acc: 0.9687     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0810 - acc: 0.9704     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0671 - acc: 0.9788     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0633 - acc: 0.9794     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0540 - acc: 0.9829     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6943 - acc: 0.5190     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6960 - acc: 0.5044     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6954 - acc: 0.5123     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6962 - acc: 0.5037     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6959 - acc: 0.5045     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6948 - acc: 0.5148     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1817 - acc: 0.9288     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1123 - acc: 0.9605     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0890 - acc: 0.9714     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0835 - acc: 0.9751     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0825 - acc: 0.9742     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0688 - acc: 0.9799     \n",
      "8060/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  15 out of  15 | elapsed:  5.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.1048 - acc: 0.9612     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0656 - acc: 0.9792     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0515 - acc: 0.9836     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0413 - acc: 0.9869     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0391 - acc: 0.9876     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0364 - acc: 0.9889     \n",
      "Best: 0.992391 using {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.978565 (0.003297) with: {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.001}\n",
      "0.987564 (0.000504) with: {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.01}\n",
      "0.992391 (0.000722) with: {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "0.991982 (0.000232) with: {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.2}\n",
      "0.827293 (0.229856) with: {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.3}\n",
      "Best: 0.992391 using {'batch_size': 20, 'momentum': 0.4, 'optimizer': 'SGD', 'epochs': 6, 'learn_rate': 0.1}\n",
      "1780/2060 [========================>.....] - ETA: 0sTest Accuracy 0.986407764328\n"
     ]
    }
   ],
   "source": [
    "##### LEARNING RATE #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.001, 0.01, 0.1, 0.2, 0.3],\n",
    "              'momentum': [.4]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1331 - acc: 0.9513     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0715 - acc: 0.9751     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0660 - acc: 0.9791     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0617 - acc: 0.9795     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0580 - acc: 0.9838     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0492 - acc: 0.9848     \n",
      "7860/8148 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1402 - acc: 0.9507     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0665 - acc: 0.9769     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0481 - acc: 0.9834     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0407 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0445 - acc: 0.9838     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0366 - acc: 0.9871     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1502 - acc: 0.9395     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0689 - acc: 0.9791     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0511 - acc: 0.9838     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0511 - acc: 0.9839     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0403 - acc: 0.9867     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0393 - acc: 0.9885     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 7.1883 - acc: 0.5203     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 8.1255 - acc: 0.4950     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 8.1650 - acc: 0.4929     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 8.1642 - acc: 0.4934     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 8.1453 - acc: 0.4940     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 8.1386 - acc: 0.4947     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1389 - acc: 0.9531     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0678 - acc: 0.9793     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0536 - acc: 0.9828     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0419 - acc: 0.9855     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0399 - acc: 0.9871     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0353 - acc: 0.9881     \n",
      "7900/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1433 - acc: 0.9426     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0783 - acc: 0.9766     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0664 - acc: 0.9783     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0544 - acc: 0.9820     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0452 - acc: 0.9876     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0432 - acc: 0.9872     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1435 - acc: 0.9472     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0713 - acc: 0.9782     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0570 - acc: 0.9823     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0507 - acc: 0.9864     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0421 - acc: 0.9867     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0378 - acc: 0.9886     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1402 - acc: 0.9497     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0718 - acc: 0.9752     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0450 - acc: 0.9839     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0420 - acc: 0.9843     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0382 - acc: 0.9850     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0318 - acc: 0.9880     \n",
      "7900/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1772 - acc: 0.9215     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0722 - acc: 0.9779     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0692 - acc: 0.9773     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0532 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0505 - acc: 0.9853     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0390 - acc: 0.9881     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.6940 - acc: 0.5031     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6941 - acc: 0.5036     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6940 - acc: 0.5009     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6941 - acc: 0.4996     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6943 - acc: 0.5014     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6944 - acc: 0.5011     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6939 - acc: 0.5123     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.6934 - acc: 0.5089     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.6938 - acc: 0.5128     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.6938 - acc: 0.5030     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.6938 - acc: 0.5060     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.6938 - acc: 0.5082     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6940 - acc: 0.5001     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5094     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6936 - acc: 0.5068     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6942 - acc: 0.4999     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6939 - acc: 0.5024     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6937 - acc: 0.5073     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1883 - acc: 0.9113     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0787 - acc: 0.9778     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0660 - acc: 0.9779     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0614 - acc: 0.9812     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0567 - acc: 0.9829     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0494 - acc: 0.9838     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1451 - acc: 0.9366     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0680 - acc: 0.9773     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0486 - acc: 0.9838     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0472 - acc: 0.9858     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0383 - acc: 0.9871     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0324 - acc: 0.9883     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.2387 - acc: 0.8810     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0726 - acc: 0.9778     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0564 - acc: 0.9831     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0578 - acc: 0.9818     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0444 - acc: 0.9874     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0419 - acc: 0.9864     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1398 - acc: 0.9467     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0771 - acc: 0.9694     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0537 - acc: 0.9794     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0492 - acc: 0.9813     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0422 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0404 - acc: 0.9861     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1651 - acc: 0.9245     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0607 - acc: 0.9809     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0500 - acc: 0.9850     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0357 - acc: 0.9876     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0385 - acc: 0.9881     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0301 - acc: 0.9898     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.2617 - acc: 0.8580     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0828 - acc: 0.9718     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0573 - acc: 0.9809     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0514 - acc: 0.9843     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0473 - acc: 0.9863     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0384 - acc: 0.9887     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1863 - acc: 0.9142     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0772 - acc: 0.9729     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0644 - acc: 0.9775     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0497 - acc: 0.9834     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0410 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0382 - acc: 0.9872     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1322 - acc: 0.9444     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0682 - acc: 0.9794     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0563 - acc: 0.9821     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0457 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0449 - acc: 0.9848     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0427 - acc: 0.9865     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1400 - acc: 0.9438     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0698 - acc: 0.9775     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0589 - acc: 0.9791     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0516 - acc: 0.9840     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0451 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0418 - acc: 0.9875     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1388 - acc: 0.9390     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0768 - acc: 0.9718     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0649 - acc: 0.9767     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0537 - acc: 0.9789     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0552 - acc: 0.9828     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0474 - acc: 0.9833     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1515 - acc: 0.9373     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0734 - acc: 0.9758     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0565 - acc: 0.9826     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0513 - acc: 0.9827     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0428 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0378 - acc: 0.9872     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1508 - acc: 0.9346     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0785 - acc: 0.9747     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0662 - acc: 0.9794     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0586 - acc: 0.9829     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0485 - acc: 0.9863     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0495 - acc: 0.9838     \n",
      "7980/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:  9.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 6s - loss: 0.1170 - acc: 0.9543     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0567 - acc: 0.9814     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0465 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0404 - acc: 0.9862     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0348 - acc: 0.9886     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0298 - acc: 0.9899     \n",
      "Best: 0.991737 using {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.989855 (0.002919) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.819684 (0.240744) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'lecun_uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.991001 (0.001620) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'normal', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.512558 (0.007718) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'zero', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.991246 (0.000305) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_normal', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.991737 (0.000835) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.991573 (0.002144) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'he_normal', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.990428 (0.001252) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'he_uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "Best: 0.991737 using {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "1860/2060 [==========================>...] - ETA: 0sTest Accuracy 0.990776697409\n"
     ]
    }
   ],
   "source": [
    "##### NETWORK WEIGHT INITIALIZATION 1 #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum, init_mode_1, init_mode_2):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation='softmax'))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'],\n",
    "              'init_mode_2': ['uniform']\n",
    "             }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1351 - acc: 0.9481     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0705 - acc: 0.9753     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0633 - acc: 0.9799     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0610 - acc: 0.9783     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0552 - acc: 0.9817     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0463 - acc: 0.9848     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1389 - acc: 0.9512     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0701 - acc: 0.9744     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0460 - acc: 0.9847     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0380 - acc: 0.9883     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0419 - acc: 0.9849     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0362 - acc: 0.9875     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1506 - acc: 0.9377     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0754 - acc: 0.9759     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0547 - acc: 0.9832     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0566 - acc: 0.9823     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0431 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0431 - acc: 0.9859     \n",
      "4074/4074 [==============================] - 1s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1276 - acc: 0.9543     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0682 - acc: 0.9773     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0516 - acc: 0.9836     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0473 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0447 - acc: 0.9867     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 2s - loss: 0.0348 - acc: 0.9885     \n",
      "7940/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1089 - acc: 0.9634     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0669 - acc: 0.9796     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0523 - acc: 0.9826     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0456 - acc: 0.9839     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0441 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0352 - acc: 0.9886     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1308 - acc: 0.9534     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0777 - acc: 0.9775     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0647 - acc: 0.9799     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0505 - acc: 0.9828     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0497 - acc: 0.9854     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0454 - acc: 0.9858     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1723 - acc: 0.9181     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0753 - acc: 0.9741     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0614 - acc: 0.9799     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0487 - acc: 0.9847     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0468 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0428 - acc: 0.9865     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1301 - acc: 0.9573     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0715 - acc: 0.9773     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0570 - acc: 0.9813     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0505 - acc: 0.9832     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0440 - acc: 0.9854     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0357 - acc: 0.9890     \n",
      "7900/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1759 - acc: 0.9253     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0707 - acc: 0.9786     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0539 - acc: 0.9825     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0442 - acc: 0.9867     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0380 - acc: 0.9886     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0330 - acc: 0.9898     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1513 - acc: 0.9362     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0735 - acc: 0.9752     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0615 - acc: 0.9799     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0503 - acc: 0.9817     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0460 - acc: 0.9822     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0448 - acc: 0.9839     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1657 - acc: 0.9282     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0537 - acc: 0.9813     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0450 - acc: 0.9845     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0370 - acc: 0.9863     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0345 - acc: 0.9899     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 2s - loss: 0.0313 - acc: 0.9890     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1402 - acc: 0.9412     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0694 - acc: 0.9740     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0622 - acc: 0.9779     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0553 - acc: 0.9821     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0483 - acc: 0.9840     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0431 - acc: 0.9861     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1445 - acc: 0.9423     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0786 - acc: 0.9742     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0695 - acc: 0.9769     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0668 - acc: 0.9782     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0563 - acc: 0.9799     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0557 - acc: 0.9786     \n",
      "7860/8148 [===========================>..] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1169 - acc: 0.9569     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0616 - acc: 0.9786     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0489 - acc: 0.9850     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0424 - acc: 0.9865     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0363 - acc: 0.9899     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0333 - acc: 0.9892     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1442 - acc: 0.9404     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0608 - acc: 0.9807     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0543 - acc: 0.9844     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0511 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0444 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0411 - acc: 0.9874     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1267 - acc: 0.9559     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0844 - acc: 0.9742     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0674 - acc: 0.9790     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0602 - acc: 0.9804     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0519 - acc: 0.9823     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0438 - acc: 0.9856     \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1176 - acc: 0.9577     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0565 - acc: 0.9817     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0491 - acc: 0.9845     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0388 - acc: 0.9865     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0347 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0292 - acc: 0.9913     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1742 - acc: 0.9177     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0635 - acc: 0.9812     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0452 - acc: 0.9843     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0450 - acc: 0.9860     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0437 - acc: 0.9865     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0349 - acc: 0.9891     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1314 - acc: 0.9541     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0631 - acc: 0.9796     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0515 - acc: 0.9826     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0447 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0443 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0330 - acc: 0.9883     \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1050 - acc: 0.9615     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0559 - acc: 0.9801     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0488 - acc: 0.9801     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0390 - acc: 0.9856     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0352 - acc: 0.9875     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0334 - acc: 0.9869     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1160 - acc: 0.9586     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0697 - acc: 0.9789     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0590 - acc: 0.9822     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0494 - acc: 0.9842     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0452 - acc: 0.9871     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0421 - acc: 0.9871     \n",
      "4074/4074 [==============================] - 2s     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1280 - acc: 0.9489     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0726 - acc: 0.9731     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0661 - acc: 0.9762     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0527 - acc: 0.9796     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0477 - acc: 0.9809     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0450 - acc: 0.9820     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1292 - acc: 0.9513     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0670 - acc: 0.9764     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0567 - acc: 0.9813     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0566 - acc: 0.9794     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0471 - acc: 0.9843     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0428 - acc: 0.9842     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1173 - acc: 0.9588     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0737 - acc: 0.9766     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0632 - acc: 0.9818     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0484 - acc: 0.9865     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0431 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0459 - acc: 0.9858     \n",
      "4074/4074 [==============================] - 2s     \n",
      "8060/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed:  9.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 6s - loss: 0.0956 - acc: 0.9642     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0514 - acc: 0.9811     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0400 - acc: 0.9858     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0368 - acc: 0.9890     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0319 - acc: 0.9902     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0323 - acc: 0.9904     \n",
      "Best: 0.992146 using {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'epochs': 6}\n",
      "0.991573 (0.000705) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'uniform', 'epochs': 6}\n",
      "0.991655 (0.001403) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'lecun_uniform', 'epochs': 6}\n",
      "0.992064 (0.000645) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'normal', 'epochs': 6}\n",
      "0.989364 (0.002719) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'zero', 'epochs': 6}\n",
      "0.990101 (0.002934) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'glorot_normal', 'epochs': 6}\n",
      "0.991001 (0.001224) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'glorot_uniform', 'epochs': 6}\n",
      "0.992146 (0.001252) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'epochs': 6}\n",
      "0.990346 (0.001852) with: {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_uniform', 'epochs': 6}\n",
      "Best: 0.992146 using {'batch_size': 20, 'optimizer': 'SGD', 'learn_rate': 0.1, 'momentum': 0.4, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'epochs': 6}\n",
      "1900/2060 [==========================>...] - ETA: 0sTest Accuracy 0.991262134418\n"
     ]
    }
   ],
   "source": [
    "##### NETWORK WEIGHT INITIALIZATION 2 #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum, init_mode_1, init_mode_2):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation='relu', input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation='softmax'))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['uniform', 'lecun_uniform', 'normal', 'zero', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform'],\n",
    "             }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.6336 - acc: 0.5736     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1152 - acc: 0.9570     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0858 - acc: 0.9708     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0776 - acc: 0.9756     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0684 - acc: 0.9785     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0608 - acc: 0.9788     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1435 - acc: 0.9437     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0717 - acc: 0.9732     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0673 - acc: 0.9771     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0607 - acc: 0.9796     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0555 - acc: 0.9802     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0550 - acc: 0.9813     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1541 - acc: 0.9366     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0779 - acc: 0.9723     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0700 - acc: 0.9761     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0626 - acc: 0.9771     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0568 - acc: 0.9794     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0574 - acc: 0.9789     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.6985 - acc: 0.5004     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6938 - acc: 0.5097     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6943 - acc: 0.4940     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6944 - acc: 0.4969     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6946 - acc: 0.4986     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6942 - acc: 0.4977     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1489 - acc: 0.9434     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0775 - acc: 0.9712     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0624 - acc: 0.9800     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0587 - acc: 0.9816     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0574 - acc: 0.9810     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0502 - acc: 0.9833     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.2483 - acc: 0.8759     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1109 - acc: 0.9620     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0891 - acc: 0.9669     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0725 - acc: 0.9750     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0711 - acc: 0.9789     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0664 - acc: 0.9771     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1215 - acc: 0.9554     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0664 - acc: 0.9790     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0476 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0396 - acc: 0.9867     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0363 - acc: 0.9888     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0341 - acc: 0.9894     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1167 - acc: 0.9551     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0670 - acc: 0.9752     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0541 - acc: 0.9817     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0528 - acc: 0.9845     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0454 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0398 - acc: 0.9869     \n",
      "4074/4074 [==============================] - 2s     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1249 - acc: 0.9572     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0610 - acc: 0.9806     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0559 - acc: 0.9825     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0446 - acc: 0.9865     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0434 - acc: 0.9840     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0375 - acc: 0.9885     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1272 - acc: 0.9512     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0755 - acc: 0.9715     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0631 - acc: 0.9773     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0531 - acc: 0.9805     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0447 - acc: 0.9847     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0412 - acc: 0.9847     \n",
      "7900/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1168 - acc: 0.9566     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0546 - acc: 0.9826     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0481 - acc: 0.9840     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0390 - acc: 0.9869     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0365 - acc: 0.9866     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0280 - acc: 0.9891     \n",
      "7900/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1143 - acc: 0.9573     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0645 - acc: 0.9786     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0552 - acc: 0.9801     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0471 - acc: 0.9839     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0460 - acc: 0.9845     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0401 - acc: 0.9859     \n",
      "7900/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1113 - acc: 0.9599     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0619 - acc: 0.9809     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0502 - acc: 0.9815     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0457 - acc: 0.9847     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0362 - acc: 0.9870     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0384 - acc: 0.9848     \n",
      "4075/4075 [==============================] - 2s     \n",
      "7940/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0977 - acc: 0.9671     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0548 - acc: 0.9806     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0426 - acc: 0.9859     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0406 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0284 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0315 - acc: 0.9893     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1238 - acc: 0.9585     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0682 - acc: 0.9785     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0564 - acc: 0.9842     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0516 - acc: 0.9827     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0410 - acc: 0.9869     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0348 - acc: 0.9865     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1663 - acc: 0.9323     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0817 - acc: 0.9691     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0657 - acc: 0.9766     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0647 - acc: 0.9764     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0601 - acc: 0.9816     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0545 - acc: 0.9822     \n",
      "7920/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.6949 - acc: 0.5031     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6933 - acc: 0.5165     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6936 - acc: 0.5133     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6937 - acc: 0.5112     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6938 - acc: 0.5068     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6935 - acc: 0.5180     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.2119 - acc: 0.9047     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1126 - acc: 0.9620     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0884 - acc: 0.9691     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0846 - acc: 0.9732     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0769 - acc: 0.9746     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0730 - acc: 0.9736     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1624 - acc: 0.9377     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0856 - acc: 0.9715     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0833 - acc: 0.9735     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0721 - acc: 0.9779     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0732 - acc: 0.9790     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0665 - acc: 0.9786     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1634 - acc: 0.9352     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0908 - acc: 0.9708     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0712 - acc: 0.9758     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0681 - acc: 0.9756     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0554 - acc: 0.9793     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0576 - acc: 0.9772     \n",
      "7920/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.6941 - acc: 0.5045     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6942 - acc: 0.5013     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6945 - acc: 0.4979     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5013     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6937 - acc: 0.5018     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6935 - acc: 0.5091     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1184 - acc: 0.9622     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0600 - acc: 0.9826     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0493 - acc: 0.9834     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0422 - acc: 0.9870     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0392 - acc: 0.9871     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0363 - acc: 0.9896     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1318 - acc: 0.9531     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0564 - acc: 0.9817     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0426 - acc: 0.9864     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0345 - acc: 0.9871     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0361 - acc: 0.9872     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0279 - acc: 0.9904     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1144 - acc: 0.9620     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0724 - acc: 0.9783     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0573 - acc: 0.9831     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0545 - acc: 0.9833     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0488 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0475 - acc: 0.9870     \n",
      "4074/4074 [==============================] - 2s     \n",
      "8149/8149 [==============================] - 1s     \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed: 11.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.1006 - acc: 0.9613     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0543 - acc: 0.9803     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0408 - acc: 0.9867     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0327 - acc: 0.9899     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0333 - acc: 0.9897     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0292 - acc: 0.9903     \n",
      "Best: 0.992391 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.988546 (0.001606) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softmax'}\n",
      "0.830484 (0.219079) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softplus'}\n",
      "0.992391 (0.001912) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.992310 (0.000705) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'relu'}\n",
      "0.991655 (0.000530) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'tanh'}\n",
      "0.826066 (0.228988) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'sigmoid'}\n",
      "0.831547 (0.224012) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'hard_sigmoid'}\n",
      "0.991737 (0.001224) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'linear'}\n",
      "Best: 0.992391 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "1940/2060 [===========================>..] - ETA: 0sTest Accuracy 0.990776697409\n"
     ]
    }
   ],
   "source": [
    "##### NEURON ACTIVATION FUNCTION 1 #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum, init_mode_1, init_mode_2, activation_1, activation_2, activation_3, activation_4):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['relu'],\n",
    "              'activation_4': ['softmax']\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.3609 - acc: 0.7941     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0905 - acc: 0.9671     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0760 - acc: 0.9740     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0767 - acc: 0.9745     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0712 - acc: 0.9763     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0599 - acc: 0.9810     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.2820 - acc: 0.8532     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0993 - acc: 0.9583     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0860 - acc: 0.9697     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0723 - acc: 0.9758     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0683 - acc: 0.9759     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0682 - acc: 0.9757     \n",
      "4074/4074 [==============================] - 3s     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.4216 - acc: 0.7422     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1232 - acc: 0.9537     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1051 - acc: 0.9642     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1013 - acc: 0.9636     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0946 - acc: 0.9675     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0904 - acc: 0.9692     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.7029 - acc: 0.4989     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6938 - acc: 0.5097     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6947 - acc: 0.4935     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6944 - acc: 0.4969     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6946 - acc: 0.4986     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6942 - acc: 0.4977     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.7022 - acc: 0.5109     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6938 - acc: 0.5029     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5112     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6937 - acc: 0.5133     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6939 - acc: 0.5096     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6939 - acc: 0.5064     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.6960 - acc: 0.5055     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5055     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6943 - acc: 0.5033     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6935 - acc: 0.5096     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5060     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6943 - acc: 0.4998     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1102 - acc: 0.9604     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0606 - acc: 0.9795     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0523 - acc: 0.9849     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0448 - acc: 0.9850     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0452 - acc: 0.9870     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0419 - acc: 0.9880     \n",
      "7900/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1131 - acc: 0.9604     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0601 - acc: 0.9794     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0544 - acc: 0.9832     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0477 - acc: 0.9859     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0435 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0417 - acc: 0.9859     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1101 - acc: 0.9611     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0562 - acc: 0.9817     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0536 - acc: 0.9816     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0453 - acc: 0.9860     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0460 - acc: 0.9845     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0400 - acc: 0.9858     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1230 - acc: 0.9559     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0645 - acc: 0.9804     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0484 - acc: 0.9843     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0467 - acc: 0.9851     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0408 - acc: 0.9880     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0368 - acc: 0.9877     \n",
      "7920/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1153 - acc: 0.9595     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0576 - acc: 0.9820     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0436 - acc: 0.9856     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0426 - acc: 0.9872     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0327 - acc: 0.9894     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0302 - acc: 0.9894     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1157 - acc: 0.9593     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0650 - acc: 0.9821     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0607 - acc: 0.9822     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0490 - acc: 0.9839     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0444 - acc: 0.9861     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0400 - acc: 0.9877     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1086 - acc: 0.9570     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0637 - acc: 0.9795     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0542 - acc: 0.9826     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0485 - acc: 0.9827     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0437 - acc: 0.9850     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0439 - acc: 0.9858     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0957 - acc: 0.9661     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0553 - acc: 0.9816     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0406 - acc: 0.9875     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0381 - acc: 0.9872     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0350 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0318 - acc: 0.9893     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1054 - acc: 0.9626     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0677 - acc: 0.9777     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0621 - acc: 0.9816     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0570 - acc: 0.9815     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0521 - acc: 0.9816     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0461 - acc: 0.9807     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.6986 - acc: 0.5018     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.6946 - acc: 0.4998     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6940 - acc: 0.4986     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.6939 - acc: 0.5048     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.6943 - acc: 0.5014     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6942 - acc: 0.5026     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.6986 - acc: 0.5031     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6933 - acc: 0.5165     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6936 - acc: 0.5137     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6937 - acc: 0.5114     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6938 - acc: 0.5067     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6935 - acc: 0.5183     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.7047 - acc: 0.5046     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6942 - acc: 0.5044     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6938 - acc: 0.5072     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6938 - acc: 0.5067     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6941 - acc: 0.5030     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6939 - acc: 0.5051     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.6961 - acc: 0.4962     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6941 - acc: 0.4958     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6943 - acc: 0.4957     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6942 - acc: 0.4951     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6943 - acc: 0.4944     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6940 - acc: 0.5038     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.7147 - acc: 0.5089     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6937 - acc: 0.5099     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6934 - acc: 0.5078     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6939 - acc: 0.5088     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5044     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6936 - acc: 0.5072     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.7008 - acc: 0.5087     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6942 - acc: 0.5018     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6945 - acc: 0.4932     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6940 - acc: 0.5002     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6937 - acc: 0.5020     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6935 - acc: 0.5085     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1168 - acc: 0.9605     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0710 - acc: 0.9780     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0620 - acc: 0.9806     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0529 - acc: 0.9838     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0485 - acc: 0.9842     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0474 - acc: 0.9861     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1221 - acc: 0.9523     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0565 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0445 - acc: 0.9840     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0372 - acc: 0.9855     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0385 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0342 - acc: 0.9881     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1194 - acc: 0.9586     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0691 - acc: 0.9801     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0619 - acc: 0.9816     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0509 - acc: 0.9859     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0451 - acc: 0.9865     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0530 - acc: 0.9854     \n",
      "8040/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed: 13.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0998 - acc: 0.9620     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0517 - acc: 0.9814     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0418 - acc: 0.9869     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0354 - acc: 0.9896     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 4s - loss: 0.0341 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0301 - acc: 0.9905     \n",
      "Best: 0.993455 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.986419 (0.000904) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'softmax', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.502741 (0.014483) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'softplus', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.992064 (0.002125) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'softsign', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.993455 (0.001365) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.991573 (0.000811) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'tanh', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.512558 (0.007718) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'sigmoid', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.512558 (0.007718) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'hard_sigmoid', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.991737 (0.001104) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'linear', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "Best: 0.993455 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "1900/2060 [==========================>...] - ETA: 0sTest Accuracy 0.988834949373\n"
     ]
    }
   ],
   "source": [
    "##### NEURON ACTIVATION FUNCTION 2 #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum, init_mode_1, init_mode_2, activation_1, activation_2, activation_3, activation_4):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],\n",
    "              'activation_3': ['relu'],\n",
    "              'activation_4': ['softmax']\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.4203 - acc: 0.7742     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.2593 - acc: 0.8616     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.2173 - acc: 0.9119     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1922 - acc: 0.9179     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1677 - acc: 0.9223     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1498 - acc: 0.9185     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.3656 - acc: 0.8326     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.2315 - acc: 0.9021     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1994 - acc: 0.9175     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1823 - acc: 0.9229     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1742 - acc: 0.9395     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1592 - acc: 0.9517     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.3742 - acc: 0.7969     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.2555 - acc: 0.8601     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.2254 - acc: 0.8985     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.1909 - acc: 0.9177     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1695 - acc: 0.9308     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.1531 - acc: 0.9436     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1258 - acc: 0.9519     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0699 - acc: 0.9750     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0525 - acc: 0.9821     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0494 - acc: 0.9845     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0452 - acc: 0.9865     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0397 - acc: 0.9867     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1225 - acc: 0.9463     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0611 - acc: 0.9762     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0515 - acc: 0.9822     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0434 - acc: 0.9842     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0399 - acc: 0.9858     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0367 - acc: 0.9867     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1346 - acc: 0.9460     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0742 - acc: 0.9762     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0607 - acc: 0.9784     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0482 - acc: 0.9826     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0482 - acc: 0.9852     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0440 - acc: 0.9864     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0977 - acc: 0.9674     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0501 - acc: 0.9856     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0361 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0315 - acc: 0.9898     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0317 - acc: 0.9904     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0250 - acc: 0.9930     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0951 - acc: 0.9682     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0534 - acc: 0.9820     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0440 - acc: 0.9850     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0372 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0334 - acc: 0.9881     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0305 - acc: 0.9888     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0972 - acc: 0.9649     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0504 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0389 - acc: 0.9887     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0343 - acc: 0.9903     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0276 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0246 - acc: 0.9920     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.1231 - acc: 0.9559     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0674 - acc: 0.9791     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0479 - acc: 0.9847     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0415 - acc: 0.9861     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0336 - acc: 0.9891     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0386 - acc: 0.9894     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1172 - acc: 0.9597     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0590 - acc: 0.9821     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0450 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0422 - acc: 0.9856     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0351 - acc: 0.9880     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0310 - acc: 0.9875     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1171 - acc: 0.9586     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0656 - acc: 0.9807     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0624 - acc: 0.9809     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0498 - acc: 0.9845     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0420 - acc: 0.9867     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0383 - acc: 0.9876     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0838 - acc: 0.9726     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0457 - acc: 0.9860     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0368 - acc: 0.9901     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0342 - acc: 0.9902     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0261 - acc: 0.9931     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0289 - acc: 0.9918     \n",
      "4075/4075 [==============================] - 3s     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0757 - acc: 0.9720     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0426 - acc: 0.9860     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0324 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0307 - acc: 0.9903     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0254 - acc: 0.9924     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0234 - acc: 0.9919     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0974 - acc: 0.9671     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0486 - acc: 0.9843     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0413 - acc: 0.9877     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0378 - acc: 0.9887     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0318 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0276 - acc: 0.9928     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1374 - acc: 0.9485     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0749 - acc: 0.9736     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0648 - acc: 0.9768     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0555 - acc: 0.9818     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0531 - acc: 0.9826     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0470 - acc: 0.9847     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.1609 - acc: 0.9361     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0708 - acc: 0.9756     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0577 - acc: 0.9827     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0493 - acc: 0.9828     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0436 - acc: 0.9871     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0389 - acc: 0.9882     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1399 - acc: 0.9470     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0822 - acc: 0.9744     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0680 - acc: 0.9780     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0619 - acc: 0.9817     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0565 - acc: 0.9836     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0553 - acc: 0.9848     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.1357 - acc: 0.9502     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0741 - acc: 0.9763     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0667 - acc: 0.9790     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0610 - acc: 0.9810     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0587 - acc: 0.9818     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0510 - acc: 0.9840     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1338 - acc: 0.9534     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0683 - acc: 0.9773     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0534 - acc: 0.9821     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0468 - acc: 0.9852     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0414 - acc: 0.9849     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0400 - acc: 0.9866     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1724 - acc: 0.9281     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0741 - acc: 0.9769     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0604 - acc: 0.9820     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0567 - acc: 0.9827     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0499 - acc: 0.9837     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0450 - acc: 0.9859     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0848 - acc: 0.9694     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0412 - acc: 0.9887     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0359 - acc: 0.9888     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0319 - acc: 0.9897     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0280 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0238 - acc: 0.9931     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0911 - acc: 0.9681     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0377 - acc: 0.9887     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0332 - acc: 0.9898     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0252 - acc: 0.9917     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0227 - acc: 0.9920     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0190 - acc: 0.9937     \n",
      "8149/8149 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0909 - acc: 0.9683     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0511 - acc: 0.9863     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0398 - acc: 0.9876     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0366 - acc: 0.9886     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0312 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0310 - acc: 0.9920     \n",
      "8100/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed: 13.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0719 - acc: 0.9753     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0376 - acc: 0.9878     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0293 - acc: 0.9903     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0256 - acc: 0.9929     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0230 - acc: 0.9928     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0206 - acc: 0.9937     \n",
      "Best: 0.992882 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.989773 (0.001335) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'softmax', 'activation_1': 'softsign'}\n",
      "0.991410 (0.002035) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'softplus', 'activation_1': 'softsign'}\n",
      "0.992800 (0.002235) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'softsign', 'activation_1': 'softsign'}\n",
      "0.992800 (0.000463) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'relu', 'activation_1': 'softsign'}\n",
      "0.992146 (0.000919) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'tanh', 'activation_1': 'softsign'}\n",
      "0.987810 (0.000646) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'sigmoid', 'activation_1': 'softsign'}\n",
      "0.990346 (0.000705) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'hard_sigmoid', 'activation_1': 'softsign'}\n",
      "0.992882 (0.001591) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "Best: 0.992882 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "1880/2060 [==========================>...] - ETA: 0sTest Accuracy 0.991262134418\n"
     ]
    }
   ],
   "source": [
    "##### NEURON ACTIVATION FUNCTION 3 #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum, init_mode_1, init_mode_2, activation_1, activation_2, activation_3, activation_4):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],\n",
    "              'activation_4': ['softmax']\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 8 candidates, totalling 24 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0899 - acc: 0.9709     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0480 - acc: 0.9845     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0391 - acc: 0.9894     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0346 - acc: 0.9893     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0300 - acc: 0.9908     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0216 - acc: 0.9937     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0905 - acc: 0.9692     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0450 - acc: 0.9856     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0334 - acc: 0.9896     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0284 - acc: 0.9915     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0283 - acc: 0.9913     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0250 - acc: 0.9919     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0836 - acc: 0.9725     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0468 - acc: 0.9860     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0341 - acc: 0.9897     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0350 - acc: 0.9902     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0298 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0292 - acc: 0.9919     \n",
      "4074/4074 [==============================] - 3s     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0847 - acc: 0.9702     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0414 - acc: 0.9874     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0301 - acc: 0.9903     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0267 - acc: 0.9917     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0253 - acc: 0.9928     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0218 - acc: 0.9939     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0862 - acc: 0.9718     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0423 - acc: 0.9875     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0320 - acc: 0.9897     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0261 - acc: 0.9913     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0284 - acc: 0.9903     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0195 - acc: 0.9930     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0981 - acc: 0.9650     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0477 - acc: 0.9855     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0318 - acc: 0.9891     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0280 - acc: 0.9920     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0263 - acc: 0.9919     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0228 - acc: 0.9930     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 6.5363 - acc: 0.4973     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 4.6459 - acc: 0.4982     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.7682 - acc: 0.4883     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6985 - acc: 0.5029     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6932 - acc: 0.5014     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6932 - acc: 0.5104     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6929 - acc: 0.5062     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6930 - acc: 0.5031     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 1.6288 - acc: 0.5055     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.8169 - acc: 0.5088     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.7277 - acc: 0.5102     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.7257 - acc: 0.5080     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.7094 - acc: 0.5080     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.7147 - acc: 0.5091     \n",
      "7940/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0182 - acc: 0.4912     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: nan - acc: 0.4825        \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4823            \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4823            \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4823            \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4823            \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4823            \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: nan - acc: 0.4889        \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4885            \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4885            \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4885            \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4885            \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: nan - acc: 0.4885            \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0150 - acc: 0.4913     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 1.1921e-07 - acc: 0.4915     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.1615 - acc: 0.4985     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4823     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4823     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4823     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4823     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4823     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0247 - acc: 0.4888     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4885     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 8.5178e-05 - acc: 0.4885     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 8.5178e-05 - acc: 0.4885     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 8.5178e-05 - acc: 0.4885     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 8.5178e-05 - acc: 0.4885     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.0996 - acc: 0.9642     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0517 - acc: 0.9828     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0415 - acc: 0.9874     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0347 - acc: 0.9902     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0304 - acc: 0.9914     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0235 - acc: 0.9923     \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0950 - acc: 0.9712     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0458 - acc: 0.9850     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0329 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0246 - acc: 0.9920     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0227 - acc: 0.9923     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0226 - acc: 0.9941     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0959 - acc: 0.9665     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0549 - acc: 0.9838     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0394 - acc: 0.9881     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0320 - acc: 0.9907     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0304 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0269 - acc: 0.9903     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 7s - loss: 1.7479 - acc: 0.5939     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6947 - acc: 0.4978     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6940 - acc: 0.4952     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6938 - acc: 0.5070     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6941 - acc: 0.4901     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.6938 - acc: 0.5015     \n",
      "8148/8148 [==============================] - 1s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 1.0262 - acc: 0.8577     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.7296 - acc: 0.4820     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.7079 - acc: 0.4820     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.7019 - acc: 0.4824     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.7230 - acc: 0.4818     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.7074 - acc: 0.4823     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.2780 - acc: 0.9438     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6096 - acc: 0.6090     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6934 - acc: 0.4969     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6939 - acc: 0.4920     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6933 - acc: 0.5021     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.6936 - acc: 0.4900     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.7512 - acc: 0.4848     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6985 - acc: 0.4795     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.6920 - acc: 0.4411     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 1.1089 - acc: 0.2299     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.8052 - acc: 0.4553     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.7002 - acc: 0.4788     \n",
      "4075/4075 [==============================] - 4s     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 3.5501 - acc: 0.4836     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 2.5879 - acc: 0.4963     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 1.2707 - acc: 0.5033     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.9850 - acc: 0.4940     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 1.1836 - acc: 0.5109     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.9981 - acc: 0.5147     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 1.2588 - acc: 0.5069     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.7161 - acc: 0.4959     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.7042 - acc: 0.5047     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6965 - acc: 0.4977     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6945 - acc: 0.5168     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.6936 - acc: 0.5166     \n",
      "7980/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  24 out of  24 | elapsed: 13.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0773 - acc: 0.9733     \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0386 - acc: 0.9881     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0305 - acc: 0.9900     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0248 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0240 - acc: 0.9925     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 5s - loss: 0.0199 - acc: 0.9932     \n",
      "Best: 0.993782 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softplus', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.987728 (0.006741) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softmax', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.993782 (0.001175) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softplus', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.498732 (0.014686) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softsign', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.487442 (0.007718) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'relu', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.487442 (0.007718) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'tanh', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.991737 (0.001408) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'sigmoid', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.501268 (0.014686) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'hard_sigmoid', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "0.502741 (0.014483) with: {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'linear', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "Best: 0.993782 using {'batch_size': 20, 'optimizer': 'SGD', 'activation_2': 'relu', 'epochs': 6, 'momentum': 0.4, 'activation_4': 'softplus', 'learn_rate': 0.1, 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'activation_1': 'softsign'}\n",
      "2020/2060 [============================>.] - ETA: 0sTest Accuracy 0.991262134418\n"
     ]
    }
   ],
   "source": [
    "##### NEURON ACTIVATION FUNCTION 4 #####\n",
    "\n",
    "def create_model(optimizer, learn_rate, momentum, init_mode_1, init_mode_2, activation_1, activation_2, activation_3, activation_4):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear'],\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0855 - acc: 0.9702     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0544 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0425 - acc: 0.9867     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0314 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0294 - acc: 0.9909     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0225 - acc: 0.9934     \n",
      "7940/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0897 - acc: 0.9678     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0470 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0333 - acc: 0.9891     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0291 - acc: 0.9891     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0207 - acc: 0.9926     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0230 - acc: 0.9925     \n",
      "8000/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0885 - acc: 0.9721     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0442 - acc: 0.9871     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0348 - acc: 0.9892     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0297 - acc: 0.9910     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0236 - acc: 0.9926     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0221 - acc: 0.9931     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0792 - acc: 0.9715     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0395 - acc: 0.9877     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0281 - acc: 0.9909     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0233 - acc: 0.9921     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0248 - acc: 0.9924     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0190 - acc: 0.9950     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0781 - acc: 0.9737     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0392 - acc: 0.9875     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0284 - acc: 0.9904     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0218 - acc: 0.9929     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0251 - acc: 0.9920     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0171 - acc: 0.9945     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0926 - acc: 0.9664     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0444 - acc: 0.9866     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0327 - acc: 0.9897     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0273 - acc: 0.9920     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0220 - acc: 0.9933     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0211 - acc: 0.9935     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0932 - acc: 0.9671     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0431 - acc: 0.9871     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0339 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0268 - acc: 0.9918     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0285 - acc: 0.9901     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0215 - acc: 0.9935     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0857 - acc: 0.9690     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0471 - acc: 0.9843     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0376 - acc: 0.9883     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0311 - acc: 0.9903     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0297 - acc: 0.9896     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0263 - acc: 0.9909     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0913 - acc: 0.9682     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0481 - acc: 0.9849     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0359 - acc: 0.9901     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0314 - acc: 0.9904     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0244 - acc: 0.9919     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0241 - acc: 0.9921     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0903 - acc: 0.9693     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0426 - acc: 0.9856     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0313 - acc: 0.9905     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0281 - acc: 0.9905     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0249 - acc: 0.9923     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0262 - acc: 0.9923     \n",
      "8148/8148 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0936 - acc: 0.9671     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0394 - acc: 0.9875     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0320 - acc: 0.9896     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0247 - acc: 0.9913     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0238 - acc: 0.9929     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0212 - acc: 0.9924     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0867 - acc: 0.9718     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0517 - acc: 0.9836     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0410 - acc: 0.9863     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0359 - acc: 0.9888     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0322 - acc: 0.9898     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0292 - acc: 0.9917     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0949 - acc: 0.9676     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0495 - acc: 0.9831     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0384 - acc: 0.9885     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0334 - acc: 0.9904     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0294 - acc: 0.9898     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0284 - acc: 0.9912     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0847 - acc: 0.9685    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0452 - acc: 0.9848     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0365 - acc: 0.9853     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0370 - acc: 0.9876     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0276 - acc: 0.9902     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0281 - acc: 0.9897     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.1073 - acc: 0.9609     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0562 - acc: 0.9827     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0447 - acc: 0.9858     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0432 - acc: 0.9854     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0343 - acc: 0.9882     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0302 - acc: 0.9898     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.1045 - acc: 0.9658     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0539 - acc: 0.9828     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0472 - acc: 0.9850     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0394 - acc: 0.9876     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0368 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0320 - acc: 0.9883     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.1032 - acc: 0.9661     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0523 - acc: 0.9827     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0391 - acc: 0.9877     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0324 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0300 - acc: 0.9908     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0273 - acc: 0.9924     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.1087 - acc: 0.9637     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0589 - acc: 0.9815     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0472 - acc: 0.9855     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0397 - acc: 0.9866     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0360 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0343 - acc: 0.9881     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.1121 - acc: 0.9599     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0651 - acc: 0.9782     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0575 - acc: 0.9806     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0471 - acc: 0.9849     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0412 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0390 - acc: 0.9867     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0941 - acc: 0.9683     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0573 - acc: 0.9811     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0461 - acc: 0.9847     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0407 - acc: 0.9866     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0421 - acc: 0.9856     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0377 - acc: 0.9863     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.1100 - acc: 0.9582     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0594 - acc: 0.9799     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0527 - acc: 0.9825     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0442 - acc: 0.9859     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0416 - acc: 0.9866     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0404 - acc: 0.9874     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.1250 - acc: 0.9558    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0652 - acc: 0.9816     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0613 - acc: 0.9807     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0477 - acc: 0.9840     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0459 - acc: 0.9859     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0492 - acc: 0.9833     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.1304 - acc: 0.9530    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0644 - acc: 0.9793     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0547 - acc: 0.9823     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0461 - acc: 0.9852     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0412 - acc: 0.9858     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0388 - acc: 0.9875     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.1351 - acc: 0.9502    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0823 - acc: 0.9742     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0647 - acc: 0.9782     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0586 - acc: 0.9794     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0596 - acc: 0.9812     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0530 - acc: 0.9844     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.1416 - acc: 0.9487    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0803 - acc: 0.9739     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0758 - acc: 0.9786     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0680 - acc: 0.9769     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0632 - acc: 0.9801     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0605 - acc: 0.9816     \n",
      "7960/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.1442 - acc: 0.9491     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0770 - acc: 0.9747     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0648 - acc: 0.9780     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0638 - acc: 0.9785     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0609 - acc: 0.9802     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0535 - acc: 0.9821     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.1572 - acc: 0.9418     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0915 - acc: 0.9720     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0773 - acc: 0.9739     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0725 - acc: 0.9766     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0693 - acc: 0.9783     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0660 - acc: 0.9790     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.2142 - acc: 0.9190     \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1369 - acc: 0.9569     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.1263 - acc: 0.9596     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1139 - acc: 0.9640     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1096 - acc: 0.9658     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0966 - acc: 0.9677     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.2149 - acc: 0.9136     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1331 - acc: 0.9559     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1318 - acc: 0.9563     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1100 - acc: 0.9627     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1141 - acc: 0.9637     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1065 - acc: 0.9667     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.2145 - acc: 0.9207     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1477 - acc: 0.9513     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1274 - acc: 0.9580     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1198 - acc: 0.9618     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1223 - acc: 0.9609     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1064 - acc: 0.9648     \n",
      "8120/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed: 19.6min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 12s - loss: 0.0685 - acc: 0.9784    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0412 - acc: 0.9872     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0331 - acc: 0.9890     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0256 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0229 - acc: 0.9932     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0205 - acc: 0.9933     \n",
      "Best: 0.993946 using {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.990673 (0.002604) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.0}\n",
      "0.993946 (0.002253) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993291 (0.001518) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.2}\n",
      "0.993210 (0.000704) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.3}\n",
      "0.991819 (0.000760) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.4}\n",
      "0.991573 (0.001158) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.5}\n",
      "0.991573 (0.001158) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.6}\n",
      "0.991737 (0.001105) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.7}\n",
      "0.990428 (0.000919) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.8}\n",
      "0.988628 (0.002523) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.9}\n",
      "Best: 0.993946 using {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2040/2060 [============================>.] - ETA: 0sTest Accuracy 0.991262134418\n"
     ]
    }
   ],
   "source": [
    "##### DROPOUT REGULARIZATION 1 #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\n",
    "              'dropout_rate_2': [0.5]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.0652 - acc: 0.9748    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0329 - acc: 0.9887     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0261 - acc: 0.9912     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0209 - acc: 0.9929     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0179 - acc: 0.9941     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0150 - acc: 0.9946     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0730 - acc: 0.9713     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0342 - acc: 0.9880     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0241 - acc: 0.9919     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0211 - acc: 0.9928     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0154 - acc: 0.9950     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0172 - acc: 0.9931     \n",
      "4074/4074 [==============================] - 4s     \n",
      "7960/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0699 - acc: 0.9784     \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0341 - acc: 0.9901     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0253 - acc: 0.9924     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0217 - acc: 0.9921     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0184 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0168 - acc: 0.9942     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0656 - acc: 0.9758    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0308 - acc: 0.9898     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0213 - acc: 0.9931     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0185 - acc: 0.9939     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0182 - acc: 0.9936     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0141 - acc: 0.9957     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0654 - acc: 0.9772    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0333 - acc: 0.9887     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0225 - acc: 0.9919     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0154 - acc: 0.9946     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0172 - acc: 0.9934     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0122 - acc: 0.9950     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0743 - acc: 0.9729    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0346 - acc: 0.9891     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0256 - acc: 0.9919     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0222 - acc: 0.9924     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0168 - acc: 0.9936     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0154 - acc: 0.9951     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.0773 - acc: 0.9751    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0366 - acc: 0.9894     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0261 - acc: 0.9921     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0193 - acc: 0.9941     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0195 - acc: 0.9930     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0139 - acc: 0.9955     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0709 - acc: 0.9762    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0402 - acc: 0.9863     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0293 - acc: 0.9892     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0239 - acc: 0.9915     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0224 - acc: 0.9924     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0175 - acc: 0.9941     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0755 - acc: 0.9728    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0406 - acc: 0.9875     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0295 - acc: 0.9924     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0240 - acc: 0.9931     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0196 - acc: 0.9939     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0161 - acc: 0.9950     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0736 - acc: 0.9739    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0363 - acc: 0.9885     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0226 - acc: 0.9929     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0197 - acc: 0.9928     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0169 - acc: 0.9950     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0191 - acc: 0.9929     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0752 - acc: 0.9748    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0348 - acc: 0.9897     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0231 - acc: 0.9933     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0217 - acc: 0.9930     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0169 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0144 - acc: 0.9952     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0739 - acc: 0.9752    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0430 - acc: 0.9866     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0318 - acc: 0.9902     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0252 - acc: 0.9915     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0244 - acc: 0.9921     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0209 - acc: 0.9935     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0795 - acc: 0.9739    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0391 - acc: 0.9888     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0319 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0261 - acc: 0.9921     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0216 - acc: 0.9934     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0195 - acc: 0.9939     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0703 - acc: 0.9747    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0342 - acc: 0.9881     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0279 - acc: 0.9908     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0250 - acc: 0.9917     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0207 - acc: 0.9931     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0159 - acc: 0.9944     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0909 - acc: 0.9690    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0479 - acc: 0.9850     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0358 - acc: 0.9896     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0302 - acc: 0.9903     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0228 - acc: 0.9923     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0208 - acc: 0.9934     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0870 - acc: 0.9703    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0466 - acc: 0.9847     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0375 - acc: 0.9892     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0298 - acc: 0.9909     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0261 - acc: 0.9920     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0218 - acc: 0.9930     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0833 - acc: 0.9739    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0378 - acc: 0.9872     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0293 - acc: 0.9899     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0233 - acc: 0.9914     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0199 - acc: 0.9935     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0155 - acc: 0.9946     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0879 - acc: 0.9677    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0525 - acc: 0.9829     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0374 - acc: 0.9890     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0298 - acc: 0.9917     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0265 - acc: 0.9930     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0214 - acc: 0.9931     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0935 - acc: 0.9689    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0512 - acc: 0.9837     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0408 - acc: 0.9869     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0334 - acc: 0.9902     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0276 - acc: 0.9918     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0239 - acc: 0.9912     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0811 - acc: 0.9715    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0455 - acc: 0.9849     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0355 - acc: 0.9886     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0309 - acc: 0.9896     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0316 - acc: 0.9893     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0267 - acc: 0.9909     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0919 - acc: 0.9685    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0489 - acc: 0.9871     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0340 - acc: 0.9896     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0314 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0287 - acc: 0.9913     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0256 - acc: 0.9917     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.1022 - acc: 0.9645    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0526 - acc: 0.9836     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0408 - acc: 0.9882     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0369 - acc: 0.9874     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0323 - acc: 0.9902     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0277 - acc: 0.9912     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.1105 - acc: 0.9590    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0486 - acc: 0.9838     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0349 - acc: 0.9885     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0342 - acc: 0.9892     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0291 - acc: 0.9912     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0221 - acc: 0.9917     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.1113 - acc: 0.9622    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0623 - acc: 0.9804     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0479 - acc: 0.9838     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0422 - acc: 0.9866     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0361 - acc: 0.9883     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0373 - acc: 0.9870     \n",
      "7980/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.1412 - acc: 0.9412    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0825 - acc: 0.9666     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0769 - acc: 0.9694     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0638 - acc: 0.9732     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0573 - acc: 0.9747     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0524 - acc: 0.9780     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.1310 - acc: 0.9487    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0721 - acc: 0.9699     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0569 - acc: 0.9762     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0526 - acc: 0.9769     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0513 - acc: 0.9755     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0470 - acc: 0.9761     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.1511 - acc: 0.9415    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0844 - acc: 0.9687     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0749 - acc: 0.9719     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0664 - acc: 0.9734     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0615 - acc: 0.9745     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0553 - acc: 0.9758     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.2580 - acc: 0.8575    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1967 - acc: 0.8844     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1793 - acc: 0.8883     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1757 - acc: 0.8870     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1689 - acc: 0.8958     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.1692 - acc: 0.8953     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.2603 - acc: 0.8645    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1880 - acc: 0.8908     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1853 - acc: 0.8924     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1651 - acc: 0.9004     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1602 - acc: 0.8997     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1592 - acc: 0.9001     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.2556 - acc: 0.8628    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1869 - acc: 0.8900     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1845 - acc: 0.8959     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1841 - acc: 0.8936     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1762 - acc: 0.8945     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.1667 - acc: 0.8959     \n",
      "8040/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed: 22.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 12s - loss: 0.0553 - acc: 0.9818    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0330 - acc: 0.9889     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0256 - acc: 0.9914     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0225 - acc: 0.9923     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0188 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0167 - acc: 0.9939     \n",
      "Best: 0.993210 using {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.979138 (0.017244) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.0, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993210 (0.002162) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992228 (0.000418) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.2, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992146 (0.000602) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.3, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.991410 (0.001219) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.4, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992637 (0.000802) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.5, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992473 (0.001408) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.6, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992800 (0.000904) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.7, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.991655 (0.000602) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.8, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.990592 (0.001751) with: {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.9, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.993210 using {'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'epochs': 6, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2000/2060 [============================>.] - ETA: 0sTest Accuracy 0.989805823391\n"
     ]
    }
   ],
   "source": [
    "##### DROPOUT REGULARIZATION 2 #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(4, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(8, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [.1],\n",
    "              'dropout_rate_2': [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.0751 - acc: 0.9752    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0382 - acc: 0.9881     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0314 - acc: 0.9904     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0289 - acc: 0.9903     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0243 - acc: 0.9920     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0194 - acc: 0.9934     \n",
      "4075/4075 [==============================] - 5s     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0730 - acc: 0.9723    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0407 - acc: 0.9855     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0303 - acc: 0.9892     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0238 - acc: 0.9910     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0206 - acc: 0.9917     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0155 - acc: 0.9946     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0742 - acc: 0.9762    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0369 - acc: 0.9888     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0270 - acc: 0.9908     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0242 - acc: 0.9924     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0193 - acc: 0.9941     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0166 - acc: 0.9948     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0593 - acc: 0.9778    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0296 - acc: 0.9898     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0216 - acc: 0.9931     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0201 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0158 - acc: 0.9950     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0151 - acc: 0.9950     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0725 - acc: 0.9732    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0322 - acc: 0.9890     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0215 - acc: 0.9928     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0158 - acc: 0.9941     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0143 - acc: 0.9957     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0123 - acc: 0.9958     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0743 - acc: 0.9724    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0327 - acc: 0.9883     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0255 - acc: 0.9909     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0227 - acc: 0.9912     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0162 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0155 - acc: 0.9947     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.0737 - acc: 0.9758    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0370 - acc: 0.9869     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0290 - acc: 0.9909     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0219 - acc: 0.9920     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0220 - acc: 0.9925     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0166 - acc: 0.9947     \n",
      "4075/4075 [==============================] - 6s     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0688 - acc: 0.9750    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0280 - acc: 0.9902     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0190 - acc: 0.9935     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0170 - acc: 0.9945     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0137 - acc: 0.9953     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0114 - acc: 0.9958     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0698 - acc: 0.9759    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0351 - acc: 0.9893     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0259 - acc: 0.9915     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0215 - acc: 0.9933     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0181 - acc: 0.9942     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0159 - acc: 0.9952     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.0661 - acc: 0.9782    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0323 - acc: 0.9904     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0253 - acc: 0.9931     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0209 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0193 - acc: 0.9941     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0180 - acc: 0.9937     \n",
      "4075/4075 [==============================] - 6s     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0636 - acc: 0.9782    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0315 - acc: 0.9892     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0236 - acc: 0.9924     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0190 - acc: 0.9929     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0141 - acc: 0.9955     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0137 - acc: 0.9948     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0658 - acc: 0.9768    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0313 - acc: 0.9903     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0259 - acc: 0.9910     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0199 - acc: 0.9930     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0176 - acc: 0.9935     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0165 - acc: 0.9944     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.0702 - acc: 0.9755    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0392 - acc: 0.9874     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0284 - acc: 0.9902     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0229 - acc: 0.9928     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0191 - acc: 0.9941     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0191 - acc: 0.9930     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0656 - acc: 0.9759    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0318 - acc: 0.9875     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0209 - acc: 0.9929     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0180 - acc: 0.9934     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0132 - acc: 0.9952     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0117 - acc: 0.9955     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0662 - acc: 0.9773    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0359 - acc: 0.9880     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0266 - acc: 0.9917     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0225 - acc: 0.9933     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0186 - acc: 0.9941     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0174 - acc: 0.9939     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 15s - loss: 0.0784 - acc: 0.9725    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.0350 - acc: 0.9891     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.0237 - acc: 0.9936     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0216 - acc: 0.9937     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.0184 - acc: 0.9939     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 9s - loss: 0.0169 - acc: 0.9950     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0643 - acc: 0.9775    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0313 - acc: 0.9883     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0227 - acc: 0.9920     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0163 - acc: 0.9942     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0148 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0115 - acc: 0.9958     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0733 - acc: 0.9736    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0343 - acc: 0.9901     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 9s - loss: 0.0269 - acc: 0.9913     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0222 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0192 - acc: 0.9942     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0141 - acc: 0.9955     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.0703 - acc: 0.9742    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0392 - acc: 0.9861    \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0287 - acc: 0.9908    \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0225 - acc: 0.9928    \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0184 - acc: 0.9945    \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 10s - loss: 0.0157 - acc: 0.9942    \n",
      "4075/4075 [==============================] - 7s     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 16s - loss: 0.0621 - acc: 0.9764    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0301 - acc: 0.9897    \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0236 - acc: 0.9919    \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0186 - acc: 0.9933    \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0128 - acc: 0.9946    \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0131 - acc: 0.9955    \n",
      "4074/4074 [==============================] - 6s     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 16s - loss: 0.0690 - acc: 0.9756    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0374 - acc: 0.9890    \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0271 - acc: 0.9914    \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0243 - acc: 0.9924    \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0203 - acc: 0.9937    \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0171 - acc: 0.9944    \n",
      "4074/4074 [==============================] - 7s     \n",
      "8080/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed: 21.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 14s - loss: 0.0569 - acc: 0.9813    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0302 - acc: 0.9903     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0241 - acc: 0.9921     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0186 - acc: 0.9936     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0173 - acc: 0.9935     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0156 - acc: 0.9946     \n",
      "Best: 0.993373 using {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.983719 (0.012666) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 1, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993373 (0.001116) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992555 (0.000418) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 10, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.991328 (0.001729) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 15, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992719 (0.000504) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 20, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993046 (0.001905) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 25, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992882 (0.000919) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 30, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.993373 using {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 8, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2020/2060 [============================>.] - ETA: 0sTest Accuracy 0.991747571427\n"
     ]
    }
   ],
   "source": [
    "##### HIDDEN LAYER NEURONS 1 #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [1, 5, 10, 15, 20, 25, 30],\n",
    "              'neurons_2': [8],\n",
    "              'neurons_3': [16]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 7 candidates, totalling 21 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.1934 - acc: 0.9103    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0873 - acc: 0.9708     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0728 - acc: 0.9742     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0717 - acc: 0.9755     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0648 - acc: 0.9767     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0670 - acc: 0.9757     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0851 - acc: 0.9677    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0454 - acc: 0.9849     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0414 - acc: 0.9840     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0392 - acc: 0.9849     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0366 - acc: 0.9861     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0372 - acc: 0.9854     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.1001 - acc: 0.9638    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0614 - acc: 0.9799     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0534 - acc: 0.9825     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0500 - acc: 0.9837     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0460 - acc: 0.9855     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0420 - acc: 0.9848     \n",
      "4074/4074 [==============================] - 6s     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 12s - loss: 0.0716 - acc: 0.9753    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0310 - acc: 0.9898     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0210 - acc: 0.9932     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0217 - acc: 0.9920     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0177 - acc: 0.9946     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0155 - acc: 0.9945     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0734 - acc: 0.9726    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0353 - acc: 0.9879     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0236 - acc: 0.9910     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0212 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0174 - acc: 0.9929     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0140 - acc: 0.9946     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0766 - acc: 0.9730    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0369 - acc: 0.9876     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0284 - acc: 0.9912     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0231 - acc: 0.9919     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0185 - acc: 0.9942     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0171 - acc: 0.9937     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.0727 - acc: 0.9739    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0372 - acc: 0.9890     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0300 - acc: 0.9905     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0226 - acc: 0.9923     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0202 - acc: 0.9936     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0165 - acc: 0.9948     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0621 - acc: 0.9779    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0296 - acc: 0.9901     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0216 - acc: 0.9928     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0183 - acc: 0.9933     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0135 - acc: 0.9955     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0116 - acc: 0.9957     \n",
      "4074/4074 [==============================] - 6s     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0750 - acc: 0.9740    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0383 - acc: 0.9881     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0268 - acc: 0.9925     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0217 - acc: 0.9937     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0191 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0173 - acc: 0.9935     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.0685 - acc: 0.9780    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0339 - acc: 0.9899     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0245 - acc: 0.9914     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0184 - acc: 0.9939     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0164 - acc: 0.9945     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0131 - acc: 0.9952     \n",
      "8040/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 10s - loss: 0.0681 - acc: 0.9759    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0339 - acc: 0.9888     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0232 - acc: 0.9913     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0163 - acc: 0.9956     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0142 - acc: 0.9946     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0115 - acc: 0.9958     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0691 - acc: 0.9753    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0347 - acc: 0.9896     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0237 - acc: 0.9924     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0212 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0166 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 4s - loss: 0.0127 - acc: 0.9961     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.0670 - acc: 0.9778    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0319 - acc: 0.9903     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0228 - acc: 0.9929     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0199 - acc: 0.9941     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0152 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0142 - acc: 0.9956     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0637 - acc: 0.9757    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0321 - acc: 0.9882     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0205 - acc: 0.9923     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0161 - acc: 0.9939     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0131 - acc: 0.9953     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0109 - acc: 0.9962     \n",
      "4074/4074 [==============================] - 6s     \n",
      "8149/8149 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0646 - acc: 0.9778    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0340 - acc: 0.9893     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0253 - acc: 0.9914     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0203 - acc: 0.9934     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0160 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0148 - acc: 0.9957     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 11s - loss: 0.0751 - acc: 0.9755    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0409 - acc: 0.9865     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0287 - acc: 0.9913     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0233 - acc: 0.9931     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0193 - acc: 0.9939     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0138 - acc: 0.9957     \n",
      "4075/4075 [==============================] - 6s     \n",
      "8148/8148 [==============================] - 2s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0648 - acc: 0.9780    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0253 - acc: 0.9919     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0188 - acc: 0.9944     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0143 - acc: 0.9947     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0123 - acc: 0.9962     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0099 - acc: 0.9961     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0708 - acc: 0.9752    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0357 - acc: 0.9879     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0263 - acc: 0.9925     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0199 - acc: 0.9934     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0187 - acc: 0.9941     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0131 - acc: 0.9950     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.0610 - acc: 0.9805    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0289 - acc: 0.9901     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0218 - acc: 0.9921     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0182 - acc: 0.9942     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0154 - acc: 0.9950     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0123 - acc: 0.9958     \n",
      "8020/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0605 - acc: 0.9783    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0232 - acc: 0.9923     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0161 - acc: 0.9945     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0124 - acc: 0.9961     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0118 - acc: 0.9961     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0098 - acc: 0.9966     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0631 - acc: 0.9783    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0330 - acc: 0.9901     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0242 - acc: 0.9921     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0192 - acc: 0.9944     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0160 - acc: 0.9952     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0126 - acc: 0.9968     \n",
      "8149/8149 [==============================] - 2s     \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  21 out of  21 | elapsed: 17.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 14s - loss: 0.0539 - acc: 0.9814    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 7s - loss: 0.0261 - acc: 0.9912     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0205 - acc: 0.9935     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0165 - acc: 0.9944     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0142 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0121 - acc: 0.9954     \n",
      "Best: 0.993864 using {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.984946 (0.001102) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 1, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993046 (0.000612) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 5, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992882 (0.001403) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 10, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993700 (0.001009) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 15, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992555 (0.001288) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 20, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993864 (0.001514) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993291 (0.000704) with: {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 30, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.993864 using {'neurons_3': 16, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2060/2060 [==============================] - 6s     \n",
      "Test Accuracy 0.992718445445\n"
     ]
    }
   ],
   "source": [
    "##### HIDDEN LAYER NEURONS 2 #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [5],\n",
    "              'neurons_2': [1, 5, 10, 15, 20, 25, 30],\n",
    "              'neurons_3': [16]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 6 candidates, totalling 18 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.0836 - acc: 0.9741    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0403 - acc: 0.9866     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0280 - acc: 0.9919     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0224 - acc: 0.9915     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0178 - acc: 0.9947     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0156 - acc: 0.9946     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0752 - acc: 0.9732    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0302 - acc: 0.9899     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0222 - acc: 0.9918     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0193 - acc: 0.9934     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0156 - acc: 0.9945     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0126 - acc: 0.9963     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0754 - acc: 0.9745    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0362 - acc: 0.9892     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0271 - acc: 0.9919     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0189 - acc: 0.9942     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0166 - acc: 0.9942     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0158 - acc: 0.9966     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.0625 - acc: 0.9782    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0308 - acc: 0.9896     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0205 - acc: 0.9935     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0185 - acc: 0.9941     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0158 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0108 - acc: 0.9963     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 12s - loss: 0.0701 - acc: 0.9745    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0342 - acc: 0.9897     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0213 - acc: 0.9930     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0155 - acc: 0.9948     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0119 - acc: 0.9960     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0124 - acc: 0.9952     \n",
      "8149/8149 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0662 - acc: 0.9768    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0302 - acc: 0.9891     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0226 - acc: 0.9926     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0179 - acc: 0.9941     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0153 - acc: 0.9960     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0134 - acc: 0.9958     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.0722 - acc: 0.9762    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0348 - acc: 0.9882     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0248 - acc: 0.9923     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0211 - acc: 0.9942     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0160 - acc: 0.9946     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0139 - acc: 0.9956     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 33s - loss: 0.0630 - acc: 0.9789    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0316 - acc: 0.9883     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0214 - acc: 0.9926     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0171 - acc: 0.9937     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0139 - acc: 0.9952     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0112 - acc: 0.9955     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0671 - acc: 0.9757    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0309 - acc: 0.9899     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0211 - acc: 0.9930     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0179 - acc: 0.9944     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0162 - acc: 0.9952     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0125 - acc: 0.9964     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.0646 - acc: 0.9784    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0348 - acc: 0.9880     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0247 - acc: 0.9920     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0202 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0161 - acc: 0.9946     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0135 - acc: 0.9945     \n",
      "8080/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0638 - acc: 0.9791    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0300 - acc: 0.9907     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0202 - acc: 0.9931     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0182 - acc: 0.9931     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0131 - acc: 0.9952     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0120 - acc: 0.9966     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0688 - acc: 0.9748    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0322 - acc: 0.9908     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0241 - acc: 0.9917     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0181 - acc: 0.9946     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0160 - acc: 0.9952     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0129 - acc: 0.9961     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.0626 - acc: 0.9810    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0278 - acc: 0.9913     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0210 - acc: 0.9934     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0183 - acc: 0.9939     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0136 - acc: 0.9946     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0120 - acc: 0.9962     \n",
      "8148/8148 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0654 - acc: 0.9745    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0293 - acc: 0.9896     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0194 - acc: 0.9939     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0154 - acc: 0.9950     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0137 - acc: 0.9956     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0115 - acc: 0.9961     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 13s - loss: 0.0648 - acc: 0.9774    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0356 - acc: 0.9898     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0259 - acc: 0.9915     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0206 - acc: 0.9930     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0174 - acc: 0.9950     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0148 - acc: 0.9955     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.0737 - acc: 0.9750    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0363 - acc: 0.9891     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0237 - acc: 0.9925     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0198 - acc: 0.9937     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0161 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0142 - acc: 0.9952     \n",
      "8120/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0677 - acc: 0.9774    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0264 - acc: 0.9904     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0193 - acc: 0.9937     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0139 - acc: 0.9946     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0129 - acc: 0.9961     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0103 - acc: 0.9963     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0670 - acc: 0.9759    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0331 - acc: 0.9885     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0247 - acc: 0.9931     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0175 - acc: 0.9946     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0160 - acc: 0.9955     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0127 - acc: 0.9951     \n",
      "8100/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  18 out of  18 | elapsed: 16.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 15s - loss: 0.0545 - acc: 0.9808    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0269 - acc: 0.9914     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0209 - acc: 0.9928     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0176 - acc: 0.9945     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0135 - acc: 0.9956     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 8s - loss: 0.0110 - acc: 0.9958     \n",
      "Best: 0.994191 using {'neurons_3': 10, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.990673 (0.003280) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994191 (0.001450) with: {'neurons_3': 10, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993128 (0.001003) with: {'neurons_3': 15, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993619 (0.000919) with: {'neurons_3': 20, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.992146 (0.001712) with: {'neurons_3': 25, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.991164 (0.002912) with: {'neurons_3': 30, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.994191 using {'neurons_3': 10, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "1940/2060 [===========================>..] - ETA: 0sTest Accuracy 0.992718445445\n"
     ]
    }
   ],
   "source": [
    "##### HIDDEN LAYER NEURONS 3 #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [5],\n",
    "              'neurons_2': [25],\n",
    "              'neurons_3': [5, 10, 15, 20, 25, 30]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 1 candidates, totalling 3 fits\n",
      "Epoch 1/12\n",
      "8148/8148 [==============================] - 12s - loss: 0.0837 - acc: 0.9742    \n",
      "Epoch 2/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0404 - acc: 0.9869     \n",
      "Epoch 3/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0281 - acc: 0.9915     \n",
      "Epoch 4/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0223 - acc: 0.9918     \n",
      "Epoch 5/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0179 - acc: 0.9947     \n",
      "Epoch 6/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0156 - acc: 0.9946     \n",
      "Epoch 7/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0148 - acc: 0.9948     \n",
      "Epoch 8/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0102 - acc: 0.9962     \n",
      "Epoch 9/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0090 - acc: 0.9972     \n",
      "Epoch 10/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0080 - acc: 0.9974     \n",
      "Epoch 11/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0063 - acc: 0.9982     \n",
      "Epoch 12/12\n",
      "8148/8148 [==============================] - 5s - loss: 0.0080 - acc: 0.9968     \n",
      "8148/8148 [==============================] - 2s     \n",
      "Epoch 1/12\n",
      "8149/8149 [==============================] - 12s - loss: 0.0664 - acc: 0.9766    \n",
      "Epoch 2/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0310 - acc: 0.9897     \n",
      "Epoch 3/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0223 - acc: 0.9929     \n",
      "Epoch 4/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0171 - acc: 0.9934     \n",
      "Epoch 5/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0140 - acc: 0.9956     \n",
      "Epoch 6/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0111 - acc: 0.9972     \n",
      "Epoch 7/12\n",
      "8149/8149 [==============================] - 6s - loss: 0.0094 - acc: 0.9963     \n",
      "Epoch 8/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0099 - acc: 0.9961     \n",
      "Epoch 9/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0072 - acc: 0.9977     \n",
      "Epoch 10/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0045 - acc: 0.9991     \n",
      "Epoch 11/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0061 - acc: 0.9979     \n",
      "Epoch 12/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0050 - acc: 0.9984     \n",
      "8020/8149 [============================>.] - ETA: 0sEpoch 1/12\n",
      "8149/8149 [==============================] - 13s - loss: 0.0689 - acc: 0.9777    \n",
      "Epoch 2/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0393 - acc: 0.9887     \n",
      "Epoch 3/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0306 - acc: 0.9914     \n",
      "Epoch 4/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0221 - acc: 0.9934     \n",
      "Epoch 5/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0210 - acc: 0.9936     \n",
      "Epoch 6/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0163 - acc: 0.9945     \n",
      "Epoch 7/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0124 - acc: 0.9960     \n",
      "Epoch 8/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0131 - acc: 0.9966     \n",
      "Epoch 9/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0106 - acc: 0.9960     \n",
      "Epoch 10/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0099 - acc: 0.9971     \n",
      "Epoch 11/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0086 - acc: 0.9969     \n",
      "Epoch 12/12\n",
      "8149/8149 [==============================] - 5s - loss: 0.0091 - acc: 0.9963     \n",
      "8020/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:  4.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/12\n",
      "12223/12223 [==============================] - 15s - loss: 0.0651 - acc: 0.9786    \n",
      "Epoch 2/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0286 - acc: 0.9916     \n",
      "Epoch 3/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0202 - acc: 0.9935     \n",
      "Epoch 4/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0181 - acc: 0.9944     \n",
      "Epoch 5/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0131 - acc: 0.9960     \n",
      "Epoch 6/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0137 - acc: 0.9949     \n",
      "Epoch 7/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0147 - acc: 0.9944     \n",
      "Epoch 8/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0115 - acc: 0.9962     \n",
      "Epoch 9/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0097 - acc: 0.9966     \n",
      "Epoch 10/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0088 - acc: 0.9973     \n",
      "Epoch 11/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0079 - acc: 0.9970     \n",
      "Epoch 12/12\n",
      "12223/12223 [==============================] - 8s - loss: 0.0076 - acc: 0.9967     \n",
      "Best: 0.993619 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 12, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993619 (0.000918) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 12, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.993619 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'epochs': 12, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "1960/2060 [===========================>..] - ETA: 0sTest Accuracy 0.991747571427\n"
     ]
    }
   ],
   "source": [
    "##### Kernel Size #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=(3, 3), activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, (3, 3), activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [5],\n",
    "              'neurons_2': [25],\n",
    "              'neurons_3': [5, 10, 15, 20, 25, 30]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.0849 - acc: 0.9745    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0491 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0460 - acc: 0.9849     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 4s - loss: 0.0412 - acc: 0.9860     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0415 - acc: 0.9874     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 3s - loss: 0.0401 - acc: 0.9863     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0882 - acc: 0.9715    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0455 - acc: 0.9840     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0404 - acc: 0.9852     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0394 - acc: 0.9852     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0364 - acc: 0.9869     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0337 - acc: 0.9881     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 11s - loss: 0.0770 - acc: 0.9742    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0503 - acc: 0.9843     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0451 - acc: 0.9860     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0421 - acc: 0.9861     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0407 - acc: 0.9864     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 3s - loss: 0.0391 - acc: 0.9869     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 13s - loss: 0.0714 - acc: 0.9756    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0375 - acc: 0.9877     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0238 - acc: 0.9925     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0214 - acc: 0.9930     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0158 - acc: 0.9945     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 5s - loss: 0.0140 - acc: 0.9948     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0672 - acc: 0.9780    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0307 - acc: 0.9892     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0196 - acc: 0.9937     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0168 - acc: 0.9945     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0125 - acc: 0.9958     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0134 - acc: 0.9951     \n",
      "8149/8149 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0775 - acc: 0.9741    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0380 - acc: 0.9880     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0286 - acc: 0.9915     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0238 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0199 - acc: 0.9945     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 5s - loss: 0.0167 - acc: 0.9941     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 15s - loss: 0.0641 - acc: 0.9782    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0270 - acc: 0.9914     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0236 - acc: 0.9920     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0185 - acc: 0.9941     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0160 - acc: 0.9947     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0117 - acc: 0.9962     \n",
      "4075/4075 [==============================] - 13s    \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 17s - loss: 0.0619 - acc: 0.9775    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0257 - acc: 0.9924     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0190 - acc: 0.9940     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0169 - acc: 0.9942     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0143 - acc: 0.9955     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0126 - acc: 0.9957     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0651 - acc: 0.9755    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0312 - acc: 0.9908     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0230 - acc: 0.9937     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0210 - acc: 0.9935     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0195 - acc: 0.9947     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0151 - acc: 0.9952     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 16s - loss: 0.0643 - acc: 0.9797    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0246 - acc: 0.9920     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0223 - acc: 0.9936     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0168 - acc: 0.9948     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0155 - acc: 0.9951     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0123 - acc: 0.9956     \n",
      "8148/8148 [==============================] - 4s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 18s - loss: 0.0611 - acc: 0.9801    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0253 - acc: 0.9933     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0192 - acc: 0.9942     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0157 - acc: 0.9950     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0162 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0100 - acc: 0.9958     \n",
      "4074/4074 [==============================] - 9s     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 17s - loss: 0.0623 - acc: 0.9794    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0300 - acc: 0.9910     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0243 - acc: 0.9935     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0206 - acc: 0.9944     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 8s - loss: 0.0178 - acc: 0.9945     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0133 - acc: 0.9960     \n",
      "8120/8149 [============================>.] - ETA: 0s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed: 11.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 17s - loss: 0.0504 - acc: 0.9843    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0234 - acc: 0.9929     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0195 - acc: 0.9939     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0165 - acc: 0.9948     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0133 - acc: 0.9961     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0127 - acc: 0.9962     \n",
      "Best: 0.994600 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.983228 (0.001272) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [1, 1], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994273 (0.000704) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [3, 3], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994600 (0.000723) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994519 (0.002028) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [7, 7], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.994600 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2020/2060 [============================>.] - ETA: 0sTest Accuracy 0.99563106692\n"
     ]
    }
   ],
   "source": [
    "##### Kernel Size #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3,\n",
    "                 kernel_size):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=kernel_size, activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, kernel_size=kernel_size, activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [5],\n",
    "              'neurons_2': [25],\n",
    "              'neurons_3': [5],\n",
    "              'kernel_size': [[1,1],[3,3],[5,5],[7,7]]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 16s - loss: 0.0684 - acc: 0.9773    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0277 - acc: 0.9913     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0208 - acc: 0.9940     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0171 - acc: 0.9952     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0171 - acc: 0.9942     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0126 - acc: 0.9956     \n",
      "8148/8148 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0717 - acc: 0.9737    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0226 - acc: 0.9931     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0184 - acc: 0.9929     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0138 - acc: 0.9956     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0127 - acc: 0.9960     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0124 - acc: 0.9962     \n",
      "8060/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0673 - acc: 0.9782    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0278 - acc: 0.9917     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0212 - acc: 0.9946     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0161 - acc: 0.9957     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0170 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0143 - acc: 0.9958     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8148/8148 [==============================] - 15s - loss: 0.0637 - acc: 0.9785    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0310 - acc: 0.9904     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0234 - acc: 0.9930     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0191 - acc: 0.9944     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0161 - acc: 0.9950     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0140 - acc: 0.9948     \n",
      "8060/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 14s - loss: 0.0563 - acc: 0.9813    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0230 - acc: 0.9920     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0161 - acc: 0.9952     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0142 - acc: 0.9960     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0126 - acc: 0.9953     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0108 - acc: 0.9962     \n",
      "8149/8149 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 16s - loss: 0.0686 - acc: 0.9775    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0324 - acc: 0.9894     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0260 - acc: 0.9931     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0217 - acc: 0.9926     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0165 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0161 - acc: 0.9951     \n",
      "4074/4074 [==============================] - 10s    \n",
      "8149/8149 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 14s - loss: 0.0640 - acc: 0.9780    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0270 - acc: 0.9913     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 8s - loss: 0.0238 - acc: 0.9924     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0188 - acc: 0.9935     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0159 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0117 - acc: 0.9961     \n",
      "8148/8148 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8149/8149 [==============================] - 16s - loss: 0.0620 - acc: 0.9773    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0258 - acc: 0.9923     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0191 - acc: 0.9936     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0163 - acc: 0.9952     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0147 - acc: 0.9953     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0126 - acc: 0.9956     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0651 - acc: 0.9755    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0312 - acc: 0.9909     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0230 - acc: 0.9936     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0211 - acc: 0.9934     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0195 - acc: 0.9948     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0151 - acc: 0.9955     \n",
      "8149/8149 [==============================] - 3s     \n",
      "Epoch 1/6\n",
      "8148/8148 [==============================] - 16s - loss: 0.0629 - acc: 0.9791    \n",
      "Epoch 2/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0273 - acc: 0.9910     \n",
      "Epoch 3/6\n",
      "8148/8148 [==============================] - 6s - loss: 0.0216 - acc: 0.9939     \n",
      "Epoch 4/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0205 - acc: 0.9939     \n",
      "Epoch 5/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0175 - acc: 0.9947     \n",
      "Epoch 6/6\n",
      "8148/8148 [==============================] - 7s - loss: 0.0142 - acc: 0.9955     \n",
      "4075/4075 [==============================] - 9s     \n",
      "8140/8148 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 15s - loss: 0.0572 - acc: 0.9805    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0239 - acc: 0.9928     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0173 - acc: 0.9939     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0141 - acc: 0.9961     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0130 - acc: 0.9951     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0094 - acc: 0.9969     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/6\n",
      "8149/8149 [==============================] - 16s - loss: 0.0609 - acc: 0.9769    \n",
      "Epoch 2/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0285 - acc: 0.9908     \n",
      "Epoch 3/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0231 - acc: 0.9936     \n",
      "Epoch 4/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0172 - acc: 0.9955     \n",
      "Epoch 5/6\n",
      "8149/8149 [==============================] - 6s - loss: 0.0165 - acc: 0.9944     \n",
      "Epoch 6/6\n",
      "8149/8149 [==============================] - 7s - loss: 0.0145 - acc: 0.9961     \n",
      "4074/4074 [==============================] - 8s     \n",
      "8149/8149 [==============================] - 3s     \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  12 out of  12 | elapsed: 12.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/6\n",
      "12223/12223 [==============================] - 17s - loss: 0.0503 - acc: 0.9842    \n",
      "Epoch 2/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0233 - acc: 0.9928     \n",
      "Epoch 3/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0195 - acc: 0.9940     \n",
      "Epoch 4/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0164 - acc: 0.9947     \n",
      "Epoch 5/6\n",
      "12223/12223 [==============================] - 9s - loss: 0.0135 - acc: 0.9962     \n",
      "Epoch 6/6\n",
      "12223/12223 [==============================] - 10s - loss: 0.0126 - acc: 0.9963    \n",
      "Best: 0.995009 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.995009 (0.002047) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994519 (0.000306) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [2, 2], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994355 (0.000723) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [3, 3], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994519 (0.001365) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [5, 5], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.995009 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 6, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2040/2060 [============================>.] - ETA: 0sTest Accuracy 0.996601940937\n"
     ]
    }
   ],
   "source": [
    "##### Pool Size #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3,\n",
    "                 kernel_size,\n",
    "                 pool_size):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=kernel_size, activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, kernel_size=kernel_size, activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [6],\n",
    "              'batch_size': [20],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [5],\n",
    "              'neurons_2': [25],\n",
    "              'neurons_3': [5],\n",
    "              'kernel_size': [[5,5]],\n",
    "              'pool_size': [[1,1],[2,2],[3,3],[5,5]]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n",
      "Epoch 1/20\n",
      "8148/8148 [==============================] - 19s - loss: 0.0684 - acc: 0.9772    \n",
      "Epoch 2/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0275 - acc: 0.9910     \n",
      "Epoch 3/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0207 - acc: 0.9939     \n",
      "Epoch 4/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0170 - acc: 0.9953     \n",
      "Epoch 5/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0173 - acc: 0.9939     \n",
      "Epoch 6/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0126 - acc: 0.9957     \n",
      "Epoch 7/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0177 - acc: 0.9942     \n",
      "Epoch 8/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0097 - acc: 0.9969     \n",
      "Epoch 9/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0088 - acc: 0.9975     \n",
      "Epoch 10/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0080 - acc: 0.9974     \n",
      "Epoch 11/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0079 - acc: 0.9980     \n",
      "Epoch 12/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0092 - acc: 0.9971     \n",
      "Epoch 13/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0063 - acc: 0.9978     \n",
      "Epoch 14/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0089 - acc: 0.9964     \n",
      "Epoch 15/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0084 - acc: 0.9971     \n",
      "Epoch 16/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0035 - acc: 0.9988     \n",
      "Epoch 17/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0087 - acc: 0.9969     \n",
      "Epoch 18/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0047 - acc: 0.9979     \n",
      "Epoch 19/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0022 - acc: 0.9993     \n",
      "Epoch 20/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0036 - acc: 0.9985     \n",
      "8100/8148 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 15s - loss: 0.0633 - acc: 0.9782    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0286 - acc: 0.9908     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0186 - acc: 0.9936     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0168 - acc: 0.9947     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0150 - acc: 0.9947     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0107 - acc: 0.9972     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0092 - acc: 0.9973     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0099 - acc: 0.9974     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0075 - acc: 0.9972     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0086 - acc: 0.9979     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0073 - acc: 0.9973     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0051 - acc: 0.9979     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0081 - acc: 0.9978     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0052 - acc: 0.9983     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0062 - acc: 0.9983     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0028 - acc: 0.9993     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0064 - acc: 0.9978     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0038 - acc: 0.9984     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0057 - acc: 0.9980     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0022 - acc: 0.9991     \n",
      "4074/4074 [==============================] - 10s    \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 15s - loss: 0.0658 - acc: 0.9763    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0293 - acc: 0.9920     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0195 - acc: 0.9936     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0210 - acc: 0.9939     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0157 - acc: 0.9952     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0158 - acc: 0.9947     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0113 - acc: 0.9967     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0117 - acc: 0.9962     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0104 - acc: 0.9968     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0075 - acc: 0.9980     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0098 - acc: 0.9969     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0084 - acc: 0.9973     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0065 - acc: 0.9973     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0072 - acc: 0.9983     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0063 - acc: 0.9974     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0072 - acc: 0.9974     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0031 - acc: 0.9990     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0044 - acc: 0.9985     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0033 - acc: 0.9985     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0067 - acc: 0.9975     \n",
      "8140/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8148/8148 [==============================] - 14s - loss: 0.0887 - acc: 0.9709    \n",
      "Epoch 2/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0373 - acc: 0.9890     \n",
      "Epoch 3/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0296 - acc: 0.9909     \n",
      "Epoch 4/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0230 - acc: 0.9931     \n",
      "Epoch 5/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0199 - acc: 0.9941     \n",
      "Epoch 6/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0159 - acc: 0.9946     \n",
      "Epoch 7/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0156 - acc: 0.9953     \n",
      "Epoch 8/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0156 - acc: 0.9948     \n",
      "Epoch 9/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0120 - acc: 0.9966     \n",
      "Epoch 10/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0138 - acc: 0.9948     \n",
      "Epoch 11/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0110 - acc: 0.9964     \n",
      "Epoch 12/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0095 - acc: 0.9972     \n",
      "Epoch 13/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0100 - acc: 0.9963     \n",
      "Epoch 14/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0087 - acc: 0.9972     \n",
      "Epoch 15/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0083 - acc: 0.9978     \n",
      "Epoch 16/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0080 - acc: 0.9979     \n",
      "Epoch 17/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0063 - acc: 0.9980     \n",
      "Epoch 18/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0072 - acc: 0.9975     \n",
      "Epoch 19/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0074 - acc: 0.9977     \n",
      "Epoch 20/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0055 - acc: 0.9979     \n",
      "8000/8148 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 13s - loss: 0.0705 - acc: 0.9719    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0289 - acc: 0.9909     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0200 - acc: 0.9945     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0149 - acc: 0.9952     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0164 - acc: 0.9947     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0125 - acc: 0.9966     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0122 - acc: 0.9960     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0117 - acc: 0.9956     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0097 - acc: 0.9969     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0101 - acc: 0.9963     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0098 - acc: 0.9971     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0084 - acc: 0.9972     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0076 - acc: 0.9975     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0063 - acc: 0.9982     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0076 - acc: 0.9971     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0046 - acc: 0.9991     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0058 - acc: 0.9983     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0037 - acc: 0.9988     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0044 - acc: 0.9988     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0088 - acc: 0.9974     \n",
      "4074/4074 [==============================] - 7s     \n",
      "8040/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 14s - loss: 0.0860 - acc: 0.9694    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0390 - acc: 0.9881     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0277 - acc: 0.9907     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0221 - acc: 0.9937     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0192 - acc: 0.9941     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0183 - acc: 0.9952     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0163 - acc: 0.9955     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0158 - acc: 0.9958     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0117 - acc: 0.9968     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0105 - acc: 0.9972     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0127 - acc: 0.9966     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0118 - acc: 0.9962     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0102 - acc: 0.9972     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0090 - acc: 0.9966     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0086 - acc: 0.9974     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0074 - acc: 0.9971     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0068 - acc: 0.9975     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0074 - acc: 0.9978     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0070 - acc: 0.9979     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0086 - acc: 0.9968     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8148/8148 [==============================] - 13s - loss: 0.0758 - acc: 0.9739    \n",
      "Epoch 2/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0378 - acc: 0.9877     \n",
      "Epoch 3/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0264 - acc: 0.9917     \n",
      "Epoch 4/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0220 - acc: 0.9936     \n",
      "Epoch 5/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0190 - acc: 0.9951     \n",
      "Epoch 6/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0197 - acc: 0.9939     \n",
      "Epoch 7/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0145 - acc: 0.9955     \n",
      "Epoch 8/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0156 - acc: 0.9950     \n",
      "Epoch 9/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0128 - acc: 0.9966     \n",
      "Epoch 10/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0120 - acc: 0.9966     \n",
      "Epoch 11/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0106 - acc: 0.9968     \n",
      "Epoch 12/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0107 - acc: 0.9963     \n",
      "Epoch 13/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0114 - acc: 0.9964     \n",
      "Epoch 14/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0091 - acc: 0.9972     \n",
      "Epoch 15/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0097 - acc: 0.9974     \n",
      "Epoch 16/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0086 - acc: 0.9972     \n",
      "Epoch 17/20\n",
      "8148/8148 [==============================] - 4s - loss: 0.0074 - acc: 0.9980     \n",
      "Epoch 18/20\n",
      "8148/8148 [==============================] - 3646s - loss: 0.0081 - acc: 0.9973   \n",
      "Epoch 19/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0058 - acc: 0.9982     \n",
      "Epoch 20/20\n",
      "8148/8148 [==============================] - 5s - loss: 0.0058 - acc: 0.9980     \n",
      "4075/4075 [==============================] - 12s    \n",
      "7980/8148 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 14s - loss: 0.0838 - acc: 0.9717    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0378 - acc: 0.9875     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0258 - acc: 0.9920     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0217 - acc: 0.9924     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0177 - acc: 0.9946     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0162 - acc: 0.9946     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0140 - acc: 0.9955     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0138 - acc: 0.9961     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0118 - acc: 0.9962     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0101 - acc: 0.9967     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0095 - acc: 0.9969     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0076 - acc: 0.9973     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0089 - acc: 0.9968     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0076 - acc: 0.9982     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0074 - acc: 0.9975     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0055 - acc: 0.9984     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0048 - acc: 0.9987     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0072 - acc: 0.9974     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0052 - acc: 0.9984     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0051 - acc: 0.9983     \n",
      "8100/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 13s - loss: 0.0853 - acc: 0.9707    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0349 - acc: 0.9898     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0276 - acc: 0.9909     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0242 - acc: 0.9934     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0209 - acc: 0.9933     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0185 - acc: 0.9944     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0169 - acc: 0.9946     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0145 - acc: 0.9963     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0130 - acc: 0.9963     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0139 - acc: 0.9955     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0123 - acc: 0.9967     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0107 - acc: 0.9971     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0101 - acc: 0.9972     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0097 - acc: 0.9969     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0075 - acc: 0.9978     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0088 - acc: 0.9971     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 5s - loss: 0.0076 - acc: 0.9979     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0065 - acc: 0.9985     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0064 - acc: 0.9979     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0064 - acc: 0.9983     \n",
      "8149/8149 [==============================] - 2s     \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed: 81.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "12223/12223 [==============================] - 17s - loss: 0.0782 - acc: 0.9731    \n",
      "Epoch 2/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0357 - acc: 0.9882     \n",
      "Epoch 3/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0265 - acc: 0.9927     \n",
      "Epoch 4/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0223 - acc: 0.9927     \n",
      "Epoch 5/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0174 - acc: 0.9949     \n",
      "Epoch 6/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0154 - acc: 0.9946     \n",
      "Epoch 7/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0148 - acc: 0.9949     \n",
      "Epoch 8/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0142 - acc: 0.9959     \n",
      "Epoch 9/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0114 - acc: 0.9963     \n",
      "Epoch 10/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0095 - acc: 0.9975     \n",
      "Epoch 11/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0100 - acc: 0.9969     \n",
      "Epoch 12/20\n",
      "12223/12223 [==============================] - 8s - loss: 0.0111 - acc: 0.9967     \n",
      "Epoch 13/20\n",
      "12223/12223 [==============================] - 8s - loss: 0.0079 - acc: 0.9975     \n",
      "Epoch 14/20\n",
      "12223/12223 [==============================] - 8s - loss: 0.0092 - acc: 0.9969     \n",
      "Epoch 15/20\n",
      "12223/12223 [==============================] - 8s - loss: 0.0084 - acc: 0.9975     \n",
      "Epoch 16/20\n",
      "12223/12223 [==============================] - 8s - loss: 0.0078 - acc: 0.9975     \n",
      "Epoch 17/20\n",
      "12223/12223 [==============================] - 8s - loss: 0.0083 - acc: 0.9972     \n",
      "Epoch 18/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0060 - acc: 0.9978     \n",
      "Epoch 19/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0064 - acc: 0.9982     \n",
      "Epoch 20/20\n",
      "12223/12223 [==============================] - 7s - loss: 0.0066 - acc: 0.9979     \n",
      "Best: 0.994600 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 60, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 20, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994600 (0.000530) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 20, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 20, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.993455 (0.001531) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 40, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 20, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "0.994600 (0.001513) with: {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 60, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 20, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "Best: 0.994600 using {'neurons_3': 5, 'activation_2': 'relu', 'learn_rate': 0.1, 'dropout_rate_2': 0.1, 'optimizer': 'SGD', 'activation_1': 'softsign', 'momentum': 0.4, 'batch_size': 60, 'neurons_1': 5, 'kernel_size': [5, 5], 'epochs': 20, 'neurons_2': 25, 'activation_4': 'softplus', 'init_mode_1': 'glorot_uniform', 'pool_size': [1, 1], 'init_mode_2': 'he_normal', 'activation_3': 'linear', 'dropout_rate_1': 0.1}\n",
      "2040/2060 [============================>.] - ETA: 0sTest Accuracy 0.994174757629\n"
     ]
    }
   ],
   "source": [
    "##### Epochs - The more the better usually. #####\n",
    "\n",
    "def create_model(optimizer, \n",
    "                 learn_rate, \n",
    "                 momentum, \n",
    "                 init_mode_1, \n",
    "                 init_mode_2, \n",
    "                 activation_1, \n",
    "                 activation_2, \n",
    "                 activation_3, \n",
    "                 activation_4,\n",
    "                 dropout_rate_1,\n",
    "                 dropout_rate_2,\n",
    "                 neurons_1,\n",
    "                 neurons_2,\n",
    "                 neurons_3,\n",
    "                 kernel_size,\n",
    "                 pool_size):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(neurons_1, kernel_size=kernel_size, activation=activation_1, input_shape=input_shape)) \n",
    "    model.add(Conv2D(neurons_2, kernel_size=kernel_size, activation=activation_2))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(dropout_rate_1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(neurons_3, kernel_initializer=init_mode_1, activation=activation_3))\n",
    "    model.add(Dropout(dropout_rate_2))\n",
    "    model.add(Dense(2, kernel_initializer=init_mode_2, activation=activation_4))\n",
    "    \n",
    "    optimizer = SGD(learn_rate, momentum)\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [20],\n",
    "              'batch_size': [20, 40, 60],\n",
    "              'optimizer': ['SGD'],\n",
    "              'learn_rate': [0.1],\n",
    "              'momentum': [.4],\n",
    "              'init_mode_1': ['glorot_uniform'],\n",
    "              'init_mode_2': ['he_normal'],\n",
    "              'activation_1': ['softsign'],\n",
    "              'activation_2': ['relu'],\n",
    "              'activation_3': ['linear'],\n",
    "              'activation_4': ['softplus'],\n",
    "              'dropout_rate_1': [0.1],\n",
    "              'dropout_rate_2': [0.1],\n",
    "              'neurons_1': [5],\n",
    "              'neurons_2': [25],\n",
    "              'neurons_3': [5],\n",
    "              'kernel_size': [[5,5]],\n",
    "              'pool_size': [[1,1]]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\"Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 3 candidates, totalling 9 fits\n",
      "Epoch 1/20\n",
      "8148/8148 [==============================] - 21s - loss: 0.0683 - acc: 0.9777    \n",
      "Epoch 2/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0296 - acc: 0.9897     \n",
      "Epoch 3/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0209 - acc: 0.9939     \n",
      "Epoch 4/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0159 - acc: 0.9944     \n",
      "Epoch 5/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0146 - acc: 0.9951     \n",
      "Epoch 6/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0135 - acc: 0.9950     \n",
      "Epoch 7/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0146 - acc: 0.9944     \n",
      "Epoch 8/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0096 - acc: 0.9968     \n",
      "Epoch 9/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0090 - acc: 0.9973     \n",
      "Epoch 10/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0088 - acc: 0.9971     \n",
      "Epoch 11/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0083 - acc: 0.9973     \n",
      "Epoch 12/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0074 - acc: 0.9966     \n",
      "Epoch 13/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0057 - acc: 0.9978     \n",
      "Epoch 14/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0074 - acc: 0.9971     \n",
      "Epoch 15/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0057 - acc: 0.9978     \n",
      "Epoch 16/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0039 - acc: 0.9983     \n",
      "Epoch 17/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0042 - acc: 0.9983     \n",
      "Epoch 18/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0057 - acc: 0.9977     \n",
      "Epoch 19/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0057 - acc: 0.9978     \n",
      "Epoch 20/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0028 - acc: 0.9990     \n",
      "4075/4075 [==============================] - 12s    \n",
      "8148/8148 [==============================] - 3s     \n",
      "Epoch 1/20\n",
      "8149/8149 [==============================] - 16s - loss: 0.0620 - acc: 0.9807    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0262 - acc: 0.9913     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0187 - acc: 0.9931     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0155 - acc: 0.9946     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0138 - acc: 0.9948     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0114 - acc: 0.9969     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0112 - acc: 0.9963     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0099 - acc: 0.9968     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0088 - acc: 0.9969     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0074 - acc: 0.9977     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0061 - acc: 0.9980     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0058 - acc: 0.9977     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0076 - acc: 0.9980     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0044 - acc: 0.9982     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0049 - acc: 0.9983     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0059 - acc: 0.9983     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0039 - acc: 0.9989     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0037 - acc: 0.9985     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0102 - acc: 0.9973     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0026 - acc: 0.9990     \n",
      "8080/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8149/8149 [==============================] - 17s - loss: 0.0650 - acc: 0.9774    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0303 - acc: 0.9903     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0197 - acc: 0.9942     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0208 - acc: 0.9940     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0155 - acc: 0.9956     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0153 - acc: 0.9951     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0095 - acc: 0.9973     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0109 - acc: 0.9971     \n",
      "Epoch 9/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0067 - acc: 0.9978     \n",
      "Epoch 10/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0060 - acc: 0.9979     \n",
      "Epoch 11/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0093 - acc: 0.9966     \n",
      "Epoch 12/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0097 - acc: 0.9960     \n",
      "Epoch 13/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0058 - acc: 0.9978     \n",
      "Epoch 14/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0053 - acc: 0.9979     \n",
      "Epoch 15/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0070 - acc: 0.9978     \n",
      "Epoch 16/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0038 - acc: 0.9984     \n",
      "Epoch 17/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0046 - acc: 0.9983     \n",
      "Epoch 18/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0046 - acc: 0.9979     \n",
      "Epoch 19/20\n",
      "8149/8149 [==============================] - 7s - loss: 0.0044 - acc: 0.9985     \n",
      "Epoch 20/20\n",
      "8149/8149 [==============================] - 8s - loss: 0.0047 - acc: 0.9982     \n",
      "8120/8149 [============================>.] - ETA: 0sEpoch 1/20\n",
      "8148/8148 [==============================] - 17s - loss: 0.0817 - acc: 0.9726    \n",
      "Epoch 2/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0336 - acc: 0.9904     \n",
      "Epoch 3/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0261 - acc: 0.9914     \n",
      "Epoch 4/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0205 - acc: 0.9942     \n",
      "Epoch 5/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0182 - acc: 0.9941     \n",
      "Epoch 6/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0142 - acc: 0.9947     \n",
      "Epoch 7/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0143 - acc: 0.9958     \n",
      "Epoch 8/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0125 - acc: 0.9959     \n",
      "Epoch 9/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0092 - acc: 0.9973     \n",
      "Epoch 10/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0113 - acc: 0.9956     \n",
      "Epoch 11/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0084 - acc: 0.9977     \n",
      "Epoch 12/20\n",
      "8148/8148 [==============================] - 8s - loss: 0.0089 - acc: 0.9972     \n",
      "Epoch 13/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0102 - acc: 0.9968     \n",
      "Epoch 14/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0077 - acc: 0.9975     \n",
      "Epoch 15/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0066 - acc: 0.9978     \n",
      "Epoch 16/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0051 - acc: 0.9982     \n",
      "Epoch 17/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0049 - acc: 0.9980     \n",
      "Epoch 18/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0050 - acc: 0.9978     \n",
      "Epoch 19/20\n",
      "8148/8148 [==============================] - 6s - loss: 0.0064 - acc: 0.9980     \n",
      "Epoch 20/20\n",
      "8148/8148 [==============================] - 7s - loss: 0.0040 - acc: 0.9984     \n",
      "4075/4075 [==============================] - 14s    \n",
      "8148/8148 [==============================] - 2s     \n",
      "Epoch 1/20\n",
      "8149/8149 [==============================] - 15s - loss: 0.0641 - acc: 0.9739    \n",
      "Epoch 2/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0275 - acc: 0.9913     \n",
      "Epoch 3/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0189 - acc: 0.9945     \n",
      "Epoch 4/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0133 - acc: 0.9961     \n",
      "Epoch 5/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0146 - acc: 0.9950     \n",
      "Epoch 6/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0126 - acc: 0.9952     \n",
      "Epoch 7/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0098 - acc: 0.9968     \n",
      "Epoch 8/20\n",
      "8149/8149 [==============================] - 6s - loss: 0.0093 - acc: 0.9972     \n",
      "Epoch 9/20\n",
      " 360/8149 [>.............................] - ETA: 6s - loss: 0.0033 - acc: 1.0000"
     ]
    }
   ],
   "source": [
    "##### Final Model #####\n",
    "\n",
    "def create_model():\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(5, kernel_size=(5,5), activation='softsign', input_shape=input_shape)) \n",
    "    model.add(Conv2D(25, kernel_size=(5,5), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(1, 1)))\n",
    "    model.add(Dropout(.1)) \n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(5, kernel_initializer='glorot_uniform', activation='linear'))\n",
    "    model.add(Dropout(.1))\n",
    "    model.add(Dense(2, kernel_initializer='he_normal', activation='softplus'))\n",
    "\n",
    "    model.compile(loss=keras.losses.categorical_crossentropy, \n",
    "                  optimizer=SGD(lr=.1, momentum=.4), \n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "    return model\n",
    "\n",
    "# fix random seed for reproducibility\n",
    "seed = 0\n",
    "numpy.random.seed(seed)\n",
    "\n",
    "# Test all combinations of the following parameters:\n",
    "param_grid = {'epochs': [20],\n",
    "              'batch_size': [20, 40, 60]\n",
    "              }\n",
    "\n",
    "my_classifier = KerasClassifier(create_model)\n",
    "\n",
    "grid = GridSearchCV(my_classifier, param_grid, cv=3, n_jobs=1, verbose=1)\n",
    "\n",
    "grid_result = grid.fit(x_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))\n",
    "    \n",
    "# Print Results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "\n",
    "# Print Test Accuracy\n",
    "print(\" Test Accuracy\", grid_result.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}