bopt_nn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian Optimization on Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNIST training on Keras with Bayesian optimization\n",
    "* This notebook runs MNIST training on Keras using Bayesian optimization to find the best hyper parameters.\n",
    "* The MNIST model here is just a simple one with one input layer, one hidden layer and one output layer, without convolution.\n",
    "* Hyperparameters of the model include the followings:\n",
    "* - output shape of the first layer\n",
    "* - dropout rate of the first layer\n",
    "* - output shape of the second layer\n",
    "* - dropout rate of the second layer\n",
    "* - batch size\n",
    "* - number of epochs\n",
    "* - validation rate\n",
    "* I used GPy and GPyOpt to run Bayesian optimization."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import GPy, GPyOpt\n",
    "import numpy as np\n",
    "import pandas as pds\n",
    "import random\n",
    "from keras.layers import Activation, Dropout, BatchNormalization, Dense\n",
    "from keras.models import Sequential\n",
    "from keras.datasets import mnist\n",
    "from keras.metrics import categorical_crossentropy\n",
    "from keras.utils import np_utils\n",
    "from keras.optimizers import Adam\n",
    "from keras.callbacks import EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Define MNIST model\n",
    "* includes data loading function, training function, fit function and evaluation function "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# MNIST class\n",
    "class MNIST():\n",
    "    def __init__(self, first_input=784, last_output=10,\n",
    "                 l1_out=512, \n",
    "                 l2_out=512, \n",
    "                 l1_drop=0.2, \n",
    "                 l2_drop=0.2, \n",
    "                 batch_size=100, \n",
    "                 epochs=10, \n",
    "                 validation_split=0.1):\n",
    "        self.__first_input = first_input\n",
    "        self.__last_output = last_output\n",
    "        self.l1_out = l1_out\n",
    "        self.l2_out = l2_out\n",
    "        self.l1_drop = l1_drop\n",
    "        self.l2_drop = l2_drop\n",
    "        self.batch_size = batch_size\n",
    "        self.epochs = epochs\n",
    "        self.validation_split = validation_split\n",
    "        self.__x_train, self.__x_test, self.__y_train, self.__y_test = self.mnist_data()\n",
    "        self.__model = self.mnist_model()\n",
    "        \n",
    "    # load mnist data from keras dataset\n",
    "    def mnist_data(self):\n",
    "        (X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "        X_train = X_train.reshape(60000, 784)\n",
    "        X_test = X_test.reshape(10000, 784)\n",
    "\n",
    "        X_train = X_train.astype('float32')\n",
    "        X_test = X_test.astype('float32')\n",
    "        X_train /= 255\n",
    "        X_test /= 255\n",
    "\n",
    "        Y_train = np_utils.to_categorical(y_train, 10)\n",
    "        Y_test = np_utils.to_categorical(y_test, 10)\n",
    "        return X_train, X_test, Y_train, Y_test\n",
    "    \n",
    "    # mnist model\n",
    "    def mnist_model(self):\n",
    "        model = Sequential()\n",
    "        model.add(Dense(self.l1_out, input_shape=(self.__first_input,)))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Dropout(self.l1_drop))\n",
    "        model.add(Dense(self.l2_out))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Dropout(self.l2_drop))\n",
    "        model.add(Dense(self.__last_output))\n",
    "        model.add(Activation('softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',\n",
    "                      optimizer=Adam(),\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "        return model\n",
    "    \n",
    "    # fit mnist model\n",
    "    def mnist_fit(self):\n",
    "        early_stopping = EarlyStopping(patience=0, verbose=1)\n",
    "        \n",
    "        self.__model.fit(self.__x_train, self.__y_train,\n",
    "                       batch_size=self.batch_size,\n",
    "                       epochs=self.epochs,\n",
    "                       verbose=0,\n",
    "                       validation_split=self.validation_split,\n",
    "                       callbacks=[early_stopping])\n",
    "    \n",
    "    # evaluate mnist model\n",
    "    def mnist_evaluate(self):\n",
    "        self.mnist_fit()\n",
    "        \n",
    "        evaluation = self.__model.evaluate(self.__x_test, self.__y_test, batch_size=self.batch_size, verbose=0)\n",
    "        return evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Runner function for the MNIST model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to run mnist class\n",
    "def run_mnist(first_input=784, last_output=10,\n",
    "              l1_out=512, l2_out=512, \n",
    "              l1_drop=0.2, l2_drop=0.2, \n",
    "              batch_size=100, epochs=10, validation_split=0.1):\n",
    "    \n",
    "    _mnist = MNIST(first_input=first_input, last_output=last_output,\n",
    "                   l1_out=l1_out, l2_out=l2_out, \n",
    "                   l1_drop=l1_drop, l2_drop=l2_drop, \n",
    "                   batch_size=batch_size, epochs=epochs, \n",
    "                   validation_split=validation_split)\n",
    "    mnist_evaluation = _mnist.mnist_evaluate()\n",
    "    return mnist_evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Bayesian Optimization\n",
    "#### bounds for hyper parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# bounds for hyper-parameters in mnist model\n",
    "# the bounds dict should be in order of continuous type and then discrete type\n",
    "bounds = [{'name': 'validation_split', 'type': 'continuous',  'domain': (0.0, 0.3)},\n",
    "          {'name': 'l1_drop',          'type': 'continuous',  'domain': (0.0, 0.3)},\n",
    "          {'name': 'l2_drop',          'type': 'continuous',  'domain': (0.0, 0.3)},\n",
    "          {'name': 'l1_out',           'type': 'discrete',    'domain': (64, 128, 256, 512, 1024)},\n",
    "          {'name': 'l2_out',           'type': 'discrete',    'domain': (64, 128, 256, 512, 1024)},\n",
    "          {'name': 'batch_size',       'type': 'discrete',    'domain': (10, 100, 500)},\n",
    "          {'name': 'epochs',           'type': 'discrete',    'domain': (5, 10, 20)}]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# function to optimize mnist model\n",
    "def f(x):\n",
    "    print(x)\n",
    "    evaluation = run_mnist(\n",
    "        l1_drop = float(x[:,1]), \n",
    "        l2_drop = float(x[:,2]), \n",
    "        l1_out = int(x[:,3]),\n",
    "        l2_out = int(x[:,4]), \n",
    "        batch_size = int(x[:,5]), \n",
    "        epochs = int(x[:,6]), \n",
    "        validation_split = float(x[:,0]))\n",
    "    print(\"LOSS:\\t{0} \\t ACCURACY:\\t{1}\".format(evaluation[0], evaluation[1]))\n",
    "    print(evaluation)\n",
    "    return evaluation[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optimizer instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  8.28040134e-02   2.94981174e-01   7.51470156e-02   2.56000000e+02\n",
      "    5.12000000e+02   5.00000000e+02   1.00000000e+01]]\n",
      "Epoch 00009: early stopping\n",
      "LOSS:\t0.0633744144346565 \t ACCURACY:\t0.9799000054597855\n",
      "[0.063374414434656495, 0.97990000545978551]\n",
      "[[  1.57578393e-02   2.54071426e-01   2.02050321e-02   6.40000000e+01\n",
      "    5.12000000e+02   1.00000000e+02   1.00000000e+01]]\n",
      "Epoch 00006: early stopping\n",
      "LOSS:\t0.08654434870812111 \t ACCURACY:\t0.972300005555153\n",
      "[0.08654434870812111, 0.97230000555515295]\n",
      "[[  1.28377149e-01   1.96911207e-03   2.11710770e-01   1.02400000e+03\n",
      "    5.12000000e+02   5.00000000e+02   5.00000000e+00]]\n",
      "Epoch 00005: early stopping\n",
      "LOSS:\t0.057901502959430216 \t ACCURACY:\t0.9818000048398972\n",
      "[0.057901502959430216, 0.98180000483989716]\n",
      "[[  1.05757625e-01   3.31146774e-02   2.07325052e-01   5.12000000e+02\n",
      "    6.40000000e+01   5.00000000e+02   5.00000000e+00]]\n",
      "LOSS:\t0.06972178800497204 \t ACCURACY:\t0.9781000047922135\n",
      "[0.069721788004972043, 0.97810000479221348]\n",
      "[[  1.71757378e-01   1.95669357e-01   1.71948271e-01   1.28000000e+02\n",
      "    5.12000000e+02   1.00000000e+01   2.00000000e+01]]\n",
      "Epoch 00002: early stopping\n",
      "LOSS:\t0.11427416277118163 \t ACCURACY:\t0.9655999943614006\n",
      "[0.11427416277118163, 0.96559999436140065]\n"
     ]
    }
   ],
   "source": [
    "# optimizer\n",
    "opt_mnist = GPyOpt.methods.BayesianOptimization(f=f, domain=bounds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Running optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.00000000e+00   3.00000000e-01   0.00000000e+00   1.02400000e+03\n",
      "    5.12000000e+02   5.00000000e+02   5.00000000e+00]]\n",
      "LOSS:\t0.06117972570937127 \t ACCURACY:\t0.9800999999046326\n",
      "[0.061179725709371267, 0.98009999990463259]\n",
      "[[  3.00000000e-01   0.00000000e+00   3.00000000e-01   1.02400000e+03\n",
      "    5.12000000e+02   5.00000000e+02   1.00000000e+01]]\n",
      "Epoch 00007: early stopping\n",
      "LOSS:\t0.06730220378376543 \t ACCURACY:\t0.9805999994277954\n",
      "[0.067302203783765432, 0.98059999942779541]\n",
      "[[  1.29882915e-01   2.45356433e-02   2.51330623e-01   5.12000000e+02\n",
      "    1.28000000e+02   5.00000000e+02   5.00000000e+00]]\n",
      "LOSS:\t0.07530965844634921 \t ACCURACY:\t0.9765999972820282\n",
      "[0.075309658446349209, 0.97659999728202818]\n",
      "[[  3.00000000e-01  -3.46944695e-18   3.00000000e-01   2.56000000e+02\n",
      "    5.12000000e+02   5.00000000e+02   2.00000000e+01]]\n",
      "Epoch 00007: early stopping\n",
      "LOSS:\t0.08680501421913504 \t ACCURACY:\t0.9733999997377396\n",
      "[0.086805014219135043, 0.97339999973773961]\n",
      "[[   0.    0.    0.  256.  512.  500.    5.]]\n",
      "LOSS:\t0.08390971436165273 \t ACCURACY:\t0.9752000004053116\n",
      "[0.083909714361652732, 0.97520000040531163]\n",
      "[[  9.68849746e-02   6.92757377e-02   1.88680672e-01   5.12000000e+02\n",
      "    6.40000000e+01   5.00000000e+02   1.00000000e+01]]\n",
      "Epoch 00008: early stopping\n",
      "LOSS:\t0.05885540647432208 \t ACCURACY:\t0.9806000024080277\n",
      "[0.058855406474322081, 0.98060000240802769]\n",
      "[[  3.00000000e-01   3.00000000e-01   3.00000000e-01   5.12000000e+02\n",
      "    6.40000000e+01   5.00000000e+02   1.00000000e+01]]\n",
      "LOSS:\t0.07554996255785226 \t ACCURACY:\t0.9768000096082687\n",
      "[0.075549962557852263, 0.97680000960826874]\n",
      "[[  3.00000000e-01   0.00000000e+00   3.00000000e-01   1.02400000e+03\n",
      "    5.12000000e+02   5.00000000e+02   5.00000000e+00]]\n",
      "LOSS:\t0.0699780025985092 \t ACCURACY:\t0.9778000026941299\n",
      "[0.069978002598509198, 0.97780000269412992]\n",
      "[[  0.00000000e+00   3.00000000e-01   0.00000000e+00   1.02400000e+03\n",
      "    5.12000000e+02   5.00000000e+02   5.00000000e+00]]\n",
      "LOSS:\t0.06794648915529251 \t ACCURACY:\t0.9771999984979629\n",
      "[0.067946489155292514, 0.97719999849796291]\n",
      "[[    0.     0.     0.  1024.   512.   500.     5.]]\n",
      "LOSS:\t0.06090784752741456 \t ACCURACY:\t0.9816000074148178\n",
      "[0.060907847527414558, 0.98160000741481779]\n"
     ]
    }
   ],
   "source": [
    "# optimize mnist model\n",
    "opt_mnist.run_optimization(max_iter=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Optimized Parameters:\n",
      "\tvalidation_split:\t0.1283771492891259\n",
      "\tl1_drop:\t0.0019691120669648177\n",
      "\tl2_drop:\t0.21171077023226453\n",
      "\tl1_out:\t1024.0\n",
      "\tl2_out:\t512.0\n",
      "\tbatch_size:\t500.0\n",
      "\tepochs:\t5.0\n",
      "\n",
      "optimized loss: [ 0.0579015]\n"
     ]
    }
   ],
   "source": [
    "# print optimized mnist model\n",
    "print(\"\"\"\n",
    "Optimized Parameters:\n",
    "\\t{0}:\\t{1}\n",
    "\\t{2}:\\t{3}\n",
    "\\t{4}:\\t{5}\n",
    "\\t{6}:\\t{7}\n",
    "\\t{8}:\\t{9}\n",
    "\\t{10}:\\t{11}\n",
    "\\t{12}:\\t{13}\n",
    "\"\"\".format(bounds[0][\"name\"],opt_mnist.x_opt[0],\n",
    "           bounds[1][\"name\"],opt_mnist.x_opt[1],\n",
    "           bounds[2][\"name\"],opt_mnist.x_opt[2],\n",
    "           bounds[3][\"name\"],opt_mnist.x_opt[3],\n",
    "           bounds[4][\"name\"],opt_mnist.x_opt[4],\n",
    "           bounds[5][\"name\"],opt_mnist.x_opt[5],\n",
    "           bounds[6][\"name\"],opt_mnist.x_opt[6]))\n",
    "print(\"optimized loss: {0}\".format(opt_mnist.fx_opt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1.28377149e-01,   1.96911207e-03,   2.11710770e-01,\n",
       "         1.02400000e+03,   5.12000000e+02,   5.00000000e+02,\n",
       "         5.00000000e+00])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opt_mnist.x_opt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}