exploring_neural_nets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/aml-spring-19/homework-5-cv5/blob/master/HW5.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "bKxgsG4-8GCp",
    "outputId": "70f11595-370b-4fc3-d1b8-a5ddd022cc13"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "\n",
    "# plots\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "# processing & model selection\n",
    "from keras.preprocessing import image\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.utils import resample\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split, StratifiedShuffleSplit\n",
    "\n",
    "# models\n",
    "from keras import regularizers\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Activation, BatchNormalization, Dropout, Input, add, Conv2D, MaxPooling2D, Flatten\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "# other\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "np.random.seed(42)\n",
    "\n",
    "# task3 data loading\n",
    "import fnmatch\n",
    "from glob2 import glob\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HCQfFRpL8kF6"
   },
   "source": [
    "# Exploring neural networks\n",
    "\n",
    "## 1. Classification with the iris dataset\n",
    "\n",
    "The first task is to run a multilayer perceptron with two hidden layers and relu activations, using the Keras Sequential Interface. We also tune for regularization strength and number of hidden units. \n",
    "\n",
    "Importing the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "3-AIU5gK8YYR",
    "outputId": "95a8d89a-ab69-4d0f-f92d-6d9309669f42"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>setosa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal_length  sepal_width  petal_length  petal_width species\n",
       "0           5.1          3.5           1.4          0.2  setosa\n",
       "1           4.9          3.0           1.4          0.2  setosa\n",
       "2           4.7          3.2           1.3          0.2  setosa\n",
       "3           4.6          3.1           1.5          0.2  setosa\n",
       "4           5.0          3.6           1.4          0.2  setosa"
      ]
     },
     "execution_count": 2,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = sns.load_dataset('iris')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EPNx0mJ32SLm"
   },
   "source": [
    "Map the target variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "H0ymJ4zw3sVY",
    "outputId": "df836ceb-3536-45c2-d917-6b44c121a211"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['setosa' 'versicolor' 'virginica']\n"
     ]
    }
   ],
   "source": [
    "print(df.species.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p3eDZt2C4lwR"
   },
   "outputs": [],
   "source": [
    "mapping = {'setosa': 0, 'versicolor': 1, 'virginica': 2}\n",
    "df.species.replace(mapping, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UKMu653l3LMH"
   },
   "source": [
    "Split our dataset into training and test sets. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TT7TiTFgFtSQ"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(df.drop('species', axis=1), df.species, test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4Bp1ip1K4tTc"
   },
   "source": [
    "Encode the target:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "c3N6qbYR3u5W"
   },
   "outputs": [],
   "source": [
    "num_species = len(df['species'].unique())\n",
    "\n",
    "y_train = to_categorical(y_train, num_species)\n",
    "y_test = to_categorical(y_test, num_species)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "j8eFwU73E58G"
   },
   "source": [
    "Make our model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ccuII1y0IN5u",
    "outputId": "bec59cdb-ef00-4a61-a35e-7f519ca80be4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(120, 4)"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nh4g4hb5Ajno"
   },
   "outputs": [],
   "source": [
    "def make_model(hidden_size, reg_strength, optimizer='adam'):\n",
    "    \"\"\"\n",
    "    Create the model to pass to the Keras Classifier\n",
    "    \"\"\"\n",
    "    \n",
    "    model = Sequential([\n",
    "        Dense(hidden_size, input_shape=(4,), activation='relu',   # first layer\n",
    "              kernel_regularizer=regularizers.l2(reg_strength)),\n",
    "        Dense(hidden_size, activation='relu',                     # second layer\n",
    "              kernel_regularizer=regularizers.l2(reg_strength)),\n",
    "        Dense(3, activation='softmax')                            # output layer\n",
    "    ])\n",
    "    \n",
    "    # compile the above model\n",
    "    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qpIERVKgYF2J"
   },
   "source": [
    "Before proceeding any further, we should initialize the cross-validation method to be used in the grid search later on. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Pahcj0WAX6ZW"
   },
   "outputs": [],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=3, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 272
    },
    "colab_type": "code",
    "id": "zfKn0mTYJB4A",
    "outputId": "a3434b01-22ba-4bef-9ca1-1b1d28ae2e70"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedShuffleSplit(n_splits=3, random_state=42, test_size=0.2,\n",
       "            train_size=None),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=<keras.wrappers.scikit_learn.KerasClassifier object at 0x7fc5af1445c0>,\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'hidden_size': [32, 64, 128], 'reg_strength': array([0.001  , 0.00316, 0.01   , 0.03162, 0.1    ])},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = KerasClassifier(make_model, epochs=10, verbose=0)\n",
    "\n",
    "param_grid = {'hidden_size': [32, 64, 128],\n",
    "              'reg_strength': np.logspace(-3, -1, 5)}\n",
    "\n",
    "grid = GridSearchCV(clf, param_grid=param_grid, cv=sss, n_jobs=-1)\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "atFAkxnTJs2U",
    "outputId": "1454b2f8-879d-4cef-88b2-20d8bfa7c940"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.92\n",
      "Best params: {'hidden_size': 128, 'reg_strength': 0.001}\n"
     ]
    }
   ],
   "source": [
    "print('Best score: {}'.format(np.round(grid.best_score_, 2)))\n",
    "print('Best params: {}'.format(grid.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ZZ81ZqHiKn7m",
    "outputId": "6576096f-4583-4f41-d6a3-319fd4bfa5c4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test score: 0.97\n"
     ]
    }
   ],
   "source": [
    "score = grid.score(X_test, y_test)\n",
    "print('Test score: {}'.format(np.round(score, 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XbLvGZO6S0II"
   },
   "source": [
    "## 2. Classification with the MNIST fashion dataset\n",
    "\n",
    "We will train a multilayer perceptron (fully connected) on the Fashion MNIST dataset. \n",
    "We'll first use 10000 samples from the training set for model selection and to compute learning curves (accuracy vs epochs).\n",
    "Then, we'll compare the following models and plot their learning curves:\n",
    "    - Vanilla Model\n",
    "    - Model using dropout\n",
    "    - Model using batch normalization and residual connections (but no dropout)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 171
    },
    "colab_type": "code",
    "id": "JouH89vCTMSN",
    "outputId": "169016e5-a190-4aab-d3bd-761fa2b43dc0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz\n",
      "32768/29515 [=================================] - 0s 9us/step\n",
      "Downloading data from http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz\n",
      "26427392/26421880 [==============================] - 4s 0us/step\n",
      "Downloading data from http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz\n",
      "8192/5148 [===============================================] - 0s 0us/step\n",
      "Downloading data from http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz\n",
      "4423680/4422102 [==============================] - 2s 1us/step\n"
     ]
    }
   ],
   "source": [
    "# loading the data\n",
    "(X_train, y_train), (X_test, y_test) = keras.datasets.fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "g9Tv8VfaTp79",
    "outputId": "143dabce-04fb-431c-a620-faed5b2ab3f4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training size: 60000, Test size: 10000\n",
      "Each image has a shape of: 28x28\n"
     ]
    }
   ],
   "source": [
    "print('Training size: {}, Test size: {}'.format(X_train.shape[0],X_test.shape[0]))\n",
    "print('Each image has a shape of: {}x{}'.format(X_train.shape[1], X_train.shape[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K_LR9acOjbaI"
   },
   "source": [
    "Before proceeding with the modeling, we have to process our data accordingly. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bRiA9YkijZe4"
   },
   "outputs": [],
   "source": [
    "# reshape X and cast to float\n",
    "X_train = X_train.reshape(60000, 784); X_test = X_test.reshape(10000, 784)\n",
    "X_train = X_train.astype('float32'); X_test = X_test.astype('float32')\n",
    "\n",
    "# scale X\n",
    "X_train /= 255; X_test /= 255\n",
    "\n",
    "# encode the target accordingly\n",
    "n_classes = len(np.unique(y_train))\n",
    "y_train = to_categorical(y_train, n_classes); y_test = to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RAjaxmvbsRRB"
   },
   "source": [
    "We also have to get a stratified sample of size 10000 from the training set for model selection."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bp0JWcmnzaX1"
   },
   "outputs": [],
   "source": [
    "X, X_valid, y, y_valid = train_test_split(X_train, y_train, stratify=y_train, random_state=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n1p6Vwa45c7A"
   },
   "outputs": [],
   "source": [
    "# sss = StratifiedShuffleSplit(n_splits=1, test_size=1/6, random_state=6)\n",
    "# for a, b in sss.split(X_train, y_train):\n",
    "#     X_sample, y_sample = X_train[b], y_train[b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5mtwzvfDuiii"
   },
   "source": [
    "### Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "t-FgjPaXum8q"
   },
   "outputs": [],
   "source": [
    "sss = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=410)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "U-ZsJhF4irWO"
   },
   "source": [
    "**Vanilla Model**\n",
    "\n",
    "We start off by tuning our vanilla model. Vanilla models are typically used to describe simple networks with 1 layer, so we'll only look at optimizing the number of hidden units in this case. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4Y0oSA_7jRci"
   },
   "outputs": [],
   "source": [
    "def make_model(hidden_size):\n",
    "    \"\"\"\n",
    "    Create the model to pass to the Keras Classifier\n",
    "    \"\"\"\n",
    "    \n",
    "    model = Sequential([\n",
    "        Dense(hidden_size, input_shape=(784,), activation='relu'), # first layer\n",
    "        Dense(10, activation='softmax')                            # output layer\n",
    "    ])\n",
    "    \n",
    "    # compile the above model\n",
    "    model.compile(optimizer='adam', loss='categorical_crossentropy', \n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uxQ9uH-pu4fH"
   },
   "outputs": [],
   "source": [
    "clf = KerasClassifier(make_model, epochs=10, verbose=0)\n",
    "\n",
    "param_grid = {'hidden_size': [256, 512]}\n",
    "\n",
    "grid = GridSearchCV(clf, param_grid=param_grid, cv=sss, n_jobs=-1).fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "MWXC2WiOwwS7",
    "outputId": "d4a4563a-7823-44dc-ec44-a787b8a71f2a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VANILLA MODEL TUNING:\n",
      "Best Score: 0.88\n",
      "Best Hidden Size: 512\n"
     ]
    }
   ],
   "source": [
    "print('VANILLA MODEL TUNING:')\n",
    "print('Best Score: {}'.format(np.round(grid.best_score_, 2)))\n",
    "print('Best Hidden Size: {}'.format(grid.best_params_['hidden_size']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QPEsXeGJy1Jf"
   },
   "outputs": [],
   "source": [
    "vanilla = grid.best_estimator_\n",
    "history_vanilla = vanilla.fit(X, y, epochs=10, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8yYMyR33yajb"
   },
   "source": [
    "**Bigger Model w/ Dropout**\n",
    "\n",
    "We can now proceed with tuning our bigger model with dropout. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E0BSW0Z9xnPB"
   },
   "outputs": [],
   "source": [
    "def make_model(hidden_size, hidden_layers, dp_p):\n",
    "    \"\"\"\n",
    "    Create the model to pass to the Keras Classifier\n",
    "    \"\"\"\n",
    "    model = Sequential()\n",
    "    \n",
    "    # input layer\n",
    "    model.add(Dense(hidden_size, input_shape=(784,), activation='relu'))\n",
    "    model.add(Dropout(dp_p))\n",
    "    \n",
    "    # add as many layers as needed\n",
    "    for i in range(hidden_layers):\n",
    "        model.add(Dense(hidden_size, activation='relu')) \n",
    "        model.add(Dropout(dp_p))  \n",
    "    \n",
    "    # output layer\n",
    "    model.add(Dense(10, activation='softmax'))\n",
    "    \n",
    "    \n",
    "    # compile the above model\n",
    "    model.compile(optimizer='adam', loss='categorical_crossentropy', \n",
    "                  metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "colab_type": "code",
    "id": "JW1GE7wCysKT",
    "outputId": "f03828f5-f798-4029-a367-a52d9264e28d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "clf = KerasClassifier(make_model, epochs=10, verbose=0)\n",
    "\n",
    "param_grid = {'hidden_size': [256, 512],\n",
    "              'hidden_layers': [1,2,3],\n",
    "              'dp_p': np.linspace(0.25, 0.75, 3)}\n",
    "\n",
    "grid = GridSearchCV(clf, param_grid=param_grid, cv=sss, n_jobs=-1).fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "colab_type": "code",
    "id": "BhgFu-qEyjvu",
    "outputId": "c59e1d34-0b52-4b79-b590-0a4304e67526"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DROPOUT MODEL TUNING:\n",
      "Best Score: 0.88\n",
      "Best Hidden Size: 512\n",
      "Best Number of Hidden Layers: 1\n",
      "Best Dropout Prob: 0.25\n"
     ]
    }
   ],
   "source": [
    "print('DROPOUT MODEL TUNING:')\n",
    "print('Best Score: {}'.format(np.round(grid.best_score_, 2)))\n",
    "print('Best Hidden Size: {}'.format(grid.best_params_['hidden_size']))\n",
    "print('Best Number of Hidden Layers: {}'.format(grid.best_params_['hidden_layers']))\n",
    "print('Best Dropout Prob: {}'.format(grid.best_params_['dp_p']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZY2GFzquylDV"
   },
   "outputs": [],
   "source": [
    "dropout = grid.best_estimator_\n",
    "history_dropout = dropout.fit(X, y, epochs=10, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CgTyq9NnIIK_"
   },
   "source": [
    "**Residual Batch Model**\n",
    "\n",
    "Moving on to our final model for this task - we introduce batch normalization and residual connections. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tgM_YfAMEVK4"
   },
   "outputs": [],
   "source": [
    "inputs = Input(shape=(784,))                 # input\n",
    "\n",
    "x1 = Dense(256, activation='relu')(inputs)   # first layer: 256 units\n",
    "\n",
    "x2 = Dense(256, activation='relu')(x1)       # second layer: 256 units\n",
    "x2 = BatchNormalization()(x2)                # batch normalization\n",
    "x2 = add([x1, x2])                           # residual connection\n",
    "\n",
    "x3 = Dense(256, activation='relu')(x2)       # third layer: 256 units\n",
    "x3 = BatchNormalization()(x3)                # batch normalization\n",
    "x3 = add([x2, x3])                           # residual connection \n",
    "\n",
    "x4 = Dense(128, activation='relu')(x3)       # fourth layer: 128 units\n",
    "x4 = BatchNormalization()(x4)                # batch normalization\n",
    "\n",
    "preds = Dense(10, activation='softmax')(x4)\n",
    "\n",
    "resbatch = Model(inputs=inputs, outputs=preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "N3M9-oHJqiDi"
   },
   "outputs": [],
   "source": [
    "resbatch.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "history_resbatch = resbatch.fit(X, y, epochs=10, validation_data=(X_valid, y_valid), verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 168
    },
    "colab_type": "code",
    "id": "09HYcAqMWoIi",
    "outputId": "85bb936b-8c0e-45a7-919f-aac71489ad55"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VANILLA RESULTS:\n",
      "Training Accuracy: 0.91\n",
      "Validation Accuracy: 0.9\n",
      "DROPOUT RESULTS:\n",
      "Training Accuracy: 0.89\n",
      "Validation Accuracy: 0.89\n",
      "RESBATCH RESULTS:\n",
      "Training Accuracy: 0.9\n",
      "Validation Accuracy: 0.89\n"
     ]
    }
   ],
   "source": [
    "for k, h in {'vanilla': history_vanilla,\n",
    "             'dropout': history_dropout,\n",
    "             'resbatch': history_resbatch}.items():\n",
    "    print(k.upper() + ' RESULTS:')\n",
    "    print('Training Accuracy: {}'.format(np.round(h.history['acc'][-1], 2)))\n",
    "    print('Validation Accuracy: {}'.format(np.round(h.history['val_acc'][-1], 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6lD-qDykIbK6"
   },
   "source": [
    "### Learning Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 301
    },
    "colab_type": "code",
    "id": "nnD9pVTZV2EG",
    "outputId": "6e79fc2f-84e6-4b34-cb84-216464a4792e"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABBYAAAEICAYAAADrxnfmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3XmcXFWZ//HPU0vvS9JZupNOQkBA\nyCIgwQVHUFFBVsHBgKDiCC6jMiLjyIijDIMj47iM/nR0BBVBFCKCw6Y4I1FEHCRhlbAayEqSTqf3\nTnfXcn5/nFPdtyu9VJruVHf6+3697qvuVrdOVXU9fe5zzj3XnHOIiIiIiIiIiIxFrNgFEBERERER\nEZGpS4kFERERERERERkzJRZEREREREREZMyUWBARERERERGRMVNiQURERERERETGTIkFERERERER\nERkzJRak6Mzss2Z2bZhfbGbOzBJh+bdmdmFxS7invSlXeD8HT3SZRESms0JjrZm9ycw274syiYiI\nTBdKLMiozOxXZnblEOvPMLNtuSTAWDnn/tU5N+mSByIixWRmL5rZbjPrMLNWM3vAzD5iZpPyf3co\n71uLXQ4RGbC//C7zE4Jm9qSZvWkCXqfTzA4a7+MWy0jvx8wuMLP7x+l19ou/s9GY2Xlm9util2O8\nmNl1ZnZVgfuO+h1PysqJTDo/As43M8tb/17gRudcughlEhGZDk5zzlUDBwBXA58Bvj/czmYW31cF\nE5GxiSQNO82sxczuMrOFBT53UM/O6co5t9Q599uXc4yhep8656qcc+tfVuGGf735w/WWCt9pV/ib\n2GJmXxuPeD6R72d/N9RvzTl3o3Pu7cUs12SmxIIU4hfALOCNuRVmNhM4Fbg+LJ9iZo+YWbuZbTKz\nKyL75n6Y7zezjWa208wuj2y/wsx+PFohzOwVZnavmTWHY9xoZjNG2N+Z2d+a2XOhxe9fwjEeCOVc\nZWYlkf0vMrPnzWyXmd1uZvMj295mZk+bWZuZfQuwvNf6GzN7KlQQ7jGzA0Z7PyIihXLOtTnnbgdW\nAu83s2XQ39rwHTO728y6gDebWa2ZXW9mTWa2wcw+l+vlEFqo/mBm3wrx7GkzOyH3OqHie3uIg8+b\n2UWRbYNaNqItiGZ2A7AIuCNUjP8h/z3k9jezfzCzHWb2kpm908xONrNnw2t+NrJ/qZn9h5ltDdN/\nmFlpZPunwzG2mtnf5L1WqZl9JfzP2W5m3zWz8pf9RYiMn9Occ1XAPGA78P+KXJ5xNd0TH8M4GfjV\nCNuPCH8Tx+Nj/d+MsK/IpKPEgozKObcbWAW8L7L63cDTzrnHwnJX2D4DOAX4qJm9M+9QfwW8EjgB\n+LyZHb6XRTHgS8B84HBgIXDFKM85ETgaeB3wD8D3gPPDc5cB5wKY2VvCsd+N/ye/AbgpbJsN3Ap8\nDpgN/AV4Q3+hzM4APgucBcwBfg/8dC/fm4jIqJxzfwI2E0n0Au8BvghUA/fjT1BqgYPwFdT3AR+I\n7P9afBybDXwBuNXM6sK2m8Lx5wN/DfxriI+jleu9wEbCyZJz7svD7NoAlAGNwOeBa/Ax+ejwnv7J\nzA4M+16Oj91HAkcAr8HHYczsJODvgbcBhwD53TOvBg4Nzz048noik4pzrge4BViSWzdSYw1wX3hs\nDUm814fnXBQaODrMbJ2ZvTrynCPN7PGQTLzZzMqGKktIPN4fknItZvaCmb0jsn2kxOMVZnaLmf3Y\nzNqBC8K6n4V1HWb2hJkdamb/GJKLm8zs7ZFjfCDyHtab2YeH+9ws0i3b/KVinWHqMt+wtNjMZprZ\nnSHJ2hLmF4TnfBEfc74VnvetsL5/rJYCkrTDflbDOBm4e5R9cM49D/wBH79y77fWzL4fkqlbzOwq\nCz0azOxgM/td+H53mtnNkedF38+s8P21m9mfgFdE9tujdd4iPTpsLxr3zOzw8HmcO8z2pWb2P+Hv\naLuFhLKNkEy2gcT0pTaQmP5A5JjXmdm3zff+6TCzB80s+v4Oi7zmM2b27si2cjP7aviO28L3Ws4Q\nvzXLu3zEzI41s4fC8x4ys2PzPr9/MZ/M7zCzX5s/p8DMysz/LprD3+9DZlY/zOf1ovlE+uPh7/v7\nZlZvZr8Mx/1f8w2+uf1PN3+pUGsow+GRbUeZ2cPheTfj/x9HX+tUM3vUBi6/fNVQZRqWc06TplEn\nfFKgFSgLy38ALhlh//8Avh7mFwMOWBDZ/ifgnDB/BfDjvH0TYfm3wIXDvMY7gUdGKIMD3hBZXgt8\nJrL8VeA/wvz3gS9HtlUBqVCe9wH/F9lm+Ir3hWH5l8AHI9tjQDdwQKQcBxf7O9SkSdPUmoAXgbcO\nsf7/gMvD/HXA9ZFtcaAPWBJZ92Hgt2H+AmArYJHtf8Jf2rYQyADVkW1fAq6LvNZVkW1vAjaPVt68\n/XcD8bBcHeLjayP7rAXeGeb/Apwc2XYi8GKY/wFwdWTboblYG2J0F/CKyPbXAy8MVW5Nmvb1FP2t\nABX4S06jv+M3ActDfeJV+B4Nud/FYiL1pLDubGALcEz4+z84Ugd5MfzG5wN1wFPAR4Yp1wX4us9F\nIZZ8NBov8Cda/4k/GTkSaALeErZdEZ77zlDu8rCuJ/x2E/heri/gk4bJ8DovRF7/FPzJruGTot3A\nqyOfyajxBvjXUM4kvrftu8JnXA38DPhFZN/fklfHJFJnC+X97/DcxcCzhPreaJ/VEOVKAjuJxNcR\nXvcw4CUi9WzgNuC/gEpgbvhOPxy2/TR8prHw3fzVMMe9Cd9QWIlvXNsC3D/C31X/54P/m3obUIpv\nRLuPUIeOfh/Aq/FJ5lOHeZ/V4b1dGspaTfgfAFyJ//82N7zGA8C/RL7/dNgniU/SdAMzw/brgGZ8\nAjoB3AjcFLZVApvwCfYEcFT4LpaE7d8O77UxfJfHhvc51GdyQeQzqwNa8P8/E/jGyhZgVuTz+wv+\n/1N5WL46bPswcAf+bzOOT67XjBAv/g+oD2XcATwc3kcZcC/whbDvofj/f28Ln9M/AM8DJWHaAFwS\ntv01/m/4qvDco8KxXxvK9P7w2qUj/eaik3osSEGcc/fjf4TvDBnA1wA/yW03s9ea2eqQ1W0DPoJv\nDYvaFpnvxp+8Fyxk524Kmdp24MdDvEa+7ZH53UMs58owH/9jA8A514kPUI1h26bINhddxl/7/I2Q\n3WsFduH/KTbuxdsTESlUIz7O5ETj0Wx8hWFDZN0GBsejLSGORbfPD9Mu51zHCM99uZqdc5kwvzs8\nFhSXI+XMbduUty1nDr6ytjYSl38V1otMFr8If5tt+JOAf89tcM791jn3hHMu65x7HH/iePwIx7oQ\n3zjykPOed85FfxPfdM5tdc7twp/MHDn0YQDY4Jy7JvxOf4TvxVlvfgyIN+AbaHqcc48C1zK4N+sf\nnXO/COXO/b5/75y7x/nxuH6G/x1e7ZxL4U90F+davp1zdznn/hLew++AXzO4d9aIzGwlvgfXu5xz\nKedcs3Pu58657hDXvsjIn2P0WHHgHOAfnXMdzrkX8Q1S7x3tsxrmkMcBj+XF13wPm7+k7Sn8Seh/\nhrLU40+kP+mc63LO7QC+HsoH/uTwAGB++G72GJAxvJ93AZ8Px/hzKHNBwt/U/zjnep1zTcDX2POz\nfCNwO/A+59ydwxzqVGCbc+6roawdzrkHw7bzgCudczvCa/wzgz/vVNiecs7dDXTie0Ln3Oac+1P4\nW7uRgb/zU/FJ6R8659LOuUeAnwNnm++B8jfA3znntjjnMs65B5xzvQV8LKcAzznnbgjH/SnwNHBa\nZJ8fOueedQO9v3NlSuETXweH11zrnGsf4bX+n3Nuu3NuC75n9IPOuUec7/F0Gz4pAP4SmrvCd5UC\nvoJPahyL7wGYxCeEUs65W4CHIq/xIeC/nHMPhjL9COgNzyuIEguyN67H/wM5H7jHORetDP4EH0wW\nOudqge+SNw7BOPhXfOZwuXOuJpRjvF5jKz4oA2Bmlfgf/BZ8ZnVhZJtFl/GV2w8752ZEpnLn3APj\nVDYREQDM7Bj8iX604hhNEuxkoJKZswgfy3IaQxyLbt8apjozqx7muV34E/achrziOcbXoLgcKSfk\nxeWwLWcnPkGxNBKTa52/dllksninc24GvsXx48DvzKwBCm6siVqIbxkdzt407PTv65zrDrNVFJZ4\njCb7cvIThzuHSC5WAZjZO8zs/0J39Vb8yfRoDUiE5x4FfAs4M5yUYmYVZvZfoYt7O76VfYYVNihi\nIUna4T6roRRyGcSrw/NX4luNK8P6A0JZXookS/8L37IPvlXagD+FLvBDjc0wB9+qPlxCdkQFNu59\nBHjAjTyo5kh/qyMlk8EnpqMDxuf/LQ/3d34A8NrcZxc+v/Pw/8Nm43+DI/1+hpNf3lyZh/wbySvT\nDcA9wE3mL/v4spklR3itsTaUZvHfea6hdKiGhZwDgEvzPqeFDP4ORqTEguyN6/HdnC5izyxnNf4f\nTo+ZvQafMR5v1fjsZJuZNQKfHsdj/xT4gJkdaf56rn/FZwNfBO4ClprZWeavPbuYwRXq7wL/aGZL\nof86uLPHsWwiMs2ZWY2ZnYpv4fuxc+6JofYLFfZVwBfNrNr8QLKfwlcCc+YCF5tZMsSqw4G7nXOb\n8F1PvxSu/3wV8MHIcx8FTjazunAC9Mm8l9+OH9dhvPwU+JyZzQnXpX4+UpZV+Gu4l5hZBX6sCKC/\nInUN8HUzmwtgZo1mduI4lk1kXISWwVvxlyH9VVg9UmPNUAm8TUSul58goyUe4WUkF0Pd6+f4Ftb6\nkHS5mwIakMLv/BfAx0JrdM6l+Bbt14YGqeNyTymgvIUkafdGoeMrOOfcKuCPDIwLswnfcjw7kiyt\ncc4tDc/Z5py7yDk3H9/F/j8tjKsQ0YS/lGC4hGxXeBwueVxI495HgEVm9vUR3uImhv8/MVIy+eXY\nBPwurwGwyjn3Ufz33MPQv5/R/p7zywsF/o2EHgP/7Jxbgu9NcCqDe/+MVX5Daa4xNNdQOlTDQs4m\n4It5n1NF6IlRECUWpGDhJPsBfAb19rzNfwtcaWYd+EC4agKK8M/4bG4b/mT/1vE6sHPuf4F/wv9T\newkfYM4J23bir1+8Gn95xCH4MSZyz70N+Dd81rEd+DMw2gA+IiKFuCPE1U34a2i/xuCBGIfyCXwl\ncT2+Z8NP8GMS5DyIj2M78V2D/9o51xy2nYu/rnQrvnvlF0J8BN/C8hj+OstfAzcz2JfwiYBWM/v7\nvXqXQ7sKWAM8DjyBv6b0KgDn3C/xY/nci79+9N68534mrP+/EJf/l8FdZkUmBfPOAGbiu8DDyI01\nTUCWwSdn1wJ/b2ZHh+MdbON8d6oCEo8vVwn+uvYmIG1+IMRRb+sXGnxuwSdc8+ue1fjW3FbzA9R+\nIW/7sMnQApO0BTE/IG2pc+6pUXcecDVwkZk1OOdewsfcr4Ykc8z8YIrHh+OfbWFQSvw1/g7/N5L/\nfm4Frgg9OZbgr6HPbW/Cn3yeb2bx0OsherJdSONeB3AScJyZXT3M+7oTmGdmnzQ/WGO1mb02bBsp\nmfxy3AkcambvDQn1pJkdY2aHh0T0D4CvmR+cNG5+kMbc32L+by3q7nDc95hZwvylOEvC643IzN5s\nZstD75l2fBIrO8rTCrEKOMXMTgg9IC7FJ6UewCer0gw0LJyFv7Q95xrgI+Z7TJmZVZofSLY6/0WG\n5SbBIDaaNGnSpEmTpomfiAw8pUmTpuJM+OTcbvyJWge+QeK8yPa/xndR7sCfpHyLMMh12H4l/qSn\nFXhdWPcR4JlwzD8DR0Ve662R514RPVZeufaIDwwe/G9BKM8ufNfxj4x03Px1+F6vL0aWE0QG9wY+\nhj/Zb8UnMm9iYGC5NzHE4I0MDLDXFd57blqE78L927D8LL41v38wPvygrs/iT8a/OcT7nYk/sW3C\nJ3c/D8QK+azy1n8c+NYofxN7PBc/OPhXw3wt8B384OFtwCMMDIL+ZXxSoDN8Lx8a5vubE76/dvzg\nj/8SfQ/4RrEXwuf/VeB3DAzeuBQ/uG4nvvfapUN9H2G+Dp+E/pdh3usy4Dfhc98GXBbWlwHfxDfw\nvRTmc4PGD/r+h3jN6xh5cOFX4hslm/CNhPcCR4Zt5fhE9Zbw2d4HlA/1W8v/3vG9jNaG561l8MCZ\nvyUyOCiDB348F/977cL/zX+TyCCRw73PsPxj4IrI8oXA/0aWzwTWhTL9Dn9ZYG7bCvzfTge+ceDm\nvM/tJPy4C63hO/gZYcDR/HIMNeVGeRUREZH9nJldgK/o/NVo+4qIyMtnZnfjEwujXgohMpXpUggR\nEREREZGJ8VtgdbELITLR1GNBRERERERERMZMPRZEREREREREZMwSxXrh2bNnu8WLFxfr5UVEhrR2\n7dqdzrk5xS7HvqJYLCKT0XSKxYrDIjIZ7W0cLlpiYfHixaxZs6ZYLy8iMiQz21DsMuxLisUiMhlN\np1isOCwik9HexmFdCiEiIiIiIiIiY6bEgojIFGBmJ5nZM2b2vJldNsT2A8zsN2b2uJn91swWFKOc\nIiIiIjL9KLEgIjLJmVkc+DbwDmAJcK6ZLcnb7SvA9c65VwFXAl/at6UUERERkelKiQURkcnvNcDz\nzrn1zrk+4CbgjLx9lgD3hvnVQ2wXEREREZkQSiyIiEx+jcCmyPLmsC7qMeCsMH8mUG1ms4Y6mJl9\nyMzWmNmapqamcS+siIiIiEwvSiyIiOwf/h443sweAY4HtgCZoXZ0zn3PObfCObdizpxpcTc3ERER\nEZlASiyIiEx+W4CFkeUFYV0/59xW59xZzrmjgMvDutZ9V0QRkf2fBtIVERlaotgFEBEZiXMOsllc\nOo1LpSGT9vNDLafT0D+fwaVTwyxnwroUZDK4VNgnky722x3OQ8AhZnYgPqFwDvCe6A5mNhvY5ZzL\nAv8I/GCfl1JEJi/nILUb+rqgrzM8hvlU9+D16T6IxSCWAItDLA4Wx5mBi+Ey4JzhsuCyQGTeZZxf\nl3VhnQvrHPRvz+LSIbZnHC6T7Z/IZP2+k1BkIN234S9Je8jMbnfOrYvslhtI90dm9hb8QLrv3fel\nFZHpzDkH6TTZ3j5cXy+urw/X20u2txeXW5db7ksNXg7b95YSCyKyB5fN9gcg19cXgtJAEBpYlxeo\n+lJhey/Zvr4QmPr2XJc7Rl/voH2yfb3Qf5LvT/5JpYr9cRSdcy5tZh8H7gHiwA+cc0+a2ZXAGufc\n7cCbgC+ZmQPuAz5WtAKLyCDZ3l6ynZ1kOzrIdHSS7ewg09FBtqOTbFdn/3ymswPX3Y3LZCCTgnQK\nl0n1z5PNhGWfVPXLkflsBjIZf8KezUDWz/vlrC9MOGd3LjdvkXn/mEsa9CcMIsvTXP9AugBmlhtI\nN5pYWAJ8KsyvBn6xT0soImPmsllcTw/Znh5cKjQ+ZbO+USqbHVjOZEIiND14OZsJSdJ0SJxmIH99\nJsTn6ONw60Pdu/9kP9SVXaQ+3V+X7o3U0UO9mlzc30cKSiyY2UnAN/AV2mudc1fnbT8A3zo2B9gF\nnO+c2zzOZRWRETjncD09voKaq8B2dvqKa2eozHZ0kOnsINvZNTCfW9/Vhdu9m2xf3/iczJthpaV+\nKkkSK8nNl2ClJcRKSolXVWOzSsNyid+WTEIigcUTWCKBJRN7LsfjWCLplxNxvz2RxJIJLB5ZTsSx\nRHh+3hRdR9wfN7d+MnLO3Q3cnbfu85H5W4Bb9nW5RKYM56CnDbqbYXeLf+xuhu5dA/M9rf5k3GX7\nT9RdX4ZMT5psT4psT24+TaY3S7Y349f1Zsn2Zsn0+cdsryPT58j2ObK9oVV/FBZ3xEscsUQWzJ/l\nmwG5c3lzfjasG5g3LBaDWNzHv1jcx8V4ie9pEPfxk3gCYpH5RDJsKwnzSSye9POJEiweC1Mc4jEf\nT8Oyxc2vi0zEwGIWls3vEzMshl+O+QlzYV14f3HDzGGxsN6cv1D3dR8e9z+BcTDUQLqvzdsnN5Du\nN4gMpOuca47uZGYfAj4EsGjRogkrsMj+JFfXzfb04Lq7ye7eTXZ3D9nd3b4Ou8dyZL57N9mesNwd\n9u3ZHZnvwe3eXey3OIglkwN16dISYsmSwcsVFcRnzhyoa5eWYiWR7bnlXN27f3uYz9XLS0qJlUaO\nXVJKrCQJpaV7Vd5Ra9Dq9iUysZxzkMn4oNbZOTgx0NHp54dNDPh1ucQA6dG78scqK4lVVxOrqiRe\nVU185kxKFi0kVllFrKI8EoBCkAkBZ1DQyg9EJZFglCwhVlriT9xt2rduyVSSzUCmD9K9voU40zt4\nPpMKy30DU3Q5nVs/3L65bdF9w5TN4M+q8icbZv1w24faf7RjhH3ipVBSAckwlVSGx+i6CkhWDqyL\nxSfkq3DOQSpFti+FS/X53lCplG+lSYUum50tuM5mXMcuXFcLrqsV192G627HdXfgdnfgdnfherpx\nvd243p7QDd98V/5srvu+4bIxHEmyJMj2Gdk+R6bXJwYooFd+rDRGrDROvCxOrKyE+IwEJWUJYmUJ\n4mUJYuVJYuVJ4mVJYmVJYuUlxMtL/GNFKbHyEp/YNPOXH5RUQklVeKwMn3nlnutLKiGenJDvoPgm\nZWKhEH8PfMvMLsD3HhtyIF3n3PeA7wGsWLFicl77ITJG/QmA7u6BqSs33xVZ5+fdHvt0hxP97khC\nYLc/8Xd793Ox0lJi5eVYRTmxsnJi5X6K180kWTZ/z20V5VhZuT+pj8cGkrWx+MByPOYbueITsD4e\n98nieHzK1aMLaZpTty/Zr7hs1geprtxJe6cPaqnU4Kkvtee6Paa+IddTyPP7Bp5bUJCMxYhVVxOv\nqup/TNbXEzv4YOLVVcSqqolVV/ntufnqamJV1cSrQjKhstIHL5HJoLcDmp/POwEP3b775yPrs/nr\n92bfvtB9vG+IY4R1bsibaIxdLOFP1uNJSJT6FuTclCgJ20qgtNqf3DvHwIXqWb+czYBL5a2PbB9q\nPW70ffK3ZzM+6VFI03qEs1KysQqyVJClnKwrIZstI+uSZDNJspm4n9KGS8fI9DlcnyObCq39vSk/\n9aTI7g5dOFMpSI/zd0Ec4tXEkqEXUzI50EuqLJJQTSZJVlWFWFrtY211FbEQV+PVVSExG+JrdTWx\nigpfCZTpoKCBdAm3/jWzKuBdGkhXJjPnXOTEvmtwMiD/ZL87LzkQ2d/l7VdwAiAWI1ZRMWiyinLi\ntbXE6uv9iX55ObHyCmLlZYPmY+W5beUD8xUVxMoGtik+7zuFJBbU7UsmBZdKke3qItPZNSgpkOns\n9C34neFa1ehyZyeZrsHLexXshpJI+CxmAVOsqmrP9SWDl8ntW1Y+KDEwUIH1iQGrqJhymcspI5v1\n3aA7Xip2SfZfu1vgpcfhpUfhpcf81PwXCmoOHkosGU7Sk5ET9kRkPqyPJX0L76j7lkZO+KMJgNIh\nthe47ySuzGT7+si0tJJpbSXT0kKmtZVsZwfZznay7W3+sbODbFdHqDh2+YRst++Cmu3pI7u7D5fK\nJQAc0B2moVnMEUtmiSVcZMqSSDpiZQ6rcsTikS7xMYclYlhpBVZehZVXYhXVWHkNVlmDVczAKmdg\n1XVYVR1WPQurnoNVzcSSJSHWlgzEXMVPefk0kK4UnUunB07oR3rs6gpJgfA43D570wsgHvc9X/MS\nAcm59X6+MqwL+9ig/SoH75NLIpSVKT4XUzYbBvDthN7OMIhvGMh3L43XxcTq9iVDcs75FqhoEiAM\nVrVXSYGuLlxPz+gvaBZalqp8K31lFfHqGpLz5vuu/5VV/dtjlRXhJL7KZzVDa9VQJ/39ldNEYmyZ\nT+f8j3Z3q7/Gtyc8RpfpgMoMVCSgogoqSqCyGsrr/EmQ7L1owqBzu3/s2DYwdeYet0N20t4RYurp\nbArJg0gSoXXDwPbahTDvCHjVOTD3MN+lPj8hMFyiIF4SRqpXJQR8jM12dfsEQevgRMFwy+nWVlz3\n8AkAAJJJ4hUVWLSSOHMGycZoxbByUAVy0Pq8ymWsvNz3lkrvhr5uHw9T3WG+y9+tINUNJdVQURem\nWT4ppO9aJgkNpCsvh0unw0CtHWTaO8h2tJNpayO79TkyW54hs/1Fsq0tZK3ST9kSsmkLSd2BRIDr\n6yv4NWMhjscrKvsf43NmU1LpG6zilZUDcTqSDIgPSgwMxHQrKVESoNiymUgSoAv6OiLznb43aP98\nJFEw3P59XYy5kSdPIWcr6vY1DQ0eCLBr2ETAoGTBEL0DMl1dhQ0EmEwOnORXVRGvrCQxZw4lBx4Y\n1lX67f2JgcpB+8cqqya+VT+bCQmB1sEJgZGSBdHl7MsYELFsBlTOhorZvrJdOcs/VswO62cNTJWz\n/XW3+7OXmzAor4PqBj/NOQyq6qF6HlTXwz+/c9+/n6nKOf85RxMILz0G7ZF/ETMPhMZXw4oP+GRC\nwxH+71f24LJZsu3t/uS/PxnQNmqiwI0QY2O1tcRn1BKfMYP4nNmUHnKIn585wz/OmDmwHLr15yqP\nEyI3LoDIFKWBdKcvl073j4WVaW/38TqXIGjvINPRTnbQY4ffJzxmR0vomiNWEiMW30kskfG9upKO\nZEUlsVkziL2inljdfGKzFxKrayBWlZfcHfRY6S8hmMQ95/Z7zvnLLPtv7RumVNee6/ZY3+kT70Ml\nB9J7McBkSdXAuDylVT55X9UAs6oGtpVG96kevP8/r9irt1xIYkHdvqYw35rVRaa5mXTzLjK78h93\nDQwGGEkGZDs7/S2rRmGlpQMn+uGkPzl/vu/GH+0dUFVJvKKcmO0mlmkjlm4m3ruDWM9WYn3biSVi\n4T7ZDmLdYL0Qa93jHtrEYpBJQHscOiPrLO737d8v7q9ZjsXzjhGL7JMYWOeywyQEIvN9HSN/GLGE\nTwCUz4CyWj8/44DBy2W1YTlvvrQGcAMjk3ftHGJ+p59veRG2rPHrhmtlT5RHEhCzBxIOFXV5yYjw\nWD5zcnTbnsiEQfU8v76q3ndZl73jHLRuHJxAeOkx6NoRdjCYfSgc8AafQJh3BDQs93/j00h/i1R7\nO5kw5SqfmfY2X9lsbw8tVaHC2dZGpq2NTHv78LeGiscHJQSSByyi7IhXkZgZEgO5KbpcU+PHExAR\nkUFcXx/p5mbSO3aQbmoK087y5X39AAAgAElEQVSB+DxEgiDbNUrXcDNiNTU+SVtTTbyinJLZ5cTm\nQDyTJpbqIE4nsWSWeFmceMOBxBYuJX7gq4kd/DpijUt9zM5mYNcLsP3PsP3J8PhnaH3cv04HkJ4J\ntgyqlkLNUqhfBnMPgmT5hH92U07/WELpMAZTOtyKN+0b/XLL/dvClOtJl2vVLzRBEN22N71h4yUD\nA/TmBksuqYIZC/dMDvTP5ycHIvPJin1etx+1xqFuX5OP6+sjvWsX6eZmMrnH5l2kd+U/7iLT3Dxs\nl6lYdTXxupnEq2t8QmDRQj9YVX6PgMq85TAIYLyycs9WrXQvtGyAXev91PKCf9y53p+URH9gyQqo\nOwhmzAfMb3OZgVt9pfvA7R74wecGGMvtk03vua5/W966QgckK6kafNI/Y9EICYG8ZEGy4uV32c2d\nFBcieuu0aPJhqMRE83P+lmp9nUMfy2Lhkovc9+kGbnKe/5h77UHbKGCfAo4zHCUM9q1s1v9283si\n7G7x2y0Ocw+HQ942kESoX+b/mU1xuUGsMh0dZNpyFcwREgS5BEJIEIzaIpVMEs9VPGtriNfUULKg\nMfQsmEEiPzkQ5mNVVep+KiL7F+d8D7dtf4amp3w9qqYRauZD7QLf+LGXJ0bZ3l7STTsHJwwGJQ/8\ncqalZc8nm4UBW6v7EwTJRQspq64hXlNNbI/H6oF4XlFCrO157KVHYMtaP7W8kDswzHklzD/B9+Br\nPNr/z0wM0zMsFofZB/tpaaQXZU8bbF83OOHw8A3+JBZ8XW7WwVC/NEzL/WPtgsl3SVm6z9dZO3f4\nempXk2+o6Gryy70deSf/mYET/kxq6MTASImD8ZR/555kSABU1YdkwBDbSioHEgVD3u1n/7jDT0FN\nGer2Nc6y2dCFvgXiSVyyyt+BrKVlhGSBTxKkd+0i294+5GGtpIT4rFkk6uqIz6rzXV5n1ZGom+Uf\nZ80iXjfwGBtrV9e+bh8st60fSCDsesFPbZsYdIJYWuOTB/OOgKVn+vncVFW/7wJd/+jq+UmHkMG0\nuE8QTKWxDMx8QqN8Bsx6RWHPSe3O6wmxa6AnRHezD8j9X4kN3ER9tMdcefbYNsL6IY8X8z0olDDY\nt3Y8PTiBsO1x6A1xJl4Cc5fA4af73/H8I/3yFGwVcZkMqZe2kdq0kb4NG+nbtJHUxk2ktm0blCAY\n7batsaoq3xpVU5tX8awZWF8TKqY1ufX+UYNUici0lO6FHU/5k+FtoQV+2xNhjKlhxEt8XaCmkWx5\nA2k3k3S6inRfKendRrozQ7qlk/ROnzBI7Wgi29Y2xHHiJGbPJjFnDsnGRsqPPJLEnDkk5s7xj3Pm\n+vm6usJ6eWXS0PQ0bH0YngtJhO3rBu4sVLPAJxCOfr9PIsw7Espqxva5RZXVwgGv91NOriFg+5MD\nyYatj8CTtw1+Xv2ywQmHuYeN72VpucauoZIEXU17JhB6hviewA9+XDXXd8mPJQZP8aSve+Svz23L\n9VKOJfzAzdHlkbbHkwO9mKPbkuVD3+53uISQYO7ljI7/MqxYscKtWbOmKK897tJ9g7uq507cuptx\nnU1kdm4jta2J1I4WUs0dpFp7SXUZqa4E6Z4Ymd4YuKErmvHKJImqUuK1FSRqq4jPqCUxq4543WwS\nc+YSnzuPRH0j8XkLic2sH79rqXraQrIgkjjI9T7IHzm/YpZPFMw8cHDioO4g3/VelWiZQsxsrXNu\n7y4qm8JWNCbdmosq/EKiHBqWhV4IR/rHOYdNqX+i2b4+Ups307dhA6lNm+jbuIm+jRtIbdxE35Yt\ng8Z8sWSS5MKFJOfN87e1qq0hXl1DvLYmtETlEgT+MV7je3fp8gKRiTedYvF+VScGfxK57YnBSYSd\nzw60HCfKoX4J1C8jO+tw0olGUqlq0tu2kd7yF9IvbSa9fTvpnbtIt3aQbush2zfE+UrMkSjLkqhK\nkKwtJ1FXQ2L2LBIN80nMW0Ri0StILF5CfP4rxh63nfODD295OPREeNj35kuF3mlltT55MD/0RGh8\ndeE9TydSb8eevRu2PxnpuWq+nt6wLCQdQuJhxqKBensmlZcoGCZJkJvPDDOoZPlMqJwLlXP8pbhV\nkfn89Rqwd1LZ2zis2lE+5/yPsXsndDWP3L28uxnX1Uy6rZNUV9xP3YmB+bDs0tEfSJxYWQ3JWbUk\nD6yjfEYl8co4iXIjXuZIlGaIJ3tJJLqJx7qwvg7obYae9YO78uwK0zORQ1vMZ/jKaqG01mdHS2v8\nY1ntwHx0XaLcd0Mb1PNgvX/PUVUNPgC94i1QF0kgzDxw2l1DLbJfqZgNZ37dJxFmHTIleu1kOjtJ\nbdwYkgYbfQ+EMJ/etm3QbbNilZUkD1hE6StfSfXb3kZy0UJKFi6iZNFCEvX1/k4FIiKy9zJpf6nl\ntj/D9if847Yn+sfecVlIx+eTKjuIdNmZpNI1pHYnSO/sJvXodlLbHiGz83/2OKyVloaeBI2UHjSX\nyjm5ngWzSdSUkihLk0j2EHetWMdWX49t3wptm6H9CehKwfP4CQZ6PtQu8JdZ1DT6qbZxYDl32UVX\ns++JkLucYctaX+8H35o+7wh49ftCEuFoXxeejCfCpdWw6LV+yslmfZKkv3fDE/77Wnc7/b2NS2v8\nCX5388Dlj/niJSEhMNv3Kq1ftmeSoHKOP07FrP2ii78UZvLXIMeTc761ffs62PEktG2J9DLYNZBA\niGTcXBbSu0OSYHcpqVQ1fbtLSXfFSXUYqdYaXGbwdcXxGj+AYenShVQ1NpJsbCQ5f76fGhuJ1dTs\nfVdY53w39t526GkPj62R+dxj2+B17ZthR2TdsGMNmA+4dQfCYacM7nUwc/F+ce20iAyhthGOOKfY\npRjEOUdm1y76NgxOGqQ2bqRv0yYyu3YN2j8+axYlCxdS+ZpjSIakQcmiRSQXLSI+c6YuPRARebl2\nt0Z6IDyBe+lxMpueJdWRId0dJ7U7SYo5pFPzSXUvINXWS3pXexgIfH2Y/O0PE/PmkWxooPSwV5Js\nmEdy3jwSDfUk6+tJzJkztnpyTjbr6/PtW3w9v32rrwu3b/XLm/7k5/Pv1BUv8eNmRQcknns4HPqO\ngXER5i6ZUj349hCLhcbBA+HwUwfW93YOXKay/Un/GVTOGUge5JIEuaRBac3kTKZI0e2/iYXeTn/9\n0/Y/h65AT/pkQjT7VlaLK60jlZlJqqeWVM8cUp0xUu1pUi29/rKFna17jNIdn1PtEweHz6e6sZFE\nSBqUNDaSmDefeNUE3ErLLAz6UTH2LlbO+RFKowmIVJfP1M44AJJl41tmEZERuFSK7rUPh8sWBicQ\nBg2CaEZy3jySixZRfcIJlBywqD+BkFy4aGJirojIdJTN4natJ/vCGlJPP0R6/VOkNr9IqrndJxC6\n46R2l5DujuEydYOeaiVZEg1lJBsaqFzaQKJhHsl5DSFx4Odj1dUTm+yNxfxJcNVcmH/UsO9xyORD\nVzPMOTSMi3CEb/WfDkqrYOExfhJ5GaZ+YiGb8V33c916doTriVpeHNinpMpnGZecQabmUDqe66b9\nj0/R+9xfSDc1gds2sG8sRqK+nuT8+ZQf/CpqGgd6GuR6HcRKp+hAcmY+eJRW+a5fIiJFkO3upvWW\nn9N83Q9Jb/VjtlgySXLBAkoWLaLimGMoWbiwP4GQXNA49sFmRURkT85B53bSz6+l/a676H3uOVLb\nm0i1dJPuMrLpvDG7YjUk6mb4+nHjov5eB8n5A0mDeF3d1OghVkjyQUT22tRKLHQ2+V4H258cGJCk\n6RlI7/bbc7dZmXckHHl+GBhmKa68gc4HHqDtjjvovPcaXG8vycZGKt/whoGEQWMjycb5JOvrsaSu\nBRIRGW/plhZafvITWm74MZnWVsqPPpr6yy6jfNkyjXcgIjIR0n1+8O2dz4bpOdyOZ+h87AXanoWO\nLWXgjHhZluSMMkoXNVI5fwHJxYeSPHg5icYDfPJg9mwNXCsiI5qcESLVEy5jiPRA2L4uct0T/rqf\n+iVwzAd9b4T6pf4eseEWaC6bZffatbT95Ie033MP2bY24jNnMuNdZ1Fz6mmUH3Xk1MiqiohMcalt\n29j1w+to+dnPcN3dVL35zcy66EIqXv3qYhdNRGT/sLsFdj43KIHAzmf9oNzhNoi9bQlat8yl7S9x\nMl3lxKvLqXvnsdSefQ5lR71B182LyMtS3MRCbnTSHesGX8rQ/PzAIIOJMn+7s0Pe3t8DgblLoWrO\nHodzztH79NO033knbXfdTfqll7CKCqpPOIHaU0+h8thj1RtBRGQf6V2/nuZrv0/bHXdANkvtqadQ\n98EPUnboocUumojI1JPNQtvGoRMIXU0D+8VLoO4VMHcJmQNPpv3pHlr/8BQ9T/0FEgmqjj+eGWed\nSdVxx6leLCLjpniJhZ3PwtULI/dTxd+6sH4pLD0z9EJY5kcujY3cPbZv8xba77yT9rvupPe5533Q\nfMMbqLn0Uqrf8mZiFRUT/GZERCRn9+OP03zNNXT872+w0lJmrlxJ3QUXULKgsdhFExGZ/Pq6fCNb\nfgKh+XlI9wzsVz4TZr8SDj0JZh8apkNwNQvpXrOW1p/fSsf/3IHr7aX0kIOZ+w//QO3pp5GYPbt4\n701E9ltF7LFgcOR5oRfCMt8rYS9uaZhuaaH9l7+k/Y472f3IIwCUv/rVNHzh81SfdBKJmTMnquAi\nIpLHOUfXHx6g+Zpr6H7wQWK1tcz+6EeYef75JOrqRj+AiMh01bUTfvmZgQRC26aBbRbzd+6afSgc\n9KZIAuFQqJw16DB9mzfTduNttP3iF6S2biVWU0PtWWcy46yzKFu2TJcAi8iEKl5iYfYhcPKX9+op\n2e5uOn5zL2133kHXHx6AdJrSQw5mziWXUHPKKWoNExHZx1wmQ8evf83Oa66hd91TJObOZe5nPsOM\ns8/WbSBFRArRtgkevsHXjRe9Hma/38/PPhTqDhrxduDZ7m7af/1r2m69je4//QnMqDz2WOZc+imq\n3/rWqXsnMxGZcibn4I0RLpWi64EHaLvjTjp+8xvc7t0k5s1j1gXvp+a00yg99FBlYEVE9rFsby9t\nv/hvmr//fVIbN1KyeDHzvngVNaedpltDiojsjfql8NnHCh480TnH7kcepe22W2m/+5dku7pILlrE\nnL+7mNozziA5X7cUF5F9b1ImFnzAfMSPm/DLX5FpaSFWW0vtaadRe9qplB99NBaLjX4gEREZV5nO\nTlpvuonmH/2ITNNOypYvZ+43v0H1CSfodpEiImMRLykoqZDavoO2//5v2m67jb4XXsAqKqg58URm\nnHUm5StWqKFNRIpqUiUWep97jrY77qT9rrtIbdmClZZSfcJbqDn1VKr+6q8wtYKJiBRFeudOdl1/\nAy0//SnZjg4qjz2WWf/+71S89rWqzIqITJBsXx+d966m9bZb6fr9/ZDNUn700cy78EKqTzxRl5yJ\nyKRR9MRC6qWXaL/rLtruuJPeZ56BWIzKY49l9ic+TvVb36aAKSJSRH0bN9L8gx/QduttuFSK6hNP\nZNaFF1K+bGmxiyYist/qWbeO1ltvo/2OO8i0tZGor2fWRRcx48x3UrJ4cbGLJyKyh6IlFjItLWw4\n/710r1kDQNkRr6L+8supecdJug2OiEiR9Tz9NM3XXEv7L3+JxePUvvOdzPrg36hCKyIyQdItLbTf\ncSett95K79NPYyUlVL/1BGrPPIvKY1+vy81EZFIrWmIhtWUr6eZmZl/8CWpPOYWSAw4oVlFERCY9\nMzsJ+AYQB651zl2dt30R8CNgRtjnMufc3XvzGs45uh96iOZrrqXr978nVllJ3QcuoO597ydZP3ec\n3omIiERlOzrYfPHf0bF6NaRSlC1dSv0/fY7aU04hPmNGsYsnIlKQoiUWSl5xEAfdfZeuzRURGYWZ\nxYFvA28DNgMPmdntzrl1kd0+B6xyzn3HzJYAdwOLCzm+y2bpXL2a5u9dw+7HHiM+axZzLrmEmeee\nQ7ymZpzfjYiIRPVt2Ej3mjXUvec91J51FmWvPLTYRRIR2WtFSyzEysuVVBARKcxrgOedc+sBzOwm\n4AwgmlhwQC4LUAtsLeTAmdZW1p92On1/+QvJBQto+MLnqT3zTGJlw983XURExk/JooUc8tvVGqRc\nRKY03bNRRGTyawQ2RZY3h3VRVwDnm9lmfG+FTxRy4NTmLVgiwfyvfIVX/OqXzDz3XCUVRESGYWYn\nmdkzZva8mV02xPZFZrbazB4xs8fN7OTRjhmrqVFSQUSmvIISCxMRREVEZFydC1znnFsAnAzcYGZD\nxngz+5CZrTGzNbtnzuDAX9xG7amnYImi3yhIRGTSilyW9g5gCXBuuPQsKndZ2lHAOcB/7ttSiogU\nx6iJBQVREZGi2wIsjCwvCOuiPgisAnDO/REoA4a8xY5z7nvOuRXOuRUzGxt1WZqISGH6L0tzzvUB\nucvSosZ0WZqIyFRXSI8FBVERkeJ6CDjEzA40sxJ8Avf2vH02AicAmNnh+MRC0z4tpYjI/m3cLkuL\n9hxralKoFpGpr5DEgoKoiEgROefSwMeBe4Cn8D3EnjSzK83s9LDbpcBFZvYY8FPgAuecK06JRUSm\nrYIuS4v2HJszZ84+L6SIyHgbrwtqc0H0q2b2enwQXeacy0Z3cs59D/gewIoVK1ThFREpkHPubnzi\nNrru85H5dcAb9nW5RESmkUIvSzsJ/GVpZpa7LG3HPimhiEiRFNJjYVyv7RURERERmYJ0WZqIyDAK\nSSwoiIqIiIjItKbL0kREhjfqpRDOubSZ5YJoHPhBLogCa5xzt+OD6DVmdgl+IEcFURERERHZr+iy\nNBGRoRU0xoKCqIiIiIiIiIgMpZBLIUREREREREREhqTEgoiIiIiIiIiMmRILIiIiIiIiIjJmSiyI\niIiIiIiIyJgpsSAiIiIiIiIiY6bEgoiIiIiIiIiMmRILIiIiIiIiIjJmSiyIiIiIiIiIyJgpsSAi\nIiIiIiIiY6bEgoiIiIiIiIiMmRILIiIiIiIiIjJmSiyIiIiIiIiIyJgpsSAiIiIiIiIiY6bEgoiI\niIiIiIiMmRILIiIiIiIiIjJmSiyIiIiIiIiIyJgpsSAiIiIiIiIiY6bEgoiIiIiIiIiMmRILIiIi\nIiIiIjJmSiyIiIiIiIiIyJgVlFgws5PM7Bkze97MLhti+9fN7NEwPWtmreNfVBERERGR4lGdWERk\naInRdjCzOPBt4G3AZuAhM7vdObcut49z7pLI/p8AjpqAsoqITFtmdhLwDSAOXOucuzpv+9eBN4fF\nCmCuc27Gvi2liMj+S3ViEZHhFdJj4TXA88659c65PuAm4IwR9j8X+Ol4FE5ERAZVZt8BLAHONbMl\n0X2cc5c45450zh0J/D/g1n1fUhGR/ZrqxCIiwygksdAIbIosbw7r9mBmBwAHAvcOs/1DZrbGzNY0\nNTXtbVlFRKYrVWZFRIpPdWIRkWGM9+CN5wC3OOcyQ210zn3PObfCObdizpw54/zSIiL7rXGrzIqI\nyD6hOrGITCuFJBa2AAsjywvCuqGcg1rJRESKacTKLKilTERkjFQnFhEZRiGJhYeAQ8zsQDMrwQfK\n2/N3MrPDgJnAH8e3iCIi0964VmbVUiYiMiaqE4uIDGPUxIJzLg18HLgHeApY5Zx70syuNLPTI7ue\nA9zknHMTU1QRkWlLlVkRkSJTnVhEZHij3m4SwDl3N3B33rrP5y1fMX7FEpGxSqVSbN68mZ6enmIX\nZVIrKytjwYIFJJPJYhdlVM65tJnlKrNx4Ae5yiywxjmXSzKoMisySSgWF2YqxWJQnVhkKlEcLsx4\nxeGCEgsiMnVs3ryZ6upqFi9ejJkVuziTknOO5uZmNm/ezIEHHljs4hRElVmRqUWxeHRTMRaLyNSh\nODy68YzD431XCBEpsp6eHmbNmqUAOgIzY9asWcpgi8iEUSwenWKxiEwkxeHRjWccVmJBZD+kADo6\nfUYiMtEUZ0anz0hEJpJizOjG6zNSYkFERERERERExkyJBREpuqqqqmG3vfjiiyxbtmwflkZEZPpR\nHBYRKb6pHIuVWBARERERERGRMdNdIUT2Y/98x5Os29o+rsdcMr+GL5y2dMR9LrvsMhYuXMjHPvYx\nAK644goSiQSrV6+mpaWFVCrFVVddxRlnnLFXr93T08NHP/pR1qxZQyKR4Gtf+xpvfvObefLJJ/nA\nBz5AX18f2WyWn//858yfP593v/vdbN68mUwmwz/90z+xcuXKMb9vEZGxKkYsVhwWERmgOvHEx2Il\nFkRk3K1cuZJPfvKT/UF01apV3HPPPVx88cXU1NSwc+dOXve613H66afv1YAx3/72tzEznnjiCZ5+\n+mne/va38+yzz/Ld736Xv/u7v+O8886jr6+PTCbD3Xffzfz587nrrrsAaGtrm5D3KiIyGSkOi4gU\n33SKxUosiOzHRsuiTpSjjjqKHTt2sHXrVpqampg5cyYNDQ1ccskl3HfffcRiMbZs2cL27dtpaGgo\n+Lj3338/n/jEJwA47LDDOOCAA3j22Wd5/etfzxe/+EU2b97MWWedxSGHHMLy5cu59NJL+cxnPsOp\np57KG9/4xol6uyIiIypGLFYcFhEZoDrxxMdijbEgIhPi7LPP5pZbbuHmm29m5cqV3HjjjTQ1NbF2\n7VoeffRR6uvrx+3e5e95z3u4/fbbKS8v5+STT+bee+/l0EMP5eGHH2b58uV87nOf48orrxyX1xIR\nmSoUh0VEim+6xGL1WBCRCbFy5Uouuugidu7cye9+9ztWrVrF3LlzSSaTrF69mg0bNuz1Md/4xjdy\n44038pa3vIVnn32WjRs38spXvpL169dz0EEHcfHFF7Nx40Yef/xxDjvsMOrq6jj//POZMWMG1157\n7QS8SxGRyUtxWESk+KZLLFZiQUQmxNKlS+no6KCxsZF58+Zx3nnncdppp7F8+XJWrFjBYYcdttfH\n/Nu//Vs++tGPsnz5chKJBNdddx2lpaWsWrWKG264gWQySUNDA5/97Gd56KGH+PSnP00sFiOZTPKd\n73xnAt6liMjkpTgsIlJ80yUWm3NuQg48mhUrVrg1a9YU5bVF9mdPPfUUhx9+eLGLMSUM9VmZ2Vrn\n3IoiFWmfUywWmRiKxYWb7rFYcVhkYigOF2484rDGWBARERERERGRMdOlECIyKTzxxBO8973vHbSu\ntLSUBx98sEglEhGZXhSHRUSKb6rGYiUWRGRSWL58OY8++mixiyEiMm0pDouIFN9UjcW6FEJERERE\nRERExkyJBREREREREREZMyUWRERERERERGTMlFgQERERERERkTErKLFgZieZ2TNm9ryZXTbMPu82\ns3Vm9qSZ/WR8iyki+7OqqqpiF0FEZFpTHBYRKb6pHItHTSyYWRz4NvAOYAlwrpktydvnEOAfgTc4\n55YCn5yAsoqIiIiIFI0a20REhlbI7SZfAzzvnFsPYGY3AWcA6yL7XAR82znXAuCc2zHeBRWRMfjl\nZbDtifE9ZsNyeMfVI+5y2WWXsXDhQj72sY8BcMUVV5BIJFi9ejUtLS2kUimuuuoqzjjjjFFfrrOz\nkzPOOGPI511//fV85Stfwcx41atexQ033MD27dv5yEc+wvr16wH4zne+w7HHHvsy37SIyMtQhFis\nODz+Io1tbwM2Aw+Z2e3OuXWRfaKNbS1mNrc4pRWRQVQnnvBYXEhioRHYFFneDLw2b59DAczsD0Ac\nuMI596v8A5nZh4APASxatGgs5RWRKWDlypV88pOf7A+iq1at4p577uHiiy+mpqaGnTt38rrXvY7T\nTz8dMxvxWGVlZdx22217PG/dunVcddVVPPDAA8yePZtdu3YBcPHFF3P88cdz2223kclk6OzsnPD3\nKyIy2SgOTwg1tonIXplOsbiQxEKhxzkEeBOwALjPzJY751qjOznnvgd8D2DFihVunF5bRIYzShZ1\nohx11FHs2LGDrVu30tTUxMyZM2loaOCSSy7hvvvuIxaLsWXLFrZv305DQ8OIx3LO8dnPfnaP5917\n772cffbZzJ49G4C6ujoA7r33Xq6//noA4vE4tbW1E/tmRURGU4RYrDg8IcatsU1E9jHViSc8FheS\nWNgCLIwsLwjrojYDDzrnUsALZvYsPtHw0LiUUkSmnLPPPptbbrmFbdu2sXLlSm688UaamppYu3Yt\nyWSSxYsX09PTM+pxxvo8EZHpTnG4KApqbFMvXpHpY7rE4kLuCvEQcIiZHWhmJcA5wO15+/wCH0Ax\ns9n4bO36cSyniEwxK1eu5KabbuKWW27h7LPPpq2tjblz55JMJlm9ejUbNmwo6DjDPe8tb3kLP/vZ\nz2hubgbo7/Z1wgkn8J3vfAeATCZDW1vbBLy7fU8DhonI3lIcHneFNrbd7pxLOedeAHKNbYM4577n\nnFvhnFsxZ86cCSuwiBTfdInFoyYWnHNp4OPAPcBTwCrn3JNmdqWZnR52uwdoNrN1wGrg08655okq\ntIhMfkuXLqWjo4PGxkbmzZvHeeedx5o1a1i+fDnXX389hx12WEHHGe55S5cu5fLLL+f444/niCOO\n4FOf+hQA3/jGN1i9ejXLly/n6KOPZt26dSMdfkrQ3XlEZCwUh8edGttEZK9Nl1hszhVnqIMVK1a4\nNWvWFOW1RfZnTz31FIcffnixizElDPVZmdla59yKIhVpSGb2evx1uieG5X8EcM59KbLPl4FnnXPX\n7s2xFYtFJoZiceGmSiwGMLOTgf/Aj5/wA+fcF83sSmCNc+5286OvfRU4CcgAX3TO3TTSMRWHRSaG\n4nDhxiMOj9fgjSIiMnE0YJiIyCTgnLsbuDtv3ecj8w74VJhERKYNJRZEZFJ44okneO973ztoXWlp\nKQ8++GCRSjTlFDRgGGjQMBEZmuKwiEjxTdVYrMSCiEwKy5cv59FHHy12MSarcb07j279KyJDURwW\nESm+qRqLC7krhIiIFNfzNHwAACAASURBVJcGDBMRERGRSUuJBRGRSU535xERERGRyUyXQoiITAEa\nMExEREREJiv1WBCRcVdVVVXsIoiITHuKxSIixTWd4rASCyIiIiIiIiIyZkosiMiEcc7x6U9/mmXL\nlrF8+XJuvvlmAF566SWOO+44jjzySJYtW8bvf/97MpkMF1xwQf++X//614tcehGR/YNisYhIcU2H\nOKwxFkT2Y//2p3/j6V1Pj+sxD6s7jM+85jMF7Xvrrbfy6KOP8thjj7Fz506OOeYYjjvuOH7yk59w\n4okncvnll5PJZOju7ubRRx9ly5Yt/PnPfwagtbV1XMstIlIsisUiIsWlODzx1GNBRCbM/fffz7nn\nnks8Hqe+vp7jjz+ehx56iGOOOYYf/vCHXHHFFTzxxBNUV1dz0EEHsX79ej7xiU/wq1/9ipqammIX\nX0Rkv6BYLCJSXNMhDqvHgsh+rNAs6r523HHHcd9993HXXXdxwQUX8KlPfYr3ve99PPbYY9xzzz18\n97vfZdWqVfzgBz8odlFFRF42xWIRkeJSHJ546rEgIhPmjW98IzfffDOZTIampibuu+8+XvOa17Bh\nwwbq6+u56KKLuPDCC3n44YfZuXMn2WyWd73rXVx11VU8/PDDxS6+iMh+QbFYRKS4pkMcVo8FEZkw\nZ555Jn/84x854ogjMDO+/OUv09DQwI9+9CP+/d//nWQySVVVFddffz1btmzhAx/4ANlsFoAvfelL\nRS69iMj+QbFYRKS4pkMcNudcUV54xYoVbs2aNUV5bZH92VNPPcXhhx9e7GJMCUN9Vma21jm3okhF\n2ucUi0UmhmJx4aZ7LFYcFpkYisOFG484rEshRERERERERGTMlFgQERERERERkTFTYkFEZBrrTncX\nuwgiIiIiMsUpsSAiMo1t7thMS09LsYshIiIiIlOYEgsiItNYOpvm8vsvJ+uyxS6KiMi0lHbpYhdB\nRORlKyixYGYnmdkzZva8mV02xPYLzKzJzB4N04XjX1QRERlvDZUN/H7L77nuyeuKXRQRkWlpQ9sG\nOvo6il0MEZGXZdTEgpnFgW8D7wCWAOea2ZIhdr3ZOXdkmK4d53KKiMgEqCur4+0HvJ1vPvzN/8/e\nnYc3dR14H/9ebZZkSd73HbABs5g1EAIEEtampEs6zbRvm2k6TZq06ZJO6HSbrpMu0zRN0y1NO2m6\nTqbbtKFJCJCwpRACCTsGA8b7bsu2ZEnWdt4/riwsY8CAbcnW+TyPHt2re3R97KaHo5/OwuG2w9Gu\njiRJUtzpD/Tz8M6H8Qa80a6KJEnSdRvJiIWbgHNCiGohhBd4DnjHjf7g/kD/jd5CkiRJGgVfXfZV\nci25bN69Wa63IEmSNM7yLHkcaDkgp6VJkjShjSRYyAPqB503hF4b6i5FUY4pivInRVEKhruRoij3\nK4pySFGUQxe6L9DobLyOKkuSNBG8853vZOHChcyaNYunn34agK1bt7JgwQIqKiq4/fbbAXA6ndx7\n773MmTOHuXPn8uc//zma1Y5LVoOVx259jC5Pl+zYStIkItvh0TcW04OTEpL4zMLPsLVmK989+F2E\nEGNTeUmSoiJe2mLdKN1nC/A/Qoh+RVE+CvwKuG1oISHE08DTAJYpFnH/tvv59cZfk2ZKG6VqSJI0\nWMs3v0l/5elRvWfCzBlkf+ELVy33zDPPkJqaitvtZvHixbzjHe/gvvvuY8+ePZSUlNDV1QXAN77x\nDZKSkjh+/DgAdrv8xjwaytPK+eziz/LogUd59uSzfHj2h6NdJUmaNKLVFst2eHQNmh68FvWLtoOK\nojwvhDg1pOj/CiEeupZ7f2jWh2hztfHbyt+SZc7iQ7M/NDqVliQJkH3i8TCSYKERGDwCIT/0WpgQ\nonPQ6S+A/7raTQtthbS52nhwx4M8s/4ZLAbLSOorSdIE8eSTT/J///d/ANTX1/P000+zcuVKSkpK\nAEhNTQVgx44dPPfcc+H3paSkjH9lJQDunn43B1sO8uRbTzI/cz7zM+dHu0qSJN0A2Q6PuvD0YABF\nUQamBw8NFq6ZoihsXryZdnc733vze6SZ0tg0ddON3laSpBgQL23xSIKFg0CpoiglqIHCPwPvH1xA\nUZQcIURz6PROoPJqNzXrzHxv1ff41Kuf4pM7P8lP1/yUBG3CNVZfkqQrGUmKOhZ27drFjh072L9/\nP2azmVWrVjFv3jxOnx7dpFgaXYqi8NVlX6Wyq5LNuzfzx01/JMU4sf5Rk6RYFI22WLbDY2K46cFL\nhil3l6IoK4Eq4GEhRP3QAoqi3A/cD1BYWAiARtHwzeXfxO6x8+V/fJk0YxrL8paN9u8gSXFJ9onH\n3lXXWBBC+IGHgJdRA4M/CCFOKorydUVR7gwV+6SiKCcVRTkKfBL40Eh++Mr8lXxj+Tc42HKQz+7+\nLP6g3MdXkiaDnp4eUlJSMJvNnD59mtdffx2Px8OePXu4cOECQHjY19q1a/nxj38cfu9EG/Y12cj1\nFiRpcpDtcNRsAYqFEHOB7ajTgy8hhHhaCLFICLEoIyMj/LpBa+CJ1U8wNXkqn971aU52nByfWkuS\nNCbiqS0eyeKNCCFeFEKUCSGmCiEeDb32ZSHE86HjzwshZgkhKoQQq4UQI45g3j7l7Xzups/xav2r\nfOP1b8gFayRpEtiwYQN+v5+ZM2fyuc99jqVLl5KRkcHTTz/Nu9/9bioqKrj77rsB+NKXvoTdbmf2\n7NlUVFSwc+fOKNdeGlhvYW/jXp49+Wy0qyNJ0nWQ7fCYGNH0YCHEwNZnvwAWXusPsRqs/HTNT0lJ\nSOFjr3yMut66666wJEnRFU9t8Wgt3nhD/t/M/0eXp4unjz1NckIyDy98ONpVkiTpBiQkJPDSSy8N\ne23jxo0R5xaLhV/9atgvdKQokustSNLEJtvhMTEm04OHk2HO4Km1T3HPS/fwwI4H+PXGX5NuSr+R\nukuSFAXx1BaPaMTCeHho3kP8U9k/8cyJZ/jVyYn7B5UkSZoMBtZbyLXksnn3ZuyeiTUcT5IkabSN\n5fTg4ZQklfDj239Mh7uDj7/ycVw+143+CpIkSWMmZoIFRVH44pIvsrZoLY8deoy/nftbtKskSZIU\nM8Zi7/SrkestSJIkRRrL6cHDmZsxl8dufYwzXWd4eNfD+AK+0fg1JEmSRl3MBAsAWo2Wb6/4Nktz\nlvKVfV9hV/2uaFdJkiYkuVbJ1U2kv9GgvdM3AuXA+xRFKR+m6P8KIeaFHr8YjZ8t11uQpOs3kdqZ\naJF/o6tbmb+Sr9z8FfY17ePL+74sQ15Jugayjbm60fobxVSwABdXw52ZOpNHdj/CoZZD0a6SJE0o\nRqORzs5O2ZBegRCCzs5OjEZjtKsyUuG904UQXmBg7/Rxcff0u1lXtI4n33qSw22Hx+vHStKEJtvi\nq5uAbXHUvKv0XXxi/if4e/XfeeKtJ6JdHUmaEGQ7fHWj2Q7HxOKNQyXqE/nJmp9wz0v38IlXP8Gz\nG55leur0aFdLkiaE/Px8GhoaaG9vj3ZVYprRaCQ/Pz/a1RipUds7/XoMrLdQ2VXJ5t2b+eOmP5Ji\nTBmNW0vSpCXb4pGZYG1xVN035z7aXG388sQvyTBl8MHyD0a7SpIU02Q7PDKj1Q7HZLAAkGJM4em1\nT/PBlz7IR7d/lN9s/A0FtoKrv1GS4pxer6ekpCTa1ZDG3xbgf4QQ/YqifBR17/TbhiuoKMr9wP0A\nhYWFI7r5wHoLH3jxA3zxtS/yo9t/hEaJuUFvkhQzZFssjTZFUfj8TZ+n093Jfx38L9JN6Wws2Xj1\nN0pSnJLt8PiK6V5hjiWHp9c+jV/4uX/7/bS7ZNokSVJcGtW904UQTwshFgkhFmVkZIy4EnK9BUmS\npOjSarR8e+W3WZi1kC+89gUONB+IdpUkSZKAGA8WAKYkT+Ent/+ETk8nD+x4gF5vb7SrJEmSNN7C\ne6crimJA3Tv9+cEFFEXJGXR63XunX41cb0GSJCm6ErQJPHnbkxTbivnUzk9xuuuGNp6QJEkaFTEf\nLIC61c4Tq56guqeaT7zyCTx+T7SrJEmSNG7Ge+/0KxlYbyHXksvm3Zuxe+xj8WMkSZKkK7AZbPx0\nzU+x6C08uONBGp2NV3+TJEnSGJoQwQLAsrxlfGv5tzjcdpjNuzfjD/qjXSVJkqRxM957p1/JwHoL\nXZ4uvvjaF+XWZ5IkSVGQnZjNz9b+DG/AywPbH5BBryRJUTVhggWADSUb+MKSL7CrYRdf2fcV2ZmV\nJEmKErnegiRJUvRNTZ7Kj27/Ec19zTz0ykO4fK5oV0mSpDg1oYIFgH+e8c98rOJjPH/+eR4/9Ljc\nl1SSJClK5HoLkiRJ0Tc/cz7fWfkdTnSe4JHdj+AL+qJdJUmS4tCECxYAHqh4gPfNeB+/OvUrnjnx\nTLSrI0mSFJfkeguSJEmx4fbC2/niki+yt3EvX9//dfnFmyRJ425CBguKovC5mz7HxuKNPPHWE/zl\n7F+iXSVJkqS4JNdbkCRJig3vnf5eHqx4kL+e+ys/PPzDaFdHkqQ4MyGDBQCNouHR5Y9yS+4tfG3/\n13il9pVoV0mSJCkuyfUWJEmSrl+328e5NieB4I2PMniw4kHuKr2Lnx//Oc+dfm4UaidJkjQyumhX\n4EbotXoeX/U49227j8/u+SxPrX2KxdmLo10tSZKkuHP39Ls52HKQJ996kvmZ85mfOT/aVZIkSZoQ\n6rtcrHl8N0a9hhnZNmbm2CjPtVGeY2NGtpXEhJF31xVF4UtLv0Snp5NvHvgm6aZ01hStGcPaS5Ik\nqZRozcFatGiROHTo0Kjcq9vTzb9s/RdaXa08s/4ZytPKR+W+kiTFH0VR3hRCLIp2PcbLaLbFDq+D\nu/9+N96Alz9u+iMpxpRRua8kSfEnntri2RXzxdee2cKp5l5ONfVyqrmXHre6AKOiQHFaIuWDwoaZ\nOTaybAkoinLZe7r9bu7bdh+VnZX8bO3PWJQdF39KSZJG0bW2w1ELFqaVzxWVx4+g147ObIyWvhbu\neeke+gP9/HrjrymyFY3KfSVJii/x1JmF0Q0WAE51nuIDL36ApTlL+dHtP0KjTNgZd5IkRVE8tcVD\n22EhBM09nnDIMPBc13VxK8nUREM4bJiZY6U8J4kpGYkR/epuTzf3bL2HDlcHz258lrKUsnH9vSRJ\nmtgmTLCQkFMqFnzyKe69pZh/vqkQm1F/w/es6anhnpfuwaw38+uNvybTnDkKNZUkKZ7EU2cWRj9Y\nAHju9HM8euBRHl74MB+e/eFRvbckSfEhntrikbbDDo+P0y0ONWho6qWypZfTLQ68fnXRXINOw/Qs\nayhosFGem0SyzcGDr3wYFPjtxt+SY8kZ619HkqRJYkyCBUVRNgA/ALTAL4QQ375MubuAPwGLhRBX\nbCGnz64QCz75M/ZXd2JJ0PG+mwq495YScpNNI637sE52nuTDWz9MriWXZzc8S1JC0g3dT5Kk+BJP\nnVkYm2BBCMEjux/hlbpX+OWGX8r1FiRJumbx1BbfSDvsDwSp7uhTg4bmiyMcOvu84TK5mXZcaU9i\n1aXxb3OeZHFhPrlJxitOpZAkSRr1YEFRFC1QBawFGoCDwPuEEKeGlLMCLwAG4KGrBQsDjejxhh5+\nvreaF443A/D2uTnct2IKs/OuPxA40HyAB3c8yKy0WTy97mlMuhsLKyRJih/x1JmFsQkWQK63IEnS\njYmntni022EhBG2O/ohpFEfb36Tb9mOCnnxcdR/BlmAKrdmQxMwcK6umZ5JhTRi1OkiSNPFdazs8\nksmvNwHnhBDVQggv8BzwjmHKfQP4DuAZ6Q8HmJOfxJPvm8/uzau4d1kxO0618vYfvsb7f/46O8+0\ncT1TNZbkLOE7K7/DsY5jfGbXZ/AFfdd8D0mSJOn6WQ1WHrv1Mbo8XXzxtS8SFMFoV0mSJCkuKIpC\nls3I6umZfHz1NH78/gW89qn7+NaKb6Ez17Fg0Qu8bW4W/f4g//NGHZv/dIw1j+/mr4cbr6vfLUmS\nBCMLFvKA+kHnDaHXwhRFWQAUCCFeuNKNFEW5X1GUQ4qiHGpvb4+4lp9i5ktvL2ff52/n8xtnUN3e\nx72/PMi67+/hDwfr6fcHRvYbhawtWst/LP0PXmt8jf/4x3/ITq0kSdI4K08r57OLP8vexr08e/LZ\naFdHkiQprm2atpHP3fQ5zjpfx5TzN/7y4DJOfG09f//EcqZmJPLp/z3CR3/zJu2O/mhXVZKkCeiG\nl+tWFEUDPA7829XKCiGeFkIsEkIsysjIGLZMkknPR2+dyp7Prub7d1eg02r47J+Psfw7O/nxznN0\nu7zDvm847yl7D5+c/0leqH6B7x78rkxhJUmSxtnd0+9mXdE6nnzrSQ63HY52dSRJkm6IoigbFEU5\noyjKOUVRPneFcncpiiIURYmp6Rzvn/l+PjLnI/yp6k88dewptBqF2XlJ/PGBZXzhbTPYVdXOuu/v\n5u/HmqJdVUmSJpiRBAuNQMGg8/zQawOswGxgl6IoNcBS4PkbbUgNOg3vmp/Pi59czm//dQkzc2x8\n9+Uz3PytV/nK305Q1+m6+k2Aj8z5CB+Y+QF+W/lbfn785zdSJUmSJOkaKYrCV5d9lVxLLpt3b8bu\nsUe7SpIkSdcltO7Yj4GNQDnwPkVRyocpZwU+BRwY3xqOzCfnf5I7p97JT478hD9V/QkArUbh/pVT\nefGTyylMNfPQ7w/z8d+9RVffyL/QkyQpvo0kWDgIlCqKUqIoigH4Z+D5gYtCiB4hRLoQolgIUQy8\nDtx5tcUbR0pRFJaXpvPrD9/E1k+v4I65Ofz+jTpWPbaTj/3uTQ7XXbmTqigKmxdv5u1T3s4PD/+Q\nP5z5w2hUS5IkSRohud6CJEmTxJiuOzZeBgLf5XnL+cbr32Bn3c7wtWmZVv784DI2r5/OtlMtrPv+\nbraeaIlibSVJmiiuGiwIIfzAQ8DLQCXwByHESUVRvq4oyp1jXcHBZmTbeOyfKnjt32/jo7dO5bWz\nHbzrJ/t4z0/38fLJFgLB4ac6aBQNX7/l66zMX8l/vv6fbKvZNp7VliRJintyvQVJkiaBcVl3bDzo\nNXq+d+v3KE8tZ/OezbxQ/QIunzoaWKfV8PHV09jyieVk2Yw88Ns3+fRzh69pOrIkSfHnqttNjpXR\n2Fqnr9/PHw7V89+vXaDB7qYkPZF/XV7CXQvyMRm0l5R3+918dPtHOdFxgp+s+QlLc5be0M+XJGny\niactzmDstpscjhCCR3Y/wit1r/DLDb9kfub8cfm5kiRNPLHYFiuK8h5ggxDiI6HzDwJLhBAPhc41\nwKvAh4QQNYqi7AIeGekW7NHQ5eni3q33Ut1TTYI2gaU5S1lVsIpVBatIN6XjCwT5yc7z/PDVs6Qk\nGvj2u+dw+8ysqNRVkqTxda3t8IQOFgb4A0G2nmzh53uqOdrQQ2qigQ8sLeKem4tIt0TuydvT38O9\nL99Lo6OR7976XZbnLUej3PAalpIkTRKx2JkdS+PdoXV4Hdz997vxBrz8cdMfSTGmjNvPliRp4ojF\ntlhRlJuBrwoh1ofOPw8ghPhW6DwJOA84Q2/JBrq4yhThaAYLAL6gj8Oth9lZv5Od9TtpdDaioDAn\nYw6rC1azumA1Lmcam/90jNMtDu5akM+XN5WTZNJHrc6SJI29uAwWBgghOFhj5+k91eyobMWg03DX\ngnw+sqKEqRmWcLk2Vxv/8tK/0OBsICcxhzum3MGmKZuYkjxlVOsjSdLEE4ud2bEUjQ7tqc5TfODF\nD7A0Zyk/uv1HMtyVJOkSsdgWK4qiA6qA21EXMj8IvF8IcfIy5XcR4yMWhhJCcLb7LLvqd7Gzbicn\nOk8AUGAtYGXeKjrbpvGX/XoyLGa+fdccVk3PjHKNJUkaK3EdLAx2vt3Jf792gT+/2UC/P8iamZnc\nt2IKN5WkoigKbr+bV+teZUv1FvY37ScogpSnlbNpyiY2lmwkzZQ2ZnWTJCl2xWJndixFq0P73Onn\nePTAozy88GE+PPvD4/7zJUmKbbHaFiuK8jbgCUALPCOEeFRRlK8Dh4QQzw8pu4sJFiwM1drXyu6G\n3eys38mB5gP4gj4sOhv+vhnY20t514zb+PId87Ea5egFSZpsZLAwRIezn9/sr+U3r9fS1eelIj+J\nj6yYwsbZ2ei06rdkHe4OXqx+kb9X/53Krkq0ipZlucvYNHUTqwtWY9QZx7yekiTFhljtzI6VaHVo\nB9Zb2Fa7jfmZ81lfvJ51RevIMGeMe10kSYo98dQWx3KwMJjL5+IfTf9gV/0udtfvpsfbgwhq0XvL\neO+sDfzrgreTaZYjGCayYFDgDQQx6i9dq06KPzJYuAyPL8Cf32rgF3svcKGjj7xkE+9ekMfi4lQW\nFKVgSdABcM5+ji3VW3ih+gVaXa0k6hNZW7SWTVM2sSh7kRyyK0mTXDx1ZiG6HVqXz8XvKn/H1pqt\nVNmrUFBYmLWQDcUbuL3odtJN6VGplyRJ0RdPbfFECRYG8wf9HGk7wnMnX2J77asEtR0AlKfO4rbC\n1awuXE1pcimKokS5piPT7ujneGM3S0rSSAx9JognQgi2nWrlv7aeprbTxYrSdDZV5LK2PEuORolj\nMli4imBQsKOylf9+7QIHa7oICtBqFGbl2lhcnMpNJaksLk4lyaTlUOshtpzfwvba7bj8LrITs7mj\n5A42Td3E1OSp4153SZLGXjx1ZiF2OrTV3dW8XPMyW2u2Ut1TjUbRsDhrMetL1rOmcI1c5FGS4kw8\ntcWx0g5fL7fXz3+8+CrPn92OOfk0QUMtAkGeJY/VBatZVbCKBVkL0Gti6wNqdbuTbada2X6qlbfq\n7AgB6RYDD66axv9bUhg339ofquniWy+d5s1aO1MzEllRmsH2U600drtJ0Gm4bUYmd1bksnpGZtz8\nTSSVDBaugbPfz1u1dg7WdPHGhS4O13fj9QcBmJZpYXFxKktKUplTYOZM7/7wegwBEWBm6kw2TVXX\nY5DfqknS5BFPnVmAivkLxdHDb0a7GhHO2s/ycs3LvFzzMjW9NWgVLUtylrC+eD23F95OUkJStKso\nSdIYi6e2OBb6xKPhjQtdbP7TUeq6W7l1Xjum5NMcbD1Af6Afq8HKyvyVrCpYxfLc5VgMlqvfcJQF\ng4IjDd1sO9nK9lMtnG/vA2B2no21M7Mpz7Xx7L4L/ONcJ9k2I5+4fRrvXVSAXjs5RyufbXXwna1n\n2FHZSpYtgYfXlPGehfnotBqCQcHhejvPH2nihePNdDi9WBJ0rCvPYlNFLstL0yft30W6SAYLN6Df\nH+B4Qw9vhIKGN2vsOPr9AOQlm7ipJJXyfAWn7iD727aF12O4OfdmNk3ZxOrC1Zh0pij/FpIk3YhY\n7cwqirIB+AHqgmG/EEJ8+zLl7gL+BCy+2oJhAKbcMvHY717kX5eXxNw3EUIIquxVbK3ZytYLW2lw\nNqBTdCzNXcqG4g2sLlyNzWCLdjUlSRoDsdoWj4VY7BNfL5fXz3deOs2v9tdSkp7Io+8uxa07HV6X\nwd5vR6fRcVP2TeHRDNmJ2WNWH48vwP7znWw71cKOyjbaHf3oNApLpqSyrjybNeVZ5CVH9t33ne/g\nsZfP8FZdN4WpZj51eynvnJ+HVjMxpnVcTUuPh+9vr+KPb9aTaNDxwKqpfPiWEkyG4fsA/kCQ16u7\n2HK0iZdONNPr8ZNs1rNxdg6bKnJYUpI2af42UiQZLIyiQFBwuqWXNy50hUY12Olw9gOQmmhgVpEL\nXdJhLrj30tnfRqI+kTWFa9g0dROLsxfHzHoMQgjsLh8NdheNdjcNdjeN3W4a7C46nF5m59lYWZrB\nzVPT5DwqKe7FYmdWURQt6hZna4EG1C3O3ieEODWknBV4ATAAD40kWMgoKReJd3+XwlQzX7xjJuvK\ns2JyTqwQglNdp3i55mW21Wyj0dmIXqPnltxbWFe8jtUFq6PyDZgkSWMjFtvisTIR+sTXat/5Dj77\np2M0dru5b8UUPrO2DL0WjnUcY2fdTnbW76SmtwaAmakzWV2wmhX5K5iZOhOt5sZC7h6Xj1fPqFMc\ndp1px+UNkGjQsmp6JutmZbGqLJMk85X7u0IIdp1p57FtZzjZ1Mu0TAufWVvGhlnZaCboh+hej4+n\ndp3nmX9cIBAUfHBpMQ/dNo3URMOI79HvD7C3qoPnjzax/VQrbl+ATGsCd8zN4c6KXOYVJMdkH0K6\nPjJYGENCCGo6XbxxoZM3LqhTKOq6XECQRFst6TkncGjfwifcZJqzePuUO9g0ZRPTUqaNeb3anf1q\nYBAODlwR525fIOI91gQdeSkmks16jjX04PIG0GoUFhQms7I0gxVlGczJS5IJpBR3YrEzqyjKzcBX\nhRDrQ+efBxBCfGtIuSeA7cBmRrDFGaht8Q+e28rXtpykqtXJitJ0vvz2ckqzrKP/i4wSIQQnOk6w\ntWYrL9e8TKurFYPGwPK85Wwo2cCt+bdi1pujXU1Jkm5ALLbFY2Ui9olHwtnv55svVvL7A3VMzUjk\ne++dx7yC5PD1Cz0X2FW/i531OznSdgSBICkhiSXZS7g592Zuzr2ZPEveiH5Wg93F9tB6CQcudBEI\nCjKtCawpz2JdeRY3T00jQXftgUUwKNh6soXHt1dxrs3JrFwb/7aujNXTMyfMB+h+f4Df7K/lRzvP\n0e3y8c55ufzbuukUpN7Yv5Mur59XKtvYcrSJXWfa8QaCFKSa2DQ3l00VuczItk6Yv5E0PBksjLOW\nHg9v1HRx8II6feJMWyc6SyWG5MNoE6tACZJvnsa7yu7k3WWbrms9hkBQ0ObwDAoKXKERB6Hzbnd4\nbYgByWY9+Skm8pPN5KWYyE8xkZdsIj9FPU8yXUxqvf4gb9XZ2VPVzt6zHZxo6kEI9R63TEtnZWk6\nK0ozyE2W0zykyS8WO7OKorwH2CCE+Ejo/IPAEiHEQ4PKLAC+KIS4a6R7p8PFttgfCPLb12t5fHsV\nfd4A99xcxKfXrk40ZwAAIABJREFUlEW0FbEoKIIcaz8WHsnQ5m7DqDWyIn8F64vXszJ/pZyiJkkT\nUCy2xWNlsvSJL2dPVTv//udjtPZ6eODWqXxqTeklH/K7PF0caD7A/qb97GvaR6urFYAiWxE356gh\nw03ZN4VHpgkhONnUGw4TTjX3AlCaaWFteRbrZmUzNy9p1EYXBIKCvx1p5IkdZ6nrcrGgMJlH1k1n\n2bTYXWctGBT87Wgjj71cRWO3mxWl6fz7hhnMzhv9dYp63D62nWzh+aNN7DvfSSAoKM20sKkilzsr\ncilOTxz1nymNPRksRFm3y8uhGjtv1HSxr6aGs869aG1voTU1gtCQpZ/D6ryNfLDiDgpT1NTWHwjS\n3OOJmKIweMpCc48bXyDyf6d0i4G8FDP5yaHQIBweqMGB5Qa2yul09vOP852hoKGd1l51+se0TAsr\nStNZWZrBkimpmA3xtx2PNPnFYmf2asGCoiga4FXgQ0KImqsFC4qi3A/cD1BYWLiwtrY2fK3T2c/3\ntlfxP2/UkWI28Mi66dy9uGBCjF4KiiCH2w6z9cJWttdup9PTiUln4tb8W9lQvIFb8m7BqDNGu5qS\nJI1ALLbFY2Wy9okH6/X4+M+/n+IPhxqYnmXle++tuOwHXCEEF3ovsL9pP/ub9vNGyxu4/W60ipZi\ny0wMvhnUNuTT2p6BomhZVJTC2vIs1pZnUzLGH2B9gSB/PNTAk6+cpaXXw7KpaTyyfjoLCmNn5yIh\nBHvOdvDtl05T2dzLrFwbn984k+Wl4xOCdDj7eel4M1uONvNGTRcAc/KSuLMilzvm5sgvKicQGSzE\nGJfXz5G6bl6uOsbelq20Bfej6LsRAQMJ3nloPTPp6srG700G1I67okCmNUEdXRARHKjnecmmyy6w\nMtqEEJxtc7Knqp09Zzs4UN1Jvz+IQathUXEKK0ozWFGaTnmObcLOOZOkwWKxM3u1qRCKoiQB5wFn\n6C3ZQBdw59VGLVyuLT7Z1MPXnj/FGzVdzMq18dU7Z7G4OHXUfqexFggGeLP1TbbWbGVH7Q7s/XbM\nOjOrC1ezvmg9t+TdgkE78nmlkiSNr1hsi8dKvPSJAXaebuNzfzlGh9PLx1dP46HV0zDoLr8mmbPf\nz6unm/jzyX/wVvsBAglVaIwNKIogQZPITdk3cWvBLSzLXUaBrWDcfg+PL8DvDtTxk53n6OzzctuM\nTP5tXRmzcqO7a9Hxhh6+vbWSf5zrpCDVxCPrprNpbm7U+uhN3W5eONbM80ebON7YA8BNxalsqsjh\nbXNySLMkRKVe0sjIYCHG9fv9/OXkXv567nnOOP9BADcAidpkpiXNoiKjgmUF85mfOTcm5wh7fAEO\n1nSx92wHe6raOd3iANQRFMunqVMmVpSlk2mV3wrGm2BQ4AsG8QUEPn8w4tgfDOL1C/zBIL7A5Y8D\nQZiakcis3KQrdjTGUix2ZhVF0aEu3ng70Ii6eOP7hRAnL1N+F9c4FWI4Qgj+fqyZb75YSXOPhzsr\ncvn822aQkzSxvm3wB/280fIG22q2saNuBz39PVj0Fm4rvI31xeu5Oedm9NrYnvIhSfEmFtvisRJv\nfeIel4+vbTnJXw43Up5j43vvrWBmzsUdftp6PeyobGPbqRb2nevEGwiSYtZz24ws1s3KYm6hnuOd\nb7KvaR/7m/bT1NcEQJ4lj2W5y8LTJsZja+K+fj/P7qvhZ7vP0+vxc8ecHB5eW8a0zPFdTLiu08V3\nt51hy9EmUhMNfOK2abx/SeF1rSsxVi509LHlaBPPH23iXJsTrUZh2dQ07qzIZd2s7DGbeun1B+n1\n+Oh1++hx++j1+Acd++h1+wcdqw+Hx0+6NYE5eUnMyUtidl4SU9IT4+5LVBksTCC+oI9z9nMcaz/G\nsY5jHGs/Fl4hV6NoKE0uZW7G3PCj2FYcMztNDGjr9bD3bAd7z6rrM3T2eQGYkW1lZZk6mmFxcWrM\nbWMnqbpdXk63OKhs7uVMi4Met0/9sB8Q+APB8PFAOOALCLyDjgcHCIHg6LUlCToNFQXJLCxKYVFR\nCgsKU0i5hlWLb0SsdmYVRXkb8ATqdpPPCCEeVRTl68AhIcTzQ8ruYhSChQEur5+ndp3nqT3VaBWF\nj62ayn0rp0zI/1/7gj4ONB9g64WtvFr3Kg6fA7POzNyMuczPnM+8zHlUZFSQqJfzQSUpmmK1LR4L\n8don3nayhS/833F63D4+tmoaCXoN20+1criuG4DCVLO6XkJ5FguLUtBpL+0DCyGoc9SF12Z4o+UN\n+nx9aBQNs9NmszR3KctylzE3Yy56zdgFyD1uH7/YW80zr13A7Qvwrvn5fOr2UgrTxvZLwk5nPz98\n9Ry/O1CLVqPwkeVT+OitU2J6lzchBKdbHGw52sSWY03Ud7kxaDWsmp7Bpopc1szMihiZHQwKHB4/\nvZ5QGHCZQOByocHQBeyH0msVkkx6bCY9NqP6bDXqaLS7qWzupT+0jl2iQcusXDVkmJNvY05e8qQP\nG2SwMMF1e7o53nE8HDQcbz+Ow6eOCrAarMxNvxg0zEmfMy5p7EgFg4JTzb3hoOFQjR1vIEiCTsOS\nKWmsLE1nZVkGpZkWuUrsOPMHgtR09nGq2cHp5t5wmNDc4wmXSTHrybAmoNNo0Os0GLTKsMd6rYJe\no0GvU9BrNaHHpcc6rfq+4Y71WgXDMMcAlc29HKqx82ZtFyebevGHAoupGYksKkplYVEKC4tTmJKe\nOCb/HcVTZxaurS2u73LxzRcreelEC/kpJr50RznrZ8Xm9pQj4Q142d+0n72NeznSdoSz3WcJiiAa\nRUNZShnzMuYxP3M+8zPnk2PJiXZ1JSmuxFNbHM994q4+L195/iRbjqqjDubmJ7F2prr4YlnWtfcX\nfUEfJzpOhIOG4x3HCYogZp2Zm7JvCgcNxbbiMfm3q9PZz1O7z/Pr/bUEgoK7FxfwidtKyU4a3ZG8\nLq+f/957gZ/tqcbtC/DeRQV8ek0pWbaJNWJYCMGR+m6eP9rEC8eaaXP0YzZoKUpLxBEKEpz9fq70\ncVVRCAUCOjUgMKoPNSzQqcdm/aVlTGqZBJ3msv8t+AJBzrU5Od7Yw4nGHo439nCq6UphQxIl6ZYJ\nsS7VSMhgYZIJiiA1PTUcbT8aDhvOdZ8jKNT/oIttxczNmEtFRgVzM+YyLXkaOk1sLKro8vo5UN3F\nnrPt7Klq53x7HwBZtgRWlGawsiyD5dPSr2n/XOnq7H1eKlt6qQyFCJUtvVS1OsM7h+g0ClMzLMzM\nsTIjx8aMbCvlOTYyrAkx9wHR7Q1wtKGbN2vt4UeP2weoQcjCohQWhsKGuflJo/INejx1ZuH62uJ9\n5zr42pZTnGl1cMu0NL6yaRZlMbw95Ug5vU6OtR/jcPthDrcd5lj7Mdx+dbpaljkrPKJhfuZ8ylLK\nYqatlaTJKJ7aYtknhhONPaRZDKM+1a7X28vB5oPsb1aDhnpHPQDZidnqtImcm1mSs4QU4+guvtjS\n4+FHO8/yvwfrURSFDy4t4sFVU0m/wTUFfIEgfzhUzxM7ztLu6Gf9rCw2r58x7lMvxkIgKDhwoZO/\nH2umtcdzcRSBSY/NqAsHAREBgUmPxaAb11ED/kCQc+1OjjeoQcPxxh4qm3vx+NR+ttmgZVauTQ0b\nQo8pGRMzbJDBQhzo8/VxsuMkxzqOqYFD+zG6POqqqyadiVlps8KjGioyKq5ri8ux0Njt5rWz7eyp\n6uC1cx30uH0oiro1UIrZgCVBhzlBhyVBS6JBR2KCjsQELYkJOvWaQT23JISuhc4Tx7lBiRW+QJAL\nHX1UNodChJZeKpt7w7t4gLr2xcxQeDAj28bMHBtTMxNjas7dtQgGBdUdztCIBvVR3aEGVnqtwqzc\nJBYVpbCoOIUFRSnXtdZHPHVm4frbYn8gyO8O1PH49iqc/X4+uLSIh9eUkWSO3eGX18of9FNlr+Jw\n22GOtB3hcNvh8BZoZp2ZORlzwiMa5PQJSRpd8dQWyz7x+Kl31LO/aT+vN7/O682v4/A6UFCYmTaT\npTlLmZU2i7KUMgqsBWg1N95Xqu9y8YNXzvKXtxow6rV8+JYS7lsx5Zr/rRRC8PLJFv5r6xmqO/pY\nVJTC5982g4VFE2dR5clscNgQHtkwJGwoz7kYNszNnxhhgwwW4pAQgkZnY8RaDZVdlfiDfgByE3Mj\n1mqYmToz6quhB4KC44097K1q50h9Nw6Pnz6vn75+P87+AH39V58TNZhJPxBAaDEbdKHwQTsogAhd\nS7h4fDG80JFoUK+Z9VrMCVoM2ssPi4qGTmd/ePpCZbP6fK7NiTegNlh6rToKoTzHxowcayhMUEch\nTHadzn7equvmUG0Xb9XaOdrQEx6dUZhqDo1qUMOG0kzrVRvxeOrMwo23xV19Xh7ffobfH6gjyaTn\nkfXT+efFhTH/j+X1anY2c7hNHdFwpP0IVfYqOX1CksZAPLXFsk8cHf6gn1Odp8KLQB5rP4ZfqH1n\no9bI1OSplKWUUZpSSllKGWUpZdc9suFcm5MndlTx92PN2Iw67l85hXtvKSFxBNvDv3Ghi2+9VMnh\num6mZVr49w0zWDMzM6b6qdKl/IEg59v7LplGMfD5ZmjYMCc/iakxFjbIYEECoD/QT2VnZUTY0NzX\nDIBeo2dm6szwOg1lKWUUJRWN6aI21yMQFLi8fvr6A4NCBz+u0LmzX32tLxRE9HkvHjv7/bi8F4/V\n6yMPKrQaJRwymA26UHChxWRQQwiTQYvZoIYTpkHPZoNa3mwIlderxwP3Meu1Vxxd4fUHqe5wUtnc\ny+lmB5WhMKHdcXEUQoY1gZk5NmZmW8MhwpR0S9R2UYg1/f4AJ5t6ebPGzqHaLt6stdPhVBcVtSbo\nmF+UwsJCNWiYV5B8yT/q8dSZhdFri0819fLVLSd540IX5Tnq9pQ3lUz+b1Lk9AlJGhvx1BbLPnFs\n8Pg9VPdUU2WvospexVn7WarsVeFRwQAZpoyIoKE0pZQpSVNG/IXdqaZeHt9+hh2VbaQmGvjYqql8\nYGnRsFM5z7Y6+M7W0+yobCPLlsBn1pZx14L8YRexlCaGQFBwftA0ihONPZwcFDaY9FrKc9W1Gmbl\n2shNNpFs1pOaaCDFbBj3RbPHJFhQFGUD8APU1ch/IYT49pDrDwAfBwKo+6jfL4Q4daV7ykZ0/LW5\n2jjefpyjHer0iZMdJ/EE1MX7dIqO4qRipiZPZWryVKYlT2Nq8lQKrYWTpiMcDArcvkA4ZBgaOrhD\nwYTbFwgHGm5vAJcvgCsUVLi8A88XjwcWcBkpo14TDh8GBxFdfV7OtzvxBdT/Txq0GkqzLKEpDGqA\nMD3besPz8+KNEIK6Lhdv1to5VGvnrVo7Z1odCAEaBWbm2NSdJ4pSWFScSn6KOW46szC6bbEQgheO\nN/PNFypp6vHw9rk5fOFtM8lNnljbU96IEU+fyJjP3Iy5WAwTf16sJI0FGSxIsaLD3RERNJy1n+V8\n93m8QfVLC62ipSSpJCJwKEspI8t8+cWN36qz8/i2Kl4710GWLYFP3FbKexcVYNBpaO5x8/3tVfzp\nzQYSDToeXD2Ve5eVROyUIE0eVwsbBjPptaSY9aSEggb1Wa8eD3o9NdEQDiRMeu11j24Z9WBBURQt\n6v7pa4EG1P3T3zc4OFAUxSaE6A0d3wl8TAix4Ur3lY1o9PmCPqq7qznbrTaQ5+znONd9jkZnIwL1\nvwu9Rk9xUjHTkqeFw4ZpydPIt+SPytyzyWBgZIXbG6BvSPgwEFYMDicGwg21/MWylgSdOhIhFCKU\npCeil6n0mOhx+zhcp4YMh2rtHKnvxhUa0VL7nbfHTWcWxqYtdnsDPLX7PE/tPo+iwMdWTeP+Cbo9\n5Wi40vSJ0uRS5mXOY1baLPKt+RRYC8g0Z8bc1sKSNN5ksCDFMn/QT11v3SWjG5r6msJlrHprOGwY\n/Dx4PZ795zt5bNsZ3qy1U5BqYkVpBn9+swEh4IM3F/HQ6mnjtt22FDsCQUFNZx8djn7sLi92l4+u\nPi/dLi9dfT712eWlO/T6wMLmwzHoNKSaDREjH1IS9aHXIkOIgbAi0aCGEWMRLNwMfFUIsT50/nkA\nIcS3LlP+fcA9QoiNV7qvbERjl9vvprqnOiJsON99PqKxTNAmUJJUEg4aBkKHPEue7BBLE44/EOR0\ni4NDNV3cu3xK3HRmYWzb4ga7uj3li8dbyEs28aU7ZrJhdnbczwt1ep0c6zgWHtFwrP0YLr8rfN2g\nMZBnzSPfogYNBdaCcOiQZ8nDqJtY24lJ0vWQwYI0ETm8Ds51n6OqKxQ4dKuBQ5+vL1wm35IfMbph\nWvI0LrSY+P72c5xs6uWd8/L4zNoyClLNUfxNpInEHwjS4/aFQwh7n/eS46GBhN3lvew2nnqtQorZ\nwMEvrR31YOE9wAYhxEdC5x8ElgghHhpS7uPAZwADcJsQ4uww97ofuB+gsLBwYW1t7UjrKcWAPl8f\n1d3VnOu+GDac6z4XHuYL6q4UJUklEaMbpiVPIycxJ+4/TEgTQzx1ZmF8OrT7znfw9S2nON3i4OYp\naXzlznJmZNvG9GdOJP6gn2ZnM/WOehqcDdQ76tVjh3o8OHQAyDRlkm/NDz/C4YMln1RjqmxrpUkh\nntpiGSxMbkIImvqawqMaBh61vbXh7eMHFosssk6hLHUqJUklFCcVU2AtiLk10KTJIRgU9HrUEQ+R\nYcTF8++8pyI6wcKg8u8H1gsh/uVK95WN6OTh8Do4330+HDQMhA7t7vZwGbPOfMn6DdOSp11x/pkk\nRUM8dWZh/NpifyDI79+o43vbqnB4fOr2lGvLSDbLIZ5XIoTA3m+PCBoGjhscDbS52yLKm3XmcNgw\ndMRDjiVHdlClCSOe2mLZJ45PgxeLHLx+Q6enM1xGq2jJt+ZTYlODhmJbcTh0SElIkX1oaUxdazs8\nklX5GoGCQef5odcu5zngpyOtgDTxWQ1W5mXOY17mvIjXe/p7wmHDwPOehj389dxfw2UsegtTkqdQ\nbCumyFZEoa2QImsRRbYizHo5BEySJgudVsM9NxezaW4uj2+v4jev1/L80SY+tmoa07IsJJn0EQ+5\nvohKURRSjamkGlOpyKi45LrH76HR2RgOHQZGPFT3VLO3YW94cTEAjaIhJzFHHekwzDQLq8E6nr+a\nJElSXDPqjJSnlVOeVh7xusProKanhpreGi70XAg/72vaF9Gm2ww2ipOKw6FDia2EkqQSdZSDVobI\n0vgbyYgFHerijbejBgoHgfcLIU4OKlM6MPVBUZRNwFeulm7IdDZ+2T32iLChuqea2t5a2lyR37xl\nmDIotBVSbCtWAwdbEUXWIgpsBSRo5c4I0tiIp2/JIHptcWVzL1/bcpLXq7uGvW42aLEZLwYNtiHB\nQ5JJR5I5dG1IuXhdJHKooAjS5mq7JHQYGO1g77dHlM8wZVCWWsaMlBnMSJ1BWWoZRdYiuVCvFBWx\n2hbLndKkaAkEAzT1NYVDh5qeGi70XqCmpyZilLBW0ZJnyYsIHQZGOsjpctK1GKvtJt8GPIHaiD4j\nhHhUUZSvA4eEEM8rivIDYA3gA+zAQ4ODh+HIRlQayuVzUe+op7a3NuJR56iL2ENYQSEnMedi2DDo\nkWvJlUN9pRsSq53ZsRLNtlgIQU2nC7tLXdG41+2jx+2jxxV6HvTo9fjD1539/iveN0GnuWQEhO2y\nAYWe1EQ92UkmLAmTY2vdkXJ6nRFrOpyzn+OM/QzV3dX4hfo3NmqNlKaUMj11OtNTpjMjdcYlq5rf\nCCEEvW4/Lb0eOpz95KeYKEw1y46vFJNtsdwpTYpVTq/zkhEONb011PXW0R/oD5ezGqwXRzgklVBs\nKw5/gWfQyqmJUqQxCRbGgmxEpWvR6+2lrrdODRp668KNZW1vLQ6fI1xuIKUdCBoGhw/Z5mz5zZt0\nVbHYmR1LE7Et9geC9Hr8l4YPQ56Hezg8lw8lrAk6spOMZCcZyUkykp1kCj2r5zk2EzaTbtJ/6PUG\nvFT3VHO66zRnus5wxn6GM11n6PX2hssUWgvDYcP0VDVwGLpmTiAoaHf009LroaXHTUuPh5beflp7\nPTT3uGnt7aelx3PJXt3JZj1z8pKYV5DM3PxkKvKTyLTJnTDiTSy2xXKnNGmiCYogzX3NatAweHpF\nT03EGj0aRaOOcrAVU5xUTKG1kEJrIQW2AnISc9Bp4it4l1RjscaCJEWdzWBjdvpsZqfPjnh9YGGz\noWFDbW8th1oP4fa7w2UNGgMF1oJLp1fYisgwZUz6DwuSNFnotBpSE9W9l69VIChwDgklOvsGPux6\naOnx0NTjoaq1nTZH/yVbMZn02nDYEBFA2C6epyYaJnR7YtAamJGqTocYIISgpa+FM/YznO46zanO\n0xxvP8X22u3hMnolEZMoAG8unr4seroz8HsyGdzV0GsVsmxGsm1GynNt3DYjk5wkI1k2I2kWA7Wd\nLo7Wd3O0oYef7DpPIKj+D5CTZGRufhIVBclU5CczJz8Jm1GOTpPGXR5QP+i8AVgytNDQndKGu9GQ\nndJGvaKSBBcDgzxLHsvzlkdc6/P1RQQNA9MrDrYcxBPwhMvpNLrwujyFtkL12VpIoa1QjhSWIsgR\nC9KkJYSg3d1+cUrFoPChzlGHL+gLlzVqjWQnZpOdmE1OYs6wz3Lv+PgQi9+SjSXZFl+eLxCk3dEf\nDhyaQ9+6N/eq5y09Hlp7PfiDkf+OGnQasm2DgwdjKHhQR0DkJBlJtySg0cRe+CCEoKvPGxpl4KGl\n10Nr6Lk59Pu29HjoHRj5oelHm9CMxtiM0dyC3tSCX9eMUNQFxjToyDYVMjW5jNnpM5ifNYuZaTNI\nNiZftS5ub4CTTT0cqe/mWEMPxxq6qem8uPXmlIxE5uUnMzc/ibkFyZTn2OT6GpNILLbFcqc0KR4M\n9J/reuvCU5TrHOpxXW9dxBbIWkVLriVXHd0QCh4GRjrkW/Ll9IoJTo5YkKQQRVHINGeSac5kcfbi\niGuBYIAWV0s4dGhwNNDS10JLXwuvNb5Gh7sDQeSHhZSElEsDB0s22Wb1ON2ULqdaSNIkotdqyE02\nkZtsumyZQFDQ6VTDBzWAcIeDh+YeD4frumnp8eANBCPep9OEvrkPBQ82ow4hQAgICkFQgECEXhs4\nV68JIcLl1GeAUJlQ2WDoS4PgkLIidN9gxGvqebfbS2tP/yV1VRTIsCSQk2SkOC2RpVPSyLKFQhOb\nkazQc2JojYpAMECto1adRhGaSnG66032Nm+F4+o9s8xZl0ylKLAWoFEu7gZiMmhZVJzKouLU8Gvd\nLi/HGnrCoxr2nuvgL4cbw3/TGTlWKvLVUQ0VBclMy7SgjcEAR5qw5E5p0qQ3uP+8KDvyM6UQgk5P\nZzhkqHPUhZ+Pth/F6XNevE9oTbQCWwFF1qKI0Q751nz5hd0kJEcsSNIwfAEfra5Wmvuaw4FDc19z\n+Ly5r5k+X1/Ee3SKjqzELLLMWeRYctTgwZxNjuViECG3c4t9sfgt2ViSbfHYGxgFEBE+9FwMH5p7\n3Dj7A2g1akdMo6gdO0VRP9RrFAWF0HPomkZRy0acD5QNlx+u7MX7DC6rKJBk0qtBR2iqwkDokWFJ\nQDcK2392ujs5Yz9DVVcVp+3q+g0Xei4QEOoaCyadiVRjKlaDlUR9Ila9FYvBgkVvCT9bDdaI80R9\nIp5+A9Wtfs40+zjW0Mvxhh4coQU+zQYts3OTqChIYm5+MvMKkslPMU3oqSrxIhbbYrlTmiRdnhCC\n7v7ucNhQ76hXRzr01lPrqKWnvyeifJY5KzzCITzSIbQNstxyPjbIEQuSNAr0Wr2617s1/7JlHF5H\nOGQY+nyk7Qgv970cXll9QKI+MTziYSBsGHyeZc6Sw8YkaZJRFIU0SwJplgRm5yVFuzpRk2ZKY5lp\nGctyl4Vf6w/0c777PGe6zlBlr8Leb6fP24fD56C5rxlntxOnz4nT6wwHEFeSmJhI1mwLBRozBI14\nvQbq3XpOnNfw6zNGRCABky6RguRUpqalMzMrkzk52RQkp2AxWLDqrXL/d+myhBB+RVEeAl7m4k5p\nJwfvlAY8pCjK4J3SrjgNQpImC0VRSDGmkGJMoSKj4pLrPf09ESMdBo531u+M2P0NINWYGu4XD/SR\ns83Z4eMMc4Zc2yEGyWBBkq6T1WDFarBSmlI67PVAMECnp/OSUQ8Dz6c6T13SkMLlG9OsRPU805wp\nG1NJkiaFBG0C5WnllKeVX7GcEAK3362GDKGgwel14vA56PP14fA6Lr7uu3jN6XViNHeTYHXS63Xg\nC3oJArVArR1etQOnI3+WRZuGTZeDTZuDRZuNRZNDoiYbo5IJQocvECQQFPiDYtBzEH9APR58PnDd\nP+g88r0X3zdwDmBJ0GMz6bAa9diMOmxGPVajDptJjzUh9Dz09dCzxaCLyfU7LscXCOLxBXD7Ani8\nQTz+AG7v1UOkaBFCvAi8OOS1Lw86/tS4V0qSJoCkhCSSEpIuWYgd1O0y6xwXp1Y0OZtocbVQ76jn\nYMvBiCkWoI7CyzBlhPvGEX3mUL9ZTlEefzJYkKQxotVow3PUhktuATx+D62uVpqcTbS6WsMBRKur\nlXpHPYdaDkVspwlqY5pmSotIbgc3qFnmLDLMGXJrIEmSJg1FUTDrzZj1ZjLJvO77+AI+NYzw9tHW\n182J5lYqW9s419lBnb2L7v5ufPouegydKIZqNLqLi5QJoaAEklF8GWgCGegCGeiCGRhEFgaRjk6j\nR6dV0Go06DUKWo1Cgl6DWaNBFzrXh64PnF98XRM+F4DD46fXo26P2uv20dTtptfjx+Hx4fEFL/8L\nok5rsSQMCh2MkSGFNeJ8SDAROjdoNXj8ATy+oPqB36d+2Pf4Lr428LonfD0Y8Vr4fb7goDKBUHBw\n8bWhi59KkhR/LAbLFUNmp9cZ0U9ucbWEj8/az/Ja42sRO8GBurBkhjkjor88eORDVmIWqcbUiLV9\npBsjP3k2YI0mAAAOmUlEQVRIUhQZdcbwlpeX0+fro7WvNaIhHWhcq3uq2de0L2KFXlC3F0o3pUeG\nDoNGPcgkV5KkeKTX6knVppJqTKXAVsDCnDkR111eP0KATqug02hweNWhu7WO2vB2xnW9ddQ6juHw\nOugH+lA7sDmJOeTbisJbGRda1edcS+6oBr1efxCHxxcOGnrdoeeBICIURlwMJ3w0dXtw9DvCZcfq\ns7xBq8Go12DUazEZtJj0WhL0Wkx6DWmJBozJg1/TYjJoMOrUska9+jDptRj1GtZ8Z2zqKEnSxGMx\nqGvrTE2eOux1IQS93t6IPvLgL+tOdZ7i1bpX8Qa9Ee/Ta/RkmjOHnW6RbkonOSGZVGMqJp1cm2ck\nZLAgSTEuUZ/IlOQpTEmectkyA+s9DG1QW12tI05yB0Y6pJnSSDOmhZ+TE5JlACFJUlwwGyK7RcnG\nZJKNyczJiAwgBhYpG9iGLRw49NZypP1IxOK+OkVHnjUvHDQU2grDK6TnJOZcc/tq0GnCa3ZcDyEE\nfd7A8KGEWw0svP5gOBgIBwUDH/wN2lAYoBkSBmjlDhySJEWFoijhqRbTU6cPW0YIgb3fHhE6tLha\nwl/eHWk7QqurFX/Qf8l7E7QJ4ZAhOSE5vJZESkLKsMdJhqS47DvLYEGSJoGrrfdwuSR34PhU5yl2\n1u+kP9B/yXs1ioaUhJRLAoc0U9olr6UYU+QUDEmSJr3Bi5TNy5wXcW1gO7bwCIdBwcOh1kMRIa9e\noy4UPBA0FNmKyLPkYTFYMOvMmHQmdQqIzkyCNmFUvjFTFAVLgg5Lgo6c+F1LVJKkOKMoCqlGdcTa\n5aZcBEWQTncnLX0tdHm66PKoU+TsHnvEcYOzAbvHfsnaD+GfhRp0XC58GAgpBr82GbbflJ8AJCkO\njDTJdfgcdLo71Yfn0ucudxd1jjo63Z14Ap5Lfw4KyQnJ4aAh1ZR6+TDCmCZXX5ckadJRFIV0Uzrp\npnQWZC2IuCaEoN3dPmhKxcWRDvub9w8b7g7QKBrMOjVkMOsjQ4fhnk0601WvmXSmuPxWTZIkaTga\nRUOGOYMMc8aIynsD3nDYYO+3q8/DHNf21nKk7Qjd/d2X3eHIpDNdDB6MyaQmqMHDQBiSZkojJSGF\nVFNqeHpGrJHBgiRJgNoZthls2Aw2SpJKrlhWCIHL74oMHoYJI050nKDT3XnJGhADbAbbJaMeBvaw\nv9rDqDXK+W6SJE0oiqKEF/VdnL044lpQBGlztdHkbMLld+HyuS55dvvduPwu3D712eV30eXposHZ\nEFFuJFtzDjBqjZcEFQPtrNVgxaK3hLfiHPw88LpFb8FqsMqtkiVJijsGrSHcpo9EUARxeB3YPXa6\n+7vDoyC6PF2Rr3m6qempocvTdclU5gEmnSkcOgyMfhh8PvQxHl/myWBBkqRrpihKuONZaCu8anm3\n3z38KIhBx2e6ztDp7sTpcyK4+spiWkWLWa92gC16i3qsSwwPIR54jrgeOh4aUkyERXkURdkA/AB1\n7/RfCCG+PeT6A8DHgQDgBO4XQpwa94pKknRdNIomvGjYjRBC4Av6Lg0mrhZSDLrm9Dpp7mumz9uH\nw+e4bMd2MIPGoAYPg8IIi/5i8DA4hIi4PiiskIGxJEmTmUbRhEcQj5TL58Leb6fL3YW9306nuzMc\nRAxM12hztXG66zRdni58Qd+w97HqraSaUtVRD8bU8MiH4cKJ5ITk65raLIMFSZLGnElnIt+aT741\n/6plgyKIx+/B6XPS5+tTO7mh46s+/H20udro8198LSiuvDUbXBxinKhPHI1fd9QpiqIFfgysBRqA\ng4qiPD8kOPi9EOKpUPk7gceBDeNeWUmSokpRFAxaAwatgWSSR+We/qCfPl8fDq8Dp8+Jw+uIOHd6\nnTh8Dpzei8d9vj5q3bXh632+vquGxjpFFw4cJEmSJMJbLedZ8q5aVgiB0+cMBw+dniEhhFsNIuqd\n9RxtP4q93z5sP3lgavO1ksGCJEkxRaNowo3ojRJC4Al4RhZKhB6v8Moo/Baj7ibgnBCiGkBRlOeA\ndwDhYEEI0TuofCKMYNiHJEnSCOg0umv+lm2ooAjS5+uLCB4c3lAYEQorBj9vZeso/gaSJEmTn6Io\n4QXdr7SV/YCgCNLb30uXp+vSEMLTxV72XtPPl8GCJEmTlqIomHQmTDoT6ab/3969xdhV11Ec/67e\nQqFEDEVT2ypFGyIhxmKDKAqGggExxfhUEow3gg/FFHnwjkZ5Qo3xxRgbqCGRS6BA0pAKaGz0yUpL\nIbQUlEuBItgxooAk0uLyYe8ph2lrz5zM/P/b2euTTHrOzJ75rTNnZs3Jv+fs/8KhPudarp3mVCNZ\nDDw7cH0v8MGJB0laC1wNzAPOKxMtIuLoZmnWwQe8i1h01OOv47oCqSIi+muWZh3cVvkUDt3W/hqu\nmdzXm6pgERFRl+2f2n438DXg20c6TtIVkrZJ2jY2NlYuYERERETMSFlYiIjovueApQPXl7TvO5Jb\ngU8d6YO219teaXvlSScNt6VSRERERMSRZGEhIqL77geWS1omaR6wBtg0eICk5QNXLwb+XDBfRERE\nRPRYzrEQEdFxtg9IuhK4l2a7yQ22d0n6PrDN9ibgSknnA/uBF4HP1kscEREREX0y1MLCEPunXw1c\nDhwAxoAv2H56irNGRPSW7c3A5gnv+87A5XXFQ0VEREREMMRLIQb2T78IOA24VNJpEw7bAay0/T5g\nI/CDqQ4aEREREREREd0zzDkWDu6fbvs1mpOCXTJ4gO0ttl9tr/6B5sRiERERERERETHDDbOwcLj9\n0xf/j+O/CPzqcB/IFmcRERERERERM8uUnrxR0mXASuDcw33c9npgfXvsy5Iem8r5I1gI/C0Zqmeo\nPT8ZkmHQqRVnF7d9+/ZXKndx7fs7GZKhaxlqz+9Kht50cQd6GLpxnydD/fnJkAyDJtXDwywsDLV/\nens28m8B59r+9xBf9zHbK4dKOU0kbUuG+hlqz0+GZJg4v9bsSqp2ce37OxmSoWsZas/vUoaa8wvL\nY+Jk6MT8ZEiGifMnc/wwL4UYZv/0FcDPgdW2900mQERERERERET8/zrqwoLtA8D4/um7gdvG90+X\ntLo97IfAAuB2SQ9K2nSELxcRERERERERM8hQ51gYYv/080eYvX6Ez5lqydConaH2fEiGcclQf35p\ntW9v7fmQDOOSoVE7Q+35kAyldeG2JkOjdoba8yEZxiXDJOfL9nQFiYiIiIiIiIgZbphzLERERERE\nREREHFYWFiIiIiIiIiJiZFUWFiRdKOkxSY9L+nqF+Rsk7ZO0s/Tsdv5SSVskPSJpl6R1FTIcI+mP\nkh5qM3yvdIaBLLMl7ZB0d6X5eyQ93J54tMr2VpJOkLRR0qOSdkv6UMHZp7a3ffztJUlXlZo/kOMr\n7c/iTkm3SDqmQoZ17fxdNb4HJfW9h9sM6eI3cqSHK/ZwOz9dTL96GNLF6eFDsqSL85i4eg+3GSbf\nxbaLvgGzgSeAU4B5wEPAaYUznAOcAewsffvb+YuAM9rLxwN/qvA9ELCgvTwX2AqcVen7cTVwM3B3\npfl7gIU1Zg9kuBG4vL08DzihUo7ZwAvAuwrPXQw8Bcxvr98GfK5whtOBncCxNCe2/Q3wnpo/F9N8\nP/e6h9sM6eI3cqSHO9LD7fxednGfenjgfu51F6eHD8mSLu5IF/e1h9uZI3VxjWcsnAk8bvtJ268B\ntwKXlAxg+/fA30vOnDD/edsPtJdfptnGc3HhDLb9Snt1bvtW/EyekpYAFwPXl57dFZLeQvOH/QYA\n26/Z/kelOKuAJ2w/XWH2HGC+pDk0RfaXwvPfC2y1/aqbbXZ/B3y6cIZSet/DbYZ0Melh6FwPQ3+7\nuE89DOni9PCAdHHnurivPQwjdnGNhYXFwLMD1/dSuEC6RNLJwAqa1dHSs2dLehDYB/zadvEMwE+A\nrwL/qTB7nIH7JG2XdEWF+cuAMeAX7dPfrpd0XIUcAGuAW0oPtf0c8CPgGeB54J+27yscYyfwUUkn\nSjoW+ASwtHCGUtLDE/S8i9PD3eph6G8X96mHIV38Jj3vYUgXQ7e6uK89DCN2cU7eWJGkBcAdwFW2\nXyo93/brtt8PLAHOlHR6yfmSPgnss7295NzD+IjtM4CLgLWSzik8fw7N0xB/ZnsF8C+gxuss5wGr\ngdsrzH4rzf/SLAPeARwn6bKSGWzvBq4D7gPuAR4EXi+ZIerocxenhw/qRA9Dv7s4Pdxffe5hSBcP\n6EQX97mHYfQurrGw8BxvXvFY0r6vVyTNpSnQm2zfWTNL+xSjLcCFhUefDayWtIfm6X/nSfpl4Qzj\nK4PY3gfcRfPUxJL2AnsHVsc30pRqaRcBD9j+a4XZ5wNP2R6zvR+4E/hw6RC2b7D9AdvnAC/SvNZz\nJkoPt9LF6eFWV3oYet7FPephSBcD6eFWurjRlS7udQ/DaF1cY2HhfmC5pGXtatAaYFOFHNVIEs1r\nh3bb/nGlDCdJOqG9PB+4AHi0ZAbb37C9xPbJND8Hv7VddEVO0nGSjh+/DHyc5uk/xdh+AXhW0qnt\nu1YBj5TM0LqUCk/5aj0DnCXp2Pb3YxXN6yyLkvS29t930ryW7ObSGQrpfQ9DuhjSw+M61MPQ8y7u\nUQ9Dujg93EoXNzrUxb3uYRiti+dMd6iJbB+QdCVwL83ZNjfY3lUyg6RbgI8BCyXtBb5r+4aCEc4G\nPgM83L6eC+CbtjcXzLAIuFHSbJoFpttsV9naprK3A3c1v7fMAW62fU+FHF8GbmofWDwJfL7k8PYP\nyAXAl0rOHWd7q6SNwAPAAWAHsL5ClDsknQjsB9ZWPnnbtEkPH5Qu7ob0cCtdDPSkhyFd3EoPd0e6\nmPTwgEl3seziJz2NiIiIiIiIiBkiJ2+MiIiIiIiIiJFlYSEiIiIiIiIiRpaFhYiIiIiIiIgYWRYW\nIiIiIiIiImJkWViIiIiIiIiIiJFlYSEiIiIiIiIiRpaFhYiIiIiIiIgY2X8B/IwAEL1QUzQAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1296x288 with 3 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,3, figsize=(18, 4))\n",
    "pd.DataFrame(history_vanilla.history).plot(ax=ax[0])\n",
    "ax[0].set_title(\"Vanilla model\")\n",
    "pd.DataFrame(history_dropout.history).plot(ax=ax[1])\n",
    "ax[1].set_title(\"Dropout model\")\n",
    "pd.DataFrame(history_resbatch.history).plot(ax=ax[2])\n",
    "ax[2].set_title(\"Batch normalization / Residuak connections model\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "m_r-WFlP9UpX"
   },
   "source": [
    "## 3. Classifying breast histopathology images\n",
    "\n",
    "Before proceeding, we should get the data. We have downloaded the dataset onto a Google Drive, where we can unzip the file and load the images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "IfNTLT8f8dCa",
    "outputId": "5126eb2f-28df-4cd0-a805-c1f14f7e8ee8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive', force_remount=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fwrbWiPZJftV"
   },
   "outputs": [],
   "source": [
    "# unzip the new zip file and store all its files into our images directory\n",
    "!unzip -q -o drive/'My Drive'/'histopathology'/IDC_regular_ps50_idx5.zip -d images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CqP4asWphi4s"
   },
   "outputs": [],
   "source": [
    "# load the images\n",
    "imagePatches = glob('images/**/*.png', recursive=True)\n",
    "\n",
    "# seperate our two classes\n",
    "class0 = fnmatch.filter(imagePatches, '*class0.png')\n",
    "class1 = fnmatch.filter(imagePatches, '*class1.png')\n",
    "\n",
    "# create our target variable\n",
    "y = []\n",
    "for img in imagePatches:\n",
    "    if img in class0:\n",
    "        y.append(0)\n",
    "    elif img in class1:\n",
    "        y.append(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "P5YPuOz0jUn3"
   },
   "source": [
    "We turn the images and the target into a dataframe so we can see the class proportions and create a test set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bFDzzdIqirmf"
   },
   "outputs": [],
   "source": [
    "images_df = pd.DataFrame()\n",
    "images_df['images'] = imagePatches\n",
    "images_df['y'] = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "id": "yOJJ2Voo4Dap",
    "outputId": "e562b26c-91cb-45a2-87db-c43e80e9357b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "y\n",
       "0    198738\n",
       "1     78786\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images_df.groupby('y').size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "OvjSOXoD6iqP"
   },
   "source": [
    "Given the imbalanced proportions of our dataset, we implement downsampling."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iJdy5Fa26veR"
   },
   "outputs": [],
   "source": [
    "images_0 = images_df[images_df.y == 0]  # all our images w/ class 0\n",
    "images_1 = images_df[images_df.y == 1]  # all our images w/ class 1\n",
    "\n",
    "# downsample the majority class (0)\n",
    "images_0 = resample(images_0, replace=False, n_samples=images_1.shape[0], random_state=42)\n",
    "# images_1 = resample(images_1, replace=False, n_samples=100000, random_state=42)\n",
    "\n",
    "df = pd.concat([images_0, images_1])    # join the two back together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "colab_type": "code",
    "id": "yugdMA6t7mx5",
    "outputId": "9274b67a-005a-4663-faa5-a5a6846dae4c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>images</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21948</th>\n",
       "      <td>images/10268/0/10268_idx5_x3501_y1101_class0.png</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83757</th>\n",
       "      <td>images/14211/0/14211_idx5_x701_y2101_class0.png</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181091</th>\n",
       "      <td>images/13616/0/13616_idx5_x1551_y701_class0.png</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  images  y\n",
       "21948   images/10268/0/10268_idx5_x3501_y1101_class0.png  0\n",
       "83757    images/14211/0/14211_idx5_x701_y2101_class0.png  0\n",
       "181091   images/13616/0/13616_idx5_x1551_y701_class0.png  0"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "k9EqVO5oNfF_"
   },
   "outputs": [],
   "source": [
    "df = df.sample(frac=1).reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_4lZAyAe8I_3"
   },
   "source": [
    "We can now process our images:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oib_0HeB8NSm"
   },
   "outputs": [],
   "source": [
    "y = df.y\n",
    "X = np.asarray([cv2.resize(cv2.imread(img), (50,50)) for img in df.images])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "atJoGwNG78-z"
   },
   "source": [
    "We create a test set and encode/process our data accordingly:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jSEN2-HuIOSy"
   },
   "outputs": [],
   "source": [
    "# encode the target accordingly\n",
    "n_classes = len(np.unique(y))                  # number of classes in our target\n",
    "y = to_categorical(y, n_classes)               # encode training\n",
    "\n",
    "# cast X to float and scale\n",
    "X = X.astype('float32')\n",
    "X /= 255"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p319S59XG_dm"
   },
   "source": [
    "Now that our data is ready, we can proceed with our tasks. \n",
    "\n",
    "**3.1**: A model without residual connections.\n",
    "\n",
    "**3.2**: Augment the data. How much has this improved the original model?\n",
    "\n",
    "**3.3**: Build a deeper model using residual connections. Show that you can build a deep model that would not be able to learn without the residual connections.\n",
    "\n",
    "#### 3.1 Simple Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "colab_type": "code",
    "id": "k0M1IlqABPDK",
    "outputId": "06d26b3d-9e5a-45ff-96c7-ba8982f47aaa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "input_shape = (50, 50, 3)\n",
    "simple = Sequential()\n",
    "\n",
    "                                                             ### Layer 1 ###\n",
    "simple.add(Conv2D(32, (2, 2), activation='relu',             # Convolutional L\n",
    "                  input_shape=input_shape))\n",
    "simple.add(BatchNormalization())                             # Batch Norm.\n",
    "simple.add(MaxPooling2D(pool_size=(2, 2)))                   # Pooling Layer\n",
    "\n",
    "                                                             ### Layer 2 ###\n",
    "simple.add(Conv2D(32, (2, 2), activation='relu'))            # Conv. Layer\n",
    "simple.add(BatchNormalization())                             # Batch Norm.\n",
    "simple.add(MaxPooling2D(pool_size=(2, 2)))                   # Pooling Layer\n",
    "\n",
    "                                                             ### Layer 3 ###\n",
    "simple.add(Flatten())                                        # Flatten the data\n",
    "simple.add(Dense(128, activation='relu'))                    # 64 units dense\n",
    "\n",
    "simple.add(Dense(2, activation='softmax'))                   ### Output ###\n",
    "\n",
    "simple.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 440
    },
    "colab_type": "code",
    "id": "g32bCdG7EDrf",
    "outputId": "ef08dce8-2397-4ce6-e0e1-0533b10be402"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 141814 samples, validate on 15758 samples\n",
      "Epoch 1/10\n",
      "141814/141814 [==============================] - 21s 145us/step - loss: 0.4027 - acc: 0.8292 - val_loss: 0.5967 - val_acc: 0.7365\n",
      "Epoch 2/10\n",
      "141814/141814 [==============================] - 17s 121us/step - loss: 0.3528 - acc: 0.8502 - val_loss: 1.2139 - val_acc: 0.6669\n",
      "Epoch 3/10\n",
      "141814/141814 [==============================] - 17s 121us/step - loss: 0.3371 - acc: 0.8576 - val_loss: 0.6629 - val_acc: 0.7253\n",
      "Epoch 4/10\n",
      "141814/141814 [==============================] - 18s 124us/step - loss: 0.3245 - acc: 0.8627 - val_loss: 0.3839 - val_acc: 0.8470\n",
      "Epoch 5/10\n",
      "141814/141814 [==============================] - 17s 121us/step - loss: 0.3112 - acc: 0.8694 - val_loss: 0.6542 - val_acc: 0.7390\n",
      "Epoch 6/10\n",
      "141814/141814 [==============================] - 17s 122us/step - loss: 0.2958 - acc: 0.8767 - val_loss: 1.3702 - val_acc: 0.6005\n",
      "Epoch 7/10\n",
      "141814/141814 [==============================] - 17s 122us/step - loss: 0.2799 - acc: 0.8847 - val_loss: 0.6503 - val_acc: 0.7796\n",
      "Epoch 8/10\n",
      "141814/141814 [==============================] - 18s 125us/step - loss: 0.2619 - acc: 0.8914 - val_loss: 0.8431 - val_acc: 0.6896\n",
      "Epoch 9/10\n",
      "141814/141814 [==============================] - 18s 124us/step - loss: 0.2435 - acc: 0.9003 - val_loss: 0.5116 - val_acc: 0.8015\n",
      "Epoch 10/10\n",
      "141814/141814 [==============================] - 17s 122us/step - loss: 0.2255 - acc: 0.9081 - val_loss: 0.9319 - val_acc: 0.7382\n"
     ]
    }
   ],
   "source": [
    "history_simple = simple.fit(X, y, batch_size=128, epochs=10, verbose=1,\n",
    "                            validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 281
    },
    "colab_type": "code",
    "id": "1Eea3u78UN2g",
    "outputId": "a29d46ca-5524-477d-e8e7-d6a05d21eae9"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VNXdx/HPLztJ2EEFgoKKCm6g\nKUrRqkULuG9FRWrdirXa+jy1Vm2tVn18Sh+rVVtcUNFWcEHUioqKC2pVUAJSZRMQFQIIEWRJQvbz\n/HEmMIRAJmGSO7nzfb9eeWXmzpk7vwzhmzPnnnuuOecQEZFwSQm6ABERiT+Fu4hICCncRURCSOEu\nIhJCCncRkRBSuIuIhJDCXVoFM7vQzKY1074fN7P/aY59iwRF4S4Jw8yOMbMPzWyjma03sw/M7HsA\nzrmJzrkfBV1jXeb9yszmmVmJmRWa2bNmdmjk8cfNzJnZwKjn7G9mLur+O2ZWZmY9o7adaGZftegP\nI6GicJeEYGbtgJeBvwGdgB7ArUB5kHXF4F7gGuBX+LoPAP4FnBLVZj3Q0CeDEuAPzVGgJCeFuySK\nAwCcc08556qdc1ucc9Occ58CmNnFZvZ+beNIb/gXZrbEzDab2e1mtl+k57/JzCaZWUak7fGRHvXv\nzOxbM/vKzC7cWSFmdqqZzTWzDZH9HbaTdn2Aq4ALnHNvO+fKnXOlkU8ZY6Ka/gM4zMyO28XPfx9w\ngZntF+sbJrIrCndJFIuBajP7h5kNN7OOMTxnKHAkcDTwW2AcMAroCRwCXBDVdi+gC/4TwU+BcWZ2\nYN0dmtkAYDxwBdAZeAiYYmaZ9bz+EKDQOfdxA3WWAv8L3LGLNiuBh/GfVkR2m8JdEoJzbhNwDODw\nIVdkZlPMbM9dPO3/nHObnHPzgXnANOfcMufcRuBVYECd9n+I9K7fBV4BRtSzz9HAQ865jyKfIP6B\nHxo6up62nYHVMf6IDwF7m9nwXbT5E3CamR0c4z5FdkrhLgnDObfQOXexcy4P3/PuDtyzi6esibq9\npZ77uVH3v3POlUTd/zqy/7r2Aa6NDMlsMLMN+E8C9bVdB3TbRX1bOefKgdsjXztrUwT8Hbgtln2K\n7IrCXRKSc24R8Dg+5OOho5nlRN3fG1hVT7sVwB3OuQ5RX9nOuafqafsWkGdm+THW8BjQATh7F23u\nBE7ADzeJNJnCXRKCmR1kZteaWV7kfk/8mPnMOL7MrWaWYWbHAqcCz9bT5mHg52Z2VGSaY46ZnWJm\nbes2dM4tAe4HnooctM0wsywzO9/MbqinfRVwC3D9zgp0zm0A7sIfQxBpMoW7JIrNwFHAR2ZWgg/1\necC1cdr/N8B3+N76RODnkU8H23HOFQA/ww+PfAcsBS7exX5/FWk7FtgAfAGcBby0k/ZP0fA4/b1A\ndQNtRHbJdLEOCTszOx6YEBnLF0kK6rmLiIRQg+FuZuPNbK2ZzdvJ42Zm95nZUjP71MyOiH+ZIiLS\nGLH03B8Hhu3i8eFAn8jXaOCB3S9LJH6cc+9oSEaSTYPh7px7D782xs6cAfzTeTOBDmYW09xfERFp\nHmlx2EcP/NzgWoWRbTvMCDCz0fjePTk5OUcedNBBcXh5EZHkMXv27G+dc10bahePcI+Zc24cfv0P\n8vPzXUFBQUu+vIhIq2dmX8fSLh6zZVbiT8+ulRfZJiIiAYlHuE8BLorMmjka2Oici3UxJRERaQYN\nDsuY2VPA8UAXMyvEnz6dDuCcexCYCpyMP5OvFLikuYoVEZHYNBjuzrkLGnjc4S9YICIiCUJnqIqI\nhJDCXUQkhBTuIiIhpHAXEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRw\nFxEJIYW7iEgIteiVmEREwq66xlFcXkVJeRWlFVUUl1dTUl61dVtJeRUDe3fmwL3aNmsdCncRSWo1\nNY6SiipKyqu3C+Di8ipKK+puiwR1RSS8a59Tse05ZZU1Db7mbWccrHAXEalPVXUNxeVVbC6r/apk\nc1lVZFslm6Juby6rorisis31BPWWyuqYXzMnI5WczDRyM9PIyUwjJzOVbu2zIrfTyM2s+3havc9p\n3ya9Gd8ZT+EuIi3KOUd5VQ2bokO3NpzL6wR1WRWbyyvrDfDSioZDOS3FaJuVRtusdHIz08jNSmPP\ndlnbgjgjjey6oZyRFhXGqVtDOjs9lZQUa4F3KD4U7iKyUxVVNWypqKa00ofplopqSiuqKa2o2na7\nspotFXUfr2ZLpR/q2FJRTUnFtnAuLq+isto1+NrZGanbBXPbrDR6dGiz9XbbrHRys/ztdllp5Gam\nR7b7EG+XlU5mWgpmrSeQ40nhLhIC5VXVlNQ5cFfbu60detgaunWDuJ5wLomEd1VNwyEcLTMthZzM\nNNqkp5Kd4b/aZKSyV7ss+uzhA7k2fNtmpUdCOWp7pg/lnMxU0lI1mW93KNxFAlBT4yitrBPGkeGG\nkqgZFnXDur7tJeXVVFQ3fBCvVlZ6CtkZaVHh64cc9mqXTpuM2lBO87fTfTjnZPr2PrTTotqlRm77\nQE9tRcMWYadwF2mAc46K6hpKy32PtrY3vN33Cj9zonbIorjOQbuSiqjwLq+itLIaF0On2AxyM7Yd\niKs9KNcpJ3vrmHBuZvrWMePoA3e127LTt4Vxm1Y2bixNp3CXUKmpcWyp3Ba2DYVx7fBDSUU1pZEQ\nrq99Y4YnaocmciIH7HIz0+iUk0HPTtlbgzo6jNtm7XgQrzagszNSk3bMWHaPwl1aXHUkgLc7KBcJ\n1XoP2lXWbtv5WHFt+1jmGNdKMSKzJXwIt8nw3zvlZNCzYzbZUcMRW79Hta/v8eyMNA1NSEJQuAvg\nA7esspryqhrKq6opq/TfyytrorbX1Num9vuW7cI5EsSVO24rr4o9gMFPZ6sdVsiJGu9t3yadbu2y\nosZ9/fhxTkYq2ZH5xdkZvidc+z06lJN5JoWEn8I9gdXOB649aOaHDPw4rh9C2DYTorxO0G4fxFG3\n63yv3d7YWRF1ZaSlbJ0hsfVgW3oaHbMz6NEhdWuvuG4QZ6dvf1Cu7vPbZKSSkaZZEyKNpXCPo4qq\nGkortoVu7XhtcXn9oVwaOeW5JGqst7h8+7Hi6kaEbnqqkZmWSmZaClnp/ntG1O32bdLJaptJZnpt\nm5Qd2m+9HXms/jZR29NTyEhN0UE6kQSjcI/Bxi2VrFhfytfrSlm+vpTl60tYvr6Uos3l23rUjZyO\nVjs0kLP1+7aDbrXDCbmZ28Z3a9vWnk2XnZG2dfw3OyOVjNSUYOcFV5RAeraf3iEigVO448ebV2/c\n4oM7EuBfry9lxXp/e0Np5XbtO0dCuHeXHHIz07eO6eZGj+1GAntriEcFeeimo336LLz0K+jWH856\nEDruE3RFIkkvacK9pLwq0uveMcALvyvd7nTotBSjR8c27N0pm1MO7cY+nbPZu1M2e3fKoWenNrTN\nav5Ff1qFqgqYdhN8/BB0HwDffAYPDIbhf4b+I9WLFwlQaMK9psaxdnN5VICXbBfg3xZXbNe+bVYa\n+3TOpl+3dgw9eK+oAM+mW/ssnfrckE2r4dmLYcVMGHQ1nPhH2LQK/nUlvPgL+HwqnHYv5HQJuFCR\n5NTqwv2bjWXMX7XRB/c6H9y1AR49xS7FoFt73/s+se+e9OyUvTXA9+mUQ/ts9b6b7OsPfbCXF8O5\n4+GQc/z2jvvAT1+CGWPh7dvh/kFwxt/hgKGBliuSjFpduL/wyUr+/NoiwB+U3LtTNvt2yeGEA7v6\nnnfnHPbulE2PDm00hS7enIOZD/ihmE694aIXYY++27dJSYXBv4L9h8Dzo+HJEXDkxfCjOyAzN5Cy\nRZKRuVgWuGgG+fn5rqCgoNHPW7lhC2s2lbF3p2w652ToJJSWUl7sD5rOew4OOhXOvB+y2u/6OVXl\nMP0O+OA+6NgLzh4HPQe2SLkiYWVms51z+Q21a3Vd2x4d2nDE3h3pkpupYG8p3y6FR06E+S/4sfXz\nJjQc7ABpmXDSbXDxK1BTDeOHwtv/A9WVDT9XRHZLqwt3aWELX4Zxx0PJWvjJC3DMfzd+FkyvwXDl\nB3D4BfDenfDIECj6vFnKFREvpnA3s2Fm9rmZLTWzG+p5fG8zm25mn5jZp2Z2cvxLlRZVUw1v/hGe\nuRC69IHR78K+xzd9f1nt/FDOeRNgYyE89AOY+SDUNG6dGRGJTYPhbmapwFhgONAPuMDM+tVpdhMw\nyTk3ADgfuD/ehUoLKvkWJpwN7/8VjrwELn0NOvSMz777ngZXzoDex8Fr18OEs2DjyvjsW2RXSr6F\nso1BV9FiYum5DwSWOueWOecqgKeBM+q0cUC7yO32wKr4lSgtauVseOg4+HoGnDEWTrvHj53HU9s9\nYeQzcOo9sGIWPDAIPpsc39cQqVVT46fn3t0X/nIgvHgVrJwTdFXNLpZw7wGsiLpfGNkW7Y/AKDMr\nBKYCv6xvR2Y22swKzKygqKioCeVKs5r9OIwfBpYCl02DAaOa77XMIP8S+Pm/ocuB8NxlMPlS2PJd\n872mJJ/Na2DiufD672C/IXDYCJj3PDx8gu/EzP6HXxcphOJ1QPUC4HHnXB5wMvCEme2wb+fcOOdc\nvnMuv2vXrnF6adltlWXw4tXw0jXQ61i44l3o3r9lXrvzfnDJq/DDm2DBi3D/9+GLt1vmtSXcFk+D\nBwfD1x/AKXfBBU/B6ffBtYvg5L9AdYWf3nvXQTD1OlizIOiK4yqWcF8JRA+45kW2RbsMmATgnJsB\nZAE677w1+O5rP0XxkyfgB7+FC5+F7E4tW0NqGvzgOrj8TchsC0+cBa9eD5VbWrYOCYfKMnj1Bnjy\nx5CzB4x+B753+bZZXlntYeDP4MoP4dLX4cDh/lPrA4P8J9dPJ/lzNFq5Bk9iMrM0YDEwBB/qs4CR\nzrn5UW1eBZ5xzj1uZn2Bt4Aebhc7b+pJTBJHS9/ywyE1NXD2Q/6XPGiVW+DNW+GjB6DLAf7Ep+4D\ngq6qdVr/pT+G0u9M/wc0Gaxd5H+n18yDo34OJ94K6VkNP69kHcydCLMfg/XLILsz9L/Qn13deb9m\nL7sxYj2JKaYzVCNTG+8BUoHxzrk7zOw2oMA5NyUye+ZhIBd/cPW3zrlpu9qnwj1ANTXw/l3w9h2w\nRz8474mE+wXmi+nwr1/4+fXH3eDn1ydLQMXDptX+xLNNhf7feNgY2Pe4oKtqPs5BwXg/tp6R66fd\nNmVNo5oa+PJdv69Fr4Crhn1PgO9dBgcMT4jfwbiGe3NQuAdkywZ44eew+FU4dIRfuTEjO+iq6rfl\nO3jlNzBvMuR9D856KPH+CCWi8mJ4bLjvgZ7we/8paMNyPw31pNv9ukBhUroepvwSFr0M+/0QznzQ\nz8jaXZtW++HK2Y/DppXQthsccZH/ap+3+/tvIoW77GjNfHhmlP+PPvRPftyxNSzh8NlkeOXXftmC\nof/rPyq3hrqDUF0FT4+EpW/66aZ9TvJj0DP+Dv++y5+c9v2r4Zhfh2Mht2XvwgtX+DnsJ90KR10J\nKXE+8b66Cpa+4XvzS97wv3sHDIf8S/0fk3i/XgMU7rK9T5/1vZus9jDin7D3UUFX1DgbV/r5ycum\nQ5+hcPrf4tM7CxPn4JVroeBROPWvPnyibVrlj2d8+jTk7uXD8NARLR5OcVFd6Rele/8e6Lw/nPso\ndDu8+V/3u6/89MlPnoCSIuiwj5/S238U5LbMDECFu3jRV0vaZzCc+1jrDcWaGpj1MLxxs79e62n3\nQr/Tg64qcXxwH7zxBxh8jV+wbWdWfOxnI62aAz3y/ZWz8hrMisSx7gt47nJf/5EX+09zGTktW0NV\nhR8GKhgPX/0bUtL972L+pf7/WTN+slS4S/1XS0oNwUVKihbD8z+D1XP9jIZhY/zaNcls/gv+3/rg\ns+Cc8Q33xmtqfA/+zT9C8Rq/qNuQW6Bdt5aotmmcg/885eekp6T5Oev96p4sH4CixX6WzdyJfnmD\nLgf6kD/8fGjTIe4vp3BPdtFXSzrjb9uulhQW1ZXw7v/Bv/8C7fL8hbl7DQ66qmAs/wj+cZo/8eyi\nKbFN/atVvtmPxc8Y63ufP7gWjr6qcftoCVs2+OMu856DfY7xU3cDPKhZr8ot/o9swXgonAVpbfz/\nu/xLoccRcevNK9yTVd2rJZ03YcerJYXJilnwwmg/p/v7v/RnusZ7LZxEtu4LePQkyOoAl70BOZ2b\ntp/1y+D1m+DzV/yFVX50Bxx0SmIcuF7+kR+G2bQSTrjRHwxOSQ26ql1b/R8oeMyfEFVZ4o8H5F8K\nh5y72weywxvu5ZuhorT1jhs3px2ulvRAcgxXVJT4P2YF42HPQ/wB42SYMlmyDh490fdqL38zPj/z\nF2/DazdC0SK/cuewMbBn3UVgW0h1lf9U8e4YaN8TznkUen4vmFqaqmwTfDYJZo2HtfMhoy0cfh4M\nvAK6HtCkXYY33N+/xx8lP3QEDPoF7Hlw/Itrjb5d6qc5fvs5DLkZBv9XYvS6WtLi1+FfV/rbI5+F\nvCODrac5VZbBP8+AVZ/4i5LHc/ZTdZX/Qzn9DijfBPmXwQm/a9llKTYs99fgXT4DDjvPrwXTmjsq\nzvmhmoLxfuGyU+9u8sJ84Q33dV/4YYe5E6Gy1M8zHXSVX/Et2cKs1sKX/YlJaRlw7vjdu6hGa7fu\nC78WffFa+PHjTTtLMdHV1PhT7Oc/73/Gg89qntcpXQ/T/9dPrcxq70+IOvKS5j9Lc97z8NJ/gavx\nC34dfl7zvl5LK10P6W38VxOEN9xrla73R6g/GgfF3/hTrAddBYf+OHnGXNcsgFmP+P983Y/wwxHx\nuqhGa1a8Fib+GL75zM/3PvKnQVcUX2/cAh/c46c7Dr6m+V9vzXx47Qb48j3o2heGj2meDkR5sZ+i\nOXeCn6J5ziPhO5s2DsIf7rWqKvwY84y/+8WCcvaAgaP9wYumHlxKZKXr/Rmbcyf6qYApab43NfSO\n5PmjFovyYnj2p/5MzeNvhOOuD8cnu4Lx8PJ/+6GSU+5quZ/JOT+v+/Xfw4av/TGdH90OnfaNz/5X\nfQKTL/MHdn/wG//vFYZpu80gecK9lnN+wZ8P/+5PFU7L8nN3B13lrwHamlVX+ZCaOxE+fxVqKmGv\nw/wc70PPhRytrlyv6kq/Rv3ciX49kFP+mhALPzXZkjfgyfNg/yFw/lPB/CyVZTBzLLx3l/89HHQV\nHHutX6q5KWpqYMbf4K3bIXcPvwpor2PiW3PIJF+4R1u7yP8C/ucZqC6HA4b5k3h6HdO6em9rFsB/\nnvQ/R8layO7iDy71vwD2OjTo6loH5/yBwffu9L8H545v+bMZ42H1p34xsE77+oubBL0uzKbV8Nat\n/qSi3D39CXKHnd+4pQw2rfbrwnz5LvQ93Z9x3NLXEmiFkjvcaxUX+fHojx+G0m/9XNNBV/sDUIn6\nka90vR9mmjvRf1RNSfOh1P9CvwhUotad6GY9ClN/49eGHzmpdX3a2Vjol++1VD/lMZHOIi0sgFd/\n69eN73EkDPtzbNMVF031awVVlfnplkdc1Lo6XgFSuEer3OJPJpgx1k8VbNsdjrrCH2hr07FlatiV\n6io/v3juBD/sUl3he+b9L/QHiFtTECWyRa/467S26w6jnovfeHFzKtvkrw60cQVc+lpiTv2tqfFz\nud+4xU9uOOx8OPEW/z7XVbnFn5Mw6xE/tHjOo02e752sFO71qamBL96CD//mPwqm58ARP/FXbAni\nqPzahTD3Sfj0Gb++R3ZnP+xy+AXQ7bCWrycZrPgYnhzhPxGNnORPC09U1ZW+1i/fgwsnw34nBF3R\nrpVvhn/f7Sc3pKTDsb/2n5RrlzJYM98fNC1a6LcPuVmTAJpA4d6Qbz7zPfnPJvurrRx0qv+Fa+6l\ncLcOuzzpV7VLSfNL2A64EPY/yc9Vl+b17RJ44mwoXeenj/Y5MeiKduScP9t4zj/hjLFNPuElEOu/\n9L3zRS9Dh739Ugabv/Hbstr7dYD2HxJ0la2Wwj1Wm1bDx+P8FLOyDf6KP4OugoNOi99shK3DLhPh\n86l+2GXPQ7YNu7TQOtASZfM3MPFc/+nptPv8H9dE8t5f4O3b/UXLf/j7oKtpmmXv+KUM1i7w9/sM\n9X+o9Pu+WxTujVVR4nvTM8bCd1/6HsdRV/phm6ZO81q7yAd69LDLoSOg/0gNuySCsk0w6Sc+hH54\nExz7m8Q4qPfps/D85X6I7qyHEqOmpqqu8seSLAUG/KR1/ywJQuHeVDXV/qDmjLGw/EPIbOcPvB71\n89iWGN3y3bZhl5Wztw279B8JfX6kYZdEU1UBU672f4DzL/VrmAS54uBXH8ATZ0LeQPjJ8xqTlh0o\n3ONh5Wwf8vP/5e8ffJa//mT3Adu3q67yl3+bO9HPyKiugD0O9h/1Dx2hj6GJzjk/Z/v9v8KBp/jT\n3oO4aPi3S/yUx9w94LJpiTGTSxKOwj2eNiyHjx7y106s2OwvozXoaj/D5j9P+ZOMir+BNp3gsMiw\ny16H6SNoa/PROD9nO+97/uLSLXlCTXERPDLEL4Z3+Zt+TXWReijcm0PZJn9h3JkPwsblfpul+pUH\n+4/0wy8admndFrwIz/3MH3MZ9Rx03Kf5X7Oi1F9Jac18uPiVcC9VLLtN4d6cqqtg0UtQ8q2/hmPu\nHkFXJPH09Qx46jy/PtGFz/ozm5tLTTVMusgP5503Afqe2nyvJaEQa7g3YiEI2So1zY+/D/yZgj2M\n9hkEl07zJ+I8doqfxtpc3rjZzwcf9icFu8SVwl2kPnscBJe/4YdlJv7YH1eJt4/G+bM5j7oSjr4y\n/vuXpKZwF9mZdt3hkqmw9yB/Ee737/Eza+Jh0VR47Xo/O2foHfHZp0gUhbvIrmS19wdWDzkH3rzF\nXymopnr39rlyjr9MXrf+ftplkPPqJbRa8ZULRFpIWiac/Qi07eaHUTavhrMf3rYgVmNsWO4vuJHT\nxU+3DGI+vSQF9dxFYpGS4odPhv4JFr4ET5zlF4FrjC0b/Ph9dblf5VEH46UZKdxFGmPQL/zVnFYW\n+HXWN6yI7XlVFfDMKFj3BZw3Eboe2Lx1StJTuIs01iFnw6jn/cqSj54E38zbdfva5Xu/+rdfFbH3\nsS1TpyQ1hbtIU/Q+Fi59FTB/bdNl7+687Ttj/DIVJ9wEh5/XYiVKclO4izTVngf7ufDtesCEc/yF\nX+qa+yS8Owb6j4If/Kbla5SkFVO4m9kwM/vczJaa2Q07aTPCzBaY2XwzezK+ZYokqPZ5vgffc6Cf\n3vjh37c9tuwdmPJL2Pd4OO0eLSQnLarBqZBmlgqMBU4CCoFZZjbFObcgqk0f4EZgsHPuOzPTNABJ\nHm06+jH4F66Aab+HTav8ZfGeuQi6HOAv5ZeaHnSVkmRimec+EFjqnFsGYGZPA2cAC6La/AwY65z7\nDsA5tzbehYoktPQsOPcxeH0vmDkWZj3sl4AeOcmfCCXSwmIZlukBRM/3Koxsi3YAcICZfWBmM81s\nWH07MrPRZlZgZgVFRUVNq1gkUaWkwLAx8KP/gbZ7+ZOUOvQMuipJUvE6QzUN6AMcD+QB75nZoc65\nDdGNnHPjgHHgl/yN02uLJA4z+P4v/ZdIgGLpua8EorsfeZFt0QqBKc65Sufcl8BifNiLiEgAYgn3\nWUAfM+ttZhnA+cCUOm3+he+1Y2Zd8MM0y+JYp4iINEKD4e6cqwKuBl4HFgKTnHPzzew2Mzs90ux1\nYJ2ZLQCmA9c559Y1V9EiIrJrusyeiEgrosvsiYgkMYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iE\nkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAX\nEQkhhbuISAgp3EVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI\n4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCKKZwN7NhZva5mS01sxt2\n0e4cM3Nmlh+/EkVEpLEaDHczSwXGAsOBfsAFZtavnnZtgWuAj+JdpIiINE4sPfeBwFLn3DLnXAXw\nNHBGPe1uB/4MlMWxPhERaYJYwr0HsCLqfmFk21ZmdgTQ0zn3yq52ZGajzazAzAqKiooaXayIiMRm\ntw+omlkKcDdwbUNtnXPjnHP5zrn8rl277u5Li4jITsQS7iuBnlH38yLbarUFDgHeMbOvgKOBKTqo\nKiISnFjCfRbQx8x6m1kGcD4wpfZB59xG51wX51wv51wvYCZwunOuoFkqFhGRBjUY7s65KuBq4HVg\nITDJOTffzG4zs9Obu0AREWm8tFgaOeemAlPrbLt5J22P3/2yRERkd+gMVRGREFK4i4iEkMJdRCSE\nFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQkhhbuI\nSAgp3EVEQkjhLiISQgp3EZEQUriLiISQwl1EJIQU7iIiIaRwFxEJIYW7iEgIKdxFREJI4S4iEkIK\ndxGREFK4i4iEkMJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhFBM4W5m\nw8zsczNbamY31PP4r81sgZl9amZvmdk+8S9VRERi1WC4m1kqMBYYDvQDLjCzfnWafQLkO+cOAyYD\n/xfvQkVEJHax9NwHAkudc8uccxXA08AZ0Q2cc9Odc6WRuzOBvPiWKSIijRFLuPcAVkTdL4xs25nL\ngFfre8DMRptZgZkVFBUVxV6liIg0SlwPqJrZKCAfuLO+x51z45xz+c65/K5du8bzpUVEJEpaDG1W\nAj2j7udFtm3HzE4Efg8c55wrj095IiLSFLH03GcBfcyst5llAOcDU6IbmNkA4CHgdOfc2viXKSIi\njdFguDvnqoCrgdeBhcAk59x8M7vNzE6PNLsTyAWeNbO5ZjZlJ7sTEZEWEMuwDM65qcDUOttujrp9\nYpzrEhGR3RBTuLeUyspKCgsLKSsrC7qUZpWVlUVeXh7p6elBlyIiIZVQ4V5YWEjbtm3p1asXZhZ0\nOc3COce6desoLCykd+/eQZcS7SENAAAHqElEQVQjIiGVUGvLlJWV0blz59AGO4CZ0blz59B/OhGR\nYCVUuAOhDvZayfAzikiwEi7cRURk9ynco2zYsIH777+/0c87+eST2bBhQzNUJCLSNAr3KDsL96qq\nql0+b+rUqXTo0KG5yhIRabSEmi0T7daX5rNg1aa47rNf93bcctrBO338hhtu4IsvvqB///6kp6eT\nlZVFx44dWbRoEYsXL+bMM89kxYoVlJWVcc011zB69GgAevXqRUFBAcXFxQwfPpxjjjmGDz/8kB49\nevDiiy/Spk2buP4cIiINUc89ypgxY9hvv/2YO3cud955J3PmzOHee+9l8eLFAIwfP57Zs2dTUFDA\nfffdx7p163bYx5IlS7jqqquYP38+HTp04LnnnmvpH0NEJHF77rvqYbeUgQMHbjcX/b777uOFF14A\nYMWKFSxZsoTOnTtv95zevXvTv39/AI488ki++uqrFqtXRKRWwoZ7IsjJydl6+5133uHNN99kxowZ\nZGdnc/zxx9c7Vz0zM3Pr7dTUVLZs2dIitYqIRNOwTJS2bduyefPmeh/buHEjHTt2JDs7m0WLFjFz\n5swWrk5EJHbquUfp3LkzgwcP5pBDDqFNmzbsueeeWx8bNmwYDz74IH379uXAAw/k6KOPDrBSEZFd\nM+dcIC+cn5/vCgoKttu2cOFC+vbtG0g9LS2ZflYRiR8zm+2cy2+onYZlRERCSOEuIhJCCncRkRBS\nuIuIhJDCXUQkhBTuIiIhpHCP0tQlfwHuueceSktL41yRiEjTKNyjKNxFJCwS9wzVV2+Abz6L7z73\nOhSGj9npw9FL/p500knsscceTJo0ifLycs466yxuvfVWSkpKGDFiBIWFhVRXV/OHP/yBNWvWsGrV\nKk444QS6dOnC9OnT41u3iEgjJW64B2DMmDHMmzePuXPnMm3aNCZPnszHH3+Mc47TTz+d9957j6Ki\nIrp3784rr7wC+DVn2rdvz91338306dPp0qVLwD+FiEgih/suetgtYdq0aUybNo0BAwYAUFxczJIl\nSzj22GO59tpruf766zn11FM59thjA61TRKQ+iRvuAXPOceONN3LFFVfs8NicOXOYOnUqN910E0OG\nDOHmm28OoEIRkZ3TAdUo0Uv+Dh06lPHjx1NcXAzAypUrWbt2LatWrSI7O5tRo0Zx3XXXMWfOnB2e\nKyISNPXco0Qv+Tt8+HBGjhzJoEGDAMjNzWXChAksXbqU6667jpSUFNLT03nggQcAGD16NMOGDaN7\n9+46oCoigdOSvwFJpp9VROJHS/6KiCQxhbuISAglXLgHNUzUkpLhZxSRYCVUuGdlZbFu3bpQh59z\njnXr1pGVlRV0KSISYgk1WyYvL4/CwkKKioqCLqVZZWVlkZeXF3QZIhJiCRXu6enp9O7dO+gyRERa\nvZiGZcxsmJl9bmZLzeyGeh7PNLNnIo9/ZGa94l2oiIjErsFwN7NUYCwwHOgHXGBm/eo0uwz4zjm3\nP/BX4M/xLlRERGIXS899ILDUObfMOVcBPA2cUafNGcA/IrcnA0PMzOJXpoiINEYsY+49gBVR9wuB\no3bWxjlXZWYbgc7At9GNzGw0MDpyt9jMPm9K0UCXuvtOcno/tqf3Yxu9F9sLw/uxTyyNWvSAqnNu\nHDBud/djZgWxnH6bLPR+bE/vxzZ6L7aXTO9HLMMyK4GeUffzItvqbWNmaUB7YF08ChQRkcaLJdxn\nAX3MrLeZZQDnA1PqtJkC/DRy+1zgbRfmM5FERBJcg8MykTH0q4HXgVRgvHNuvpndBhQ456YAjwJP\nmNlSYD3+D0Bz2u2hnZDR+7E9vR/b6L3YXtK8H4Et+SsiIs0nodaWERGR+FC4i4iEUKsL94aWQkgW\nZtbTzKab2QIzm29m1wRdUyIws1Qz+8TMXg66lqCZWQczm2xmi8xsoZkNCrqmoJjZf0f+n8wzs6fM\nLPTLsraqcI9xKYRkUQVc65zrBxwNXJXE70W0a4CFQReRIO4FXnPOHQQcTpK+L2bWA/gVkO+cOwQ/\nMaS5J30ErlWFO7EthZAUnHOrnXNzIrc34//j9gi2qmCZWR5wCvBI0LUEzczaAz/Az2TDOVfhnNsQ\nbFWBSgPaRM7DyQZWBVxPs2tt4V7fUghJHWgAkVU4BwAfBVtJ4O4BfgvUBF1IAugNFAGPRYapHjGz\nnKCLCoJzbiXwF2A5sBrY6JybFmxVza+1hbvUYWa5wHPAfznnNgVdT1DM7FRgrXNudtC1JIg04Ajg\nAefcAKAESMpjVGbWEf8JvzfQHcgxs1HBVtX8Wlu4x7IUQtIws3R8sE90zj0fdD0BGwycbmZf4Yfr\nfmhmE4ItKVCFQKFzrvbT3GR82CejE4EvnXNFzrlK4Hng+wHX1OxaW7jHshRCUogsqfwosNA5d3fQ\n9QTNOXejcy7POdcL/3vxtnMu9L2znXHOfQOsMLMDI5uGAAsCLClIy4GjzSw78v9mCElwcDmhLrPX\nkJ0thRBwWUEZDPwE+MzM5ka2/c45NzXAmiSx/BKYGOkILQMuCbieQDjnPjKzycAc/CyzT0iCZQi0\n/ICISAi1tmEZERGJgcJdRCSEFO4iIiGkcBcRCSGFu4hICCncRURCSOEuIhJC/w8EXnbQh3HTaAAA\nAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history_simple.history['acc'], label='train')\n",
    "plt.plot(history_simple.history['val_acc'], label='test')\n",
    "plt.title(\"Simple CNN\")\n",
    "plt.legend()\n",
    "plt.ylim(0,1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "wC6x_2Aek7Z0",
    "outputId": "20c52e6e-5592-4e09-dc5f-bfccfc9e01d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.91\n",
      "Validation Accuracy: 0.74\n"
     ]
    }
   ],
   "source": [
    "print('Training Accuracy: {}'.format(np.round(history_simple.history['acc'][-1], 2)))\n",
    "print('Validation Accuracy: {}'.format(np.round(history_simple.history['val_acc'][-1], 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gzw-wkdaqVvp"
   },
   "source": [
    "As we can see, our model has a good training accuracy, but appears to overfit a little. The validation accuracy in particular fluctuates a lot."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6VbKLT4XlZZ5"
   },
   "source": [
    "#### 3.2 Augmenting the data\n",
    "\n",
    "We're now going to try data augmentation to improve the results of the same CNN - this technique will likely lead our network to overfit less. Here are the options we have: \n",
    "\n",
    "- Mirroring\n",
    "- Rotations\n",
    "- Translations\n",
    "- Scaling (zooming in/out)\n",
    "\n",
    "When taking a look at the data, it makes sense to apply all these transformations to the images we have, since they don't have a \"normalized\" size, orientation, etc. If we were working on the digits dataset, it wouldn't make sense to apply rotations for instance, but here we don't have that problem and we can apply pretty much what we want, since a quick look at the images tells us that they don't seem to be standardized in their scale, orientation, centering, etc.\n",
    "\n",
    "We're going to do what's called \"online\" data augmentation, i.e. data augmentation \"on the fly\", whenever we train a batch. Otherwise, the dataset we'd create would be way too large. \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 373
    },
    "colab_type": "code",
    "id": "NURgBtVgla-T",
    "outputId": "688fc51b-80fc-47e2-87bb-9b856ac29d58"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "4432/4432 [==============================] - 424s 96ms/step - loss: 0.3368 - acc: 0.8577 - val_loss: 0.3522 - val_acc: 0.8510\n",
      "Epoch 2/10\n",
      "4432/4432 [==============================] - 422s 95ms/step - loss: 0.3216 - acc: 0.8653 - val_loss: 0.4719 - val_acc: 0.7976\n",
      "Epoch 3/10\n",
      "4432/4432 [==============================] - 423s 96ms/step - loss: 0.3152 - acc: 0.8679 - val_loss: 0.7301 - val_acc: 0.7205\n",
      "Epoch 4/10\n",
      "4432/4432 [==============================] - 422s 95ms/step - loss: 0.3100 - acc: 0.8707 - val_loss: 0.3152 - val_acc: 0.8698\n",
      "Epoch 5/10\n",
      "4432/4432 [==============================] - 420s 95ms/step - loss: 0.3064 - acc: 0.8721 - val_loss: 0.7266 - val_acc: 0.7322\n",
      "Epoch 6/10\n",
      "4432/4432 [==============================] - 416s 94ms/step - loss: 0.3028 - acc: 0.8738 - val_loss: 0.3887 - val_acc: 0.8270\n",
      "Epoch 7/10\n",
      "4432/4432 [==============================] - 416s 94ms/step - loss: 0.3006 - acc: 0.8745 - val_loss: 0.4369 - val_acc: 0.8059\n",
      "Epoch 8/10\n",
      "4432/4432 [==============================] - 416s 94ms/step - loss: 0.2972 - acc: 0.8762 - val_loss: 0.4415 - val_acc: 0.8218\n",
      "Epoch 9/10\n",
      "4432/4432 [==============================] - 415s 94ms/step - loss: 0.2957 - acc: 0.8765 - val_loss: 0.7114 - val_acc: 0.7176\n",
      "Epoch 10/10\n",
      "4432/4432 [==============================] - 427s 96ms/step - loss: 0.2936 - acc: 0.8774 - val_loss: 0.3588 - val_acc: 0.8596\n"
     ]
    }
   ],
   "source": [
    "data_aug = image.ImageDataGenerator(\n",
    "      horizontal_flip=True,                       ## Mirroring (horizontal)\n",
    "      vertical_flip=True,                         ## Mirroring (vertical)\n",
    "      rotation_range=90,                          ## Rotation\n",
    "      width_shift_range=0.2,                      ## Translation (horizontal)\n",
    "      height_shift_range=0.2,                     ## Translation (vertical)\n",
    "      zoom_range=[1,2],                           ## Zooming in\n",
    "      validation_split=0.1\n",
    "      )\n",
    "\n",
    "train_generator = data_aug.flow(X, y, batch_size=128, subset='training')\n",
    "val_generator = data_aug.flow(X, y, batch_size=128, subset='validation')\n",
    "\n",
    "history_aug = simple.fit_generator(train_generator, \n",
    "                                   steps_per_epoch=len(train_generator)*4, \n",
    "                                   # We choose to get 4 times as much data on \n",
    "                                   # each epoch\n",
    "                                   epochs=10, verbose=1,\n",
    "                                   validation_data=val_generator,\n",
    "                                   validation_steps=len(val_generator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 281
    },
    "colab_type": "code",
    "id": "oJn-8fnh7USO",
    "outputId": "a2221c84-343c-4c3b-ffbc-fb8f6f6e28ac"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcFNW5xvHfOxsji4CACwwKURSR\nGNGJYtxwBzUYlxBRNDELyU2M3htjYhY1mo24RY3GqImaRKJR1EgiKpogxijGAY1hk02QARVkX2SZ\n6ff+cWqYnmGWBrqnemqe74f5UF11uup0dffTVaeqTpm7IyIiyVIQdwVERCT7FO4iIgmkcBcRSSCF\nu4hIAincRUQSSOEuIpJACvc8ZmYXmdnEHM37QTP7SS7mHScz29fM1ptZYRNl3MwOaMl6tXY78nkx\nsz7ROi7Kdb2kcQr3mJnZsWb2ipmtMbOVZvYvM/skgLuPdffT4q5jfRZcbmbTzWyDmVWa2WNm9vFo\n+oPRl/vItOccYGae9vhFM9tkZr3Txp1iZgt3pW7u/q67d3T36rTlfHlX5pkE+fqDZmZDzKwy7nok\nkcI9Rma2O/A34FfAHkAv4Hpgc5z1ysDtwBXA5YR6Hwj8BTgzrcxKoLktvQ3ANbmooEhbp3CP14EA\n7v6wu1e7+0fuPtHd3wIwsy+Y2cs1haOtr6+b2VwzW2dmPzaz/aMt/7Vm9qiZlURlh0Rb1N83sw/N\nbKGZXdRYRczsLDN708xWR/M7tJFy/YBvACPd/R/uvtndN0Z7GWPSiv4eONTMTmji9d8BjDSz/Ztb\nUWZ2vZn9KhoujvYYbooe7xbtBeyR3iRgZj8FjgPujJpq7kyb5SnRelxtZneZmTWy3CPN7NWo3Htm\ndmfaOt6u+SF9T8HMCs3slmj9v2Nml6WXj8r+JFrf683sr2bWzczGRu/n62bWJ23e/c3s+WgP720z\nG5E27cHodTwdfTZeq1mvZvZSVOw/0XI+F41v9D03s0FmNi2a15+B0ibem0Izuzl6nQuo+yOPmV1q\nZrOieS0ws69G4zsAzwA9o3qtN7OeTa1z2QHurr+Y/oDdgRWEIBwGdK03/QvAy2mPHXgqet4hhC38\nvwMfAzoDM4HPR2WHAFXArUA74ATClvJB0fQHgZ9Ew4OAZcBRQCHweWAh0K6BOn8NWNTM63qQsNV+\neU39gQPCx21bmReBL0f1eygadwqwsJF5ngT8Nxr+FDAfeC1t2n+i4T7ReipKX069eTlhj6kLsC+w\nHBjayHKPAAYDRdG8ZwH/29Cy6i8vWlczgTKgK/BCA3WbB+yf9v7NidZDEfAH4IGobAdgMXBpNG0Q\n8CEwIG2drwCOjKaPBR6p95oPSHvc6HsOlACLgP8DioHzga01n5dGPhOzgd6EPblJ9V7nmdFrNMLn\ncCNweNrntDLTda6/zP+05R4jd18LHEv4ItwHLDez8Wa2VxNPu9Hd17r7DGA6MNHdF7j7GsJW0KB6\n5a/xsHU9GXgaGMH2RgP3uPtrHvYgfk/44RjcQNluwHsZvsR7gH3NbFgTZX4OfNrMDmlmXq8C/cys\nG3A88Dugl5l1JATG5AzrVGOMu69293cJYXRYQ4Xcfaq7T3H3KndfSHhNTe2NpBsB3O7ule6+ChjT\nQJkH3H1+2vs3391fcPcq4DFq38+zCD98D0R1eQN4HPhs2ryedPd/R88d29hrijT1ng8mhPpt7r7V\n3ccBrzfzOm9z98XuvpLwnm7j7k9Hr9Gjz+FEwh5Vg3ZxnUtE4R4zd5/l7l9w9zJgINATuK2Jp3yQ\nNvxRA487pj1e5e4b0h4viuZf337AldFu8GozW03YCmuo7Apgnybqt427bwZ+HP01VmY5cCdwQzPz\n+gioIHzJjyeE+SvAMexcuL+fNryRuuttGzM70Mz+Zmbvm9la4GdA9wyX0ZOwtV1jcQNlMn0/9wOO\nqvceXQTsnVY+o9eUNr/G3vOewBJ3T+9VcFET86r/OuuUNbNhZjYlak5aDZxBE+twF9e5RBTuecTd\nZxN2rwdmaZZdo3bNGvsCSxsotxj4qbt3Sftr7+4PN1D270CZmZVnWIcHCM0f5zZR5ibgRMLueFMm\nE5pgBhG2JCcDpxOaIl5q5Dm72u3p3YQmh37uvjvwfULzAoRmLoD2aeXTw/Y9QpNMjd7svMXA5Hrv\nUUd3/59dmF9j7/l7hL2i9OMQ+zYxr/eo+9q2lTWzdoQ9jJuBvdy9CzCB2nXY0PvT1DqXDCncYxQd\nILvSzMqix72BkcCULC7mejMrMbPjCLv2jzVQ5j7ga2Z2lAUdzOxMM+tUv6C7zwV+DTxs4aBtiZmV\nmtkFZnZ1A+WrgOuA7zZWQXdfDdwCfKeZ1zIZuASY6e5bqG23fyfaA2jIB4RjEjurE7AWWG9m/YFt\nYRotcwkwKjqo+EVC23KNR4ErzKyXmXWhiXWQgb8BB5rZxRYOKBeb2SfN7OAMn19/PTT1nr9KOF5z\nebSccwk/oI15NCpbZmZdgfTPQQmhHX85UBU10aWf3vsB0M3MOqeNa3SdS+YU7vFaRzig9ZqZbSCE\n+nTgyizN/31gFWFrfSzwtWjvoA53rwC+QmgeWUU4yPeFJuZ7eVT2LmA14eDmOcBfGylfszXYlNuB\n6mbKvALsRu1W+kxgE41vtdfM93wzW2VmdzQz/4Z8G7iQ8F7dB/y53vSvAFcRmqsOiepY4z5C+/Jb\nwBuELdYqmn+d23H3dYRQvIDwfr4P/IIQnJn4EfD7qAlmRFPvefTDeW70eCXwOeCJJuZ9H/Ac8B9g\nWnrZqN6XE34AVhHW5fi06bMJn48FUd160vw6lwxY3WY1SQozG0I4C6WsubLSMqKt1t+4+35x10WS\nT1vuIjli4fz7Myycc9+L0Dz1ZNz1krah2XA3s/vNbJmZTW9kupnZHWY2z8zeMrPDs19NkVbJCFcc\nryI0y8wCro21RtJmNNssY2bHA+uBP7j7dmdxmNkZwDcJpzcdRTiv96gc1FVERDLU7Ja7u79EOKjS\nmLMJwe/uPgXoYmYZnQctIiK5kY0uOXtR9wKGymjcdmdHmNlowpVxdOjQ4Yj+/ftnYfEiIm3H1KlT\nP3T3Hs2Va9H+lt39XuBegPLycq+oqGjJxYuItHpm1tTVwttk42yZJdS9Oq0sGiciIjHJRriPBy6J\nzpoZDKxx90w7lhIRkRxotlnGzB4mdMvZ3cIdU64j9BiHu/+GcNXdGYQr3DYSuiQVEZEYNRvu7j6y\nmelOuHmDiIjkCV2hKiKSQAp3EZEEUriLiCSQwl1EJIEU7iIiCaRwFxFJIIW7iEgCKdxFRBJI4S4i\nkkAKdxGRBFK4i4gkkMJdRCSBFO4iIgmkcBcRSSCFu4hIAincRUQSSOEuIpJACncRkQRSuIuIJJDC\nXUQkgZq9QbaI5Ja74w4pdxxwByeMSx+uMz16jkfPT0Xlon+hrNf+X51yqt1xd6pT4XHKw1/tcDQ+\nFQ17zXADZbaNg1Q071RUftt4rzuc/nqJXgew7TWF4e2n1Yzw2sHtyjY2jTrLSF9ntXVKbRsfrUcP\nz0il0td97fqteR9SNe9D+rwbep/qLCOMu/SYPpzUf6+d+rxkSuEu23j0ZayK/qqrna2pFNUpZ2t1\niqrqmmlpw9WpUHbrFqqtqE6ApOoNb/tLsa1cTVikD28LpLTAqAmI9OHqmgBL1Q7XDbG6y04PpepU\n9HrTgytVG2jVacuqG2a14+oGJHXLpLb/UlPni+91Qk2aZwa2bTgMWZ1p0aO6/2HbHtu2eRREAwVW\nd1woaxRY7TzDcJhJQUHdcVaz7Gi4dh61w+nLsGh4S1Xu33iF+07YWp1i4+ZqNlVVs7U6VRuI1SH4\nGnwchWF1KpU2LTzeWl0bqvUfV6XNvyZk06fVjq9dVk2ZmnG1y9p+Wt0A37kP3BH2NveX3MQ/U4dy\n5davsZmSLK/xugoLar9chRaGC8woiMYXFti2aYUFRkEBUblQprBe2YKoXGH0xSspKtg2vn6Zgpph\nY9u8ti2vZjlpz6350m8LEuqGBvVCoU5YWG0gFaQN1wkVQj3CMrafV/31UmA1r4W04bT1UvO6CmrX\na93npZXZ9tpJW39p6zJaH4UFURCnBXBt4Eb/16yn9Gk1A7JTEh/u7s5HW6tZv7mKjZuj/7dUs2Fz\nFRu2VIX/N9c8rh2/cXN13ek1w1uq2VKVarH6F0VftOLCEDhFBUZRoVFUUPdxYUEBxYXR44ICigsL\nKC2ufV5x9JztyxdE86uZV73y9Z5XVFiQVtbovuw1Dpl8I9UlnTjroykM6VnNwlN/i5d2paBOgIQv\na4HVhmj9QCloaNjqhqy+8NLqVW2GonY5X0yrC/e3Klfz6vwV9cI4+j8tmLeF+JaqjHd9SwoL6NCu\nkPYlRXRsV0T7doV0KCmiR6d2dCgpokM0rmNJEe3bFVFaXEBxTcimBW5xoUXBWxucDT1OD+zCgtrg\nrJmWvjuYl+ZMhMlfhj0+RuElT8Gil+n45NcY+OwIGDUOuuwbdw1F8suH82DseXDqj2HA8JwuqtWF\ne8Xcpdzy3By2WjEdSopoX1JYJ4j37FRKh+5FdCipCelC2rcLwdyhpDD6v4gO7aLhdrVlS4p08lDG\nZj4F474Eew2AUU9Ch24w8DzouBc8ciH89hS46DHY5xNx11QkPyyZBmPPB6xFNnxaXbhfXPoyl3b+\nMRz8aWzgedD3BChsdS+jdXvrUXjya9DriBDgu3WpndbnWPjic/DQ+fDAGTDiD3DAyfHVVSQfzJ8E\nfx4F7feAi/8C3fbP+SJb3aZqcdkgrP9Z2Ky/wkPnwi0Hwt++BQv/BamWawtvs6b+Hp4YDft9Ci5+\nsm6w19jzYPjy89C1D/xpBLz5pxavpkjemP4EjP1s+D586fkWCXYA85jOxSovL/eKioqdn8HWTTDv\nBZj+OMx5FrZuhE77wCHnhuaBXofXHnaX7JjyG3j2u7D/yfC5h6CkfdPlN60NWyvvTIaTfgjHfVvv\nibQt/74PJlwF+x4NIx9ueGNoB5nZVHcvb7Zcqw33dFs2hICf/gTMnQjVW6DLfjAwCvq9BipUdtXL\nv4QXfgT9z4Lz78/8aH/VFhh/Gbz1ZzjiC3DGLWpGk+RzhxfHwOQxcNAZ4TtTvFtWZt22wj3dR6vh\n7Qlhi37+JPBq6H5gCPmB50H3ftlfZpK5w4s/h8m/COvvnHugsHjH5/H3G+DlW+HAoeGDXtIhN/UV\niVuqOmytV/wOBo2Cs27P6gZN2w33dBs+hFnjwxb9wpcBh70/HkLqkHOh6365XX5r5w7PXwOv/AoO\nGwXD74CCwp2f3+u/DR/6noNg5J+hY4/s1TVfuMN/Hgl7Kh33gj36Qte+ob11j77QoYf2IpOsanM4\nJjXzL3Ds/8HJ12X9/Va417f2vbDCpz8Ola+HcWWfDEE/4DOw+z4tV5fWIJWCZ64KgfzJr8CwG8O1\n17tq9tPhFMpOe8Oox1vs4FKL+GAmPH0lvPsK7PGx8EVfu6RumZKOIehr/vaIgr9r33B63I7uFeWT\nVAo+Wgnr3gvft3Xvwbr3w//dD4QjRye7SW7zOnjkonCM6bSfwqcuy8liFO5NWbUQZjwZgv79/wIW\nTuEbeC4cfHY4Z7stS1XD+G/Cm2PhU98MF1xkc+tj8evw8OfC8IWPQlmzn9P8tnl9aLaa8usQ3qde\nD4MuCT+GWzfB6ndh1Tuw8p3w2Usfrt5cOx8rgM5lIejTQ79muLRzPK/PHTatqQ3qdfWCe937tX+p\nrds/v7QLbFoNPQ+Hz9wNe/Zv+deQa+uXh3PYP5gOZ98Fn7ggZ4tSuGdq+RyY8QT8dxysmAtWCPuf\nGLbo+58Z3xcqLtVbw27ljCfghKthyNW5aUZYMR8eOi8Ewvn3Q/8zsr+MXHOHWX+FZ68OW+iDRsEp\n10OH7pk9P5WC9e83HPqr3oGNK+qW322P7Zt5aoY77bNze1ZbNtQL6Qa2ute9D1Ufbf/c0s5huZ32\nrvf/PrWPO+4FRSWhafTpK2HLejjx+3D0N5OzFb9qIfzxXFi7NFzXceBpOV1cVsPdzIYCtwOFwG/d\nfUy96fsCvwe6RGWudvcJTc0zb8K9hnv41Z3+ePhb/S4UlkC/08IW/YFDk38QsGozPHYpvP00nPKj\n0GaYS+uXh/Pg33sTzrgZPvml3C4vm1YugAnfgXnPh7OxzrwV9j0qu8vYtLY26FctjII/Gl69OJws\nUKOoNJwhVj/0SzvD+g/qBfjS2i3tzWu3X25x+7oBnR7eu/eMQnvv5k+FrW/9Mnj6W+EHsdcRYSu+\nx0E7vXrywgczQrBXbQoX9PU+MueLzFq4m1khMAc4FagEXgdGuvvMtDL3Am+4+91mNgCY4O59mppv\n3oV7OndYMjUK+ifC1lVx+xDwA8+DA06B4tK4a5ldWzaGc9Ln/z20rx/11RZa7obwgzL3OTjuSjjp\nmvw+4Lh1E/zrdvjnLaF9/MQfxNOWXL0V1izePvRXLgzDW9Zv/5zCkga2stP/j4K7XafcvQfu4Xs1\n4dvhM3fSD+Doy3btQH1cFr0amheLO8DFT4SL91pApuGeySfySGCeuy+IZvwIcDYwM62MA7tHw52B\npTtW3TxjFtqBy8rhtJ/Au6+GD+TMp0JzRbvdw/neh42EPsfldxhlYvM6eHhkOKNo+K/g8Etabtkl\nHeCCP4Utun/eAmuWhDoU5bbb4J0y7+8hlFYuCGdbnf7TsCUbh8LicNB2j49B/WPS7qFJZ+U7sHlN\n2MrevSfs1jX+z6oZfPz88L15+lvw/LUw62/wmV+3rtOU334GHvsCdO4drtTu0jvuGm0nky3384Gh\n7v7l6PHFwFHufllamX2AiUBXoANwirtPbWBeo4HRAPvuu+8RixYtytbraBnVW8OR8OlPhF3LzWuh\n7EgYEl21GfcXZ2d8tDocCFoyLZzDfuhn46mHO7x0M0z6CXxsCIz4I5Tu3tyzWsaaJfDc98PZVnvs\nD2feDPufFHetWj/3cKxrwrdDs8ZJ18Dg/8n/rfg3HoLxl4dO8S4a1+InYGSzWSaTcP9WNK9bzOxo\n4HfAQHdvtLOXvG6WycTWTfDmQ/DPX8LaSuhVHg4+HnBK6wn5DSvgj5+BZbPCQc0cd0GakTfGwl8v\nhx4HhzbMOE9Rrd4Kr90TLuJKVYXuE465vEX64m5T1r0Pf/1fmPMM9B4ctuLz9RTZl2+DF64LP+4j\n/gjtOrZ4FTIN90wOry8B0vc5yqJx6b4EPArg7q8CpUCGpwy0UsWl8Mkvw+XT4KxfhoNWY8+H+06C\nt5/N//unrfsAHjwTPpwTmkXyIdgBBl0UTo9c9U7oNnjZ7HjqsehVuOcEmPiD0Ena16fACVcp2HOh\n096h35Vz7oHls+DuY2DK3fnVEWAqBc/9IAT7wPPCRXgxBPuOyCTcXwf6mVlfMysBLgDG1yvzLnAy\ngJkdTAj35dmsaN4qagflX4RvToNP3wEbPwwHWe4dArMn5GfIr6mEB4bB6kUhSHN86tYOO+BkuHRC\nOGf6/tNCj58tZcOH8JdvwANDw7ndnxsb1tEefVuuDm2RWTg3/OuvQd/jw+mlD54ZTpmNW/VWeOrr\n8Oqd4eD5ub/Nz2NC9WR6KuQZwG2E0xzvd/efmtkNQIW7j4/OkLkP6Eg4uPodd5/Y1DxbfbNMY6q3\nhsvP/3lzOHth70PhhO+Gc+bzoblm5Tvwh+Ghrf2ix2DfwXHXqHGrFoW9oVUL4dx74ZBzcresVAqm\n/T50jrZlfTiD44TvJP/013zkHrqJfvZ74Qf+lB+Fq6SzcYX0jtqyMRw4nfscnPhDOD7+nk11EVPc\nqreGm1q8dFNoYtjr42G3vv+n4/mQQrhg6w9nh+6RL34ydIuc7zauDHd2evdVOP1ncPQ3sr+MpW+G\nMzeWTIX9joUzb0nmVZStzZol4fjLvBfC+3L2nS27B7VxJTx8Qeiu5MxbofzSllt2ExTu+aK6CqaP\nCyG/Yh7seUgI+YPPbtmQf396OHgK4U4wew9suWXvqq2b4ImvhE7gBn899NuRjXW3aQ3846fw+n3Q\nvluY76EjYt8ykzTu4eyU574fusU49Xoo/1LuvztrloQrqFfOh/N+lz/HpFC4559UdThXfvKNoZuD\nHv3h+KtCU0OuT/1aMi3ctaqoFC4ZDz0OzO3yciGVCl/w1+6GAWfDOffu/IVk7vDfx8IBsg3Lw4Hx\nk36YlRspSI6sqQz9Hc3/R2iTH35n7np1/XAu/PGc0HQ58k9heXlE4Z6vUtWh07LJN8KHb0P3g0LI\nDzw3NyH/7pRwi6/duoRgb+0HBl+5M5zBsu+n4IKx4Z6UO2L526GPk4X/DB1ZnXVr6IJY8p97OC7y\n3A8Bh9N+DEdcmt09rSVTw/fFCkKvpXl4g3eFe75LpcJFMZNvDKd/desXhfx52buUfcHk0GbYaR/4\n/PjQ42ASTH883KC7a18YNS6zO8lv2RCaxl65M/SJcvJ14c5Q+X7BjGxv9bthK37Bi+GCt+G/yuwz\n0Jz5/4BHRoWO3y5+Mm/PtVe4txapVGhLnnwjLJsRroA8/ir4+Gd3LeTnPh/6iunaFy55Cjrtlb06\n54OFL4cDrUWl4SrBfQ5tvOzsp+GZ74a+WD5xIZx6QzJvFNKWuMPUB2DiNYDB6T+Bwz+/81vx/x0X\nNhh69A8bDJ32zmp1s0nh3tqkUqE3xsm/CH3Md+0bTrs69HM7fgOHmeNh3BdDR0YX/yW5/dMvmwUP\nnR/6Cv/cH7fvEmDVQnjm6nDlY4+Dw1kwfY6JpaqSI6sWhXv0vvNS6AJk+B07vof62r3wzHfCxWoX\n/Cnvj70o3Fsr93AP2BfHwPtvhW5cj/82fGJkZiH/1mPw5FfDaY4Xjcv7D+ouW/teOBd++exwkO2w\nkaHr4ld+FfqqsYLQLcTg/2nddzmSxqVS4X6lz18XmtlO/1noW7+5rXh3mPQzeOlGOOhMOP93WbuJ\ndS4p3Fs7d5jzbNiSX/pGaFM87srQrNDY1XHT/hA6NNrvGLjwkdB1a1uwaW1ognpncriCcP6kcEbS\ngLPh9J9D515x11Bawsp34KnLYNHLcMCpYSu+sV47U9XhwPrUB3JyE+tcUrgnhXtoP588JhzJ79w7\n3ERj0Ki6/Zy8dm+45+n+J4VL5nf0RgqtXdWWsHv+1p9Dk9YZN0O/U+KulbS0VCrc9/eF66CgGIb+\nHA67sO5WfPp1E8d+C06+tlVd26BwTxr30J/45DHhirnde4WQP/yS0MnSC9eFXcvPPtB2O7dyh0X/\nCnf5aQW715JDKxeEPoLefQX6nQ6fvj30MLppbTgQv/CfYa/u6K/HXdMdpnBPKndYMAle/AUsnhLu\nq/nRynDziHPvVbuySI1UCv59D7xwfWjKPPk6mPogLJsZbvF36Ii4a7hTFO5J5x7amF/+JXQ7INwa\nT+dsi2xvxXz4y9fDxlBx+3AT636nxl2rnZbN2+xJPjILF3B8bEi89RDJd932D11I/+eR0KdSHl51\nmgsKdxFJvoLCcCOYNiSmvmdFRCSXFO4iIgmkcBcRSSCFu4hIAincRUQSSOEuIpJACncRkQRSuIuI\nJJDCXUQkgRTuIiIJpHAXEUkghbuISAIp3EVEEkjhLiKSQAp3EZEEUriLiCSQwl1EJIEU7iIiCaRw\nFxFJIIW7iEgCZRTuZjbUzN42s3lmdnUjZUaY2Uwzm2Fmf8puNUVEZEcUNVfAzAqBu4BTgUrgdTMb\n7+4z08r0A74HHOPuq8xsz1xVWEREmpfJlvuRwDx3X+DuW4BHgLPrlfkKcJe7rwJw92XZraaIiOyI\nTMK9F7A47XFlNC7dgcCBZvYvM5tiZkMbmpGZjTazCjOrWL58+c7VWEREmpWtA6pFQD9gCDASuM/M\nutQv5O73unu5u5f36NEjS4sWEZH6Mgn3JUDvtMdl0bh0lcB4d9/q7u8AcwhhLyIiMcgk3F8H+plZ\nXzMrAS4Axtcr8xfCVjtm1p3QTLMgi/UUEZEd0Gy4u3sVcBnwHDALeNTdZ5jZDWY2PCr2HLDCzGYC\nk4Cr3H1FriotIiJNM3ePZcHl5eVeUVERy7JFRForM5vq7uXNldMVqiIiCaRwFxFJIIW7iEgCKdxF\nRBJI4S4ikkAKdxGRBFK4i4gkkMJdRCSBFO4iIgmkcBcRSSCFu4hIAincRUQSSOEuIpJACncRkQRS\nuIuIJJDCXUQkgRTuIiIJpHAXEUkghbuISAIp3EVEEkjhLiKSQAp3EZEEUriLiCSQwl1EJIEU7iIi\nCaRwFxFJIIW7iEgCKdxFRBJI4S4ikkAKdxGRBFK4i4gkkMJdRCSBFO4iIgmkcBcRSSCFu4hIAmUU\n7mY21MzeNrN5ZnZ1E+XOMzM3s/LsVVFERHZUs+FuZoXAXcAwYAAw0swGNFCuE3AF8Fq2KykiIjsm\nky33I4F57r7A3bcAjwBnN1Dux8AvgE1ZrJ+IiOyETMK9F7A47XFlNG4bMzsc6O3uTzc1IzMbbWYV\nZlaxfPnyHa6siIhkZpcPqJpZAXArcGVzZd39Xncvd/fyHj167OqiRUSkEZmE+xKgd9rjsmhcjU7A\nQOBFM1sIDAbG66CqiEh8Mgn314F+ZtbXzEqAC4DxNRPdfY27d3f3Pu7eB5gCDHf3ipzUWEREmtVs\nuLt7FXAZ8BwwC3jU3WeY2Q1mNjzXFRQRkR1XlEkhd58ATKg37tpGyg7Z9WqJiMiu0BWqIiIJpHAX\nEUkghbuISAIp3EVEEkjhLiKSQAp3EZEEUriLiCSQwl1EJIEU7iIiCaRwFxFJIIW7iEgCKdxFRBJI\n4S4ikkAKdxGRBFK4i4gkkMJdRCSBFO4iIgmkcBcRSSCFu4hIAincRUQSSOEuIpJACncRkQRSuIuI\nJJDCXUQkgRTuIiIJpHAXEUkghbuISAIp3EVEEkjhLiKSQAp3EZEEUriLiCSQwl1EJIEU7iIiCaRw\nFxFJoIzC3cyGmtnbZjbPzK5uYPq3zGymmb1lZn83s/2yX1UREclUs+FuZoXAXcAwYAAw0swG1Cv2\nBlDu7ocC44Abs11RERHJXCZb7kcC89x9gbtvAR4Bzk4v4O6T3H1j9HAKUJbdaoqIyI7IJNx7AYvT\nHldG4xrzJeCZhiaY2WgzqzBblGHDAAAIlklEQVSziuXLl2deSxER2SFZPaBqZqOAcuCmhqa7+73u\nXu7u5T169MjmokVEJE1RBmWWAL3THpdF4+ows1OAHwAnuPvm7FRPRER2RiZb7q8D/cysr5mVABcA\n49MLmNkg4B5guLsvy341RURkRzQb7u5eBVwGPAfMAh519xlmdoOZDY+K3QR0BB4zszfNbHwjsxMR\nkRaQSbMM7j4BmFBv3LVpw6dkuV4iIrILMgr3lrJ161YqKyvZtGlT3FXJqdLSUsrKyiguLo67KiKS\nUHkV7pWVlXTq1Ik+ffpgZnFXJyfcnRUrVlBZWUnfvn3jro6IJFRe9S2zadMmunXrlthgBzAzunXr\nlvi9ExGJV16FO5DoYK/RFl6jiMQr78JdRER2ncI9zerVq/n1r3+9w88744wzWL16dQ5qJCKycxTu\naRoL96qqqiafN2HCBLp06ZKraomI7LC8Olsm3fV/ncHMpWuzOs8BPXfnuk8f0uj0q6++mvnz53PY\nYYdRXFxMaWkpXbt2Zfbs2cyZM4fPfOYzLF68mE2bNnHFFVcwevRoAPr06UNFRQXr169n2LBhHHvs\nsbzyyiv06tWLp556it122y2rr0NEpDnack8zZswY9t9/f958801uuukmpk2bxu23386cOXMAuP/+\n+5k6dSoVFRXccccdrFixYrt5zJ07l2984xvMmDGDLl268Pjjj7f0yxARyd8t96a2sFvKkUceWedc\n9DvuuIMnn3wSgMWLFzN37ly6detW5zl9+/blsMMOA+CII45g4cKFLVZfEZEaeRvu+aBDhw7bhl98\n8UVeeOEFXn31Vdq3b8+QIUMaPFe9Xbt224YLCwv56KOPWqSuIiLp1CyTplOnTqxbt67BaWvWrKFr\n1660b9+e2bNnM2XKlBaunYhI5rTlnqZbt24cc8wxDBw4kN1224299tpr27ShQ4fym9/8hoMPPpiD\nDjqIwYMHx1hTEZGmmbvHsuDy8nKvqKioM27WrFkcfPDBsdSnpbWl1yoi2WNmU929vLlyapYREUkg\nhbuISAIp3EVEEkjhLiKSQAp3EZEEUriLiCSQwj3Nznb5C3DbbbexcePGLNdIRGTnKNzTKNxFJCny\n9wrVZ66G9/+b3Xnu/XEYNqbRyeld/p566qnsueeePProo2zevJlzzjmH66+/ng0bNjBixAgqKyup\nrq7mmmuu4YMPPmDp0qWceOKJdO/enUmTJmW33iIiOyh/wz0GY8aMYfr06bz55ptMnDiRcePG8e9/\n/xt3Z/jw4bz00kssX76cnj178vTTTwOhz5nOnTtz6623MmnSJLp37x7zqxARyedwb2ILuyVMnDiR\niRMnMmjQIADWr1/P3LlzOe6447jyyiv57ne/y1lnncVxxx0Xaz1FRBqSv+EeM3fne9/7Hl/96le3\nmzZt2jQmTJjAD3/4Q04++WSuvfbaGGooItI4HVBNk97l7+mnn87999/P+vXrAViyZAnLli1j6dKl\ntG/fnlGjRnHVVVcxbdq07Z4rIhI3bbmnSe/yd9iwYVx44YUcffTRAHTs2JGHHnqIefPmcdVVV1FQ\nUEBxcTF33303AKNHj2bo0KH07NlTB1RFJHbq8jcmbem1ikj2qMtfEZE2TOEuIpJAeRfucTUTtaS2\n8BpFJF55Fe6lpaWsWLEi0eHn7qxYsYLS0tK4qyIiCZZXZ8uUlZVRWVnJ8uXL465KTpWWllJWVhZ3\nNUQkwfIq3IuLi+nbt2/c1RARafUyapYxs6Fm9raZzTOzqxuY3s7M/hxNf83M+mS7oiIikrlmw93M\nCoG7gGHAAGCkmQ2oV+xLwCp3PwD4JfCLbFdUREQyl8mW+5HAPHdf4O5bgEeAs+uVORv4fTQ8DjjZ\nzCx71RQRkR2RSZt7L2Bx2uNK4KjGyrh7lZmtAboBH6YXMrPRwOjo4Xoze3tnKg10rz/vNk7roy6t\nj1paF3UlYX3sl0mhFj2g6u73Avfu6nzMrCKTy2/bCq2PurQ+amld1NWW1kcmzTJLgN5pj8uicQ2W\nMbMioDOwIhsVFBGRHZdJuL8O9DOzvmZWAlwAjK9XZjzw+Wj4fOAfnuQrkURE8lyzzTJRG/plwHNA\nIXC/u88wsxuACncfD/wO+KOZzQNWEn4AcmmXm3YSRuujLq2PWloXdbWZ9RFbl78iIpI7edW3jIiI\nZIfCXUQkgVpduDfXFUJbYWa9zWySmc00sxlmdkXcdcoHZlZoZm+Y2d/irkvczKyLmY0zs9lmNsvM\njo67TnExs/+LvifTzexhM0t8t6ytKtwz7AqhragCrnT3AcBg4BtteF2kuwKYFXcl8sTtwLPu3h/4\nBG10vZhZL+ByoNzdBxJODMn1SR+xa1XhTmZdIbQJ7v6eu0+LhtcRvri94q1VvMysDDgT+G3cdYmb\nmXUGjiecyYa7b3H31fHWKlZFwG7RdTjtgaUx1yfnWlu4N9QVQpsONICoF85BwGvx1iR2twHfAVJx\nVyQP9AWWAw9EzVS/NbMOcVcqDu6+BLgZeBd4D1jj7hPjrVXutbZwl3rMrCPwOPC/7r427vrExczO\nApa5+9S465InioDDgbvdfRCwAWiTx6jMrCthD78v0BPoYGaj4q1V7rW2cM+kK4Q2w8yKCcE+1t2f\niLs+MTsGGG5mCwnNdSeZ2UPxVilWlUClu9fszY0jhH1bdArwjrsvd/etwBPAp2KuU861tnDPpCuE\nNiHqUvl3wCx3vzXu+sTN3b/n7mXu3ofwufiHuyd+66wx7v4+sNjMDopGnQzMjLFKcXoXGGxm7aPv\nzcm0gYPLeXWbveY01hVCzNWKyzHAxcB/zezNaNz33X1CjHWS/PJNYGy0IbQAuDTm+sTC3V8zs3HA\nNMJZZm/QBrohUPcDIiIJ1NqaZUREJAMKdxGRBFK4i4gkkMJdRCSBFO4iIgmkcBcRSSCFu4hIAv0/\nC2K5G0DT7CkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(history_aug.history['acc'], label='train')\n",
    "plt.plot(history_aug.history['val_acc'], label='test')\n",
    "plt.title(\"Simple CNN with augmented data\")\n",
    "plt.legend()\n",
    "plt.ylim(0,1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "o5lxKigJ7Uyg",
    "outputId": "79ea6db2-d588-4658-b925-e0dd1e5d02c0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Accuracy: 0.88\n",
      "Validation Accuracy: 0.86\n"
     ]
    }
   ],
   "source": [
    "print('Training Accuracy: {}'.format(np.round(history_aug.history['acc'][-1], 2)))\n",
    "print('Validation Accuracy: {}'.format(np.round(history_aug.history['val_acc'][-1], 2)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ed4436FT7nz4"
   },
   "source": [
    "At the expense of a much increased training time, we can see that the same model with augmented data does a little bit better in terms of validation accuracy. The plot also show us that the validation accuracy fluctuates less and generally stays higher. The final accuracy numbers tell us that we indeed overfit less, as the training accuracy has decreased and the validation accuracy is much increased."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "V_y9szwNDXIr"
   },
   "source": [
    "#### 3.3 Deeper model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jr8MEXFeDer7"
   },
   "source": [
    "Here is our deeper model with residual connections. The architecture is inspired by resnet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yWiPqKRO1UWP"
   },
   "outputs": [],
   "source": [
    "inputs = Input(shape = (50, 50, 3))\n",
    "\n",
    "                                                             ### First layer ###\n",
    "x1 = Conv2D(32, (3, 3), activation='relu')(inputs)           # Conv Layer\n",
    "x2 = BatchNormalization()(x1)                                # Batch Norm.\n",
    "x4 = MaxPooling2D(pool_size=(2, 2))(x2)                      # Pooling Layer\n",
    "                                                          \n",
    "                         \n",
    "x5 = Conv2D(32, (3, 3), padding='same', \n",
    "            activation='relu')(x4)                           # Conv Layer\n",
    "x6 = BatchNormalization()(x5)                                # Batch Norm.\n",
    "\n",
    "x8 = Conv2D(32, (3, 3), padding='same', \n",
    "            activation='relu')(x6)                           # Conv Layer\n",
    "x9 = BatchNormalization()(x8)                                # Batch Norm.\n",
    "\n",
    "x11 = add([x4, x9])                                          # Residual connection\n",
    "\n",
    "x12 = Conv2D(32, (3, 3), padding='same', \n",
    "             activation='relu')(x11)                         # Conv Layer\n",
    "x13 = BatchNormalization()(x12)                              # Batch Norm.\n",
    "\n",
    "x15 = Conv2D(32, (3, 3), padding='same', \n",
    "             activation='relu')(x13)                         # Conv Layer\n",
    "x16 = BatchNormalization()(x15)                              # Batch Norm.           \n",
    "\n",
    "x80 = add([x11, x16])                                        # Residual connection\n",
    "\n",
    "x99 = Conv2D(32, (3, 3), padding='same', \n",
    "             activation='relu')(x80)                         # Conv Layer\n",
    "x98 = BatchNormalization()(x99)                              # Batch Norm.\n",
    "\n",
    "x97 = Conv2D(32, (3, 3), padding='same', \n",
    "             activation='relu')(x98)                         # Conv Layer\n",
    "x96 = BatchNormalization()(x97)                              # Batch Norm.           \n",
    "\n",
    "x18 = add([x80, x96])                                        # Residual connection\n",
    "\n",
    "x19 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x18)                         # Conv Layer\n",
    "x20 = BatchNormalization()(x19)                              # Batch Norm.\n",
    "\n",
    "x22 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x20)                         # Conv Layer\n",
    "x23 = BatchNormalization()(x22)                              # Batch Norm.\n",
    "\n",
    "x26 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x23)                         # Conv Layer\n",
    "x27 = BatchNormalization()(x26)                              # Batch Norm.\n",
    "\n",
    "x28 = add([x27, x20])                                        # Residual connection\n",
    "\n",
    "x29 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x28)                         # Conv Layer\n",
    "x30 = BatchNormalization()(x29)                              # Batch Norm.\n",
    "\n",
    "x31 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x30)                         # Conv Layer\n",
    "x32 = BatchNormalization()(x31)                              # Batch Norm.\n",
    "\n",
    "x40 = add([x32, x28])                                        # Residual connection\n",
    "\n",
    "x41 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x40)                         # Conv Layer\n",
    "x42 = BatchNormalization()(x41)                              # Batch Norm.\n",
    "\n",
    "x43 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x42)                         # Conv Layer\n",
    "x44 = BatchNormalization()(x43)                              # Batch Norm.\n",
    "\n",
    "x45 = add([x44, x40])                                        # Residual connection\n",
    "\n",
    "x46 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x45)                         # Conv Layer\n",
    "x47 = BatchNormalization()(x46)                              # Batch Norm.\n",
    "\n",
    "x48 = Conv2D(64, (3, 3), padding='same', \n",
    "             activation='relu')(x47)                         # Conv Layer\n",
    "x49 = BatchNormalization()(x48)                              # Batch Norm.\n",
    "\n",
    "x33 = add([x45, x49])                                        # Residual connection\n",
    "\n",
    "                                                             ### Last layer ###\n",
    "x34 = Flatten()(x33)                                         # Flatten the data\n",
    "x35 = Dense(64, activation='relu')(x34)                      # 64 units dense\n",
    "\n",
    "preds = Dense(2, activation='softmax')(x35)                  ### Output ###\n",
    "\n",
    "deep_res = Model(inputs=inputs, outputs=preds)\n",
    "deep_res.compile(optimizer='adam', loss='categorical_crossentropy', \n",
    "             metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rQypXJJgDpBh"
   },
   "source": [
    "And here is the same model without residual connections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "O9EKSyKTDtRE"
   },
   "outputs": [],
   "source": [
    "input_shape = (50, 50, 3)\n",
    "deep_nores = Sequential()\n",
    "\n",
    "                                                             ### Layer 1 ###\n",
    "deep_nores.add(Conv2D(32, (3, 3), activation='relu',         # Convolutional L\n",
    "                 input_shape=input_shape, padding='same'))\n",
    "deep_nores.add(BatchNormalization())                         # Batch Norm.\n",
    "deep_nores.add(MaxPooling2D(pool_size=(2, 2)))               # Pooling Layer\n",
    "\n",
    "for i in range(6):                                           ### Layers 2-7 ###\n",
    "  deep_nores.add(Conv2D(32, (3, 3), activation='relu',\n",
    "                       padding='same'))                      # Conv. Layer\n",
    "  deep_nores.add(BatchNormalization())                       # Batch Norm.\n",
    "  \n",
    "for i in range(9):                                           ### Layers 8-16 ###\n",
    "  deep_nores.add(Conv2D(32, (3, 3), activation='relu',\n",
    "                       padding='same'))                      # Conv. Layer\n",
    "  deep_nores.add(BatchNormalization())                       # Batch Norm.\n",
    "\n",
    "                                                             ### Layer 17 ###\n",
    "deep_nores.add(Flatten())                                    # Flatten the data\n",
    "deep_nores.add(Dense(64, activation='relu'))                 # 64 units dense\n",
    "\n",
    "deep_nores.add(Dense(2, activation='softmax'))               ### Output ###\n",
    "\n",
    "deep_nores.compile(optimizer='adam', loss='categorical_crossentropy', \n",
    "                   metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aJJ5oIVtD0yc"
   },
   "source": [
    "Let's now train them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 370
    },
    "colab_type": "code",
    "id": "DbxCKF_KDwjm",
    "outputId": "443af767-4bb4-4b3d-f165-4d09277e457c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 141814 samples, validate on 15758 samples\n",
      "Epoch 1/10\n",
      "141814/141814 [==============================] - 110s 779us/step - loss: 0.4136 - acc: 0.8314 - val_loss: 0.4475 - val_acc: 0.8035\n",
      "Epoch 2/10\n",
      "141814/141814 [==============================] - 103s 726us/step - loss: 0.3477 - acc: 0.8540 - val_loss: 0.5230 - val_acc: 0.6894\n",
      "Epoch 3/10\n",
      "141814/141814 [==============================] - 103s 726us/step - loss: 0.3315 - acc: 0.8606 - val_loss: 0.5062 - val_acc: 0.7763\n",
      "Epoch 4/10\n",
      "141814/141814 [==============================] - 103s 724us/step - loss: 0.3180 - acc: 0.8671 - val_loss: 0.5878 - val_acc: 0.7549\n",
      "Epoch 5/10\n",
      "141814/141814 [==============================] - 103s 725us/step - loss: 0.3100 - acc: 0.8710 - val_loss: 0.3904 - val_acc: 0.8346\n",
      "Epoch 6/10\n",
      "141814/141814 [==============================] - 103s 724us/step - loss: 0.2989 - acc: 0.8752 - val_loss: 0.9218 - val_acc: 0.6654\n",
      "Epoch 7/10\n",
      "141814/141814 [==============================] - 103s 725us/step - loss: 0.2867 - acc: 0.8798 - val_loss: 0.4705 - val_acc: 0.7722\n",
      "Epoch 8/10\n",
      "141814/141814 [==============================] - 103s 726us/step - loss: 0.2739 - acc: 0.8846 - val_loss: 0.7544 - val_acc: 0.6560\n",
      "Epoch 9/10\n",
      "141814/141814 [==============================] - 103s 724us/step - loss: 0.2615 - acc: 0.8888 - val_loss: 0.6169 - val_acc: 0.7583\n",
      "Epoch 10/10\n",
      "141814/141814 [==============================] - 103s 726us/step - loss: 0.2410 - acc: 0.8970 - val_loss: 0.5752 - val_acc: 0.7901\n"
     ]
    }
   ],
   "source": [
    "history_res = deep_res.fit(X, y, batch_size=128, epochs=10, verbose=1,\n",
    "                            validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 370
    },
    "colab_type": "code",
    "id": "Jx9G5aGuD0NU",
    "outputId": "8cd8416b-932f-4c3b-f22f-5ac9fdeb74c5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 141814 samples, validate on 15758 samples\n",
      "Epoch 1/10\n",
      "141814/141814 [==============================] - 78s 549us/step - loss: 0.4050 - acc: 0.8263 - val_loss: 0.8116 - val_acc: 0.5982\n",
      "Epoch 2/10\n",
      "141814/141814 [==============================] - 71s 497us/step - loss: 0.3597 - acc: 0.8471 - val_loss: 0.8282 - val_acc: 0.7086\n",
      "Epoch 3/10\n",
      "141814/141814 [==============================] - 70s 496us/step - loss: 0.3447 - acc: 0.8542 - val_loss: 2.5615 - val_acc: 0.6171\n",
      "Epoch 4/10\n",
      "141814/141814 [==============================] - 71s 499us/step - loss: 0.3364 - acc: 0.8586 - val_loss: 0.7981 - val_acc: 0.7068\n",
      "Epoch 5/10\n",
      "141814/141814 [==============================] - 71s 499us/step - loss: 0.3318 - acc: 0.8614 - val_loss: 1.1259 - val_acc: 0.7180\n",
      "Epoch 6/10\n",
      "141814/141814 [==============================] - 70s 497us/step - loss: 0.3253 - acc: 0.8638 - val_loss: 1.7328 - val_acc: 0.5802\n",
      "Epoch 7/10\n",
      "141814/141814 [==============================] - 71s 498us/step - loss: 0.3199 - acc: 0.8669 - val_loss: 1.3938 - val_acc: 0.6237\n",
      "Epoch 8/10\n",
      "141814/141814 [==============================] - 71s 498us/step - loss: 0.3140 - acc: 0.8693 - val_loss: 0.9398 - val_acc: 0.6846\n",
      "Epoch 9/10\n",
      "141814/141814 [==============================] - 71s 498us/step - loss: 0.3082 - acc: 0.8726 - val_loss: 1.1575 - val_acc: 0.6551\n",
      "Epoch 10/10\n",
      "141814/141814 [==============================] - 71s 498us/step - loss: 0.3025 - acc: 0.8742 - val_loss: 0.5720 - val_acc: 0.7692\n"
     ]
    }
   ],
   "source": [
    "history_nores = deep_nores.fit(X, y, batch_size=128, epochs=10, verbose=1,\n",
    "                               validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 301
    },
    "colab_type": "code",
    "id": "vILlPIaaD3hB",
    "outputId": "222fd4fd-96ed-40f0-ec61-e2dc29e9fd51"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAEICAYAAABcYjLsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XeYVOXZx/HvvR2WKkWlKAhYiRUR\no8augIrGCvaKKSaaqFET30QTE41JjJqoid1olNhFxYa9oQJWBKQqRQSW3pYtz/vHfZY9u+yyw+7s\nzuzO73Nde005Z855zpnZe+55zlMshICIiIiISKbLSnUBRERERETSgRJjERERERGUGIuIiIiIAEqM\nRUREREQAJcYiIiIiIoASYxERERERQImxNAIz+7WZ3b2J5Web2TtNWaZov9uY2Sozy65l+TVm9lCS\n9hXMrG8ytpVO6jqHIpI+0jUWJ5OZnWZmL29i+Rtmdn4S9nOQmc1t6HbSUV3nMNMoMU6Qmc02s7Vm\nttLMlpnZe2b2IzNLyTk0s63N7B4z+zYq0xQzu9bMCqPlwcw+j5fPzK4zs/uj+72idcZU2+5DZnZN\nQ8oWQvhTCOH8avvJacg2kyGE8E0IoU0IoSzVZWkuos/9YRWPdQ4l1RSLE5eusTiZQgj/DSEckepy\nNBc1fQ50DqtSYrx5jgkhtAW2BW4ArgDuaepCmNkWwPtAK2DfqEyHAx2APrFVuwHD69jcPmb2/UYp\naCNoaUFdROpFsbgFUVyXdKLEuB5CCMtDCKOBU4CzzKw/gJnlm9lfzewbM/vOzP5lZq0qXmdmR5vZ\nJ7Fajl1jy2ab2VVm9qWZLTWz+8ysoJYi/BJYCZweQpgdlWlOCOHiEMJnsfVuBK6tI+jcCPwxkeM2\ns6/NbK/o/mnRr85dosfnmdnT0f14k4S3ottl0SX4fWPb+2t0rLPMbMgm9jvbzK4ws8+A1WaWY2bd\nzOwJM1sUvf7nsfUHmtl4M1sRvQ83Rc9X+aVsZr3N7M2olucVoHNsGxtdNovXnkb7eD96L781s3+a\nWV6C53GL6P2dHx3/07FlF5jZdDNbYmajzaxbbFmIasamRfu9zcwsWna2mb1T2zk1s/axWq15UY1V\ndrX9To7OxZdmtqeZPQhsAzwbvXe/quEcdovKuSQq9wWxbV5jZo+a2X+i7U4yswGx5VdEZVlpZlPN\n7NBEzp9IBcXilMTiy8zsMzNbbmb/i5+bTcWvatupiCPnmdk3wGvR84Oi92OZmX1qZgfFXnO2mc2M\n4sUsMzst9vw7sfUON6+1X25m/wQstqxKc7ka4tk5sTg408wu3OQbUfWYdjGzV6Jj/87Mfh09n29m\nN5vH+/nR/fxo2UFmNtfMLjWzhVF8Pie2zfvN4/zzUZk+MLM+seU7xvY51cxOji1rZWZ/iz4ry82/\nH1pRw+eghnP4fTP7KHrdRxb7wWbeNOUPZvZuVKaXzaxztKzA/EpHUfQefmRmWyZ6DtOFEuMGCCF8\nCMwFDoieugHYHtgd6At0B34LYGZ7APcCFwKdgH8Doyv+QSKnAUfiNQ3bA1fXsuvDgCdDCOV1FPFJ\nYAVw9ibWuR3Y3mKXyzfhTeCg6P6BwEzgB7HHb9bwmorlHaJL8O9Hj/cBpuLJ6I3APWZmNby+wgjg\nKLwmphx4FvgUP8eHApeY2ZHRurcAt4QQ2uHn8tFatvkwMCEqwx+Aszax/+rKgF9Er903KsNPEnzt\ng0BrYBegK/B3ADM7BLgeOBnYGvgaGFXttUcDewO7RusdGVu2qXN6P1CKfy73AI4AKi6xngRcA5wJ\ntAOGAUUhhDOAb/DauTYhhBtrOJZR+P9AN+BE4E/RcVQYFq3TARgN/DPa5w7ARcDeUS3bkcDsTZwz\nkVopFjdpLD4ZGAz0xuPQ2ZBw/KruQGAn4Egz6w48D1wHbAFcBjxhZl3Mm6XcCgyJ4sX3gU+qbyxK\n0J7E36/OwAxgvzrKELcQj7HtgHOAv5vZnnW9yMzaAmOBF/FY2Bd4NVr8G2AQ/lncDRhI1c/TVkB7\n/DN6HnCbmXWMLR8OXAt0BKYT/XiKzskr+PdY12i9281s5+h1fwX2ws/VFsCv8O/O2j4HFceyBf4+\n3Ir/f9wEPG9mnWKrnRqdn65AHv5egX+Htgd6Rq/9EbB2U+cuLYUQ9JfAH/6lfVgNz4/DP/gGrAb6\nxJbtC8yK7t8B/KHaa6cCB8a2/6PYsqHAjFrKMi2+bi3rBPyfcygeoPLwgHN/tLxXtE4OntCNi55/\nCLimlm2eB4yO7k/GE6tR0eOvgT2j+9cAD1XfT2w7ZwPTY49bR+tstYlzf27s8T7AN9XWuQq4L7r/\nFh5IOldbJ37M2+CJYmFs+cOxch8EzE3kMxAtuwR4qvr5r2G9rfHg1LGGZfcAN8YetwFKgF6xbe4f\nW/4ocGVd5xTYEigGWsWWjwBej+6/BFycyOe+2jnsif9AaBtbfn3sM3YNMDa2bGdgbXS/L/4ldBiQ\n2xT/w/prGX+1/R+iWNxUsfj02OMbgX9F9zcZv6ptp6Is28WeuwJ4sNp6L+HJViGwDDiBWByLHcM7\n0f0zK85f9NjwH0znVz8ftZ2Tatt+mig2UsN3Qmy9EcDHtSybAQyNPT4SmB3b5tpq78lCYFB0/37g\n7mqfxSnR/VOAt6vt69/A7/BKz7XAbps499U/BxXn8Azgw2qveR84O7r/BnB1bNlPgBej++cC7wG7\nNvT/PJV/qjFuuO7AEqALHlQmRJcQluG/HrtE620LXFqxLFreE/91WWFO7P7X1ZbFFeEJVp1CCGPw\nwLCpS0J3A1ua2TF1bO5N4AAz2xrIxhOz/cysF/4rcaNf8JuwIFbGNdHdNptYP35utgW6VTuXv8YT\nQPAvje2BKdGlnKNr2F43YGkIYXXsua8TLbyZbW9mz5nZAjNbAfyJWFOMTegJLAkhLK2lTBvKEEJY\nhb/X3WPrLIjdX0PVc1bbOd0WyAW+jZ2vf+O/9ivKNCOBstdU3iUhhJWx576uo7wFZpYTQpiO/5i4\nBlhoZqOslsuuIglSLG6aWFxbDEokflVXPa6fVO192R/YOorTp+A1kN9GTQt2rGF73eLbDJ6tzalh\nvRqZ2RAzGxc1TViGJ6KJxvXaYmiV88LGn6eiEEJp7HGtcb3asm3xdunx83UaXhnSGSjYRJk2pXp5\nK8qcyPfQg/iPmVFRs5EbzSy3HmVIKSXGDWBme+MflneAxfgvtF1CCB2iv/YhhIoPzBzgj7FlHUII\nrUMIj8Q22TN2fxtgfi27Hgv80BLvhf0bPHFsXdPCEMJ6vIb1D8TaY9Ww3nT8n+BnwFshhBX4P8hI\n/NdmTZcTQ4JlrEt8O3Pw2p/4uWwbQhgalXNaCGEEnvj9GXg8uuwU9y3Qsdrz28TuryZ2vszb43aJ\nLb8DmAL0C95k49ds4txVK/sWZtahhmXz8WBXsc9C/HLUvAS2W9c+i/Ea9Irz1S6EsEtseZ9aXrup\n928+fixtY89tk2h5QwgPhxD2x4854O+VyGZTLG7SWFyb+sSv6nH9wWrvS2EI4QaAEMJLIYTD8R8i\nU4C7atjet8Teu6hJSPy9rBLX8SSyYt184Am8CcKWIYQOwBgSj+vb1bKsynlh05+nzTEHeLPa+WoT\nQvgx/j+wjprjel2fg+rlhQTjegihJIRwbQhhZ7wJx9F4LX6zosS4HsysXVQLOQq/LPN5FIjuwtsk\ndY3W6x5r93oX8CMz28dcoZkdVS2p+KmZ9Yja+PwG+F8tRbgJbwP1gJltG9vXTRbrRFIhhPAG8AWb\nbkP7IP4Lc3Adh/8m3ja0og3bG9UeV7cIbzpQW9Cojw+Bleadt1qZWbaZ9Y++HDGz082sS/SeLIte\nU+WLIoTwNTAe7xCTZ2b7A/Famq/w2s2jol+8VwPxNoht8TaDq6Kaix8nUvAQwrfAC3hbsI5mlmtm\nFW2+HgHOMbPdoyD9J+CDEHXqqa9ony8Df4s+u1lm1sfMDoxWuRu4zMz2ij6bfSs+V8B31PLehRDm\n4JfNrjfvdLErXltf51jQZraDmR0SHec6PJGpq52mSBWKxSmPxXENjV8PAceY2ZFRTC8w75zWw8y2\nNLNjo2S7GFhFzfHieWAXMzvevEPdz4klv3hN+g/Mx2NvjzfBq5CHx/hFQKl5J8REhzB7DtjazC4x\n72zX1sz2iZY9Alxt3la6M97WPRnj5T+Ht0k/I/oeyTWzvc1sp+h/4F7gJvMO0tnmnewqjm9Tn4Mx\n0XZPNe/ofgreDO65ugpkZgeb2feiiqQVeFOaZhfXlRhvnmfNbCX+S+03eFA8J7b8Crxx/Djzy+tj\ngR0AQgjjgQvwzkdLo/XOrrb9h/EEZiZ+CeS6mgoRQliC/xorAT6IyvQqsDzabk2uxhvg1yj4uLS/\n3dQ6kTfxpPCtWh5X3+4avLPAu+aXewbVsf06RWU9Gu/MMAv/dXw3fgkR/AtlkpmtwjviDQ8h1NQB\n4FS8vfISvF3Wf2L7WI63nbob/6W8Gr8MWuGy6PUr8S/a2r44a3IG/t5NwduTXRLtcyzwf3itxbf4\nr/26hnhK1Jl44P8S//w9TnQJOITwGP4ePYwfz9NUfg6ux4P6MjO7rPpG8bZ1vfBahqeA30XHUZd8\nvIPUYrymqytVv6RENkWxOA1icbXtNyh+RT+0j8Vr1Bfh7+3leJ6ShY8AMh+P1wdSQ2VECGExcBIe\nW4qAfsC7seWv4LH6M7zj9XOxZSvxRPpR/HNxKt5hOJGyr8SH6TsGj2fTgIOjxdfhlTCfAZ8DE6nl\n87Q5on0egZ/j+dF+/0xlBc5l0f4+ws/Zn4Gsuj4HIYQi/Pv1Uvwc/go4Ojq3ddkK/25Zgbd9fxP/\nodesmDfBkVQzs9l4B4FEkgoREWkEisUimU01xiIiIiIiJJAYm9m95gNPf1HLcjOzW80H9f7MEhjz\nT0REGo/itohI/SRSY3w/m+4EMARvx9MP7xF7R8OLlXlCCL106U5EkuR+FLfrRbFYJLPVmRiHEN7C\nG27X5ljgP8GNAzqYj60oIiIpoLgtIlI/m5q3PVHdqTqA9tzouW+rr2hmI/HaCQoLC/facceaxucW\nEUlvEyZMWBxC6FL3mmkrobitmC0iLUWicTsZiXHCQgh3AncCDBgwIIwfP74pdy8ikhRmlvAsic2Z\nYraItBSJxu1kjEoxj6ozy/Sg4TN1iYhI41HcFhGpQTIS49HAmVEv50HA8mimLRERSU+K2yIiNaiz\nKYWZPQIcBHQ2s7n4DGG5ACGEf+HTBw7FZ/lZQ9XZh0REpIkpbouI1E+diXEIYUQdywPw06SVSERE\nGkRxW0SkfjTznYiIiIgISoxFRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIs\nIiIiIgIoMRYRERERAZQYi4iIiIgASoxFRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWERE\nREQEUGIsIiIiIgIoMRaRDBZCoKw8pLoYIiKSJnJSXQARkfooKw+sWlfKyuISVq4rjf5KKm+Lqz+3\n8f1VxaXcMnwPjtmtW6oPR0RE0oASYxFpdCEESsoCxaVlrC8tpzj68/tlrNqQxJayqiJ5LfbkdUX1\n56OkdvX6sjr3m5NltC3IoW1BbnSbQ4+OrWkX3W9bkEufLm2a4AyIiEhzoMRYJAOEEDwZLSlnXWnZ\nhtt1JWUUl5b7bUk568s8US0uqZq4Vt6PLS8rj9armuxWeVxSFm2znLCZLRYKcrMqE9p8T2K3bFew\nIaFtk+/JbbsNSW8ubTYkvP58fk4WZtY4J1VERFocJcYiTai8PHiiWC1BrXIbT1ar3a7bkLTW8trS\nMtZFy9eVeGK6LkpqG8IMCnKyyc/NIi87i/zcLPJzsmP3s2jXyhNR/8smr+J+bhb52Vnk52aTn5NV\n+XxO5ePWeZXJbEVym5utLhAiItK0lBhLxoknpxW1oRvVdNaQmBbHbtdV3Ma2sdH6NTzX0AS1Iqks\nyM2mIEpO47ftWuVSkJu1IYnNj24LcrIpiBLT+G38tXmxZXnVEtecLFPNq4iIpER5eaAs6iydZUZe\nTuNVnCgxliZR0cZ0fZknhxv+yioTRn9cTklZebVL87FL+hWX+ze6hF9z29V4E4CK50rKGjYKQbz2\ntHqimR/Vfm5RWJlY5ldbnh9LXOMJbE2JbsVtQa7XzmZlKTkVEREXQqC0PFBSVk5JadjwHVqy4bs0\nVN4vK6ekLFBSWu1x7Hu3tDy+PNS6rZKyQHmUqFbclpUHQmBDAht/vnJdKu9vSHY3fq48BMrLK7cV\nd/qgbbjuuO812jlVYpyByssDa0rKWFNcypr1ZaxeH90WV96uLSljdXEZa9eXUlwtma34h4onnFUS\n3mrrV7w+WfKys6peks+tekm/IDeL9tFl/byaLu1Xu8yfV/0yf/S4eoJakcjmZqv2VESkOaq4YhhP\nAEvi98uqJn813S+NKnlKK54vL6eszBPU0nLfdvxxWblXDJWV+3Nl5eVVHpdG5Smrdr+krDy2TuVr\nS2PbbmhFz6bk5fj3Y262kZudRW62f0fmZhvZWVlkZ0G2GVlZVnmbZeRmGVnm9+PLs7Mq7kPWRq9j\no3XNNn4+K8vo3719ox0zKDFOaxUdpqokretLWVNcxpr1VR9XJLdrqj2ueF3FstXFZawtqbs3f1xe\n9M9Q8U9S0/02+Tnkta5lvSjZjC/Lja2Tv+H57CrrVCS6VS7rq9ZURKTRVVzlq0gES6KErCRK3CqS\nworkrDRK4ko2St5qTgRrSgp9nfJaE8HqyWL8cTy5LS0PGypxKhLg9bH9NKbsWHKYnWXkZHtTtJws\nIzvbyM3Kqlwn2+/nZBk52UZebjbZ+TlR4mnkZGWRs+F+5bbir83NziIvlrj6d2vscXYWeTnVHmdn\nkVvxXFbsfnZlIlyRmGYiJcZJUlpWHtXCVk1W19aQvMZrZitqbldXT2qLy1hTUrZZ/8QFuVkU5uXQ\nKi+bwrwcWuf7bec2+RTmVzyfTeu8HArzK29b5VZ9XJiXQ+u8bArzc9SrX0SkkcSHMdzQdKyklvux\n0WI2rB/reBtvOlYSS1prS2arJrsbr9vUE99kGVUSv9oSwYokMjurclleThaFGxI/Xzcn26Ikb+P7\nFQlgTffj26j1fk4WuVlWZXsVZdP3ZfOnxLgWJWXlfF20hukLVzFj0SpmLlrNinUlG2pd47Wvq4tL\nKd6MpgLZWUZhlHi2jhLV1nnZdG1bQOtOVZPaVnnZG5LU+POtY89XbCNbNakiIkkTgl8yX13scX5V\ncSmro4qM+OM168s23F9V7BUb1RPW2pLbzR3GsLrcbKvsz5BTcak7i5wokatIMAtys8iJaiMraiJz\no8SzItHLqUj0qi2v3FbF8xXLouezs6rUkMZrPD2prVoDWpn8RjWi0SVykXSQ8YnxmvWlzFi4mumL\nVvrtwlVMX7SKr4tWV2m7s2W7fLYozKd1XjZtC3LYql1BZZJaJVmN1b7mZdM632/jtbh52aqFFRFp\nLCEEFq0spmj1+lgC61fjVm9IYL2CY0OyW+zJbUWlR8XzpQnWnOZlZ234LvArcdnk53jlRdXOuFlV\nEtmK/g0bhjDMrTqcYdX1qy7Py8lShYhIkmVMYly0qpgZi6LEN0p+Zyxcxbxlazesk51lbNupNX26\ntOHwnbekb5c29O3ahj5d29AmP2NOlYhIs7B8TQkzF69i1uLVzF68mpmLV2+4X9fMiPFEtk2+J7MV\nlR6F+Tm0yY+u1EWVG/5cTuVzVV6b06jDR4lI02lR2V55eWD+8rUbkt8Zi1ZtuL90TcmG9Qpys+jT\npQ0DenVkeJee9O3qCfC2nQoV3ERE0sia9aXMXrzGE96i1cxctJpZUTIcj+tZBj23aE2vToXs3WsL\nencupGvbfCWyIrJZmmVivL60nK+Lqtb+Tl/o7YDjIy50bJ1L365tGNx/K/pEtb99u7ahW/tWas+U\nzkKA6a9Cx17QuW+qSyMijWx9aTlzlq5h1iKv8Z1VtHrD/QUr1lVZd6t2BfTuXMjg/luzXedCencu\npFfnQrbZorWSXRFpsGaVGF/22KdM/GYpXxetqdJjtnuHVvTp2oZ9enfypg9dCunbtQ2d2uSnsLRS\nL/MmwEu/gW/ehzZbwgWvQfseqS6ViDRQxRW9WVFzh/jf3KVrq8T0jq1z6d25kO/37RQlv23o1dlr\ngwvVrE1EGlGzijClZeVs37UtQ/pv5bW/XdqyXRcFyhZh+Tx49Vr47H9Q2AUO/S28/Xd4ZDic+xLk\nFaa6hCKymR4bP4exk7+LmkGsqTLRT+u8bHp3LqR/9/YM260bvaPa396dC+nQOi+FpRaRTNasMsqb\nh++R6iJIshWvgndvgff+AaEc9v8l7P8LKGgHW+0KD58MT46Ekx+ELF0mFWlOvK/Hanp1KuSgHbpW\nSX67ts3X6DwiknaaVWIsLUh5GXz6CLz6B1i1AHY5Hg67BjpuW7lOv8PhiD/CS1fB69d5LbKINBtX\nDd2Jq4bulOpiiIgkTImxNL1Zb3uyu+Bz6D4ATnkQeg6sed1BP4ZFU+Dtv0HnHWC3U5q2rCIiIpIx\nEro2bWaDzWyqmU03sytrWL6Nmb1uZh+b2WdmNjT5RZVmr2gGjDoNHjga1i6DE+6B88fWnhQDmMHQ\nv0KvA2D0z2DOh01XXpFmSjFbRKR+6kyMzSwbuA0YAuwMjDCznautdjXwaAhhD2A4cHuyCwrAC1fA\nR/c0yqalEa1dCi9eBbcNhJlveJOIiz6C753oiW9dcvLg5P9Au24w6lRY9k2jF1mkuUqrmC0i0swk\nUmM8EJgeQpgZQlgPjAKOrbZOANpF99sD85NXxEjpelgyE57/JTx7sT+W9FZWAuP+BbfuAR/8C3Y/\nDX42EQ64FHJbbd62Wm8Bpz7q7/sjI7zTnojUJD1itohIM5RIYtwdmBN7PDd6Lu4a4HQzmwuMAX5W\n04bMbKSZjTez8YsWLdq8kubkwYhRPmrBhPv9cvzKBZu3DWkaIcDUF+D2QfDiFT66xIVvw7Bboe2W\n9d9ul+3hpPtg4WR48gIoL6/7NSKZJz1itohIM5Ss8a9GAPeHEHoAQ4EHzWyjbYcQ7gwhDAghDOjS\npcvm7yUrGw77HZx4n3fcuvMgmDuhoWWXZPr2M/jPMB9/GPNa3jOfga36J2f7fQ+FwTfA1DE+7rHI\n5lq9GKaMgVd+B/cNhdnvprpEqdA0MVtEpJlJZFSKeUDP2OMe0XNx5wGDAUII75tZAdAZWJiMQm6k\n//HQuZ+3N71vCBxzM+x+aqPsShK0cgG8dh18/BC06gBD/gIDzoHs3OTva+AFPlLFuzdDlx303kvt\nysv8CsOcD2DuR367ZKYvy8qFrXeD0nWb3kbzk34xW0SkmUgkMf4I6GdmvfHgOhyonol8AxwK3G9m\nOwEFQONed9vqezDyTXjsLHj6x/Dtp3DEdY2TiEntStbC+//0WerK1sO+P4UfXAatOjbePs1gyJ+h\naDqM/jl07A3b7tt4+5PmY+0ymDs+SoQ/9CtK61f6ssKuPgLKXmdDz308Kd7ctu7NQ3rGbBGRZqDO\nxDiEUGpmFwEvAdnAvSGESWb2e2B8CGE0cClwl5n9Au/UcXYIIdS+1SRpvQWc/hS88n8w7nb4bhKc\n9AAUdmr0XTeJEODLp+HTUdB2K+jUz2vKO/WFDttCdgqHoS4vhy+egLHXwIq5sOPRcPjvoVOfptl/\ndi6c/ADcfRj87zS44PWqk4NIy1deDkXTfAi/ihrhRVN8mWXBlv193Oue+3hC3GHbxEZBaebSOmaL\niKQ5S1UsHDBgQBg/fnzyNvjJIz5aRZstYfh/Yetdk7ftVFgyE56/DGa8Cu17wvpVPuxZhaxc2GK7\nKFHuE0ua+zX+D4NvxsFLv4Z5E7zW7cg/Qa/9G3eftVk8He4+BNp1h3Nf8qmkm6MQ4OMH/XM85AY/\nr1JV8Ur/zM35qDIRXrfMl7XqCD0GQs+9PRHutifkt2m0opjZhBDCgEbbQRpKeswWEdlcCz6HtltD\nYefNfmmicbvlzHy3+wgftWDU6XDPEXDcbdD/hFSXavOVFsO7t8Bbf4XsPBhyI+x9vnc8XF3kzQeK\npsHiaX5/8TT46iUoL6ncRquOniB36gud+1YmzVtsBzn59S/b0tleQzzpKf9gHncH7DocspLVh7Me\nOvf1MY4fPB6eOB9GPOLnqjlZvxqe+yV8Nsrf83sHw3G3wy4/THXJUicEWDqrMgme8yEsnAQhGomk\ny06w8zBPgnsM9M96Kj+HIiLSuD5/HJ65CHY8Ck5svDktWk5iDNB9L7jwTfjfGfD4uT5CwqG/bT6J\n0sw34PlLPeHd5XiviW23deXywk7+t80+VV9XVgrLvvaZ5eJJ84zX4NOHK9ezLOiwjScRnfpVTZrb\nbl37ZeZ1y31K5nF3gGXDgVfCfj+HvMKkn4J62e4gGPoXH+P6ld/CkX9MdYkSt3AyPHoWLP4KDvo1\n7HkmPHomPHa2LzvwysxI+ErWwvyPo2YRH3r74NVRk9e8ttBjL/jB5d4kovsA7+ApIiItX3mZj0L1\n7i2wzb4w+PpG3V3LSowB2nSFs56FF37loxZ89wWccHfjdgZrqJXfwcu/gc8f845kpz/pw5IlKjsn\nak7RBzii6rJ1K6Ja5njSPA2+fg9K1lSul9cm2kbfqm2Z502A1/8EaxbDbqfCof/nM9Clm73Pg0VT\nvSNglx08wUx3nzzsP4TyCuHMpz3BBzj7OXjuF/Dmn2Hhl3Dcvxq1WUBKLZ/rxzrjNSgv9ee26AN9\nD/MkuMdA6LpT8/lxKyIiybN2qV8Nnj4WBpwLg//s81o0opaXGIOftGNu9nbGY34Fdx0Cwx+Brjum\numRVlZfB+Hvh1T9A6VqvHdz/F5BbkLx9FLSD7nv6X5V9l8PK+ZW1yxXNMuZ+BF88iffHiWy7Hxz5\nOHTbI3nlagxH/smT/ud+6c1GUtXuuS7r18ALl/vQdr0O8B9ubbeqXJ6TD8feBlvuAi9fDfceCcMf\nbnmdCyc/65fFykth0E9g2+9Dj73r1XZMRERamIVTYNQIWDYHjr7Zh4BtAi2n811tvhnnTStK1sDx\nd3rblHQw/2OvKZv/cdQU4G94u5trAAAgAElEQVTetCEdlKz1zn9F070muc8hzac3/9plcM/hPonD\nBa96gpxOFk/zphILJ/uwdgdeuenRRaaPhcfO9XVOeciTx+Zu/RrvvDnhPv+xdcI9TTeaSZKo852I\nSCOa/Bw8dSHktoZTHoRtBjV4k4nG7ZbfeHGbQTDyDei8vU8I8sYNqZ1KeN1yGHO512KvmO9JwRlP\np09SDD6265a7wM7HepOO5pIUg7c9HTEKCPDwcD/f6eKzx+DfB8Kq7+D0x+GQq+secq/vYXDBa94U\n6IFjfDr05uy7SXDXwZ4U73cxnPtys0uKRUSkkZSXw+vX+zCsnbf3/C0JSfHmaPmJMUD77nDOC7Db\nCHjjenj0DB/6qSmF4D0q/7k3fHiXjzRx0UfwvRObV+LZHHTqAyc/CEtmwGPneOfEVCpZB89eAk+e\n7817LnzbE95Ede4L578KvQ/0IQnHXA5lJXW/Lp2EAB/cCXce7G3GznjKx71u5LZiIiLSTBSv9Pzs\nzRu8T9M5L3j+1sQyIzEGb7d73B0w+AaY+oJPDFE0o2n2XTQDHjwOnjjPR3+44DUfRaGgfdPsPxP1\nPgCOusnHgX756tSVo2gG3HNYZQ3pWc/W7x+9VQc47THY9yL48E546HhYsyT55W0Mq4vgkRHernq7\nA+FH73rzHBEREfDvyrsP8/xs8A0+ZGky+1tthpbZ+a42ZjDox9B1Zx8O666D4YR7od9m1N5tjpJ1\n8M5N8M7fIacAhv7Ve1Wqh33T2OssH6li3G0+xvWAc5t2/5Oegmd+5u/3iP/BDoMbtr2sbB+KruvO\n8Nwl3hxnxKj061QaN/NNbye2psiD3T4/0hUSERGpNG0sPHGuDwd7xlNegZJCmVNjHLfdgTDydZ9R\n7uGT4J2b/VJvMk1/Fe7Y14fc2mmYN5sYeIGS4qZ2xB+g3xHe/GDWW02zz9Ji399jZ3vS+qN3Gp4U\nx+1xGpz9vE8McvdhMPXF5G07WcpKfDKY/xzrHTjPf9V/lCopFhER8Lzrnb/Df0/0fGzk6ylPiiFT\nE2OAjr3gvJe9g9nY33kzh/Vr6nxZnVZ86wnRQ8cD5h3rTryn6nBc0nSysqNRD/r66CSN3Xxm6Wwf\nXu3DO2HQT+HsMdChZ/L303OgB5FO28Ejwz24pGiEmY0smeXn4J2/w55n+KQ7zX2KdhHZtNJiH4JU\nJBHr13jeNfYa2OU4z8c69kp1qYBMa0pRXV4hnHgfbL0bjL3WZx875b/1Gy+2rBQ+uhteuw7K1vss\nZvtdnLI2MhJT0M6bHNx1CDx8Cpw/tnFmTpv8HDz9E79/yn9hp6OTv4+49j3gnBfhmZ96cPluEgz7\nh48qkiqfPerjSGdlwUkPeMATkealvNw7ya4pqva32Ps2VDxevTi6vwTWr/QJqs54Mv2GyZT0svRr\nH3ViwRdw2DWw3yVpdTUxsxNj8Ddj/1/Alv3h8fO83fFJD3jnrUTNneBtPhd8Bn0O9Y51GoIqvWzR\n28cB/s+xXqN/2uN1D5WWqNL1npiOuw223h1Out/31xTyWsOJ98KWO/uPsqIZMPy/TT87YfFKbz7y\n6SPQcxCccJdPPy6SKiGk1ZdtyoTgza7WxJLYKkltDX9rl0KoZVjT3NbQulPlX6e+ftuqA3zwL7jv\nKO9knE5DkEr6mPU2PHaWVyae9hj0OzzVJdpIy5/gY3Msnu5jHRdN97m4B47cdGBduxRe/T2Mvw/a\nbAlDboCdj1MwTmcTH4TRF/l7O/QvDd/esjmeaM8b79s84jqfuS4VpjwPT470Nr3DH4YeezXNfudN\n9EtiS2fDD34FP7g8eT860pQm+EhzCz6HR8+CNl3hmFt8mvhM8M04+OgeHys9XrNbVlzz+lk5VZPc\n1ltEt52rPlcYPW61hf8Yr82CL7zyISsbzhyd3h2DpWmF4E0MX7zKf0wNf7jJfzwlGreVGFe3boX3\nop86BnY/HY7628bNIULwS8Yv/8aDzsAL4eBf+yV7SX8vXw3v/cNHCRl4Qf2389VL/lkpK4Vj/wG7\n/DB5Zayv7yZ5m+OV33mzit1Oabx9lZfD+//wH4dttvJa4pYwM18ClBinsU8e8St4BR08IVy/Gg78\nlV+uzc5Ndekax7rlftVq/L2ewG7Rx28LO1VLfDtXTYAL2ie/ImfhFPjPMG9vfOYzsFX/5G5fmp+S\ndfD8L+GT/8IOQ+GH/05JvqTEuCHKy300iTdvgO57+SX4ikvTi77yN3j2277s6L97G2VpPsrL/MrA\ntFfg9Cegz8Gb9/qyEnjtD/DuLbDV97zpTTo1nVld5NNOf/2Ot3M/9HfJHw1l5QJ46kcw83UfdWXY\nrT47X4ZQYpyGSou9Nmr8PdDrAG9iBPDCr3zoxC37+4/F7numtpzJ9uVob8a0eiEM+gkcdBXkt0lt\nmRZP95k6S9d6B/Ruu6e2PJI6K+bD/06HeRPgwCvhwCu8D0oKKDFOhsnP+pd/XiEcf5cP9/XuLX4p\n6dDfwV5na/i15qp4JdxzBKyY50OJde6X2OuWz4PHz4U542Cvc3xs3nTsYFlW4gnB+Huh35Fwwt3J\n+4X+1cvw9I+9Jm7IDbDnWRnXfEiJcZpZPs9/DM4bD9//ucfneHOeyc/B85d68rjvRZ48bqpJQHOw\nfJ7/j095zn+gH3NreiX9S2bCA8OgeAWc/lTTNe2S9PHNBz6T3frV8MN/wU7HpLQ4SoyTZeFkr11c\nMtMf73qKtyNt0zW15ZKGW/q1j1RR0N5Hqmi9xabXnzYWnhrpl4WG3erTeae7j+6GF67wS6sjHmlY\nzXZpMbzyO/jgDq99O+GejG1DqMQ4jcx803+slq7z2bJ2Prbm9dYug1d+CxMf8NEThv1j8zpZp4vy\ncq8VH3stlJfCwVd5TXE6NhNZ9g3cf7S3dz79Cdhmn1SXSJrKhAf8x2j7Hv7d03WnVJdIiXFSrV0K\nb/8N+h6eFoNPSxJ9M84v+W0zCE5/suYvl7JSeON6ePuvPuvcSQ/4THrNxay3vDYtBDj5AdjuoM3f\nxqKvPPn47nOfve6wa9OzpryJKDFOAyH4FbxXr4VO/bzJWyL/l7PegtE/h6Wz/Krf4b/3H8fNwXdf\nwrMXw9wPYbuDvSlfU42AU1/L53mMXbnARyHotV+qS9T8rZjvVwNnv+uVE933gm57eifTVF/FLl0P\nL13llTJ9DvEmTWnSzE6JsUiiPnkEnv6RN404+u9VmwWsXODD+H39DuxxOgz5S/O8BLtkll/5WDQ1\nsRFXKoQAE/8DL17p4yMfe3tyZ/FrppQYp9i6Fd6cZ8pz3ul12D83r13t+jXwxp/g/dt8RKGjboId\nhzZeeRuqZB289Rd492ZP4o+8HnY9ufk0YVq5wJPjZXPg1FH1+3Ge6UKAuR/5kHhfPuN9Zbbeza9m\nF6/wdXILvT13tz08We6+J3TYtuk+J6sW+mgw37zXeP1bGkCJscjmGHuNz9Q25EbY50J/buYb8MT5\nULwKjr4Jdj81lSVsuOKVPpzb1DGw55kw9G+Qk1f7+muXeu3Ul89A7wO9J3G7rZuuvGlMiXEKLZzs\nnXmWzPIp3wf9pP5f/PMmwDM/g4WTYJfj/f+/TZfklrehZr/j/4dF02G3EXDEH320ieZm1UIfym3J\nTJ8Aqd9hqS5R81BaDJOe9oR4/kTIb+8ziu59vl8tKC+HJTP8szxvot8u+LxyiL7Wnbw2uSJR7rZn\n43zG538Mo07zZjPH/jMtmxoqMRbZHOXl3klg6hgY8T8PQG/cAJ239+YHadA+KinKy+H1P3qzkG32\nhZMfrDlIfjPOfxSs/BYOuRq+f3HKehKnIyXGKfL54zD6Zz5W90n3J+eyfOl6b5Lx1o3e0XrwDd6X\nJNW1sWuWeJvojx/0qXKP/rtfmm7OVhfBg8f6lauT/wM7DEl1idLXyu9gwn0+LvXqhd5caJ8L/cdR\nXVdHStf7j715E/1v/kRYNKVy0pb220D3PSqbYHTbHfLb1r+snz3q/5eFXXyCqTQdqUuJscjmKl4F\n9w72drQAuw73caxTPfRRY/j8cZ9KurCLd4zY6nv+fFmpJ81v/tkvwZ1wj3qT10CJcRMrK/Ekcdzt\nPrPiSfcn/+rFwin+5T73Q+h7mCeiqZi9MQSY9KR3ml2zBL7/Mx/iqjk24arJ2qXw4PE+U+yJ99be\nWTJTzZsAH/wbvngSykug3xHer2O7gxtWOVG8Cr791JPkitrlZV9HC83bJ3ffK2qGsad3sK5rsqqy\nUhj7O3j/n7Dt/l6JVNi5/mVsZEqMRepj2Rxvu/i9k7y5QaprjRrTvIl+6WvdMh9Kp9ue8OQF8M37\nXmM29K+atKYWSoyb0MoFPrvkN+/DPj/25hONNQJDeZl3Ghp7rf/vH3YNDDiv6a6WLPsGnr8Mpr3k\nCcoxt8LWuzbNvpvSuuXw0ImeoB1/Z1pedm9SZSXeZO2Df/sPs7y2sMdp3hekMcfIX10UJcpRsjx/\nIqxe5Muy8zw5rmiC0X0vr7Wu+F9YswQeP8ebHA68EI78Y3qOjBKjxFhE6rZygSfH88b75WnwjkiN\nOWNeC6DEuIl8/Z4nxcUrfXi1pkqgln0Dz14CM171Guph/2jckWjKyzwpeu06f3zI1X7ZPI06LiVd\n8Up4+BT/wXPs7bD7iFSXqOmtXlzZXGLlt7DFdp5k7n5qaiolQoDlcyuT5HkTYf4nsH6lL89rG3Xu\n290nlln5rV9Z2eP0pi9rPSgxFpHElKyDF6+AohlwzC3pNYtfmlJi3MhCgHF3+PTtHXv5UGxb7tw0\n+46X4dNRPiJLyRpvzrDfxcmvFVvwuTfhmP+xXzY/6m+pacKRCutX+xT2s972seH3PDPVJWoa337q\nP4Q+f9w7yfU5xJtL9D08/fpylJfB4mlVa5a/+8I79Z38IPTcO9UlTJgSYxGRRqLEuBEVr/JEcdKT\nsOPRPmlHKscZXrUwNq309+DYf3gzh4Zav8bb8r/3D59caMiffWSMltx8qyYla/2q1YxX/UfB3uen\nukSNo6zUhxf84N8+nFlua+9It8+F3r63OSktBsuuOrtkM5Bo3G5eRyUiIi3X4mk+FNvir7x9736X\npD5RbNPVO/v1P9Fn8rrrEO8Qd9BVPrZ3fcx4HZ67BJbOhj3O8ElG6pp5s6XKbQXDH4bHzvLzW1YC\ng36c6lIlz5olPtvih3fDirl+NeCIP3rzg1YdUl26+qmrU14zp8RYRERS78vR8PRPfGztM55Kv0kg\ndjoaeu3vo2O8ewtMftY7x23OtNKri+Dl38Cnj/g07Wc91zynpU623AK/LP/Eud50pbQY9r8k1aVq\nmO8m+djDnz3q05X3/gEMvRG2H9yy2463AEqMRUQkdcpK4bXfe7LZfS8f37Z9j1SXqmatOnhb2P4n\nwLM/hweOTmxa6RA8QXrpKh+R4YDL4AeXZ/S06hvJyYMT74OnLvQhwMrWw4G/SnWpNk95GUx9wRPi\n2W9DToGP8LPPhbDlLqkunSRIibGIiKTGqkU+5NPst31YtMHXN4/LtNsdCD9+v3Ja6a9e8t75NU1Y\nsWQWPP9LmPEa9Njba5mbuiNhc5GdCz+8E7JyfSKisvVw8G9S35ymLmuXwscPwYd3+ogm7XrAYdd6\nZ8JMbSLTjCkxFhGRpjfnI3j0TFi7BI67o/lNuZ7XGo64Dnb5oU8r/cjwqtNKl5XCuNvg9eshK8fH\nBR9wri6j1yU7xztcZufAW3/x5Piwa9MvOS4rga/f9U6Znz3qI5dsu59/JnY4qtl1TJNKeudERKTp\nhOCTaLx4FbTrBue90rwnsei+F4x8A9692RO5ma97U4nPRvlQbDsM9aS4ffdUl7T5yMqGY/4B2fne\nxKZ0vV9NSHVyvG45TB8LU8bAtFegeLk3l+h/IuwzMm2nQpbNo8RYRESaxvo18NwvPGnsdyQc/29o\n1THVpWq4nDxvD7vTMB9q7uXfQJutvEPZTsekPqFrjrKyfPi27Dz44A6vOR7616Yf53f5XG83POV5\nmP2OT9PcupO/rzsO9U6ieYVNWyZpVAklxmY2GLgFyAbuDiHcUMM6JwPXAAH4NITQzK6LiYi0DGkZ\ns5fMhP+d4b31D/6N16qm22QGDdV1Rzj3xcr2xM11OK50YeY1xdm58N6tnhwfc2vjfm5C8Jr+qWM8\nGV7wmT/fqa8PI7fjUf7eqklMi1VnYmxm2cBtwOHAXOAjMxsdQvgytk4/4CpgvxDCUjPr2lgFFhGR\n2qVlzJ76Ajx5oSc6pz0O/Q5r1N2lVFY29Ds81aVoOcx81I+cfG+qUl4Kx96W3MS0rMRrg6eO8c/q\n8jmAQc+B3r55h6GNOyW4pJVEaowHAtNDCDMBzGwUcCzwZWydC4DbQghLAUIIC5NdUBERSUj6xOzy\nMnjjek9ott7Nh2Lr2KtRdiUtmBkccrU3q6gYreKH/27Y9Nzrlns74aljYNrYqL1wK+hzsE//vf2R\nPrmLZJxEEuPuwJzY47nAPtXW2R7AzN7FL91dE0J4sfqGzGwkMBJgm20yZC54EZGmlT4xe/TP4JP/\n+ixfQ/+mcXulYQ78lSfDY6/x5PiEe719d6KWzfEa4aljYu2FO8POx3it8HYH+2gjktGS1fkuB+gH\nHAT0AN4ys++FEJbFVwoh3AncCTBgwICQpH2LiMjmaZqYvdfZfjl6r7MbWFyRyP6/8Jrjl37tw/2d\n/EDtY1+H4G2Ep4zxZHhDe+F+ai8stUokMZ4H9Iw97hE9FzcX+CCEUALMMrOv8KD7UVJKKSIiiUqf\nmN1zoP+JJNO+P/XkeMxlMOpUOOUhyG3ly0rXw9fvRMnwC7BiLt5eeB9vL7zjUdC5X0qLL+ktkcT4\nI6CfmfXGg+twoHrv5aeBEcB9ZtYZv0w3M5kFFRGRhChmS8s38AJPjp+9GB4+BfY4A756IRpfeEVl\ne+GDroTtB/ukKyIJqDMxDiGUmtlFwEt4W7R7QwiTzOz3wPgQwuho2RFm9iVQBlweQihqzIKLiMjG\nFLMlY+x1lifHz/wEZr0ZtRce5jPPbXeQ2gtLvVgIqWnqO2DAgDB+/PiU7FtEpCHMbEIIYUCqy9GU\nFLMlbX37GZSshR4D1F5YapVo3NbMdyIiItJ8NecpxSXttLBph0RERERE6keJsYiIiIgISoxFRERE\nRAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYRERERAZQYi4iIiIgA\nSoxFRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYRERERAZQY\ni4iIiIgASoxFRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYR\nERERAZQYi4iIiIgASoxFRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIi\nIgIoMRYRERERAZQYi4iIiIgACSbGZjbYzKaa2XQzu3IT651gZsHMBiSviCIisjkUs0VE6qfOxNjM\nsoHbgCHAzsAIM9u5hvXaAhcDHyS7kCIikhjFbBGR+kukxnggMD2EMDOEsB4YBRxbw3p/AP4MrEti\n+UREZPMoZouI1FMiiXF3YE7s8dzouQ3MbE+gZwjh+U1tyMxGmtl4Mxu/aNGizS6siIjUSTFbRKSe\nGtz5zsyygJuAS+taN4RwZwhhQAhhQJcuXRq6axER2UyK2SIitUskMZ4H9Iw97hE9V6Et0B94w8xm\nA4OA0erMISKSEorZIiL1lEhi/BHQz8x6m1keMBwYXbEwhLA8hNA5hNArhNALGAcMCyGMb5QSi4jI\npihmi4jUU52JcQihFLgIeAmYDDwaQphkZr83s2GNXUAREUmcYraISP3lJLJSCGEMMKbac7+tZd2D\nGl4sERGpL8VsEZH60cx3IiIiIiIoMRYRERERAZQYi4iIiIgASoxFRERERAAlxiIiIiIigBJjERER\nERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYRERERAZQYi4iIiIgASoxFRERERAAlxiIiIiIi\ngBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYRERERAZQYi4iIiIgASoxFRERERAAl\nxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYRERERAZQYi4iIiIgASoxF\nRERERAAlxiIiIiIigBJjERERERFAibGIiIiICKDEWEREREQEUGIsIiIiIgIoMRYRERERARJMjM1s\nsJlNNbPpZnZlDct/aWZfmtlnZvaqmW2b/KKKiEgiFLNFROqnzsTYzLKB24AhwM7ACDPbudpqHwMD\nQgi7Ao8DNya7oCIiUjfFbBGR+kukxnggMD2EMDOEsB4YBRwbXyGE8HoIYU30cBzQI7nFFBGRBClm\ni4jUUyKJcXdgTuzx3Oi52pwHvFDTAjMbaWbjzWz8okWLEi+liIgkSjFbRKSektr5zsxOBwYAf6lp\neQjhzhDCgBDCgC5duiRz1yIispkUs0VEqspJYJ15QM/Y4x7Rc1WY2WHAb4ADQwjFySmeiIhsJsVs\nEZF6SqTG+COgn5n1NrM8YDgwOr6Cme0B/BsYFkJYmPxiiohIghSzRUTqqc7EOIRQClwEvARMBh4N\nIUwys9+b2bBotb8AbYDHzOwTMxtdy+ZERKQRKWaLiNRfIk0pCCGMAcZUe+63sfuHJblcIiJST4rZ\nIiL1k1Bi3FRKSkqYO3cu69atS3VRGlVBQQE9evQgNzc31UUREam3TInZoLgtkinSKjGeO3cubdu2\npVevXphZqovTKEIIFBUVMXfuXHr37p3q4oiI1FsmxGxQ3BbJJEkdrq2h1q1bR6dOnVp0gDUzOnXq\nlBE1LCLSsmVCzAbFbZFMklaJMdDiAyxkxjGKSGbIlHiWKccpkunSLjEWEREREUkFJcYxy5Yt4/bb\nb9/s1w0dOpRly5Y1QolERGRTFLdFJJmUGMfUFmBLS0s3+boxY8bQoUOHxiqWiIjUQnFbRJIprUal\niLv22Ul8OX9FUre5c7d2/O6YXWpdfuWVVzJjxgx23313cnNzKSgooGPHjkyZMoWvvvqK4447jjlz\n5rBu3TouvvhiRo4cCUCvXr0YP348q1atYsiQIey///689957dO/enWeeeYZWrVol9ThERNJNKmI2\nKG6LSHKpxjjmhhtuoE+fPnzyySf85S9/YeLEidxyyy189dVXANx7771MmDCB8ePHc+utt1JUVLTR\nNqZNm8ZPf/pTJk2aRIcOHXjiiSea+jBERDKG4raIJFPa1hjXVUvQFAYOHFhlzMpbb72Vp556CoA5\nc+Ywbdo0OnXqVOU1vXv3Zvfddwdgr732Yvbs2U1WXhGRVEmHmA2K2yLSMGmbGKeDwsLCDfffeOMN\nxo4dy/vvv0/r1q056KCDahzTMj8/f8P97Oxs1q5d2yRlFRERxW0RaRg1pYhp27YtK1eurHHZ8uXL\n6dixI61bt2bKlCmMGzeuiUsnIiLVKW6LSDKpxjimU6dO7LfffvTv359WrVqx5ZZbblg2ePBg/vWv\nf7HTTjuxww47MGjQoBSWVEREQHFbRJLLQggp2fGAAQPC+PHjqzw3efJkdtppp5SUp6ll0rGKtDRm\nNiGEMCDV5WhKmR6zIfOOV6QlSTRuqymFiIiIiAhKjEVEREREACXGIiIiIiKAEmMREREREUCJsYiI\niIgIoMRYRERERARQYlzFsmXLuP322+v12ptvvpk1a9YkuUQiIrIpitsikkxKjGMUYEVEmhfFbRFJ\npvSd+e6FK2HB58nd5lbfgyE31Lr4yiuvZMaMGey+++4cfvjhdO3alUcffZTi4mJ++MMfcu2117J6\n9WpOPvlk5s6dS1lZGf/3f//Hd999x/z58zn44IPp3Lkzr7/+enLLLSKS7lIQs0FxW0SSK30T4xS4\n4YYb+OKLL/jkk094+eWXefzxx/nwww8JITBs2DDeeustFi1aRLdu3Xj++ecBWL58Oe3bt+emm27i\n9ddfp3Pnzik+ChGRzKG4LSLJlL6JcR21BI3t5Zdf5uWXX2aPPfYAYNWqVUybNo0DDjiASy+9lCuu\nuIKjjz6aAw44IKXlFBFJCymO2aC4LSINl76JcYqFELjqqqu48MILN1o2ceJExowZw9VXX82hhx7K\nb3/72xSUUERE4hS3RaSh1Pkupm3btqxcuRKAI488knvvvZdVq1YBMG/ePBYuXMj8+fNp3bo1p59+\nOpdffjkTJ07c6LUiItI0FLdFJJlUYxzTqVMn9ttvP/r378+QIUM49dRT2XfffQFo06YNDz30ENOn\nT+fyyy8nKyuL3Nxc7rjjDgBGjhzJ4MGD6datmzpxiIg0EcVtEUkmCyGkZMcDBgwI48ePr/Lc5MmT\n2WmnnVJSnqaWSccq0tKY2YQQwoBUl6MpZXrMhsw7XpGWJNG4raYUIiIiIiIoMRYRERERAdIwMU5V\n046mlAnHKCKZIVPiWaYcp0imS6vEuKCggKKiohYdgEIIFBUVUVBQkOqiiIg0SCbEbFDcFskkaTUq\nRY8ePZg7dy6LFi1KdVEaVUFBAT169Eh1MUREGiRTYjYobotkirRKjHNzc+ndu3eqiyEiIglQzBaR\nliahphRmNtjMpprZdDO7sobl+Wb2v2j5B2bWK9kFFRGRxChmi4jUT52JsZllA7cBQ4CdgRFmtnO1\n1c4DloYQ+gJ/B/6c7IKKiEjdFLNFROovkRrjgcD0EMLMEMJ6YBRwbLV1jgUeiO4/DhxqZpa8YoqI\nSIIUs0VE6imRNsbdgTmxx3OBfWpbJ4RQambLgU7A4vhKZjYSGBk9XGVmU+tR5s7Vt5sBMvGYITOP\nOxOPGZrfcW+b6gJsgmJ2esjE487EY4bMPO7meMwJxe0m7XwXQrgTuLMh2zCz8Zk2FWsmHjNk5nFn\n4jFD5h53ulPMrr9MPP3a0MoAAAOESURBVO5MPGbIzONuycecSFOKeUDP2OMe0XM1rmNmOUB7oCgZ\nBRQRkc2imC0iUk+JJMYfAf3MrLeZ5QHDgdHV1hkNnBXdPxF4LbT0Ed9FRNKTYraISD3V2ZQian92\nEfASkA3cG0KYZGa/B8aHEEYD9wAPmtl0YAkeiBtLgy7rNVOZeMyQmcediccMmXvcSaeYnTYy8bgz\n8ZghM4+7xR6zqZJARERERCTBCT5ERERERFo6JcYiIiIiIjSjxLiuKU5bIjPraWavm9mXZjbJzC5O\ndZmaipllm9nHZvZcqsvSVMysg5k9bmZTzGyyme2b6jI1NjP7RfTZ/sLMHjGzglSXSZIn0+K2YrZi\ndqrL1BRaetxuFolxglOctkSlwKUhhJ2BQcBPM+S4AS4GJqe6EE3sFuDFEMKOwG608OM3s+7Az4EB\nIYT+eEexxuwEJk0oQ+O2YnZmyaiYDZkRt5tFYkxiU5y2OCGEb0MIE6P7K/F/uu6pLVXjM7MewFHA\n3akuS1Mxs/bAD/DRAgghrA8hLEttqZpEDtAqGku3NTA/xeWR5Mm4uK2YrZid2lI1mRYdt5tLYlzT\nFKctPtjEmVkvYA/gg9SWpEncDPwKKE91QZpQb2ARcF90OfJuMytMdaEaUwhhHvBX4BvgW2B5COHl\n1JZKkiij47ZidouXcTEbMiNuN5fEOKOZWRvgCeCSEMKKVJenMZnZ0cDCEMKEVJelieUAewJ3hBD2\nAFYDLbpNppl1xGsQewPdgEIzOz21pRJpOMXsjJBxMRsyI243l8Q4kSlOWyQzy8UD7H9DCE+mujxN\nYD9gmJnNxi+9HmJmD6W2SE1iLjA3hFBRu/Q4HnRbssOAWSGERSGEEuBJ4PspLpMkT0bGbcVsxewW\nrsXH7eaSGCcyxWmLY2aGt1+aHEK4KdXlaQohhKtCCD1CCL3w9/m1EEKL+jVakxDCAmCOme0QPXUo\n8GUKi9QUvgEGmVnr6LN+KBnQeSWDZFzcVsxWzE5hkZpKi4/bdU4JnQ5qm+I0xcVqCvsBZwCfm9kn\n0XO/DiGMSWGZpPH8DPhvlETMBM5JcXkaVQjhAzN7HJiI9+b/mBY8zWimydC4rZidWTIqZkNmxG1N\nCS0iIiIiQvNpSiEiIiIi0qiUGIuIiIiIoMRYRERERARQYiwiIiIiAigxFhEREREBlBiLiIiIiABK\njEVEREREAPh/qSodluQxFg4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1,2, figsize=(12, 4))\n",
    "ax[0].plot(history_res.history['acc'], label='train')\n",
    "ax[0].plot(history_res.history['val_acc'], label='test')\n",
    "ax[0].set_title(\"Deep CNN with residual connections\")\n",
    "ax[0].set_ylim(0,1)\n",
    "ax[0].legend()\n",
    "\n",
    "ax[1].plot(history_nores.history['acc'], label='train')\n",
    "ax[1].plot(history_nores.history['val_acc'], label='test')\n",
    "ax[1].set_title(\"Deep CNN with no residual connections\")\n",
    "ax[1].set_ylim(0,1)\n",
    "ax[1].legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f18kJSXkBfB6"
   },
   "source": [
    "As we can see, removing the residual connections makes the performance decrease - although not by much. We expect the difference in performance to be even higher with an even deeper model."
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "HW5.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}