notebooks/output_randomized_smoothing_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aPIA-10zdv4P"
   },
   "source": [
    "## ART Randomized Smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "CGDOyI0HgDfx",
    "outputId": "2d61711f-6f8a-41b5-f05c-1085fd00fa13"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras.backend as k\n",
    "from keras.models import load_model\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout\n",
    "\n",
    "from art.config import ART_DATA_PATH\n",
    "from art.defences import GaussianAugmentation\n",
    "from art.attacks import FastGradientMethod\n",
    "from art.classifiers import KerasClassifier\n",
    "from art.utils import load_dataset, get_file, compute_accuracy\n",
    "from art.wrappers import RandomizedSmoothing\n",
    "\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FqXvuMM9dv4U"
   },
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z9OztmSidv4V"
   },
   "outputs": [],
   "source": [
    "# Read MNIST dataset\n",
    "(x_train, y_train), (x_test, y_test), min_, max_ = load_dataset(str('mnist'))\n",
    "\n",
    "num_samples_test = 250\n",
    "x_test = x_test[0:num_samples_test]\n",
    "y_test = y_test[0:num_samples_test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xDCzquK1dv4X"
   },
   "source": [
    "### Train classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G-mh9wSAHm-Z"
   },
   "outputs": [],
   "source": [
    "# create Keras convolutional neural network - basic architecture\n",
    "def cnn_mnist(input_shape, min_val, max_val):\n",
    "  \n",
    "    model = Sequential()\n",
    "    model.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, activation='relu'))\n",
    "    model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "    classifier = KerasClassifier(clip_values=(min_val, max_val), \n",
    "                                model=model, use_logits=False)\n",
    "    return classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tGbe8Cjmdv4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/sharon/anaconda/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /Users/sharon/anaconda/envs/tensorflow/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From /Users/sharon/anaconda/envs/tensorflow/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3298: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 3\n",
    "\n",
    "# # Construct and train a convolutional neural network\n",
    "# classifier = cnn_mnist(x_train.shape[1:], min_, max_)\n",
    "# classifier.fit(x_train, y_train, nb_epochs=num_epochs, batch_size=128)\n",
    "\n",
    "# import trained model to save time :)\n",
    "path = get_file('mnist_cnn_original.h5', extract=False, path=ART_DATA_PATH,\n",
    "                url='https://www.dropbox.com/s/p2nyzne9chcerid/mnist_cnn_original.h5?dl=1')\n",
    "classifier_model = load_model(path)\n",
    "classifier = KerasClassifier(clip_values=(min_, max_), model=classifier_model, \n",
    "                             use_logits=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "qH0lH14Ddv4d",
    "outputId": "058fe69b-70af-46ad-8d71-bc7f923f2202"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/sharon/anaconda/envs/tensorflow/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "Epoch 1/3\n",
      "469/468 [==============================] - 30s 63ms/step - loss: 0.2744 - acc: 0.9172\n",
      "Epoch 2/3\n",
      "469/468 [==============================] - 30s 63ms/step - loss: 0.0821 - acc: 0.9745\n",
      "Epoch 3/3\n",
      "469/468 [==============================] - 30s 64ms/step - loss: 0.0609 - acc: 0.9807\n",
      "Epoch 1/3\n",
      "469/468 [==============================] - 31s 66ms/step - loss: 0.3985 - acc: 0.8738\n",
      "Epoch 2/3\n",
      "469/468 [==============================] - 32s 69ms/step - loss: 0.1520 - acc: 0.9509\n",
      "Epoch 3/3\n",
      "469/468 [==============================] - 32s 67ms/step - loss: 0.1150 - acc: 0.9633\n"
     ]
    }
   ],
   "source": [
    "# add Gaussian noise and train two classifiers\n",
    "sigma1 = 0.25\n",
    "sigma2 = 0.5\n",
    "\n",
    "# sigma = 0.25\n",
    "ga = GaussianAugmentation(sigma=sigma1, augmentation=False)\n",
    "x_new1, _ = ga(x_train)\n",
    "\n",
    "classifier_ga1 = cnn_mnist(x_train.shape[1:], min_, max_)\n",
    "classifier_ga1.fit(x_new1, y_train, nb_epochs=num_epochs, batch_size=128)\n",
    "\n",
    "# sigma = 0.5\n",
    "ga = GaussianAugmentation(sigma=sigma2, augmentation=False)\n",
    "x_new2, _ = ga(x_train)\n",
    "\n",
    "classifier_ga2 = cnn_mnist(x_train.shape[1:], min_, max_)\n",
    "classifier_ga2.fit(x_new2, y_train, nb_epochs=num_epochs, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XYJN3rCpdv4h"
   },
   "outputs": [],
   "source": [
    "# create smoothed classifiers\n",
    "classifier_rs = RandomizedSmoothing(classifier, sample_size=100, scale=0.25, alpha=0.001)\n",
    "classifier_rs1 = RandomizedSmoothing(classifier_ga1, sample_size=100, scale=sigma1, alpha=0.001)\n",
    "classifier_rs2 = RandomizedSmoothing(classifier_ga2, sample_size=100, scale=sigma2, alpha=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kukXRDcedv4j"
   },
   "source": [
    "### Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "id": "jcPkXptcdv4k",
    "outputId": "ee65b562-0839-483b-9b1f-b7c911e3131a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 prediction(s) abstained.\n",
      "4 prediction(s) abstained.\n",
      "Original test data (first 250 images):\n",
      "Original Classifier\n",
      "Accuracy: 0.996\n",
      "Coverage: 1.0\n",
      "Smoothed Classifier, sigma=0.25\n",
      "Accuracy: 0.9959349593495935\n",
      "Coverage: 0.984\n",
      "Smoothed Classifier, sigma=0.5\n",
      "Accuracy: 1.0\n",
      "Coverage: 0.984\n"
     ]
    }
   ],
   "source": [
    "# compare prediction of randomized smoothed models to original model f\n",
    "x_preds = classifier.predict(x_test)\n",
    "x_preds_rs1 = classifier_rs1.predict(x_test)\n",
    "x_preds_rs2 = classifier_rs2.predict(x_test)\n",
    "acc, cov = compute_accuracy(x_preds, y_test)\n",
    "acc_rs1, cov_rs1 = compute_accuracy(x_preds_rs1, y_test)\n",
    "acc_rs2, cov_rs2 = compute_accuracy(x_preds_rs2, y_test)\n",
    "\n",
    "print(\"Original test data (first 250 images):\")\n",
    "print(\"Original Classifier\")\n",
    "print(\"Accuracy: {}\".format(acc))\n",
    "print(\"Coverage: {}\".format(cov))\n",
    "print(\"Smoothed Classifier, sigma=\" + str(sigma1))\n",
    "print(\"Accuracy: {}\".format(acc_rs1))\n",
    "print(\"Coverage: {}\".format(cov_rs1))\n",
    "print(\"Smoothed Classifier, sigma=\" + str(sigma2))\n",
    "print(\"Accuracy: {}\".format(acc_rs2))\n",
    "print(\"Coverage: {}\".format(cov_rs2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hqea3xvMdv4n"
   },
   "source": [
    "### Certification accuracy and radius"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D6Va8ST8dv4n"
   },
   "outputs": [],
   "source": [
    "# calculate certification accuracy for a given radius\n",
    "def getCertAcc(radius, pred, y_test):\n",
    "\n",
    "    rad_list = np.linspace(0,2.25,201)\n",
    "    cert_acc = []\n",
    "    num_cert = len(np.where(radius > 0)[0])\n",
    "    for r in rad_list:\n",
    "        rad_idx = np.where(radius > r)[0]\n",
    "        y_test_subset = y_test[rad_idx]\n",
    "        cert_acc.append(np.sum(pred[rad_idx] == np.argmax(y_test_subset, axis=1))/num_cert)\n",
    "    return cert_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iPWY6KFMdv4p"
   },
   "outputs": [],
   "source": [
    "# compute certification\n",
    "pred0, radius0 = classifier_rs.certify(x_test, n=500)\n",
    "pred1, radius1 = classifier_rs1.certify(x_test, n=500)\n",
    "pred2, radius2 = classifier_rs2.certify(x_test, n=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "colab_type": "code",
    "id": "ZZv5wDHSdv4s",
    "outputId": "a6fbe7ba-dfbb-47bd-8e56-794fb689cb14"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPXV+PHPmckybGELCCQgQfZNxAAqAloUUREEtYK1xValtVq11q1Wqdq6PHWpbUV9UFHcwL0ERWjdfqKP7KJF1IKIsqgge0ICWc7vjzsZQjLJXELuTHLnvF+vvGa5d+4cLsmc+d7zXURVMcYYYwACiQ7AGGNM/WFJwRhjTIQlBWOMMRGWFIwxxkRYUjDGGBNhScEYY0yEJQVjjDERlhSMMcZEWFIwxhgTkZLoAA5VZmamdu7cOdFhGGNMg7J8+fIfVLVNrP0aXFLo3Lkzy5YtS3QYxhjToIjI1272s8tHxhhjIiwpGGOMibCkYIwxJqLB1RSMMQcUFxezceNGioqKEh2KqSdCoRDZ2dmkpqbW6vWWFIxpwDZu3EizZs3o3LkzIpLocEyCqSrbtm1j48aN5OTk1OoYnl0+EpEZIrJFRFZVs11E5O8islZEPhGRgV7FYoxfFRUV0bp1a0sIBgARoXXr1ofVcvSypvAkMLqG7acD3cI/U4CHPYzFGN+yhGAqOtzfB8+Sgqq+B2yvYZdxwFPqWAS0EJH2XsWzaUsxj/5zJ2VltvyoMcZUJ5E1hSxgQ4XHG8PPfVt5RxGZgtOaoFOnTrV6s9lv5/P6e3tY/MU++vVuFDWbdmmXwin9Q4TS7JuXMXXpjDPO4LnnnqNFixbV7jN16lSGDx/OKaeccsjHf/fdd7n33nt57bXXDidMQwMpNKvqdGA6QG5ubq2+6h+/cwmfhLJY93UGGzYUkpoepGOnEK1bpSACpWXw0Zf7yFtUQHpq9UmhXYsgp/QP0TojWO0+jdIgrZpjCNAqI0ggYInH+J+qoqrMmzcv5r633357HCIysSQyKWwCOlZ4nB1+zhMnZBdz/Ht38qrk8vXm/azJ7Mvnhf0P+TifAm/9v92HFUvn9qlMPLUZTRsH6JWTTstm1ScYY+q7+++/nxkzZgBwySWXcPbZZ3PaaacxZMgQli9fzrx58xgxYgTLli0jMzOTP/3pTzzzzDO0adOGjh07cuyxx3Lttddy0UUXMWbMGM4991w6d+7M5MmTmTt3LsXFxbz44ov07NmTJUuWcNVVV1FUVESjRo144okn6NGjR4LPgL8kMinkAVeIyGxgCLBLVatcOqozY8YgY8YwAWDNGvSmm1g+9zu+y+oNZ50FJw6DYOwP55JSZcMPJZSUVbODwuYdpXyztTTq5rIyZfO2Yu5+yim3hNKEc05uxvmnZtC0sY0lNIfh6qth5cq6PeaAAfDAA9VuXr58OU888QSLFy9GVRkyZAgjRoxgzZo1zJw5k+OOO+6g/ZcuXcrLL7/Mxx9/THFxMQMHDuTYY4+NeuzMzExWrFjBQw89xL333stjjz1Gz549WbhwISkpKbz55pvcdNNNvPzyy3X6T052niUFEZkFnARkishG4I9AKoCqPgLMA84A1gJ7gZ97FUsV3bohL75I7uLFcP31cPN06N7dSQ6tWzt/XI0aHdZbbN1dyt59Va907StWFnxUyNLP9zEgJ5WCHSU8u2A3c97bQ58u6TRpFGDCyc3onZN+WO9vTDy8//77jB8/niZNmgAwYcIEFi5cyJFHHlklIQB88MEHjBs3jlAoRCgU4qyzzqr22BMmTADg2GOP5ZVXXgFg165dTJ48mTVr1iAiFBcXe/CvSm6eJQVVnRRjuwKXe/X+rgwZAu++C6+/DjffDA8/DHv3wocfwssvQy1HBAK0qaHm0K19Ci+2SOHfHxfRNBTk6AFN2L+nhK07S1n91X7eXraXFs0CNA4FOHtEU8YOa1ZtjcKYiBq+0cdbeZI4HOnpzhejYDBISUkJALfccgsnn3wyr776KuvXr+ekk0467PcxB7PrFSIwZozT7C4ogIcegrlznedWr/boLYXzTmjExBMbM7BLGm1apbC/USrSMp3MIxvRul0aqY2DpKYKD720k4k3b+KROTtZ/uU+Vn61n0+/Kaak1LrWmsQbNmwY//znP9m7dy8FBQW8+uqrDBs2rNr9hw4dyty5cykqKiI/P/+Qewvt2rWLrKwsAJ588snDCd1Uo0H0Poqryy5zbm+8Efr3h9mz4dxz6/xtRISR/UOA00Pjs40l/LDHqUOUlsL7n+/jm62ltE8Lsv37/bywYDevvrOHVkek0SQjhbbNgwzqmkagQloPBoST+6bTJGS53sTHwIEDueiiixg8eDDgFJpbtmxZ7f6DBg1i7Nix9O/fnyOOOIJ+/frRvHlz1+93/fXXM3nyZP785z9z5plnHnb8pipxruI0HLm5uRqXRXZ++AHOPhuWLoV582DkSO/fs4IyVTZvL6W0zEkaH31RxCtv7WHTlhKOaJ1CCVC4v+r/XWazAB1aHbh01bZlChNHZdCuteV/P/rss8/o1atXosM4JPn5+TRt2pS9e/cyfPhwpk+fzsCBNstNXYr2eyEiy1U1N9Zr7ZOiOpmZzmWkESOc5PD22zBoUNzePiBCdoUP8s5tUxk7tClvL93La+/ns69YodJl2x92l7JlRylpAQgEBFVlxRf7mPd/+Yw5sSkXjm5Oq+bW/dUk1pQpU1i9ejVFRUVMnjzZEkI9Y0mhJi1bwvz5cOKJMGqUcznpRz+CqVOdWkScBQPCqUOacOqQ6EW8TdtLuHX2bk7snc4xOal075DK7vxSnn5jN3kL83l72V7uu6otR2WnxTlyYw547rnnEh2CqYFdfI6lQwf4179g+HCnZ9Ktt8IttyQ6qqiyWqUwsEsqC1fv4++v53PTMzv5/NtSrrmgFY//oT1pqcL1/9jCZ+v3JTpUY0w9ZS0FN7p2hTlzQBWmTIE77nAuL119daIjq+IXI5tyxsBS9hSVMW95EU+9W0BRsXLq0SHuvbItv/vbFi7/y/f075pO41D1rZ3+XUOcfVJTQmn2vcGYZGJJ4VCIwCOPwPbt8NvfwrffQnY2TJgA4W5yiZaeKhzZ1vlv7ZWdyvR/5fPCB3uZv6KQFk0DXP3T1qxZV8TiT4soilKoBthfrCxatZOX3t7NT09vzhlDm5KaYuMkjEkG1vuoNoqKYPx4p94AzujnqVPhhhsSUmuoSUmp8saKInYWlPHfzcV8t7OMCcc14vSBNY/Y/s/aIh7L28V/1u6jfesgE0dl0Lp5kC5ZadaTqR5piL2PjPes91G8hUJON9U9e5zWwk03we9/D2Vlzv16JCUonDXISQClZcqD8/JZ8FERJ/cLEaphlHS/riEe+G06S1YX8ficnfx11g4AggE4dUgT2rYMclR2GkP7N7IZX43xEUsKtSUCGRnOz4svwuTJ8Ic/QLducN55iY4uqmBAGJMb4u5X9rBw9T5OPTpU4/4iwpA+jRjUK8RXm4vZX6wsWFTAGx/mU+zMOkDXjqlcMrYFg3qHbAUwY3zAqoh1IRCAGTMgNxeuvBJ27Up0RNU6ql0qPTqkMG95IUvW7GNnQZkz5qEGgYBwVHYavXLSuXpSKxb8vRP/frAjN05uTf7eMm6ctpWp//sD+2Mcx5jDtXPnTh566KHI4/Xr19O3b99aH+/WW2/l3nvvrYvQXJk/fz49evSga9eu3H333VH32bBhAyeffDK9e/emT58+/O1vf4ts69y5M/369WPAgAHk5sa8ElQrlhTqSmqqU4TessWZXK8emzisMc0bB3j03wVcN3Mn1z+1M2ZiqCwYEEYNacLMP3bg0rNb8MEnhfx5hiUG463KSaEhKS0t5fLLL+eNN95g9erVzJo1i9VR5ldLSUnhvvvuY/Xq1SxatIhp06YdtN8777zDypUr8aq2akmhLh17LFx8MUyf7tQb6qns1ilMPT+Dy09vysj+6ezdp2zcVlKrY6WmCJNGZXD5uS14/+NCfnbbZh6fs5Nn3tjFxi02rXEyKCgo4Mwzz+Too4+mb9++PP/886xfv56ePXty0UUX0b17d37yk5/w5ptvMnToULp168aSJUsAZ4Gevn370rdvXx6oNMtrtG033ngjX375JQMGDOC6664DnA/bSy+9lD59+jBq1CgKCwsBeOaZZxg8eDADBgzgl7/8JaWlztxid9xxB927d+fEE0/kiy++cPVv/Pjjjxk+fDi9e/cmEAggIkydOvWQztOSJUvo2rUrXbp0IS0tjYkTJzJnzpwq+7Vv3z4yyrtZs2b06tWLTZs8W3+sCqsp1LULL4RHH4UFCzyZSK+uBEQYkJNGx8wgb32yj29+KOWodrWfKvycH2WQ0yGNx/N28uwCZ2W6J1/fxYSTmvGrCS2sGB0Hs98vYMMP0Rd3qq2OmUEmnljzNNjz58+nQ4cOvP7664Azk+mOHTtYu3YtL774IjNmzGDQoEE899xzvP/+++Tl5XHnnXdyyy23RF2g55hjjql28Z67776bVatWsTK8mND69etZs2YNs2bN4tFHH+XHP/4xL7/8MsceeyzPP/88H3zwAampqfz617/m2WefpU+fPsyePZuVK1dSUlJS4yI/5YqKijj//PN56qmnGDx4MLfccgtFRUXcdtttkX2GDRvGnihfBO+9997ImtObNm2iY8cDi01mZ2ezePHiGt97/fr1fPTRRwwZMgRw6nyjRo1CRPjlL3/JlClTanx9bVhSqGsnnACtWkFeXr1OCuVaNQ3QOF3q5MNkYM8QA3u2A2D77lKemLuTl97ew/5i5adnODNhpgSheVObf8lP+vXrx+9+9ztuuOEGxowZw7Bhw9ixYwc5OTn069cPgD59+jBy5EhEhH79+rF+/fpqF+g55phjqt02duzYKu+fk5PDgAEDAGdBnvXr17Nz506WL1/OoPB8ZYWFhbRt25bt27czfvx4GjduDBD1eJW9+eabDBw4MDITbP/+/Zk/f/5BHSsWLlxY29NXrfz8fM455xweeOABMjIyAGdRo6ysLLZs2cKpp55Kz549GT58eJ2+ryWFupaSAmee6SzcU1LiPK7HRIROmUG+2Vq7y0fVaZUR5JoLWtGscYDZ/95D3sL8yLZfjm/B+adm1On7GWJ+o/dK9+7dWbFiBfPmzePmm29m5MiR/OxnP4sskgMQCAQijwOBQGTRnLpQ8X2CwSCFhYWoKpMnT+auu+46aN/Kl6jcWLVqVSS5AaxYsaLKJH5uWgpZWVls2LAhsm3jxo2RtSEqKy4u5pxzzuEnP/lJZAW68mMAtG3blvHjx7NkyZI6TwpWU/DCuHHOqOf33090JK50ykxh0/bSOl+4R0S49OwW3D4lk99OaslvJ7Ukt1eIGXN3suF7qzf4xebNm2ncuDEXXngh1113HStWrHD1upoW6KluW7NmzaJ++FY2cuRIXnrpJbZs2QLA9u3b+frrrxk+fDj//Oc/KSwsZM+ePcydO/eg10S7dt+6dWs++eQTAP773//yyiuvMHHixIP2WbhwIStXrqzyU54QwFlLYs2aNXz11Vfs37+f2bNnR22pqCoXX3wxvXr14pprrok8X1BQEPm3FxQU8K9//euwel5Vp35/jW2oTjsNmjd35kgaMaLejXKurGNmkJJS+G5n6UHTddcFEeHEAY0jj4f2b8zk2zdz77Pbue+qtqQE6/e5MbH95z//4brrriMQCJCamsrDDz/s6nXRFug55phjYm4bOnQoffv25fTTT+fyy6Ov6Nu7d2/+/Oc/M2rUKMrKykhNTWXatGkcd9xxnH/++Rx99NG0bds2cnmprKyMtWvX0qpVqyrHmjRpEnl5efTt25fMzExmzZpF69atD+0k4fQqevDBBznttNMoLS3lF7/4BX369IlsP+OMM3jsscdYt24dTz/9dKTrKcCdd95Jz549GT9+PAAlJSVccMEFjB49+pDjiMWmufDKtGlwxRUwaxZU+lZR35RPuT1pmLM8KEDzxuLZYLR/Ly7grpnbOGlgY/7wi9YErQhdazbNRd1YtWoVM2bM4P777090KHXCprmoj371K3jySWcw2+DB0KVLoiOqVrsWQdJTYNbCvcxauBeAM48NcfaQxjFeWTunDmnCjj2lPPLKTkLpwrU/aWW9k0xC9e3b1zcJ4XBZUvBKMAhPPw1Dh8Kpp8IHH0C7domOKqpgQLhyTDO+3eH0QHplUSE7Cso8fc8fn5LB3qIynpq3m0bpwhXntbRpMoypB6zQ7KWePZ2J8zZvrncT5VXWvUMqI/qEGNEnRKM0IR5XFSef2ZzzRjbj1XfzmZFXf6cGMSaZWFLw2pAhcMkl8MwzTnJoAESIS1IQEX41oQVnndiUZxfs5u6Z2/h+e912jTXGHBpLCvFwzTVQWgoVJraqzwQoi1P/AxHhqoktmTQqg3eWF3DJn79l+666HZVrjHHPkkI85OQ402k//DB8/32io4kpEIhPS+HA+znjGR6+oR2F+5SX3qm/80YZ43eWFOLltttg3z649tpERxJTPFsKFXXJSmPEwMbkvbeH/L3eFrqNMdFZUoiXHj3g+uud2sI77yQ6mhqJCIkav3LBaRnsLVLumrmNzT9YfcGYeLOkEE833QQdO8KttyY6khoFJDEtBYCjstO4ZGxzln9exORbN/PArO3s2GM1BmPixZJCPDVq5BSd33sPFi1KdDTVilfvo+pcMLo5z9zWnjOHNuX1D/K56r7v2b7bEoNJjpXXID4rrFXH06QgIqNF5AsRWSsiN0bZ3klE3hGRj0TkExE5w8t46oVLLoGWLeGeexIdSbUCCU4KAJktUrh6Uiv++tsj+GFnKb+55ztufmQrby4pSGxgJqGSYeW1cl6vsFYdz5KCiASBacDpQG9gkoj0rrTbzcALqnoMMBFomP/bh6JpU7jsMnj1Vfjmm0RHE1WiWwoV9T0qnTsua0PLjCDrNu3nzie3sWBRfuwXmrixldfccbvyWqJ5Oc3FYGCtqq4DEJHZwDigYmpUoHxi/eZAwxjddbguvhjuvNMpOtfDkc6JrClEc0yPEA9e1479xcpND23hnqe30+PIdDq3r/1KcX704Is7+HLj/jo95lHZaVxxXssa97GV1+p+5bV4rLBWHS+TQhawocLjjcCQSvvcCvxLRH4DNAFOIRl06QLDhsFTT8Hvf1/vptZ2Wgr1KCuEpaUKN/8ikx/ftIk3/i+fy86p+cPKxIetvFb3K6/FY4W16iR6QrxJwJOqep+IHA88LSJ9VfWgTuoiMgWYAtCpU6cEhOmByZOd+sLSpc4sqvWIiFD/UoKjRbMgx/drxL8XF3Dp2S1sPYYKYn2j94qtvFb3K6/FY4W16nhZaN4EdKzwODv8XEUXAy8AqOqHQAjIrHwgVZ2uqrmqmtumTRuPwo2zc8+FUAhq8UvqNalnl48qG318U3bml7HoP4WJDsVgK69B3a68Fq8V1qrjZUthKdBNRHJwksFE4IJK+3wDjASeFJFeOElhq4cx1R/Nm8PvfuesznbppXDyyYmOKKI+9D6qyeDeIVplBJi/qOCgVd1MYtjKa+64XXmtqKgoLiusVcfTldfCXUwfAILADFW9Q0RuB5apal64N9KjQFOcovP1qvqvmo7ZYFZec6OwEPr0gZQUmDMH6skKWvfO2U1pGdwwPiP2zgky/dUdvPDWHl64I4tWzYOJDidhbOW1umErrx3g6TgFVZ2nqt1V9ShVvSP83FRVzQvfX62qQ1X1aFUdECsh+E6jRvDYY/Ddd9C3Lzz6aKIjAup/SwGcS0hlZfBvG7dg6oCtvHaAjWhOtB/9CNatg0GDnG6qZYmfCM6pKdTvrNCpXSq9c9KY/2E+23eXUlJav+M1pqGwpFAfZGbCb34D69fD++8nOhoCEp+V1w7X6Sc05evvSjj3xk1ccMtmXn13D4tWFbLFFuoxptYS3SXVlDv7bGe081NPQZy6nlVHqP+XjwBOO64JwQAU7lPeWlrAP17YAUCPI9N4+Ib6uR62F1TV1rc2EYdbJ7akUF80aeJ0U33hBWeFtvCgnUSoT9Nc1CQlKIw+vikAZ49oylebi3n57T38e0kBxSVKaor/PyhDoRDbtm2jdevWlhgMqsq2bdsIhUK1PoYlhfrkkkvgySdh5kz49a8TFkYgUL/HKUQjInTJSiO3V4g3Pixg/bfFdOuYluiwPJednc3GjRvZujU5enKb2EKhENnZ2bV+vSWF+uSEE+C44+C++2DKFKeragI0lMtH0XQNJ4K1G/cnRVJITU0lJycn0WEYH7FCc30i4qzOtm4dvPJKQsNoaC2FclltUgilC2s31O3EcMYkC0sK9c3YsdC9O/zlLwn7up7I5TgPVyAgHJWVytqNxYkOxZgGyZJCfRMMwrXXwvLlCVvLub5NnX2ounZM48uN+ylryP8IYxLEkkJ99NOfwhFHJGx1tobS+6g6XbPT2FukfLvNxisYc6hiJgUROUtELHnEUygEV14J8+dDeHbGeAoI9XbqbDd6HukUmJetLkpwJMY0PG4+7M8H1ojIX0Skp9cBmbDLLnPGKiSgtSBSL2bbqLUuWan06pzGC2/uptSmvzDmkMRMCqp6IXAM8CXOFNcfisgUEWnmeXTJrGVLp1vqrFlxX8u5IUyIVxMR4YLRGXy7rZQ3Pixgx55SduwppbikAf+jjIkTV5eFVHU38BIwG2gPjAdWhJfRNF65+mrn9t574/q2zsprDfsD9Pi+jcjpkMr9z23nnBs2cc4Nm7jy3u8THZYx9V7M0VEiMhb4OdAVeAoYrKpbRKQxsBr4h7chJrFOneAXv4CHH3YW4qmwJKCXGnqhGZyuqX+8NJOPPnfqCss+L+L/PimkaH8ZoTQrkRlTHTdDZs8B/qqq71V8UlX3isjF3oRlIu66yxnIdtll8N57zhwUHmvoXVLLdToilU5HpALQMiPIBx8X8s13JXTv5P+RzsbUlptPmFuBJeUPRKSRiHQGUNW3PInKHNC6tVNs/uAD+P3v4/KWfmgpVNYly0kO6zbZSGdjauImKbwIVOyLUhp+zsTLRRc5E+T95S/wD++v1jXkaS6q0z4zhbRU4avNNtLZmJq4SQopqhr5ehW+b+3veBKBv/8dTjzRqS94LID/WgrBgHBkuxTWf2tJwZiauEkKW8PFZgBEZBzwg3chmaiCQTj1VPj8c9izx9O3kkD9X46zNnI6pFlLwZgY3CSFXwE3icg3IrIBuAH4pbdhmahyc52v8CtWePo2ARrGcpyHKqdDKj/sLGXP3gY8Ms8Yj7kZvPalqh4H9AZ6qeoJqrrW+9BMFbm5zu3SpZ6+jR8LzeAkBYAFi/LZ8L21GIyJxtUqLiJyJtAHCJUv+aeqt3sYl4mmbVtn7MKyZZ6+jV+6pFbWNTuNQAAeemknM1/fxQt3ZtEo3cYsGFORmwnxHsGZ/+g3OItynQcc6XFcpjq5uZ4nBb+2FFo1D/Lk1PZc/9NWFBQqC1cWJjokY+odN1+TTlDVnwE7VPU24Higu7dhmWoNGgRffgnbt3v2Fn5NCgDZbVM57bgmdGiTwvwP8xMdjjH1jpukUD7/8F4R6QAU48x/ZBJh0CDn9sMPPXsLCU+d3VBXX4tFRBh9XBNW/ncf7ywrYMmnhSz7rJB9+60AbYybpDBXRFoA9wArgPXAc14GZWowdKgzpfZrr3n2FoFI3cizt0i4UUOakBKEP83Yxo3TtnL9P7by8jvedvU1piGosdAcXlznLVXdCbwsIq8BIVXdFZfoTFWhEJx2GuTlwbRpnsyFFM4JlKl/l+Zr2yqFJ6a2Z1e+0zr44/Qf+OY7W6nNmBr/5lW1DJhW4fE+Swj1wLhxsHmzZ+MVypOCjxsKAGS1SaV3Tjq9c9LJbpvC5h8sKRjj5ovgWyJyjpT3RTWJd8YZTgthzhxPDh8obykk0SX29pkpbN5qYxeMcZMUfokzAd4+EdktIntEZLfHcZmaZGbCsGEwezaUltb54QNJ0lKoqENmCtt3l1G4L4kyoTFRuBnR3ExVA6qapqoZ4ccZbg4uIqNF5AsRWSsiN1azz49FZLWIfCoiVsB269e/hrVrndpCHYtcPkqirNChjVNe+26bXUIyyc3NymvDoz1fedGdKK8L4tQjTgU2AktFJE9VV1fYpxvwe2Coqu4QkbaHEnxSmzABunSB//kfOPvsA5/kdaDCqHWc8Yr+1yHT+VPYvLWEnA42CbBJXm6mubiuwv0QMBhYDvwoxusGA2tVdR2AiMwGxuEs4VnuUmCaqu4AUNUtLuM2KSlwzTVwxRXQsqUzrfYLL0Djxod96ECF3kfJorylYMVmk+xiJgVVPaviYxHpCDzg4thZwIYKjzcCQyrt0z18zA+AIHCrqs6vfCARmQJMAejUqZOLt04SF18M330H338Pjz8O557rFJ9TUw/rsMl4+ahZ4wBNGgmbt1pSMMmtNt3QNwK96uj9U4BuwEnAJODR8EC5g6jqdFXNVdXcNm3a1NFb+0AoBH/6E0yfDo88Am+8AXfeediHTcakICJ0yEzhW2spmCTnpqbwDw50RAkAA3BGNseyCehY4XF2+LmKNgKLVbUY+EpE/ouTJLydG9qPLr0U3n4b7roLLrgAunWr9aHKqwjJdPkIoEObVNZs2M+ufKdHV6P0AGmpyVFTMaacm5bCMpwawnLgQ+AGVb3QxeuWAt1EJEdE0oCJQOWuMv/EaSUgIpk4l5PWuQvdVHH//ZCeDldeeViHKR8knWxJIbttCpu3ljD++k2Mv34TP75pE/uLk+wkmKTnptD8ElCkqqXg9CoSkcaquremF6lqiYhcASzAqRfMUNVPReR2YJmq5oW3jRKR1UApcJ2qbjucf1BSa98e/vAHuOEGZ3rt8kV5DpFQsfdR8phwcjMymwcpLYM1G/azYFEB320vodMRh1ejMaYhcTWiGWhU4XEj4E03B1fVearaXVWPUtU7ws9NDScE1HGNqvZW1X6qOvtQ/wGmkl/9CjIy4J57an2IQBLWFABaNgsybkQzJpzcjNHHNwHgexu3YJKMm6QQUtXIxPPh+4ff79F4IyMDLrsMXnoJ1tXuSpwkYZfUyo5o5TQeHzXoAAAU0ElEQVSit+yo+xHjxtRnbpJCgYgMLH8gIscCtmRVfXbllc44hvvvr9XLk7WlUFFmiyABge+3W0vBJBc3SeFq4EURWSgi7wPPA1d4G5Y5LB06wIUXwowZsHXrIb88GbukVpYSFFo3D7Jlu7UUTHJxM/fRUqAncBnwK6CXqi73OjBzmK69FgoLnTUXDpElBccRrVPYYi0Fk2RiJgURuRxooqqrVHUV0FREfu19aOaw9OoFY8bAo48e8kvLV14rS/Ks0LZl0C4fmaTj5vLRpeGV1wAIz1N0qXchmTozYoSzGM+OHYf0MmspOI5olcLWnaWUJnPF3SQdN0khWHGBnfDspzaNZEPQo4dz+8UXh/SySO+jOg6noWnbKkhJKWzfbXUFkzzcJIX5wPMiMlJERgKzws+Z+q5nT+f2888P6WXW+8gR6ZZqxWaTRNwkhRuAd3AKzZfhDGa73sugTB3JyXFmTK1tSyHJmwpHtAoCWLHZJBU3U2eXAQ+Hf0xDkpICRx11yEkhGZfjjKZtS+fPY+V/99GsSfTvT80aB+hxZHo8wzLGU25mSe0G3AX0xllkBwBV7eJhXKau9OhR65ZCss19VFmTRgFaNw8y9/185r6fX+1+T9/Wnqw2Nj+S8Qc3E+I9AfwR+CtwMvBzarcOg0mEHj1g3jwoKXFaDi4c6JLqZWANw0PXH8H31dQUPllbxGNzdlFQaCfK+IebT4lGqvqWiIiqfg3cKiLLgakex2bqQs+eUFwM69dD166uXmJdUg9o0zKFNi2j/5nk73WKLmWWPY2PuPnGv09EAsAaEblCRMYDTT2Oy9SV8m6pr7wCa9a4eoklBXeCTh2a0iQvyBt/cZMUrsKZFfVK4FjgQmCyl0GZOtSrl9MD6YYbYOBA2L075kuSdeW1QxUMV+RtcJvxE1dzH6lqvqpuVNWfq+o5qrooHsGZOtCyJfznP/DEE5CfDy+/HPMl5SuvWUuhZsHweSq1YQzGR6xgnAx69IDJk511m596KubuB1oKlhVqUp48raVg/MSSQrIQgZ/9DN59F77+Osau5ctxxiGuBiwYDPfSspqC8RFLCsnkwgud25deqnG3gK285krk8pElBeMj1XZJFZF/UMOgVlW90pOIjHc6d4a2bWMOZrO5j9yJFJpL7UQZ/6ippbAMWI4zinkgsCb8MwCbJbXhysmJuXazdUl1x7qkGj+qtqWgqjMBROQy4ERVLQk/fgRYGJ/wTJ3r0gUWL65xF0sK7pSP/LZCs/ETNzWFlkBGhcdNw8+Zhignxyk0l1Q/86fVFNwpbylYodn4iZtpLu4GPhKRd3B6Kw4HbvUyKOOhLl2cjvUbNjgJIooDvY8sK9TEBq8ZP3IzdfYTIvIGMCT81A2q+p23YRnPlCeCr76qISk4t/YFuGY2eM34UczLR+GlOE8BjlbVOUCaiAz2PDLjjS7hGc9rKDZbTcGd8nEK1lIwfuKmpvAQcDwwKfx4DzDNs4iMt7KznSm0v/qq2l0iNQVrKtTIxikYP3JTUxiiqgNF5CMAVd0hItYltaFKSYFOndy1FOIUUkNVPs2FJU/jJ25aCsUiEiT8GSEibbDLzQ1bly6uWgp2+ahmVmg2fuQmKfwdeBVoKyJ3AO8Dd3oalfFWjAFsEll5zT7sahIZvGaFZuMjbqbOfha4Hmed5m+Bs1X1RTcHF5HRIvKFiKwVkRtr2O8cEVERyXUbuDkMOTmwdSsUFETdbC0Fd6ylYPyo2qQgIhnh21bAFmAW8Bzwffi5GoUvOU0DTgd6A5NEpHeU/ZrhLORT8zBbU3c6dXJuN2yIutl6H7ljhWbjRzW1FJ4L3y7HmQep/Kf8cSyDgbWquk5V9wOzgXFR9vsT8D9AkdugzWHq2NG5rS4phG8tKdQsECifOttOlPGPmnof3R2+7aWqtfnAzgIqfups5MAAOABEZCDQUVVfF5HravEepjbKWwrffBN1c6RXjX3WxRQMWE3B+EtNLYW/hW//z4s3FpEAcD/wOxf7ThGRZSKybOvWrV6Ek1yyspxrRNZSOGzBoFhNwfhKTS2FYhGZDmSLyN8rb3SxnsImoGOFx9nh58o1A/oC74Z7u7QD8kRkrKoedHlKVacD0wFyc3PtL/BwpaZC+/bVthQi01xYVogpGLCagvGXmpLCGJzpLU7DqSMcqqVANxHJwUkGE4ELyjeq6i4gs/yxiLwLXFs5IRiPdOxYbUshYMtxuuZcPrITZfyjpvUUfgBmi8hnqvrxoR5YVUtE5ApgARAEZqjqpyJyO7BMVfNqHbU5fJ06wcfR/1vFps52LRAQLCcYP6lpOc7rVfUvwCUiUuXX3s1ynKo6D5hX6bmp1ex7UsxoTd3p2BFee81pDpRngTAbp+BeMAhlVmg2PlLT5aPPwrd2OcePOnaEwkLYtg0yMw/aZOMU3AsGrNBs/KWmy0dzw3f3Vh7BLCLneRqV8V7FAWyVkoKtvOaeFZqN37iZ++j3Lp8zDUn5ALYoPZAOtBQsK8QSDIoVmo2v1FRTOB04A8iq1CU1A6h+gV/TMJS3FObPd8Yt5B6YdiqyHGci4mpgAoIVmo2v1NRS2IxTTyjC6ZJa/pOH003VNGRt2kCrVvDII3D88bBnz0GbRezykRvWUjB+U1NN4WMRWQWcpqoz4xiTiYdAAD79FGbOhBtvhE2boGfPA5sF1K6Vx2Q1BeM3NdYUVLUU6GgrrflUu3Zw3HHO/U2bDtokYispuWGD14zfuFmO8yvgAxHJAyIT8Kvq/Z5FZeKnQwfndvPmg54OiHVJdSMYFFuO0/iKm6TwZfgngDNfkfGT9u2d20pJQSwpuBIQu3xk/CVmUlDV2wBEpLGq7vU+JBNXTZtCRkaUloLYhHgu2Cypxm9ijlMQkeNFZDXwefjx0SLykOeRmfjp0MFaCrVkhWbjN24Grz2A0wV1Gzi9koDhXgZl4iwrq2pSwJKCG8GAdUk1/uImKaCqledYtinA/KRDhyq9jwIBG6fgRjBoLQXjL24KzRtE5ARARSQVuIoDk+UZPyi/fFRhxlRrKbjjFJrtRBn/cNNS+BVwOc6ay5uAAeHHxi86dIDiYmfG1DCrKbhjXVKN37jpffQD8JM4xGISpeJYhfCMqc40F5YVYrFCs/EbN72PZopIiwqPW4rIDG/DMnEVZQCb0yU1QfE0IDb3kfEbN5eP+qvqzvIHqroDOMa7kEzcZWU5txWSgl0+csdaCsZv3BSaAyLSMpwMEJFWLl9nGop27ZzbOXNgrzM+MdB0Iqr23xyLrbxm/MbNX/19wIciUr762nnAHd6FZOIuPR169YK8POcHkPtHUVZ+WclUKxjACs3GV9wUmp8SkWXAj8JPTVDV1d6GZeJu5UrYvdu5P3gwgdJSu3zkgrUUjN+4uj4QTgKWCPwsLe3AWs2NGiFaitraazEFglBqQzmNj7ga0WySTHo6UlZmLQUXrKVg/MaSgqkqPZ1AWal1SXXBeh8Zv7GkYKoKhZAyqym4YYVm4zeWFExV6emItRRcsVlSjd9YUjBVpacj1vvIlfJZUtVOlvEJSwqmqkhNwT7oYgkGnFllrVVl/MKSgqkqPZ1AaYm1FFwIhv+CrFuq8QtLCqaqcE3BkkJsgUhLwU6W8QdLCqaq9HQCJSV2ScSFYNC5tZaC8QtPk4KIjBaRL0RkrYjcGGX7NSKyWkQ+EZG3RORIL+MxLqWnI2V2+ciN8pqCDWAzfuFZUhCRIDANOB3oDUwSkd6VdvsIyFXV/sBLwF+8isccglDIeh+5FKkp2FgF4xNethQGA2tVdZ2q7gdmA+Mq7qCq76jq3vDDRUC2h/EYtyJdUi0rxBIMhlsKNlbB+ISXSSEL2FDh8cbwc9W5GHgj2gYRmSIiy0Rk2datW+swRBNVejoBLaXMvv7GFL56ZKOajW/Ui0KziFwI5AL3RNuuqtNVNVdVc9u0aRPf4JJRejqiZZTZdfKYIoVmSwrGJ7xcWmsT0LHC4+zwcwcRkVOAPwAjVHWfh/EYt8pnSbWvvzFZodn4jZcthaVANxHJEZE0YCKQV3EHETkG+F9grKpu8TAWcyjS0wloGWofdDHZ4DXjN54lBVUtAa4AFgCfAS+o6qcicruIjA3vdg/QFHhRRFaKSF41hzPxFG4p2OWj2CKFZjtXxic8XZldVecB8yo9N7XC/VO8fH9TS6EQAS2wloILVmg2flMvCs2mngkXmi0pxGYtBeM3lhRMVTZLqms2eM34jSUFU1WkpZDoQOo/G7xm/MaSgqkqPR1RtRHNLlhLwfiNJQVTVfngNUsKMQUsKRifsaRgqgrXFCwnxBZZec0KzcYnLCmYqiIthUQHUv/Z4DXjN5YUTFWhkDOi2ZJCTNYl1fiNJQVTVXg5Tvuci80KzcZvLCmYqtLTCahiOSG28jWarUuq8QtLCqaq8nEKKomOpN4rbylYq8r4hSUFU1VqqpMUEh1HA2CD14zfWFIwVYkQECjDWgqxWE3B+I0lBROVBAKoJYWYLCkYv7GkYKKSgNjlIxes0Gz8xpKCiUqCAbt85IK1FIzfWFIwUQUCgtqvR0w2eM34jf3Vm6gkEKBMrKUQi7UUjN9YUjBRBYIBVOzXI5bIhHhWUzA+YX/1Jiop/7CzCZBqFAw6t9ZSMH5hScFEFQhfF7GcULNw7rSagvENSwomKrGk4IqIEAhYS8H4hyUFE5WkONdFLCnEFrSkYHzEkoKJqvzykV0ViS0YFBu8ZnzDkoKJSoLWUnDLWgrGTywpmKjKk4L1PootGBBbo9n4hiUFE1XAagquWUvB+IklBROVWE3BtUBArEuq8Q1LCiYqaym4FwxCaWmiozCmblhSMFFFuqTap11MQWspGB/xNCmIyGgR+UJE1orIjVG2p4vI8+Hti0Wks5fxGPcCqSkAlBXtT3Ak9V8wAGVWUzA+4VlSEJEgMA04HegNTBKR3pV2uxjYoapdgb8C/+NVPObQRFoK+/YlOJL6zwrNxk+8bCkMBtaq6jpV3Q/MBsZV2mccMDN8/yVgpIjN11wfSIrTUtD91lKIJRiwwWvGP1I8PHYWsKHC443AkOr2UdUSEdkFtAZ+8DAu44KkpEAJ3D/re4K6KdHh1Gvfb23B198GGXPlnkSHYnzu7M7buOSaQZ6+h5dJoc6IyBRgCkCnTp0SHE1y6HVST4bMWkqJWl+EWPaHSti8v2miwzBJoEWG9x/ZXr7DJqBjhcfZ4eei7bNRRFKA5sC2ygdS1enAdIDc3Fxrp8dBq+7ZXPLH7ESHYYyJMy+/Bi4FuolIjoikAROBvEr75AGTw/fPBd5WtZ7xxhiTKJ61FMI1giuABUAQmKGqn4rI7cAyVc0DHgeeFpG1wHacxGGMMSZBPL1AparzgHmVnpta4X4RcJ6XMRhjjHHPqojGGGMiLCkYY4yJsKRgjDEmwpKCMcaYCEsKxhhjIqShDQsQka3A17V8eSY2hUZFdj4OZufjADsXB/PD+ThSVdvE2qnBJYXDISLLVDU30XHUF3Y+Dmbn4wA7FwdLpvNhl4+MMcZEWFIwxhgTkWxJYXqiA6hn7HwczM7HAXYuDpY05yOpagrGGGNqlmwtBWOMMTXwZVIQkdEi8oWIrBWRG6NsTxeR58PbF4tI5/hHGT8uzsdFIrJVRFaGfy5JRJzxICIzRGSLiKyqZruIyN/D5+oTERkY7xjjxcW5OElEdlX4vZgabT+/EJGOIvKOiKwWkU9F5Koo+/j/90NVffWDM033l0AXIA34GOhdaZ9fA4+E708Enk903Ak+HxcBDyY61jidj+HAQGBVNdvPAN4ABDgOWJzomBN4Lk4CXkt0nHE8H+2BgeH7zYD/Rvlb8f3vhx9bCoOBtaq6TlX3A7OBcZX2GQfMDN9/CRgpIhLHGOPJzflIGqr6Hs7aHdUZBzyljkVACxFpH5/o4svFuUgqqvqtqq4I398DfIazjnxFvv/98GNSyAI2VHi8kar/sZF9VLUE2AW0jkt08efmfACcE24OvyQiHaNsTxZuz1eyOF5EPhaRN0SkT6KDiZfwJeVjgMWVNvn+98OPScEcurlAZ1XtD/ybA60ok9xW4EyNcDTwD+CfCY4nLkSkKfAycLWq7k50PPHmx6SwCaj4TTc7/FzUfUQkBWgObItLdPEX83yo6jZV3Rd++BhwbJxiq4/c/P4kBVXdrar54fvzgFQRyUxwWJ4SkVSchPCsqr4SZRff/374MSksBbqJSI6IpOEUkvMq7ZMHTA7fPxd4W8NVJB+KeT4qXRMdi3MtNVnlAT8L9zI5Dtilqt8mOqhEEJF25bU2ERmM83nh1y9PhP+tjwOfqer91ezm+98PT9doTgRVLRGRK4AFOD1vZqjqpyJyO7BMVfNw/uOfFpG1OIW2iYmL2Fsuz8eVIjIWKME5HxclLGCPicgsnF41mSKyEfgjkAqgqo/grCl+BrAW2Av8PDGRes/FuTgXuExESoBCYKKPvzwBDAV+CvxHRFaGn7sJ6ATJ8/thI5qNMcZE+PHykTHGmFqypGCMMSbCkoIxxpgISwrGGGMiLCkYY4yJsKRgTB0Izyj6Wvj+2Giz0RrTEPhunIIxdSk8oElUtczta8JjPyoPmDSmQbCWgjGViEjn8PoTTwGrgMdFZFl4jv3bKuw3WkQ+F5EVwIQKz18kIg+G7z8pIudW2JYfvm0vIu+F1ylYJSLD4vYPNKYG1lIwJrpuwGRVXSQirVR1u4gEgbdEpD/OXPuPAj/CGd36/CEe/wJggareET5u47oM3pjaspaCMdF9HZ4vH+DH4dbAR0AfoDfQE/hKVdeEp3545hCPvxT4uYjcCvQLz99vTMJZUjAmugIAEckBrgVGhqcWfx0IHcJxSgj/nYlIAGf1u/IFbobjzLD5pIj8rO5CN6b2LCkYU7MMnASxS0SOAE4PP/850FlEjgo/nlTN69dzYCrysYQnnBORI4HvVfVRnOnK/bfWr2mQrKZgTA1U9WMR+QgnCWwAPgg/XyQiU4DXRWQvsBBnXd/KHgXmiMjHwHzCLRCc2UmvE5FiIB+wloKpF2yWVGOMMRF2+cgYY0yEJQVjjDERlhSMMcZEWFIwxhgTYUnBGGNMhCUFY4wxEZYUjDHGRFhSMMYYE/H/AcUeky8qhD6dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a538a8668>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot certification accuracy wrt to radius\n",
    "rad_list = np.linspace(0,2.25,201)\n",
    "plt.plot(rad_list, getCertAcc(radius0, pred0, y_test), 'r-', label='original')\n",
    "plt.plot(rad_list, getCertAcc(radius1, pred1, y_test), '-', color='cornflowerblue', label='smoothed, $\\sigma=$' + str(sigma1))\n",
    "plt.plot(rad_list, getCertAcc(radius2, pred2, y_test), '-', color='royalblue', label='smoothed, $\\sigma=$' + str(sigma2))\n",
    "plt.xlabel('radius')\n",
    "plt.ylabel('certified accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "randomized_smoothing_mnist.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}