batch_normalization/batch_normalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.distributions import uniform\n",
    "\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploring batch normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Pytorch dataset and loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = datasets.MNIST(\n",
    "    root='../datasets',\n",
    "    download=True,\n",
    "    train = True,\n",
    "    transform=transforms.ToTensor()\n",
    ")\n",
    "\n",
    "loader = DataLoader(\n",
    "    dataset,\n",
    "    num_workers=4,\n",
    "    batch_size=512,\n",
    "    shuffle=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining custom batch norm module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomBatchNorm(nn.Module):\n",
    "\n",
    "    def __init__(self, in_size, momentum=0.9, eps = 1e-5):\n",
    "        super(CustomBatchNorm, self).__init__()\n",
    "        \n",
    "        self.momentum = momentum\n",
    "        self.insize = in_size\n",
    "        self.eps = eps\n",
    "        \n",
    "        U = uniform.Uniform(torch.tensor([0.0]), torch.tensor([1.0]))\n",
    "        self.gamma = nn.Parameter(U.sample(torch.Size([self.insize])).view(self.insize))\n",
    "        self.beta = nn.Parameter(torch.zeros(self.insize))\n",
    "            \n",
    "        self.register_buffer('running_mean', torch.zeros(self.insize))\n",
    "        self.register_buffer('running_var', torch.ones(self.insize))\n",
    "        \n",
    "        self.running_mean.zero_()\n",
    "        self.running_var.fill_(1)\n",
    "\n",
    "\n",
    "    def forward(self, input):\n",
    "        \n",
    "        X = input\n",
    "\n",
    "        if len(X.shape) not in (2, 4):\n",
    "            raise ValueError(\"only support dense or 2dconv\")\n",
    "        \n",
    "        # dense layer\n",
    "        elif len(X.shape) == 2:\n",
    "            if self.training:\n",
    "                mean = torch.mean(X, axis=0)\n",
    "                variance = torch.mean((X-mean)**2, axis=0)\n",
    "                \n",
    "                self.running_mean = (self.momentum * self.running_mean) + (1.0-self.momentum) * mean\n",
    "                self.running_var = (self.momentum * self.running_var) + (1.0-self.momentum) * (input.shape[0]/(input.shape[0]-1)*variance)\n",
    "            \n",
    "            else:\n",
    "                mean = self.running_mean\n",
    "                variance = self.running_var\n",
    "                \n",
    "            X_hat = (X-mean) * 1.0 /torch.sqrt(variance + self.eps)\n",
    "            out = self.gamma * X_hat + self.beta\n",
    "\n",
    "        elif len(X.shape) == 4:\n",
    "            if self.training:\n",
    "                N, C, H, W = X.shape\n",
    "                mean = torch.mean(X, axis = (0, 2, 3))\n",
    "                variance = torch.mean((X - mean.reshape((1, C, 1, 1))) ** 2, axis=(0, 2, 3))\n",
    "                \n",
    "                self.running_mean = (self.momentum * self.running_mean) + (1.0-self.momentum) * mean\n",
    "                self.running_var = (self.momentum * self.running_var) + (1.0-self.momentum) * (input.shape[0]/(input.shape[0]-1)*variance)\n",
    "            else:\n",
    "                mean = self.running_mean\n",
    "                var = self.running_var\n",
    "                \n",
    "            X_hat = (X - mean.reshape((1, C, 1, 1))) * 1.0 / torch.sqrt(variance.reshape((1, C, 1, 1)) + self.eps)\n",
    "            out = self.gamma.reshape((1, C, 1, 1)) * X_hat + self.beta.reshape((1, C, 1, 1))\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example neural net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleNet, self).__init__()\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(28 * 28, 64),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(64, 128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 10)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleNetBN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SimpleNetBN, self).__init__()\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(28 * 28, 64),\n",
    "            CustomBatchNorm(64),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(64, 128),\n",
    "            CustomBatchNorm(128),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(128, 10)\n",
    "        )\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment of training with and without Batch Norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_without_batch_norm = SimpleNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "net_with_batch_norm = SimpleNetBN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_fn = nn.CrossEntropyLoss()\n",
    "opt = optim.SGD(net_without_batch_norm.parameters(), lr=0.01)\n",
    "opt_bn = optim.SGD(net_with_batch_norm.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   512] loss: 0.058  loss_bn: 0.060\n",
      "[1, 20992] loss: 2.301  loss_bn: 2.086\n",
      "[1, 41472] loss: 2.292  loss_bn: 1.734\n",
      "========================================\n",
      "[2,   512] loss: 0.057  loss_bn: 0.035\n",
      "[2, 20992] loss: 2.271  loss_bn: 1.297\n",
      "[2, 41472] loss: 2.259  loss_bn: 1.132\n",
      "========================================\n",
      "[3,   512] loss: 0.056  loss_bn: 0.024\n",
      "[3, 20992] loss: 2.222  loss_bn: 0.888\n",
      "[3, 41472] loss: 2.194  loss_bn: 0.784\n",
      "========================================\n",
      "[4,   512] loss: 0.054  loss_bn: 0.018\n",
      "[4, 20992] loss: 2.123  loss_bn: 0.639\n",
      "[4, 41472] loss: 2.068  loss_bn: 0.592\n",
      "========================================\n",
      "[5,   512] loss: 0.050  loss_bn: 0.013\n",
      "[5, 20992] loss: 1.924  loss_bn: 0.501\n",
      "[5, 41472] loss: 1.836  loss_bn: 0.477\n",
      "========================================\n",
      "[6,   512] loss: 0.042  loss_bn: 0.011\n",
      "[6, 20992] loss: 1.609  loss_bn: 0.417\n",
      "[6, 41472] loss: 1.480  loss_bn: 0.404\n",
      "========================================\n",
      "[7,   512] loss: 0.032  loss_bn: 0.010\n",
      "[7, 20992] loss: 1.248  loss_bn: 0.373\n",
      "[7, 41472] loss: 1.137  loss_bn: 0.353\n",
      "========================================\n",
      "[8,   512] loss: 0.026  loss_bn: 0.009\n",
      "[8, 20992] loss: 0.971  loss_bn: 0.327\n",
      "[8, 41472] loss: 0.911  loss_bn: 0.323\n",
      "========================================\n",
      "[9,   512] loss: 0.022  loss_bn: 0.008\n",
      "[9, 20992] loss: 0.797  loss_bn: 0.291\n",
      "[9, 41472] loss: 0.766  loss_bn: 0.293\n",
      "========================================\n",
      "[10,   512] loss: 0.018  loss_bn: 0.007\n",
      "[10, 20992] loss: 0.699  loss_bn: 0.273\n",
      "[10, 41472] loss: 0.679  loss_bn: 0.271\n",
      "========================================\n",
      "[11,   512] loss: 0.016  loss_bn: 0.006\n",
      "[11, 20992] loss: 0.630  loss_bn: 0.255\n",
      "[11, 41472] loss: 0.610  loss_bn: 0.253\n",
      "========================================\n",
      "[12,   512] loss: 0.014  loss_bn: 0.005\n",
      "[12, 20992] loss: 0.577  loss_bn: 0.244\n",
      "[12, 41472] loss: 0.555  loss_bn: 0.234\n",
      "========================================\n",
      "[13,   512] loss: 0.014  loss_bn: 0.006\n",
      "[13, 20992] loss: 0.533  loss_bn: 0.229\n",
      "[13, 41472] loss: 0.520  loss_bn: 0.222\n",
      "========================================\n",
      "[14,   512] loss: 0.013  loss_bn: 0.006\n",
      "[14, 20992] loss: 0.493  loss_bn: 0.210\n",
      "[14, 41472] loss: 0.492  loss_bn: 0.211\n",
      "========================================\n",
      "[15,   512] loss: 0.014  loss_bn: 0.006\n",
      "[15, 20992] loss: 0.471  loss_bn: 0.202\n",
      "[15, 41472] loss: 0.473  loss_bn: 0.206\n",
      "========================================\n",
      "[16,   512] loss: 0.011  loss_bn: 0.006\n",
      "[16, 20992] loss: 0.451  loss_bn: 0.190\n",
      "[16, 41472] loss: 0.451  loss_bn: 0.194\n",
      "========================================\n",
      "[17,   512] loss: 0.012  loss_bn: 0.005\n",
      "[17, 20992] loss: 0.435  loss_bn: 0.184\n",
      "[17, 41472] loss: 0.434  loss_bn: 0.185\n",
      "========================================\n",
      "[18,   512] loss: 0.011  loss_bn: 0.005\n",
      "[18, 20992] loss: 0.416  loss_bn: 0.175\n",
      "[18, 41472] loss: 0.410  loss_bn: 0.172\n",
      "========================================\n",
      "[19,   512] loss: 0.008  loss_bn: 0.003\n",
      "[19, 20992] loss: 0.394  loss_bn: 0.169\n",
      "[19, 41472] loss: 0.411  loss_bn: 0.170\n",
      "========================================\n",
      "[20,   512] loss: 0.011  loss_bn: 0.004\n",
      "[20, 20992] loss: 0.393  loss_bn: 0.162\n",
      "[20, 41472] loss: 0.395  loss_bn: 0.162\n",
      "========================================\n",
      "========== Epoch 20 ==========\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3gc1dX48e/Z1apZxbIkXCQX2RhsXLGFjakmNjUGEgKhE0LxSwLJDwh5X1oCIQESIIWEAPFLCyUQcCAUU0I3fsHGvWFc5SIXSe7qbc/vj9kVsqyVdqVdrXZ9Ps8zj3dn7sycWVlHd+/cuVdUFWOMMbHPFe0AjDHGhIcldGOMiROW0I0xJk5YQjfGmDhhCd0YY+KEJXRjjIkTltCN6WZEZICIVIiIu40yKiKHB3m8u0Xk+fBFaLorS+gRJCJJIvKkiGwSkXIRWSwiZ7YoM0VEvhaRKhH5WEQGhnD8jSIyNfyRR46I3CQiO0Rkn4g8JSJJAcodJSILRGSPb/lARI5qtl1E5Hcissu3PCAiEmQM3TrBqepmVU1T1UYAEflERK6JdlydFcLPPlFEZvr+f6uITG6x/UYR2SAi+0Vkm4j8UUQSuuQiujlL6JGVAGwBTgYygV8AL4vIIAARyQFe9a3vBSwA/hmNQLuCiJwO3ApMAQYBg4FfBSi+DTgf53PJAd4AXmq2fTrwHWAMMBqYBvxXJOI2nRfizx5gDnAZsKOVbW8C41Q1AxiJ83/gp+GMN1ZZQo8gVa1U1btVdaOqelX1LaAIGO8rch6wUlVfUdUa4G5gjIgM6+y5ReRaEVknIrtF5A0R6edbL74aTamvprRMREb6tp0lIl/5vk1sFZFbOhtHCz8AnlTVlaq6B/g1cGVrBVV1r+9zU0CARqB5E8MPgN+rarGqbgV+H+hYoRCR4b4a8V4RWSki5zTb1urnIyI5IvKWb5/dIvKZiBz0uyUivxKRv/hee0SkUkQe8L1PEZEaEckSkUG+mmmCiNwLnAg84muGeaTZIaeKyFrfN5i/tvMNJVlE/umLfZGIjGkW10YRucX3f2Gfr1xy5z7Jg4Tys69T1T+p6hycn3vL7etVda8/fMDLgf83Dl2qaksXLUBvoAYY5nv/MPBYizIrgO/5Xt8KvNXG8TYCU1tZ/y1gJzAOSAL+Asz2bTsdWAj0xPllGA709W3bDpzoe52FUwtq7bwnAHvbWE4IsN9S4MJm73MABbLbuMa9QAPOL+2dzdbvAyY2e18IlAf5c7gbeL6V9R5gHXA7kOj7HMuBI9v6fID7gcd9+3twErAE+Lks970+DlgPzGu2banv9SDf55Lge/8JcE2LYynwlu/nOAAoA85o43rrcb7xeIBbcCoWnmb/j74E+uF8I1oFXBftn72vXDEwuZX1lwD7fccoA8ZE+/e7OyxWQ+8iIuIBXgD+rqpf+1an4SSm5vYB6QCq+ltVndaB010KPKWqi1S1FrgNmORr6qn3HX8YTtJZparbffvVA0eJSIaq7lHVRa0dXFXnqGrPNpY5AeJqeb3+1+mBLkRVe+I0V90ALG7nWGnBtqMHcKzvuL9Vp5b4EU7SvNi3PdDnUw/0BQaqar2qfqa+rNPCF8BQEckGTgKeBPJEJA2nWe7TEOP9rTrfZDYDHwNj2yi7UFVnqmo98Acg2Xe9fn9W1W2quhunSaPVY3Xlz74tqvoPdZpcjsD5Y1rSkePEG0voXcD39fs5oA4nMflVABktimfg1Ao7ox+wyf9GVSuAXUCeL0k9AvwVKBGRGSLij+F7wFnAJhH5VEQmdTKOllper/91m9erqpU4v7TPishhbRyrIkAiDVY/YIuqeput2wTk+V4H+nwexKnZ/8d3s+7WANdRjXOf5GSchP4p8DlwPB1L6M3bl6twkmYgW5rF4cWp+fbr4LE6okM/+/ao6lpgJfBoZ44TLyyhR5ivxvgkTnPL93w1JL+VODd0/GV7AEN86ztjG9DUW8Z33GxgK4Cq/llVxwMjcGo4P/etn6+q5wKHAf8GXg5wTSf62nMDLScGiOuA6/W9LlHVXUFckwtI5Zvk2tqxwvG59W/R/j2Abz63Vj8fVS1X1Z+p6mDgbOBmEZkS4Byf4jSvHA3M970/HZgAzA6wTziGRO3vf+G7vnyc6w1JlH727UnA+b055FlCj7zHcNqpz/bV0Jp7DRgpIt/z3YT6JbCsWZNMMDwiktxsSQD+AfxQRMaK0zXsPpy22o0icoyITPQ1AVXitOk3itNV7FIRyfT90dlPKzekAHxNCmltLJ8FiPVZ4GpxuiRmAXcCz7RWUEROFZGjRcTt+wbxB2APTvuu/1g3i0ieODd8f9b8WL4bfVe28bm5WnxuScA832fy376blpNxEvRLbX0+IjJNRA73/fH2r2/1s8NJ4FcAX6lqHb72caBIVcsC7FOC0yukM8aLyHm+/x83ArXA3FAP0hU/e2jq8uu/MZvo+xmJb9s1/m9q4nRlvQ34MNRriUvRbsSP5wWnlqw4SbOi2XJpszJTga+Bapxf7kHNtt0OvNPG8Tf6jt98+Y1v23U4N91247QD5/vWTwGW+eLYidOun4ZzE/BdnKS5H6f22OoNrk5+JjfjJKj9wNNAUrNtK/2fDXCB73OpwLnp9TYwullZAR7wXd9u32vxbUvE+So/LEAMd7fyuRX7to3ASbr7gK+A7zY7ZqufD3CT72dRidOU8Ys2rj8Np839rmbXUUqzm+McfFN0ErDGd+4/+9YpcHizfZ7x/+wDXO9MnC6x5Tj3IsY1276RZjfXCXDTuKt+9m383x7k2/a07ziVvnIPAsnR/n3vDov/F8CYuCEiJwDXq+rF7RY2Jo5YQjfGmDhhbejGGBMnLKEbY0ycsIRujDFxImojlOXk5OigQYOidXpjjIlJCxcu3Kmqua1ti1pCHzRoEAsWLIjW6Y0xJiaJyKZA26zJxRhj4oQldGOMiROW0I0xJk7YtE3GmKDV19dTXFxMTU1NtEOJe8nJyeTn5+PxeILexxK6MSZoxcXFpKenM2jQIDo39Lxpi6qya9cuiouLKSgoCHo/a3IxxgStpqaG7OxsS+YRJiJkZ2eH/E3IEroxJiSWzLtGRz5nS+jGGBMnrA3dGNNxM2aE93jTp7dbRES4+eab+f3vfw/AQw89REVFBXfffXd4Y2nDlVdeybRp0zj//PO77JzBsIRuom7GwgOTwvTx7f9Sm0NXUlISr776Krfddhs5OTkh79/Q0EBCQnymvvi8KmNM3EpISGD69On88Y9/5N577z1g26ZNm7jqqqsoKysjNzeXp59+mgEDBnDllVfSq1cvFi9ezLhx40hPT6eoqIjt27ezZs0a/vCHPzB37lzeeecd8vLyePPNN/F4PNxzzz28+eabVFdXc9xxx/G3v/2tW99DsDZ0Y0zMuf7663nhhRfYt2/fAetvuOEGrrjiCpYtW8all17KT3/606Zta9as4YMPPmhqqlm/fj2zZs3i9ddf57LLLuOUU05h+fLlpKSkMGvWrKbjzZ8/nxUrVlBdXc1bb73VdRfZAZbQjTExJyMjgyuuuII///nPB6z/4osvuOSSSwC4/PLLmTNnTtO2Cy64ALfb3fT+zDPPxOPxMGrUKBobGznjjDMAGDVqFBs3bgTg448/ZuLEiYwaNYqPPvqIlStXRvjKOscSujEmJt144408+eSTVFZWBizTvHmkR48eB2xLSkoCwOVy4fF4msq6XC4aGhqoqanhxz/+MTNnzmT58uVce+213f4JWUvoxpiY1KtXL77//e/z5JNPNq077rjjeOmllwB44YUXOOGEEzp8fH/yzsnJoaKigpkzZ3Yu4C7Q7k1REXkKmAaUqurINsodA8wFLlTV7n/lxpjOC6KbYST97Gc/45FHHml6/+c//5mrrrqKBx98sOmmaEf17NmTa6+9llGjRjFo0CCOOeaYcIQcUaKqbRcQOQmoAJ4NlNBFxA28D9QATwWT0AsLC9UmuDBg3RZjyapVqxg+fHi0wzhktPZ5i8hCVS1srXy7TS6qOhvY3U6xnwD/AkqDjNMYY0yYdboNXUTygO8CjwdRdrqILBCRBWVlZZ09tTHGmGbCcVP0T8D/qGpjewVVdYaqFqpqYW5uq3OcGmOM6aBwPClaCLzk6/KTA5wlIg2q+u8wHNsYY0yQOp3QVbVp9HUReQZ4y5K5McZ0vWC6Lb4ITAZyRKQYuAvwAKhqu+3mxhhjuka7CV1VLw72YKp6ZaeiMcbElJZdTjsrmC6rbrebUaNGoaq43W4eeeQRjjvuuIDl9+7dyz/+8Q9+/OMft3ncyZMn89BDD1FYeGCPwE8++YRTTjmFN954g7PPPhuAadOmccsttzB58uT2L6oL2ZOixpiYkpKSwpIlS1i6dCn3338/t912W5vl9+7dy6OPPtqpc+bn5x80smMoGhvb7TMSFpbQjTExa//+/WRlZQFQUVHBlClTGDduHKNGjeL1118H4NZbb2X9+vWMHTuWn//85wA88MADjBo1ijFjxnDrrbc2He+VV15hwoQJHHHEEXz22WdN68eMGUNmZibvv//+QTF8+OGHHH300YwaNYqrrrqK2tpaAAYNGsQ999zDCSecwCuvvMLkyZO56aabOOmkkxg+fDjz58/nvPPOY+jQodx5551h+TxsPHRjTEyprq5m7Nix1NTUsH37dj766CMAkpOTee2118jIyGDnzp0ce+yxnHPOOfz2t79lxYoVLFmyBIB33nmHf//738ybN4/U1FR27/7mucmGhga+/PJL3n77bX71q1/xwQcfNG278847ufPOOzn11FOb1tXU1HDllVfy4YcfcsQRR3DFFVfw2GOPceONNzbF5B/x8fHHHycxMZHZs2fz8MMPc+6557Jw4UJ69erFkCFDuOmmm8jOzu7UZ2M1dGNMTPE3uXz99de8++67XHHFFagqqsrtt9/O6NGjmTp1Klu3bqWkpOSg/T/44AN++MMfkpqaCjiDfPmdd955AIwfP75pCF2/E088EeCAmvvq1aspKCjgiCOOAOAHP/gBs2fPbtp+4YUXHnCMc845B3CG6B0xYgR9+/YlKSmJwYMHs2XLlo5+JE2shm6MiVmTJk1i586dlJWV8fbbb1NWVsbChQvxeDwMGjSo1eFuVTXgrEP+IXXdbjcNDQ0Hbb/jjju49957m6awa28srLaG7PW/9r9v7Xyhshq6MSZmff311zQ2NpKdnc2+ffs47LDD8Hg8fPzxx2zatAmA9PR0ysvLm/Y57bTTeOqpp6iqqgI4oMmlPaeddhp79uxh6dKlAAwbNoyNGzeybt06AJ577jlOPvnkcF1eyKyGbozpsGiMjOlvQwenhvz3v/8dt9vNpZdeytlnn01hYSFjx45l2LBhAGRnZ3P88cczcuRIzjzzTB588EGWLFlCYWEhiYmJnHXWWdx3331Bn/+OO+7g3HPPBZw28qeffpoLLriAhoYGjjnmGK677rrwX3SQ2h0+N1Js+FzjZ8Pnxg4bPrdrhX34XGOMMbHBEroxxsQJS+jGmJBEq5n2UNORz9kSujEmaMnJyezatcuSeoSpKrt27SI5OTmk/ayXizEmaPn5+RQXF2MzjkVecnIy+fn5Ie1jCd0YEzSPx0NBQUH7BU1UWJOLMcbECUvoxhgTJyyhG2NMnLCEbowxccISujHGxIl2E7qIPCUipSKyIsD2S0VkmW/5XETGhD9MY4wx7Qmmhv4McEYb24uAk1V1NPBrILyzxhpjjAlKu/3QVXW2iAxqY/vnzd7OBULrCW+MMSYswt2GfjXwTpiPaYwxJghhe1JURE7BSegntFFmOjAdYMCAAeE6tTHGGMJUQxeR0cATwLmquitQOVWdoaqFqlqYm5sbjlMbY4zx6XRCF5EBwKvA5aq6pvMhGWOM6Yh2m1xE5EVgMpAjIsXAXYAHQFUfB34JZAOP+mbSbgg0PZIxxpjICaaXy8XtbL8GuCZsERljjOkQe1LUGGPihCV0Y4yJE5bQjTEmTlhCN8aYOGEJ3Rhj4oQldGOMiROW0I0xJk5YQjfGmDhhCd10C/OK53HfZ/fR6G2MdijGxCxL6KZbWLVzFZv2bWLVzlXRDsWYmGUJ3XQLpZWlAMzfOj/KkRgTuyyhm26hrKoMgMU7FlNdXx3laIyJTZbQTdRV11ezv3Y/w3OGU9tYy6y1s6IdkjExyRK6iTp/7fyEASeQkZTBiytejHJExsQmS+gm6soqnYTeO603hX0LmbVmFpV1lVGOypjYYwndRF1plXNDNDc1l4KsAmoba9myf0uUozIm9lhCN1FXWllKZlImyQnJZCZlArCjYkeUozIm9lhCN1FXWllKbg9n0vCMpAwAtpdvj2ZIxsQkS+gm6soqyzisx2HANwndaujGhM4SuomqyrpK9tXua0roqZ5UEt2JltCN6YB2E7qIPCUipSKyIsB2EZE/i8g6EVkmIuPCH6aJV+t2rwPgsFQnoYsIfdL6sKPSEroxoQqmhv4McEYb288EhvqW6cBjnQ/LHCr8Cd3fhg44Cd1q6MaErN2Erqqzgd1tFDkXeFYdc4GeItI3XAGa+NZUQ/c1uYAldGM6Khxt6HlA807Dxb51BxGR6SKyQEQWlJWVheHUJtat37Oe9MR0khOSm9b16WEJ3ZiOCEdCl1bWaWsFVXWGqhaqamFubm5rRcwhpqSyhMzkzAPW9UnrQ1llGQ3ehihFZUxsCkdCLwb6N3ufD2wLw3HNIaCssoz0xPQD1vVJ64OiTUMCGGOCE46E/gZwha+3y7HAPlW1p0JMUEorS0lLTDtgXZ+0PoD1RTcmVAntFRCRF4HJQI6IFAN3AR4AVX0ceBs4C1gHVAE/jFSwJv6UVZUxMHPgAessoRvTMe0mdFW9uJ3tClwftojMIaO2oZb9tftJS7IaujHhYE+KmqjZWbUT4KA29N5pvQFL6MaEyhK6iRr/xBYtE3qqJ5WMpAxL6MaEyBK6iRp/L5b0pPSDttnj/8aEzhK6iZrSSmdii5a9XMCeFjWmIyyhm6gJ1OQCltCN6QhL6CZqyirLcIubFE/KQdv69Ohjk1wYEyJL6CZqyqrKyEnNwSUH/zfsk9aH8rpymyzamBBYQjdRU1ZVdsAoi835+6KXVJZ0ZUjGxDRL6CZqyirLDhgHvTl/X/SSCkvoxgTLErqJmrKqMnJTD07oMxbOYM7mOU1ljDHBsYRuoqa0srTVhA7fdGW0EReNCZ4ldBMV9Y317K3ZG7DJxd+V0T88gDGmfZbQTVT4E3WgGnpSQhIel8eaXIwJgSV0ExX+RB2ohg7OkACW0I0JniV0ExX+tvFA3RbBaUe3NnRjgmcJ3URFUw09QJML+BK61dCNCZoldBMV/pp3m00uielWQzcmBJbQTVSUVpbiEhe9UnoFLJOWmGa9XIwJgSV0ExVlVWVkp2S3Oo6LX3pSOpX1lVTXV3dhZMbErqASuoicISKrRWSdiNzayvYBIvKxiCwWkWUiclb4QzXxpKwq8GP/fv6+6NaObkxw2k3oIuIG/gqcCRwFXCwiR7UodifwsqoeDVwEPBruQE18Ka0sbbOHC9jTosaEKpga+gRgnapuUNU64CXg3BZlFMjwvc4EtoUvRBOPSipKmkZUDMRq6MaEJpiEngdsafa+2LeuubuBy0SkGHgb+ElrBxKR6SKyQEQWlJXZL+mhbEfFDnr36N1mGX8N3W6MGhOcYBK6tLJOW7y/GHhGVfOBs4DnRA6+26WqM1S1UFULc3Pbbj818au6vpryuvKgE7o1uRgTnGASejHQv9n7fA5uUrkaeBlAVb8AkoGccARo4o9/0or2mlxSPakkuBKsycWYIAWT0OcDQ0WkQEQScW56vtGizGZgCoCIDMdJ6PZbaFrln/zZP4lFICJCTmqO1dCNCVK7CV1VG4AbgPeAVTi9WVaKyD0ico6v2M+Aa0VkKfAicKWqtmyWMQb4Zhai9ppcwBkawGroxgQnIZhCqvo2zs3O5ut+2ez1V8Dx4Q3NxKtgm1zAGRrAEroxwbEnRU2X8ze5tNcPHZwauvVyMSY4ltBNlyupKKFXSi88bk+7Za0N3ZjgWUI3Xa6ksv2HivxyU3PZU7OH+sb6CEdlTOyzhG66XDAPFfn5x3vZVb0rkiEZExcsoZsuF2oNHezhImOCYQnddLmSipKQa+jW08WY9llCN12qqr7Keey/nYeK/Pw9Yfx9140xgVlCN13Kn5iDbXLpl94PgO0V2yMWkzHxIqgHi4wJF/9DRYu2L6Kusa7d8plJmSQnJLO93BK6Me2xGrrpUv6HijKSMtop6RAR+qX3Y1uFDbFvTHssoZsu5W9yCTahA/RN62s1dGOCYAnddCl/k4t/NqJg9E3va23oxgTBErrpUjsqdtDD0wO3yx30Pv3S+rGt3JpcjGmPJXTTpUoqS8hMzgxpn77pfdlfu5/KusoIRWVMfLCEbrrUjoodITW3gNOGDtZ10Zj2WEI3XWrr/q30TO4Z0j5NfdHtxqgxbbKEbrpMfWM9W/ZvISc1tOlm+6ZbDd2YYFhCN11my/4teNVLdmp2SPv5a+h2Y9SYtllCN12maE8RADkpodXQs5KzSHInWZOLMe0IKqGLyBkislpE1onIrQHKfF9EvhKRlSLyj/CGaeJB0V5fQg+xyUVE6JPWx5pcjGlHu2O5iIgb+CtwKlAMzBeRN3wTQ/vLDAVuA45X1T0i0v5kkeaQU7SniARXAlkpWSHv2y/d+qIb055gaugTgHWqukFV64CXgHNblLkW+Kuq7gFQ1dLwhmniQdHeIgZkDsAlobf02dOixrQvmNEW84Atzd4XAxNblDkCQET+D3ADd6vquy0PJCLTgekAAwYM6Ei8JoYV7S2ioGdBSPvMWDgDgN1Vu9m4d2MEojImfgRTVZJW1mmL9wnAUGAycDHwhIgc1NlYVWeoaqGqFubm5oYaq4lxRXuKGNRzUIf2zUzOpKq+iur66vAGZUwcCaaGXgz0b/Y+H2jZmFkMzFXVeqBIRFbjJPj5YYnSxLzKukpKKkuCr6F/NvuAt5mDnOECdlTsoCArtFq+MYeKYBL6fGCoiBQAW4GLgEtalPk3Ts38GRHJwWmC2RDOQE1s8zeXFGQVUFFXcXCBFgm8pZ5Jzhe+beXbLKEbE0C7TS6q2gDcALwHrAJeVtWVInKPiJzjK/YesEtEvgI+Bn6uqrsiFbSJPf4ui6G2ofv5B/SyG6PGBBbUFHSq+jbwdot1v2z2WoGbfYsxB/E/VFSQVcDy0uUh7+8f/2XLvi3tlDTm0GVPipouUbS3iJSEFHr36N2h/Xt4epCSkML6PevDHJkx8cMmiTZdomiv08NFpLVOU63wKgXLt4AqRaMHICLk9si1hG5MGyyhmy5RtKco6JuZuZt2ctxrC+i9ybkNs2NQDv+3fQ+5WbC+aBHMmAHTp0cyXGNikjW5mIhTVTbs2cDgnoPbLZtZup9z/vI+aXuq+PjiSXxy8bFk7iznO398jwHlbjY27qJRvV0QtTGxxxK6ibiNezdSXlfOqN6j2i17zKwlNCa4ePWWM1k7YTBrJgzh5f+ZRnVaEid/sZ16GtnSuLsLojYm9lhCNxG3rGQZAKN7j26zXO7mXQxetoVlk4dTnZ7StL42LZlPL57E6E01AKxrLItcsMbEMGtDNxH31OKnEIQvt37ZlNwPosqEtxZTnZbE8snDD9q89ci+DF5bABSxftsKpkY2ZGNiktXQTcQVlxeTm5pLckJywDL91pWQt7aERaeOpD7Z02qZolOPIakB1q/9ErTlcELGGEvoJuK27t9KXkZem2WGzi+iNiWRVccNDVhGkzwUeDNZ794Pr74a7jCNiXmW0E1EVdZVUlpZ2mZCd9U3MGhFMRtH5uNNcLd5vCFp/Vl/WALccQc0NIQ7XGNimiV0E1Ery1aiKPnp+QHL5H35NUnVdRSN6R+wjN+QhMNY30vQ1avh738PZ6jGxDxL6Cai/DdB8zMCJ/SCDxdRl+yh+Mi+7R5vSEIuFa56Sk8aB3ffDTU14QrVmJhnCd1E1LKSZSS5k8hOzW51uzQ0MujTJWwakdducwvAELczMcr6m66E4mJ4/PFwhmtMTLOEbiJqaclS8jLyAs4j2m/BapL3VbJhTHBTEh6e4EvoBT1h6lS47z4oLw9bvMbEMkvoJmJUlWUly1pvbvlsNnw2m8HPz6IuKYHiYf2COuYgdzYuhLW718K990JZGfzpT2GO3JjYZAndRMyGPRvYW7OX/hkBbnaq0v/rbRQf2ZdGT/vNLQBJ4uHIhD4s2bEEJkyA73wHHnoISkvDGLkxsckSuomYz7d8DsCQrCGtbs/YWUHa3iq2HtEnpOOOS+jPou2LnDf33w/V1fCzn3UqVmPigSV0EzFfFH9BRlIGfdNb772St3YHANuGhpjQPQPYWr6V0scegtmz4dRT4fnn4aOPOh2zMbHMErqJmM+3fM7EvImBb4iu3UFFZgr7ctNDOu44j3MDdXGDbzq6M8+EIUPgRz+C2tpOxWxMLAsqoYvIGSKyWkTWicitbZQ7X0RURArDF6KJReW15SwvXc5x/Y9rvYBX6beuxKmdBzuLkc9Yj9Mmv6h+s7MiMREeewzWrIGbbrJxXswhq92ELiJu4K/AmcBRwMUiclQr5dKBnwLzwh2kiT1fbv0Sr3qZlD+p1e29duwlpaKWbUNDn2O0pyuVwe6cbxI6OM0ut9ziJPY//rGjYRsT04KpoU8A1qnqBlWtA14Czm2l3K+BBwB7dM/w+ZbPEYSJ+RNb3d5vbQkAW0NsP59RNZsZVbPpKal8WrfmwI2/+x1873tOYn/55Q7FbUwsCyah5wFbmr0v9q1rIiJHA/1V9a22DiQi00VkgYgsKCuzSQri2T9X/pO+6X15eWXriTVvzQ725aRTmdWjQ8cf4M6izFvBXm/VNytdLnjuOTj2WLjwQvj5z6GurkPHNyYWBZPQW2vgbGqkFBEX8Eeg3X5jqjpDVQtVtTA3Nzf4KE1M8aqXor1FDM5qfQ5RaWik7/oStnagucVvgLsXAEvqtxy4ISUFPvzQuUH60EMwaRK88461q5tDQjAJvRho/mRIPrCt2ft0YCTwiYhsBI4F3nbFtlAAABhcSURBVLAbo4eulaUrqaqvCtj/PHttMYm1DWw/vOMJvb8voR/Qju6XkgKPPuqMmV5aCmedBQMHwjXXOG3sM2Z0+LzGdGfBJPT5wFARKRCRROAi4A3/RlXdp6o5qjpIVQcBc4FzVHVBRCI23d4HGz4AYFjOsFa391myDoAdgw/r8DkyXMlkSSrz6otaLzBjhjMswG23wRVXOE0vTz4Jt9/u1ODr6zt8bmO6q3bnFFXVBhG5AXgPcANPqepKEbkHWKCqb7R9BHOoeX/D+/Tu0ZteKb1a3d5nyTrKs3pQ2TO1U+c5MqE3H9etwateXIFq3QkJcPzxTtPLypXwn/84N0xnz3ba2Y/yddiaPr1TsRjTHQQ1SbSqvg283WLdLwOUndz5sEysqm2o5dNNnzKh34TWC6jSZ8k6tg7u/D2UYQl9mFtfxPKGrYzxtDM5hssFo0bByJGwfDm88go8/DB861tOzxhj4kBQCd2YYM0tnktVfRXDc4e3uj19605Sd+1nxylHdvpcwxKcLo8f1H7dfkL3E4HRo2H4cKeN/aOPYMMGOPts6B/kMYzppuzRfxNW7294H7e4OTK79YQdjvZzvyxXKsPcffigblXoO3s8TpPLf/0X7NgBEyfCwoWdjsmYaLKEbsLq/Q3vMzF/IimelFa391myjtr0VPb0zgzL+aYmDWN23VrqtIMTRo8bB//9306CP+kkeKvNRymM6dYsoZuw2VO9hwXbFjC1YGrAMn2WrGPHmCHgCm38lkCmJg2nSuuYW7eh4wfJy4O5c51mmO98B55+OiyxGdPVLKGbsHlv/Xt41ctpQ05rdXvynnKyNu5gx9jDw3bOyYlH4kL4oO7rzh2ob1/45BOYMgWuugrOO8/p+uhfjIkBltBN2Mz8aiZ90/oyqX/rA3L1XroeIKwJ/Z818xnkzubvVV8wo2p25w6WlgZvvgkXXwyvveYs9oSpiSGW0E1Y/GXeX3hzzZsckX0ETyx64uACn82mz6xPaHS72Ll7y8HbO2G8ZwCbvbvZ0bi/8wdLTHQmyzjpJHj3XafPuiV1EyOs26IJi5VlK6lrrGN83/EBy/QuKqNsQHbQ84cGq9AzkJk1i5hfv7HjB2nZrHLJJc6N0g8/BK/XefAoxHHbjelqVkM3YbFo+yLSEtM4vFfrzSnuugZyt+xmR0H4B2Xr6UrlCHdv5tdvRMNVmxaBCy5wxln/5BO49VarqZtuzxK66bSahhqWlSxjbJ+xuF2t175zt+zC3eiNSEIHOCZxECXe8m+mpQsHEecp0pNPhgcegHvvDd+xjYkAS+im095Z+w61jbWM6zMuYJk+G5zx70silNDHJfTHjYt/VH8Z3gOLwEUXweWXwy9+YV0aTbdmCd102mMLHqNncs+AoysC9CkqY3fvTGp7JEUkhh6uJEYm9OPZ6rlUaZgntXC5nJEaTz0Vrr0W3nsvvMc3JkwsoZtOWb1zNe9veJ+TBp4UsLkFr5feRWWUhGFArrZMTRpGmbecJ6vmhP/gHg/MnOkM7nX++bBsWfjPYUwnWUI3nfLo/EfxuDycOODEgGWyNmwnqaY+Yu3nfkPdh3G8ZwgPVvyn40MBBDJjBrz0ktNHPSHBGcyrpCS85zCmkyyhmw6rqKvgmaXP8P0R3ycjKSNguXAOyNUWEeGOtLPY4t3D89XzInOSrCy4/nrYudMZJqC6OjLnMaYDLKGbDnti0RPsr93P9cdc32a5PkvWUZmRQnmvjk0IHYozkkYwLmEA91W8Q41GaFaiAQOch4/mzYOrr7bujKbbsIRuOmRn1U5+9emvmFIwhWPzjw1cUJV+C9ewY8hhXfJgjohwf8Z3Wd9Yxr0Vb7e/Q0eVlTk19BdfdP41phuwhG465Bcf/YLy2nIePuNhpI1EnbGllB5le9nWiQmhQzGjajYbG3dyrKeA31a8y/L6rZE72emnw7HHwhtvODdMjYkyS+gmZEt2LOFvC//G9cdcz4jDRrRZtt+C1QBdltD9LkgeR09XKtfse5YGbYzMSUTgsstgyBCnn/qXYe4Db0yIgkroInKGiKwWkXUicmsr228Wka9EZJmIfCgiA8MfqukOahpquPLfV5KTmsPdk+9ut3y/BWuozMlkX2565INrJs2VzF8yLuLL+o08UBnBfuMeD/zoR87wu+ecA5s2Re5cxrSj3cG5RMQN/BU4FSgG5ovIG6r6VbNii4FCVa0SkR8BDwAXRiJgEx0zFjqDV/1z5T9ZWrKU64+5nle+eqXtnVTpt3A1W48ZFpWBrS5MLuS15MXcVf4mZySNZJxnQGROlJ4Os2bBpEnw7W/DnDnQs2dkzmVMG4KpoU8A1qnqBlWtA14Czm1eQFU/VtUq39u5QH54wzTdwfKS5XxU9BHfGvQtRvce3W75zE0lpO7az7bCzk8I3REiwmOZl5LrSueyvU9FrtcLwGefORNjfP21Mz9pTU3kzmVMAMEk9Dyg+YhHxb51gVwNvNOZoEz3U1FXwbPLniU/PZ/zhp8X1D7+9vPt44+IZGgBzaiazcyahZyfPI5VDdv5dfmsyJ5w2DC48kpYs8ZpW2+MUNu9MQEEk9Bb+67casdbEbkMKAQeDLB9uogsEJEFZWVlwUdpou6lFS9RUVfBlUdficftCWqffgtXU9E7i/35kX1CtD0jPf041lPAA5XvsbQ+vJNrHGTCBGfY3X/9C665xpK66VLBJPRioH+z9/nAtpaFRGQqcAdwjqrWtnYgVZ2hqoWqWpibG91fchO8f331L+Zvm8+0odPon9G//R0AVOm7cA3bxh/ZLSaGuCB5PFmuHlyz77nI9XrxmzoV7r4bnnnGefDIkrrpIsEk9PnAUBEpEJFE4CLgjeYFRORo4G84ybw0/GGaaKmsq+Qn7/yE/hn9OePwM4LeL/ul10ndXc7WTIHPOjnXZxikuZI4N2k0C+o3cfHeJzo//2h7+vZ1xnv5+9+dm6U2RIDpAu0mdFVtAG4A3gNWAS+r6koRuUdEzvEVexBIA14RkSUi8kaAw5kY89DnD7G9YjsXj7w48GiKrRjw1VZUYMvwfhGMLjSFnoGMTsjj9ZqllHnLI3/CadPgvPNg/nxnkoxtB32xNSasgppTVFXfBt5use6XzV5PDXNcJspmLJzBnuo93DfnPsb3Hc+QXkNC2n/AV1sp659NTXpyhCIMnYhwScoE7i5/kxeqv+T2Hme1+ZRrWJx+OvTuDc89B+PHO6M2nn12ZM9pDln2pKgJ6PXVr+NVb9C9WvySd+/nsM272DSirc5Q0ZHlSuW7yUezqmEHf4t0s4vf2LHwxReQm+s8fHT55c5YMMaEmSV006pt5duYWzyXUwadQk5qTkj79v98JaKweXj3S+gAJyUO5aiEvvy//S+zqH5z15x05EhYsADuussZV33gQKc3zKOPOrV2Y8LAErpp1etfv05SQlJIN0L9Bn62jMrMFHblZ0Ugss5ziXB1ynEc5krn/D1/Y4+3smtOnJjo9H5ZtswZ/2XmTPjVr2DpUhuC14SFJXRzkHnF81hSsoTThpxGWmJaSPu66hvIn/sVm4f36xbdFQNJcyXzcta1bGnczbTdf6Xc24VPdg4fDj/5ibO4XE4t/fTTnadMjemEoG6KmkOHqnL7R7eTnpjOlIIpIe/fZ8k6Eitr2HxU92xuaW5S4hBe6nktF+79X7695y+8nfUT0lwRuonbWrPKyJFOcv/0U3j3XRg9Gm65Be68E1JTIxOHiWtWQzcH+Neqf/FR0UecNfQskhNCT26Hv/sl9SlJbD2ibwSiC68ZVbPZpRVclXIcc+rWc+buv7DP28X9xd1u+Na3YPVqZ77S+++Ho45yxlg3JkSW0E2T8tpybnz3Rsb2GcvJA08OeX93TR2DP1jIhqnjaEiKnS9/hYkDuTb1eObWb2DK7j+wy1vR9UH07u08hDR7NqSlwbnnOv3Y16zp+lhMzLKEbprc9cldbCvfxmPffiykh4j8Cj5eTGJlDWu+PSkC0UXWeM9Arks9iaX1xRxd9hv+VPlB1wYwY4azrFrlTEL94INOch8xAm66yZmU2ph2WEI3gNOr5eF5D3PtuGvbniO0DUe89QX7+2WzfdzQMEfXNUZ58vhx6sls8+7jkcpPqPS2OiRR5LndkJEBv/ylM8Xdww9DQYHzfteu6MRkYoIldMMnGz/hwpkXUtivkN+f/vsOHaPHjt3kffk1a799rNNzI0aN8PTj6tTj2dC4k/P2PE5tJMdQb09GhvMQ0l13wZlnwq9/DQMGwA03OG3uxrQQu795ptMavY08vuBxzn7xbIb0GsKFIy7kH8v/0TQ7USiGvj0XUY3J5paWxnsGcHnKRP5T9xWX7H0y8qMztqdvX3j5ZVi+HC68EP73f52x1ydNcro82hgxxscS+iGorrGOV1a+wsQnJvKjWT+isF8h/7nsPyH3Ofdz19Qx4pVP2Tb+CMqjPPZ5uByfOIQ/ZXyfV2sW84O9z1CnDdENaMYM+PxzpwnmN79x2tjLy5329rw8Zxz2e+6BRYvsIaVDmGiUfviFhYW6YMGCqJz7ULV532YeX/A4Ty5+ktLKUrJTsvnusO9S2K+wU4NUjfzHBxz3h1d48/Gb2e6fbq4bDJkbDu/UrODftUs5MfFw/tXzOnLdXTvZdZtUYft2WLLEedp00yZnXVaWM3PSRRfBMcd06we8TOhEZKGqFra6zRJ6/Fu4bSG/+7/fMfOrmQCM7j2akwaexFG5R+GSzn1JS6iu5aJz72DPkH7MeuzmbzbESUIH+LJuI89XzyPXlc6DGd/jwuTO/QGMmP37YeVKp5b+1VfQ0AD9+sGJJ8Ljj9vE1XHCEvohqMHbwOtfv85fvvwLn276lMykTI7NP5aTB55Mdmp22M4z5pl3mfjIa7z+09MoKYiP5pbWjEsYwLX7nmdJwxYmegq4Pe1MpiWN6vQfxIiprnYGA/vsM6fm3qOHc4P1+uudJ1RNzLKEfoho8DYwZ/McXln5CjNXzaS0spSBmQO5YcINTB8/nZdWvBTW86WW7eX8i+6hdGQB754/JqzH7o686uWL+iLeqlnGbq1imLsP/5N2OpemTMQjoffb7zKbNjk3Tl98EWprnZupV18N3/0u9OoV7ehMiCyhx7FH5z/KqrJVLNi+gGUly6iqr8Lj8jC692gm5E1gdO/REalFSkMj0677Azmrt/Das7ext3ht2M/RXTWql4X1m3mv9iuKvXvoJanck34OV6eeQLIEN4F2l5s+3Xk46dlnnRusq1c7/d1POQWmTIETToCjj3Zq8qZbs4QeZ+oa65izeQ7/XPFPXlj+ApX1laR6UhndezRjeo/hqNyjOjQOSygm/mkmY55/nw9/czXrz5gQV23mwVJVVjRs453aFaxv3ElfVyY/7fEtpqeeSC9XN06Mqk6tffFi52bq9u3fbMvLg8GD4bDDnAk5MjMhPd1pf+/ZE3JyID/f6Q+f3o1uEB9CLKHHMK96WbtrLQu2LWB56XIW71jMnM1zqKqvooenByNyR3BM3jEclXsUCa4uGD9FlVEvfMCkP81k5QWT+b//udhZfwgmdD9VZU1jCe/UrmRVww4ScXNJygR+kDKJkxKHdt92dr+KCjj8cFixwhk7pqjImVGptNS50Vof4OGq/HynPf7oo51ukxMmODdhTURZQo8x5bXlzFo7i1dXvcpba96iusEZAdAlLkbkjuDkgSczZfAUThtyGs8ve77L4nLX1HHSb55j6LtfUjR5LB/edw3eRF8TwyGc0Jvb2riHj2pXs7RhK+VaQ29XBmcmjWBq0nAmeQZT4M7pnj1k2lJf79xkrapykv/u3c6yfTsUFzvt816vUzY/HyZOdJL82LFOwu/fP6afHu5uOp3QReQM4GHADTyhqr9tsT0JeBYYD+wCLlTVjW0d81BM6F71IsgBv9CqSnVDNat3rmbe1nm8u+5dZq2dRYO3gYykDEYdNoohWUMY2HMgfdL6dE0tvIXE/ZUMf20OI/75ET3K9rHgunNYfHgPcMVYYupCddrAkvpiljUUs7J+O1XUAdBDEjk6YQBHJvRmgLsXee6eZLvSyHb1oKekkuVKpbcro3vfZG2prs5J7EVF3yzNBxNLSnJmaBowwEnuvXs7S3a202fe36zTowckJzvlk5LA43GWWPsDGGGdSugi4gbWAKcCxcB84GJV/apZmR8Do1X1OhG5CPiuql7Y1nEjldC96qXB24BXnRqDILjE1bSICKraVK7eW099Yz313noavA1Ni1e9NHob8aq3afFdK25xk+BKIMGVgNvlxqteahpqKK8tZ3vFdrbu38r6PevZsGcDm/dtZsv+Leyv3U9NgzMrToIrAY/Lg8ftoaahhrrGuqb489LzODL7SMb3G8/grMGR/bru9eJqaMRd10BCbT2eymoSK6pJ2V1O2o7dZG4ppffS9eR8vRl3QyPFE4ax+KqznIeHrEYetEb1ss27jw0NZWzx7kEV1jSWUOotD7hPriudge5eFLhz6O/OIs/VkxxXGlmuVFIlkURJwIMbF0KCuHEjeHCTJB6SJYEUSSRFPCSSgDsaTT7V1U6S37EDSkqcJhy3G7ZscZK9v0YfjMREJ8GnpjpJPz3d+SOQmen00snKcpaePZ3xb9LSnLKJiZDgqwCpQmOj822joeGbxet1FhFncbudfRITv1n8f2D8i3+9/w+Ox+Ps53Z3yR+fzib0ScDdqnq67/1tAKp6f7My7/nKfCEiCcAOIFfbOHhHE/prq17j8tcuB0BRVJVGbaTR20hjtMfcaMbj8pCTmkOvlF5kJWeR4kkh0Z0I0BRrozbicXlI9aSSnZJNQVYB2SnZYflKPvmXT1Pw8WLnjSri/1GoIl7F1ej9Zl0ADQkuyvpnU1KQy/pxA9mVZ13cwqleG9mn1VRpHZVaS5XWU+mtZZ9Ws9dbzW6tZJe3gv3eGqrp+CBhbly+5Ztvh/emn8uNPaaG61JC4/U6TTeVlU4zTlWVU8uvqXGSbH29k3wbG79JvPX1zlJb6yz+JqCqKuc4tVEaGbMll+ubxf9Hwr/AN//efLMzVEMHdDahnw+coarX+N5fDkxU1RualVnhK1Pse7/eV2Zni2NNB6b73h4JdHTIuBzgUBkg+lC6Vji0rteuNX5F8noHqmqrT/EF0yDbWnWx5V+BYMqgqjOA0Ifya3kykQWB/kLFm0PpWuHQul671vgVresNpnGtGOjf7H0+0HK8zqYyviaXTGB3OAI0xhgTnGAS+nxgqIgUiEgicBHQcgbbN4Af+F6fD3zUVvu5McaY8Gu3yUVVG0TkBuA9nG6LT6nqShG5B1igqm8ATwLPicg6nJr5RZEMmjA028SQQ+la4dC6XrvW+BWV643ag0XGGGPCyx7fMsaYOGEJ3Rhj4kTMJ3QRuUVEVERyoh1LpIjIgyLytYgsE5HXRCTupp4RkTNEZLWIrBORW6MdTySJSH8R+VhEVonIShH5f9GOKdJExC0ii0XkrWjHEkki0lNEZvp+X1f5HszsMjGd0EWkP86QBJujHUuEvQ+MVNXROMMw3BbleMLKN7zEX4EzgaOAi0XkqOhGFVENwM9UdThwLHB9nF8vwP8DVkU7iC7wMPCuqg4DxtDF1xzTCR34I/DftPIQUzxR1f+oNk07PxfnWYB4MgFYp6obVLUOeAk4N8oxRYyqblfVRb7X5Ti/9HnRjSpyRCQf+DbwRLRjiSQRyQBOwun1h6rWqererowhZhO6iJwDbFXVpdGOpYtdBbwT7SDCLA/Y0ux9MXGc4JoTkUHA0cC86EYSUX/CqXiFMCJXTBoMlAFP+5qXnhCRLp3ppOvHYg2BiHwA9Gll0x3A7cBpXRtR5LR1rar6uq/MHThf11/oyti6QFBDR8QbEUkD/gXcqKr7ox1PJIjINKBUVReKyORoxxNhCcA44CeqOk9EHgZuBX7RlQF0W6ra6nBwIjIKKACW+kaPywcWicgEVd3RhSGGTaBr9RORHwDTgClx+BRuMMNLxBUR8eAk8xdU9dVoxxNBxwPniMhZQDKQISLPq+plUY4rEoqBYlX1f9uaiZPQu0xcPFgkIhuBwpajO8YL3wQjfwBOVtWyaMcTbr7xf9YAU4CtOMNNXKKqK6MaWISIUwv5O7BbVW+MdjxdxVdDv0VVp0U7lkgRkc+Aa1R1tYjcDfRQ1Z931fm7dQ3dNHkESALe930jmauq10U3pPAJNLxElMOKpOOBy4HlIrLEt+52VX07ijGZ8PgJ8IJv3KsNwA+78uRxUUM3xhgTw71cjDHGHMgSujHGxAlL6MYYEycsoRtjTJywhG6MMXHCEroxxsQJS+jGGBMn/j8q+9GGaazo+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xT1fvA8c/pouxCoey9N0IZCiqKLGWJspSpiCi4/SIqQ3CAqKA4QFBA/TFUUAEBkVFEFBRQQDaIjApCyyirBUrv74+TNEmTtmmb5jbJ8369+uq9596kT0J5enPuOc9RhmEghBDC9wWZHYAQQgjPkIQuhBB+QhK6EEL4CUnoQgjhJyShCyGEn5CELoQQfiLThK6UqqCUilFK7VVK7VZKPeXinDZKqQSl1HbL19jcCVcIIUR6Qtw4Jxl4zjCMP5RShYFtSqnVhmHsSXPez4ZhdPZ8iEIIIdyR6RW6YRgnDcP4w7J9EdgLlMvtwIQQQmSNO1foqZRSlYGbgN9cHL5ZKbUDOAE8bxjGbhePHwoMBShYsGDT2rVrZzVeIYQIaNu2bYs3DKOkq2PK3an/SqlCwE/A64ZhfJPmWBEgxTCMS0qpu4H3DMOokdHzRUdHG1u3bnXrZwshhNCUUtsMw4h2dcytUS5KqVBgMTAvbTIHMAzjgmEYlyzbK4BQpVSJHMQshBAii9wZ5aKAT4G9hmFMSeec0pbzUEo1tzzvGU8GKoQQImPu9KG3AvoDfymltlvaXgIqAhiGMQO4H3hMKZUMJAJ9DCnjKIQQXpVpQjcMYyOgMjnnA+ADTwUlhMibrl+/TmxsLElJSWaH4vfCw8MpX748oaGhbj8mS6NchBCBLTY2lsKFC1O5cmUsvawiFxiGwZkzZ4iNjaVKlSpuP06m/gsh3JaUlERkZKQk81ymlCIyMjLLn4QkoQshskSSuXdk5332uYS+6/QuRq8bTfyVeLNDEUKIPMXnEvqBMwd4/efX+ffCv2aHIoQwgVKK5557LnX/7bff5pVXXvFqDIMGDWLRokVe/Znu8LmEHhEeAcD5pPMmRyKEMEO+fPn45ptviI/P3qf05ORkD0eUd/jcKJei+YoCkHA1weRIhBBmCAkJYejQoUydOpXXX3/d4djRo0d56KGHiIuLo2TJksyZM4eKFSsyaNAgihcvzp9//kmTJk0oXLgw//zzDydPnuTAgQNMmTKFzZs3s3LlSsqVK8eyZcsIDQ1lwoQJLFu2jMTERG655RY+/vjjPH0PwfcSerhO6CcvnjQ5EiEC3NNPw/btmZ+XFY0bw7vvZnra8OHDadiwISNHjnRoHzFiBAMGDGDgwIHMnj2bJ598ku+++w6AAwcOsGbNGoKDg3nllVf4+++/iYmJYc+ePdx8880sXryYyZMnc++997J8+XK6d+/OiBEjGDtWL+/Qv39/vv/+e7p06eLZ1+xBPtflElUwCoDt/3n4F0kI4TOKFCnCgAEDmDZtmkP7pk2beOCBBwCdgDdu3Jh6rGfPngQHB6fud+rUidDQUBo0aMCNGzfo2LEjAA0aNODIkSMAxMTE0KJFCxo0aMC6devYvdupiGye4nNX6EXyFaFEgRIkJieaHYoQgc2NK+nc9PTTT9OkSRMGDx6c7jn23SMFCxZ0OJYvXz4AgoKCCA0NTT03KCiI5ORkkpKSePzxx9m6dSsVKlTglVdeyfMzZH3uCp1Llyj9Tzznv/wMlIKaNeH552HkSDh3Tp8jZWSE8HvFixenV69efPrpp6ltt9xyCwsXLgRg3rx5tG7dOtvPb03eJUqU4NKlS3lyVEtavpfQp02jwHVItH62OHgQ3nkH3noLihfXST4oSH9XClq1gv/9z9SQhRC547nnnnMY7TJt2jTmzJlDw4YN+eKLL3jvvfey/dwRERE88sgjNGjQgO7du9OsWTNPhJyr3F7gwtOyvcBFSgptXiyDce0qP72bzZEuXbroPwC1amXv8UIEqL1791KnTh2zwwgYrt7vHC9wkacEBZG/YRMu1ammu1bsv06fhqFDYerUjJ9j2TKoXdt2Fa8UzJoF338Pmzd753UIIYSH+dxNUYAqEVWI+SeGazeuERYcZjtQsiR8/LHefvppW3tCAqxdC7t3g2UIkpOhQx33w8KgcmX4808oUMCj8QshRG7wvSt0oGmZply9cdX9sehFi0KPHjBmjO1q/uBBWLwYXn4ZKlRwfsy1a3DgABQsqK/gb74Z9uyBRBldI4TIm3wyoVvHop+6fCr7T1K9uk7yr70Gx45BSgokJcHPP0NkpPP5mzdDvXr6ar19e5g0Kfs/WwghcoFPJvRShUoBcPryac89qVKQLx+0bg3x8bareMtkAwerV8OLL8Izz8D1656LQQghcsAnE7r1Ct2jCd2V6tVh5UpbN82bbzoef/dd3dfeqJGMfRdCmM4nE3qpgvoK3ev1XEaOtCX3IkVs7Tt32sa+v/SSd2MSIsAEBwfTuHFjGjVqRJMmTfj1118zPP/8+fN89NFHmT5vmzZtcDWUev369SilWLZsWWpb586dWb9+fZZjz20+mdDzh+YnqmAURxOOmhdEQgKcOgW33+7YPnEi1KgBc+fCb7+BH5fqFMIM+fPnZ/v27ezYsYOJEyfy4osvZni+uwk9I+XLl3eq7JgVN27cyNHPd5dPJnSASkUrceT8EXODiIqC9ev1FfuXX9raDx2CwYOhZUvIwordQoisuXDhAsWKFQPg0qVLtG3bliZNmtCgQQOWLFkCwKhRo/j7779p3Lgx/7PMGp88eTINGjSgUaNGjBo1KvX5vv76a5o3b07NmjX5+eefU9sbNWpE0aJFWb16tVMMa9eu5aabbqJBgwY89NBDXL16FYDKlSszYcIEWrduzddff02bNm145plnuO2226hTpw5btmyhR48e1KhRg9GjR3vk/fDJcegAFYpWYE/cHrPDsOnVS3+98AJMnux4TCl47z148klzYhMiFzz9w9Mer3rauHRj3u2YcdGvxMREGjduTFJSEidPnmTdunUAhIeH8+2331KkSBHi4+Np2bIlXbt2ZdKkSezatYvtllK/K1eu5LvvvuO3336jQIECnD17NvW5k5OT+f3331mxYgXjx49nzZo1qcdGjx7N6NGjadeuXWpbUlISgwYNYu3atdSsWZMBAwYwffp0nrbMgwkPD0+t+DhjxgzCwsLYsGED7733Ht26dWPbtm0UL16catWq8cwzzxDpaoRdFvjsFXpUgai8ua7om2/q8eppPfWUTux5vPymEHmdtctl3759/PDDDwwYMADDMDAMg5deeomGDRty11138e+//3LqlPPQ5jVr1jB48GAKWCYMFi9ePPVYjx49AGjatGlqCV2rW2+9FcDhyn3//v1UqVKFmjVrAjBw4EA2bNiQerx3794Oz9G1a1dAl+itV68eZcqUIV++fFStWpXjx49n9y1J5bNX6CUKlOBs4llSjBSCVB77u1Snju3maVCa2OrXlxExwi9kdiXtDTfffDPx8fHExcWxYsUK4uLi2LZtG6GhoVSuXNlluVvDMNJddchaUjc4ONjlUnUvv/wyr7/+OiEhIanPlZGMSvZat637nlgaL49lQveVKFCCFCOFc4nnzA4lfUrB1atw4oRz+99/w65d5sQlhJ/Yt28fN27cIDIykoSEBKKioggNDSUmJoajR/WgicKFC3Px4sXUx7Rv357Zs2dz5coVAIcul8y0b9+ec+fOsWPHDgBq167NkSNHOHToEABffPEFt6cdKOFFPnuFXr5IeQCOJRwjskDO+p1yVVgYlCkDX32l+9itqlfX348cgUqVTAlNCF9k7UMHfYX82WefERwczIMPPkiXLl2Ijo6mcePG1K5dG4DIyEhatWpF/fr16dSpE2+99Rbbt28nOjqasLAw7r77bt544w23f/7LL79Mt27dAN1HPmfOHHr27ElycjLNmjVj2LBhnn/RbvK98rkWO0/tpNGMRiy8byG96/fO/AF5wf79uspjWl9+qcsQhPjs31cRIKR8rnf5f/lci+rF9RXu4XOHTY4kC2rV0v3nCxY4tvfuDc2a6YJgQgiRTT6b0AuE6jvUL63zwZmZffroYmD2tm+HcePMiUcI4Rd8NqH7PKXgr78c2yZNkhEwIs8zq5s20GTnfZaEbqb69SEuzrEtKMj01dSFSE94eDhnzpyRpJ7LDMPgzJkzhIeHZ+lxfnEXLjklmZAgH30pJUrA1q0QbXeP45ln4JtvwG6CghB5Qfny5YmNjSUu7YWI8Ljw8HDKly+fpcf4aBbUpt8znceWP8bpy6cpW7is2eFkX9Omej3UqChb288/w9dfQ8+e5sUlRBqhoaFUqVLF7DBEOjLtclFKVVBKxSil9iqldiulnnJxjlJKTVNKHVJK7VRKNcmdcB2VLlQaMKGMbm4oWdL5itx+3LoQQmTCnT70ZOA5wzDqAC2B4UqpumnO6QTUsHwNBaZ7NMp0WOui9/u2nzd+XO679Vb44w/HWutKyapIQgi3ZJrQDcM4aRjGH5bti8BeoFya07oBnxvaZiBCKVXG49GmYb1C3xe/L7d/lPfcdJNeMMOeXTEgIYRIT5ZGuSilKgM3Ab+lOVQOsC8VFotz0kcpNVQptVUptdUTN1WqFKtCkAqibZW2OX6uPCVtKYC2bfWCGpcvg5cK5QshfI/bCV0pVQhYDDxtGMaFtIddPMRpXJNhGDMNw4g2DCO6ZMmSWYs0HZ1rdibuih/ecT92zHE/IgIKFdJleIUQwgW3ErpSKhSdzOcZhvGNi1NigQp2++WBEy7O87hCYYXYeWonJy565cd5T4UKrts//NC7cQghfIY7o1wU8Cmw1zCMKemcthQYYBnt0hJIMAzDK0NP1h9ZD8CoNaMyPtEXrVgBU1y85f/+6/1YhBB5njtX6K2A/sCdSqntlq+7lVLDlFLWOpErgMPAIWAW8HjuhOusSRk9QvJM4hlv/Ujv6dRJTzJKa/x478cihMjzMp1YZBjGRlz3kdufYwDDPRVUVnzW/TMiJ0f691Tk9u3hxx9t+4sWwcyZ5sUjhMiTfL6WS/H8ej3A6yl+PFZ72TKwXzz23DldV/37782LSQiR5/h8Qge4t/a9/jFbND1hYXDwoGPb/v3QpQuMHm1OTEKIPMcvEnrZwmWJvRDr390uxYrBY485t7/+uvdjEULkSX6R0KtEVCHhagLP/fic2aHkro8+gjMubv768x8yIYTb/CKhW0e6TN081eRIvKB4ceel6kb54ZBNIUSW+UVCv6PKHWaH4F2hoY77kydDzZrmxCKEyDP8IqHb8+t+dHuLF0O1arb9gwfh7Fnz4hFCmM5vEnrH6h0BSLiaYHIkXtKjh74yt9e7tzmxCCHyBL9J6NYSAM+scjGz0l/dey88Z3cjeM0auUEqRADzm4ReLLwYAHO3zzU3EG9SCkaOdGxbtMicWIQQpvObhL6s7zKzQzBHVBTceadtv1cvqZkuRIDym4TetGxTyhUuR7uq7cwOxfvS1nV54gm9GIYQIqD4TUIHPR599eHVJKckmx2Kd1Wr5th3Pn06tG5tXjxCCFP4VUJfdkB3uyz4a4HJkeQB27fD+fNmRyGE8CK/Suh317gbgHVH1pkciUmuXnXcd1UmQAjht/wqob/d7m0gwEa62AsLc9zfudOcOIQQpvCrhG6tjQ5wIyVAR3rMnm3b7tEDDh0yLxYhhFf5VUIvVahU6vb5pADtPx482HG/Rg1z4hBCeJ1fJXSAz7t/DvjpGqPuSlsC4MoVc+IQQniV3yX0wvkKA7A5djNxl+NMjsYkCxc67t9/vzlxCCG8yu8S+rUbulb4wO8GEvV2lMnR5BErVzqXCBBC+B2/S+gXr140O4S8Ie2Il7feMicOIYTX+F1Cb1aumdkh5A2XLkFiomPbpk3mxCKE8Aq/S+gNSzWkZqSs3kNoKISHO7bdcos5sQghvMLvEjrA0j5LzQ4h79q71+wIhBC5xC8TesmCJVO3A2ZJuvRs2+a4X7cu7N9vTixCiFzllwm9eP7itKncBoCHlj5kbjBma9JE96fb++8/c2IRQuQqv0zoAPfVuQ8I4Lou9goWhAcesO3/+KN5sQghco3fJvSTF0+aHULeYl/j5Y03nCcfCSF8nt8m9CdbPGl2CHlLvnywfr1tv29f00IRQuQOv03opQqV4rU7XgPgavLVTM4OELff7rj/00/mxCGEyBV+m9DBVnFx5raZmZwZQOz70rdsMS8OIYTH+XVCv3RNj+4YHTPa5EjykPz5bdv/+x/cCNC68UL4Ib9O6JPumgTAhasX2HV6l8nR5BFpa7qkHacuhPBZmSZ0pdRspdRppZTLjKiUaqOUSlBKbbd8jfV8mNlTNLxo6nbfxXITEIBixRz3hw6Vq3Qh/IQ7V+hzgY6ZnPOzYRiNLV8Tch6W58zuqofr7Tq9i60ntpocTR5RqZJte8cOndSFED4v04RuGMYG4KwXYskVg2+yLck2+ZfJJkaShzz4oOP+7NmQkmJOLEIIj/FUH/rNSqkdSqmVSql66Z2klBqqlNqqlNoaF+f91YQSkxMzPykQvPoqnE+z5ur06ebEIoTwGE8k9D+ASoZhNALeB75L70TDMGYahhFtGEZ0yZIl0zst11hHvQS8oCAoWhQGDLC1jRgBly+bF5MQIsdynNANw7hgGMYly/YKIFQpVSLHkXlQu6rtAFh/ZL25geQ1n33muL98uTlxCCE8IscJXSlVWimlLNvNLc95JqfP60mlCpVK3d4Tt8fESPK43r3hjz/g6FGzIxFCZIM7wxYXAJuAWkqpWKXUw0qpYUqpYZZT7gd2KaV2ANOAPkYeK0LeoVqH1O16H9Uj9kKsidHkcU2bQuXKZkchhMgGZVbujY6ONrZu9d4wQjVepW7/PPhnWlds7bWfnacp5bo9b/1NFkJYKKW2GYYR7eqYX88UtVcgtEDqdv6Q/BmcGWDWrZPKi0L4iYBJ6OdeOJe6ffKS1EpPdccdMH++c/uVK96PRQiRIwGT0MOCw/i066cAdFnQxeRo8qCuXR3327UzJw4hRLYFTEIHuHzNNs76zY1vmhhJHrRkieP+r7+aE4cQItsCKqHfMGxFqEatHWViJD7Cftk6IUSeF1gJPUWqCmaoaVPH/YcfNicOIUS2BFZCNyShZ2jrVufhitKXLoTPCKiEXjRfUYd9WWvUDWvWSCVGIXxEQCX0IU2G8OHdH1KigC41czRBpri7Zdw4WQRDCB8QUAk9OCiYx5s9zgP19ULJtT6olbqQtLBz/jw0bmzbf+01qFjRvHiEEG4JqIRuVSOyRup2sTeLZXBmgCpaFN57z7HtxAmIjzcnHiGEWwIyod9X5z6zQ8j7IiKc20yoYS+EcF9AJnT7croiHVWquG6XvnQh8qyATOhBKojV/Ven7iclJ5kYTR5VuLCu5xIS4tieKMv4CZFXBWRCB7ir6l0UDisMQJu5bcwNJq/Knx9q1XJsu/VWc2IRQmQqYBM6wGPRjwHw27+/mRxJHrZypeP+9u3mxCGEyFRAJ/SL1y6mbv936T8TI8nDKlSA69fNjkII4QZJ6BZl3inDGz+/YWI0eVhICNx9t21fKZgzR4YxCpHHBHRCLxRayGF/3PpxJkXiAz76yHH/oYfg3nvNiUUI4VJAJ/RJd03i5VtfTt1PTkk2MZo8rlIlGDDAsW3jRnNiEUK4FNAJvWh4UZ6/5Xmzw/Ad77/v3Pbxx96PQwjhUkAndICIcBczIoVrRYo4J/Bhw3TZ3XfekUlHQpgs4BM6wPBmw1O3i04qypMrnyTxukygcWnoUOjVy7GtWTN4/nlYtMicmIQQgCR0APIF50vdvnD1Au///j4TfppgYkR5XGio435wsP5+5Yr3YxFCpJKEjuup/1JWNwPh4Y771q4WpbwfixAilSR0oFXFVmaH4FveSGe8viR0IUwlCR14oMEDzOw80+wwfEdUFHTp4ty+fr3XQxFC2Cgj7aLAXhIdHW1s3brVlJ+dHjXe8QrTGGfOe+MTDh2CGjWc20uUgLg478cjRIBQSm0zDCPa1TG5Qrez+eHNDvvHE46bFIkPqF7ddY2X+HiIifF+PEIISej2WpRv4bB/+NxhkyLxESEhrqf/33mnrvVyyy3ej0mIACYJPY2i+Yqmbr+64VUTI/ER33wDGzY4tz/0EGza5P14hAhgktDTUHYjNdb+s5Z75t/Db7G/yUSjjGS06EWy1McRwlskoWdixcEVtPy0JQ8vfdjsUPK2H3903V66tHfjECKASUJ305YTW8wOIW9r1851+5kz3o1DiACWaUJXSs1WSp1WSu1K57hSSk1TSh1SSu1USjXxfJjeExYcBsB7Hd9zaD909hCTNk4yIyTf8dNPZkcgREBz5wp9LtAxg+OdgBqWr6HA9JyHZZ61A9byQqsXGNJkiNOxt39924SIfMhtt+nhjGmtXg07d8KCBd6PSYgAkmlCNwxjA3A2g1O6AZ8b2mYgQilVxlMBelv9qPpMumsS4SHhTsfOJEr3QaaiXcx3aN8eGjWCBx4AkyayCREIPNGHXg6wn4ETa2lzopQaqpTaqpTaGpfHZxMGqSD6N+zv1H7y4kkTovEjU6aYHYEQfssTCd1VRSaXl2GGYcw0DCPaMIzokiVLeuBH565pnaY5tV26dsmESHzI1KkZH3/+edi3T49dT5ShoEJ4UogHniMWqGC3Xx444YHnNZ2r1YyazmxKteLVKFGgBKv7rzYhqjyudGldEuDCBYiMdH1OnTr6e9eusGSJ92ITws954gp9KTDAMtqlJZBgGIbf9ktcvHaR7f9tZ83hNWaHkneFhEDx4pmft3QprJH3UQhPcWfY4gJgE1BLKRWrlHpYKTVMKTXMcsoK4DBwCJgFPJ5r0QrfNMR5xFAqVwtPCyGyJdMuF8Mw+mZy3ACGZ3SOLzv3wjkuX7tMp3md+Ov0X2aH43tKloSZM+GTT1wfj4+HY8d090zBgt6NTQg/IzNFMxERHkG5IuWY12Oe2aH4nkOHYO9evZLRqlWuz/n1V6hUCapW9W5sQvghSehuKlWolFPbun/WmRCJD6lWzXZjtH37jG+Anj4ti0wLkUOS0N0UVTCKU8+fcmhr+3lbk6LxUV276pK6vXu7Pl6wIDzxhHdjEsKPSELPgqiCUU5t/7fz/0hKTjIhGh/VsiUsXKhLAbjywQfw5ZewZQv89593YxPCx0lCz6KpHRwnzvT/tj9PrXzKpGh8WPny6R976y1o3hzKlIE9e7wXkxA+ThJ6Fj3d8mmOPn3UoW1XnMtClCIjwcHpH9u2zbad0eIZQggHktCzoWLRig77vx7/1aRIfFhGCd3euXNw8aK+aQq6K+ZXeb+FcEUSejYNbTLUYX/MujEmReKjrAm9QIGMzzMMKFIESpWChATdFdOqVe7HJ4QPkoSeTR93+dhh/7WfXzMpEh+VLx+MHQubN0NKCjz6aOaPcbUq0ttvSz0YISyUYVJ96ujoaGPr1q2m/GxP6bOoD1/u/tKpPWVsisNi08INc+fC4MHun2/9vbW+z1JnXQQIpdQ2wzBcLDwgV+g58nHnj1221/2oLskpstp9lli7XurWde/8Bx/MvViE8FGS0HOgaHhRl+374vex9vBaL0fj4xo10t+ffhoGDcr8/PnzoVs3235KCixfrvvbr13LlRCFyOskoefQn4/+6bK947yMlmEVTmrV0iNahgyBDz/UI1mefz7jxyxdatsODobOnfWImD59IDlZ15IRIoBIQs+hRqUapXts24ltssJRVkRE6D7xAgXg5pv1BKPMVkBy5dtvITQUatSA2FjHY+nNUBXCD0hCzyGlFJ1rdnZ5LHpWNE0+buLliPxM1645e3xcnC7fu2wZLF6su3a+dL6RLYQ/kITuAcv6Lkv32MGzB70YiR8qVixnj9+9Ww+J7NoV/vhDt/0lde2Ff5KE7iGjbx3No01dj6Uev368l6PxI8WKQVIOip/172/bfuMN/T0lRd84vXEjZ7EJkcdIQveQV+98lQ/v/tDlsVd+eoX5f82XoYzZlS+fHmc+fz7cdlvOn2/iRP2cbdrA2rW2sgJC+DhJ6B4UHJR+fZIHv3mQ51Y958Vo/FDfvvDTT/Cn65FFWbZxI9x1ly4rsHu3HmVz5oxnnlsIE0hC97Cv7v+K9QPXuzw27fdpAKQYKTL6JScaN7YNafRUed2PP4bixaFECc88nxAmkITuYT3r9eT2yrdneM64mHEUnliYhKQEL0XlhyZNgiNHoE4dzzxfYqJtOyVFf/3+u+7qqVwZunf3zM8RIhdJQs9lMQNjHPZnbZvF5zs/B+Bc0jkzQvIPwcF6cWnQXSdvvQX79mX/+T75xLZdo4YeFdOihV5B6ehR5wJgR49mXj8mPl5qzAivkoSey+qVrOewP/T7oRxLOAbAN3u/oeH0hszbOc+M0PxHq1a6C6ZWLZ1Aq1bV7fZJOisOH9ZlBACefNLWnpSku2Y2b9ZX7W+/nf5z7N0LJUvC9OmZ33RNTIRL0gUnck4Sei6xrj9asmDJdM957sfn+Ov0X/T7tp+3wgosrVt79vnGj4dhw+DZZ/X+yJG2dU8NA156yTYT9cAB/X30aH3TddGi9J+3QgUoXNizsYqAJAk9lxx84iCnnj8FwJjbMl/8YsjSIXKj1NNCQuDNNz33fJMm6e+bNtnaZs+Gnj316JuJE/VM1AsXbGPcz1m61X76Kf3nlZE1wkOkHroXpBgpJCQl0PyT5hw6m37BqMeiH+Ojez7yYmR+qmpV+OcfXZyrWjV9c/OXX2xX1mYYPlz3x7siNd1FFkg9dJMFqSCK5S9G7RK1Mzwv5kgMi/csdrpSv3L9CicvnszNEP1L2sUvmjeHZ57R7RcuwGuveb+eS0qKd3+eyF1r1+bJmcaS0L2oXVUXS6jZ2Re/j/u/vp8B3w5waL/r87soO6VsboYWOAoXhpdf1mPZvSnZMkvYMHQ1yLZt4fHHnc9r0QKekwloedoPP+gJaW+9ZXYkTiShe9ETzZ8g7n9xmZ63ZP8SHl32KB/8rj+ib4rdlMkjhIO0V+iu1Kxp698GfeOyVi29/b//2YZEesqsWTquhQuhRw9Yt06PgNm/34TqDhAAACAASURBVDHu33+HKVPcf9433tBrswrv+fdf/f1g3iu8Jwndi5RSlChQgrDgsAzPSzFSmPnHTJ5Y+YRDu1n3O3xWZuu6RkToei6gE+7q1XqY4sSJ8P77no8nKAgeeMCxrbZdN9zevbbtkyf1OPbMvPwyvPoqXLnifGzPHv0eeGo2rcjzJKGb4NTzpzj9/Gm2PLKF62OuZ3iuGm9LSlLcy019++rv7pTe7dVLf69eXQ8ffO89PWmpXTvXs0NrZ3wfJEfq2c1ZKFsWmjXTif34cf2a5s2DDRts57zyim17wgTn51u4UH//+mtbW0ICnDgBn35qa4uP1zNuM5qYdeqUvrEs8jRJ6CaICI+gZMGSRJeNJiQohMGN3Vvt/mjCUa7dkPUyM/X66zpxFXW95quDYcP0xJ+0JQTCw3Vft5U12T3h+KkpVx05ohN7xYo6OffrB7fblZUYb1eW2Tr0ccYM/eli71490gdsk6TWrNGfSsqV00v9LV2q25Yu1a9v8uT0Y2nWzPPj+n1dHvzEHGJ2AALeavcWc7bPyfS8Gu/XoFLRSuwbsY/wkHAvROajgoL0YtHuUErP6EzP+vW2/vXTp3XxLuvNzMy6dHLL+vW2riKrTz7RsVnHytvbsgViYpzHwlsX2Z45U39XyrbAdliabsHjx3Matf8w69/dDXKFngdEFoh0+9yjCUcZ9N0gwl4NY+3htbkYlQD0FbG1m6VkScf/zLt2OZ47ejSMG5f7Md1xh+uk4iqZW/3wgx6u6Yp1SGVQkL56j4jQ5Q+6dNFX8PasV6UXLrh+rn378uRwvmwzjDx5JZ4etxK6UqqjUmq/UuqQUmqUi+ODlFJxSqntlq8hng/Vv0UVjKJmZE0OPXGIVf1W0bBUw3TP/XL3l1xPuc5dX9zFnyf/pMeXPXh2lYmTZgJV5cqO++XKOfZr5yUZdacMG6a/L1mi+9MTE6FzZ/j+e30vwd7u3boefdGitj56q/37dddVTv+oKZV3uneCghzfg7fegocfNi+eTGSa0JVSwcCHQCegLtBXKVXXxalfGobR2PKVzapIgevU86fYP2I/1YpXo3219kxo4+ImlwtNZjbh233fMnXz1FyOUDhJ2y1xyy36u68OI4yzG1JrP+Imxq5iaIMGtpusjz+uh15ar/Ctw/nSu3lqHXVjvR9hGHr27DkXVUezewN26VK4+WbPTuRaa/dJeORIzz1vLnDnCr05cMgwjMOGYVwDFgLdcjcs0a12N2Z2npmlxxw4c4A9cXuo9UEtHvzmwdT2xOuJGTxKZFuI3S2okSOhoeVT1bhxuiLjvn2639o+Of34o3dj9IQ773TcP2mZtXzunE7qzzyj961dE//8A9ev6+RtX5FywQL9/fXXYdkyfVP3iSf0J4Rr1/T547Ow/m5KinPffp8++r3PyTq07sqDXTHuJPRygP27FmtpS+s+pdROpdQipVQFV0+klBqqlNqqlNoaF5f5BJtA90jTRxh3u/sfX2t9UIt6H9XjwJkDzP9rPgD/t/P/KPBGAfbH73c4d/qW6cReiPVovAFHKfjoI12PfeJEW3tQkJ7xWauWXgP1llt0Iv/8c+cuDNAf4du29V7cOfX3347706bpJH3XXXr/6FFdRwf0JC2l9Je1D////k/Xm7cm73PnbFfp9l1Wjz2mRyuldfkybN2qn69iRdtoHtDvPejRQenN5ExJyXoydhVHZi5edPzU4wWZFudSSvUEOhiGMcSy3x9obhjGE3bnRAKXDMO4qpQaBvQyDONO18+oBVJxrpy4kXKDzbGbaT0n632KDUs1ZOcpXc71655fc3/d+wGIvxJPybdKUqdEHfYMl0knXjdkiB4HPnUqHDtmmxlqvdH58ss6QVrt25e749/N1qkTREbqRJ9W9er6ar56dX2136SJrkn/zTe2c376Sf/xDA2FKlUcb9ha81u/fnrk07ff6pLHt9ziXreO9d9k5EhdudP+ZnS7dq4/ccXF6U8o0dH604xhwPnzukuqXj3n87Moo+Jc7gxbjAXsr7jLAyfsTzAMw77+5yzAgzVLA1twUDCtKrbK1mOtyRz0HwbDMDh9+TQhQfqffV/8Pk5ePMkfJ/+gY/WOGS5yLTzok09cL77Rrh0ULAhDh+o+bGvSqlVLjxzZuBHuv9/xqq94cTh71jtx55aVK9M/duiQvtE6aBDMnavbypd3POfGDShdOuOfMS/NIjK//qq7ZcLD9R+A7dv1p6mUFP2p4tFH9cpVVq66cFav1n+Qw8Icf35UlPO51klu06frUUrWMhMe5k6XyxaghlKqilIqDOgDLLU/QSlVxm63K7AX4VHWJGy1bsC6LD1+5aGVtP28LaXfKc2BM3rxBQODslPK0nlBZ26bexsAW/7dwuFzhz0TtMiaH3/UV5AVK8LixY7HgoJ0wlm6FO6+W7dFRemuhUBgTeYAsWm6CjOqNf/zz443Ne3lz68//RQtqoennj2rE/uUKTrhvvSS7dzkZD2eP61KlaBMGf3vdvWqrcvHnn28jz2Wq5+2Mk3ohmEkAyOAVehE/ZVhGLuVUhOUUl0tpz2plNqtlNoBPAkMyq2AA9X1MdeJ/188L7R6gXtq3EPTsk1Tj10dfTXTx3+24zNijujRCrfMvsXp+K/Hf+XfC//S/JPmVJtWjX3x+9h0fBPXblyTGjJmGTBAD4W017Klbdp+yZJQ1zLgbMsWyJfP+TnSuwk71Y9GRWV0I/W222x9+6506GDbjox0PNd+XH9ysi7DnJ5583TtfVf/V3bvTv9xHiYLXPio5JRkQl8N5ZEmjzCzy0ziLscR9baLj3pZMLvrbB5a+pBT+6S2k3ih9QskpySTYqRkWlxMeMHUqXDffXp26KJF0L+/rrdSxvJhecgQXXAMdP0X+5IBoK9GixfX29auB9B/IOyLeV26BIUK5e5rCUQ5yLsZ9aFLQvdh55POUyiskEN3zJJ9S1h+cDmz/pjl0Z91dfRVKkytwI2UG8SPdKMKoDBHQoLu2+3c2ZakAW66SXcn2J/32GMwf75OLtabfSkpug//k0/04xMTbcd++UUvyC1yLpcSukz992ER4RFOfevdandjZpesjV/PTIdqHWjxSQtOXz7NmcQzvLbhNdR4xWsbXmPI0iFcue6idKswR9Gi+sZpeJpaP19/rceL58+v94OCdL902huqStmGDlpv5D3wADz0kG3ilL1Tp1x3RfTu7Z0yCL5q2rRceVpJ6H5qca/FxAyM4adBPzG/x3zmdMu8+Fd6WpRrwfb/bFd3Y2LGpH7/9M9PGbJ0CGq8chrrLvKQ6tX1zb6pU3Uyz59fD/OzJu0PPoDfftPb1loswZZRT/PmOZbbtRcVpWePptWokf5DEBqqbzxmZSTOu++6f66vmj49V55WulwChHXsuTcY4/Tv1IEzB1AoakTWcDpn4a6F1I+qT/2o+l6JSWTBjRu6EuOLLzp3sYSH69EcoG/Knj6tk/X48brEbv/++ti1azqZ29u9G+pb/r3Ll3cerWJ19arrG7z+ZNQox8loWSBdLoLQoFCnti2PuBiG5QEVp1bkyvUr1PqgFjU/qMnyA8tTx8Fb9V3clwbTXVzZCfMFB+vCXK76y61dNo0awVdf6e3ixfXCIP36wTvv6CqUaZM56Ek1p0/rbprjx+GPP5zP2bVLj+s+dixrMRcsmPHxK1d0SYC8Ijh35nxIQg8QocGO/8Fm3DOD6LIu/8iTLzhnV0fHLxynzoe2BSM6L+hMyKshBE3Qv27ufCq8fO0yKYZzgaWbP72ZhbsWungErDi4gq4Luro8ZmUdg2914uIJriZnPuxTWFStqr//8otzTXaAZ5/NeDZkyZK2iTc33aT/AEyeDAMH6vHi1sdWsJvLGBenbyKmnRxk9fff+uem54sv9B+iFi30sM/s6tEj+49Ny9V4dU88ba48q8hz0ibpqIKOQxyfbvE0AD3r9uTcC+eYfk/O+viOJbi+wlLjFXvibMPiJv48kceXP873B75Pbbt07RKFJhZibIxj1cIUI4XNsZvpu7ivy+e+Z/49LDuwLN2l+pbsW0KtD2qxeI9t0k65KeXo+XVPt19XwFuxQk9uyuyK2F3PPqtnZs6d61wEbONGeOopPTQTnNdjtapaVZdLsLKWA7bq18+2/eijjsdKlXJ+vl69dIngbdv0J4Zjx3R9msWL3VsFyx25tEiGJPQAERwUjDHO4MCIA9xR+Q7aVXMsEvV2+7cZc9sYPrrnI/KH5ufWirem+1yFwnI2Lrn+dFu/+UvrXmL61ul0WdCFhKQE7v/qfjYe2wjAzG0zCX01lNc2vMbla5fZHOv8kfno+aP8evxXh7a4y3Euk7q1FIL1Bq/1E8CyA8ty9HoCSqlSeuELb2jVyvkG6fHjOtGmlS+frbLjBx/oYZvgWAYY9GStOXN0nRVwXGbQatYsaNxY142pV09/WqhYUR978UXn8+1vGNsXWctoTdsQd6quZJ0k9ABTI7IG6wauc0rKwUHBTLhjAiUK6KuhiPAIh+NVIqqkbnev7WLxZA+IeDOCxXsX88BifSUWd0Un5jExYyg0sRCtZtv6dB/7/jGOnD9C5fcq02p2K4fFtMtOKUufRX2cnj9I6V93ayK/kWJbWUdmw/qI8uV1oj12TFdZtB8989xzumsmOFiX6l23znmafVCQrgtj7fKwXilbl+MrUiTj5Qv79tWfGPbbjeh6yG4ynv0KTw/aSlg7yaXfN0nowqUi+Wy/1MY4g8NPHWZCmwl83/d7FLm7puK5JBcLHqQxY9sMqrxXJd3ji/cu5smVTzq0jY4ZDcDe+L2o8cph8pX9Hwt3XL52mdV/r3br3L9O/ZXrfzDGxYxz+KPm9ypU0CtGpXcVXKiQLoKVHvtl91JSbDd47eu3uFKxou7Tr1lT71vLIW/aZFuM29qtM26c3t66FWbP1vcKJlgWrkl23S2YU5LQA9yzLZ+lXGHn8vaF8xV2ahtz+xjuqXlP6pUuwB2VM/hPY7L3f3+fSu9W4tM/PuXajWup7d/u0x+zh68Yntq2KXYTz//4PGq84uLVixw9f5QZW2dgGAaHzh7ixEWHAqM8suwR2v9fe6dCZscSjjksKLLq0Coazmjo1iLgOTFhg3srXAkL+4SulB5ZYxjwwgvuP0dSkq1SZMuWtqJp77+vS/SWKAEzZkDTpjB4sL5XMGaM/gSRS+uuSkIPcO90eIfYZ12PBz753ElOPnfSqb1j9Y4AbBu6jbUD1nLi2RN829uxL7Jrra7EDIwhZawHlwLLhmMJxxiybAj5Xst85M47m94BoMikIlR+rzKPLX+Mjcc2UuP9GpSbUo6kZF1CNTklmQW79Oo7f5/Viz2cvnyasTFjqfRuJe6Zfw9xl+PovrA7yw/qq7Y/T/6Z7s+9cPUCaw7rj+rnk86ne1PXHdl57JkrZ3jmh2cc/uj5PfuEnl358rkefhga6vpmq1VISK4l9NzpmRd+oXQh1zWm+9TvQ4dqHSiWX3/cLVO4DN1rd6dbrW4s2b8EgLG3jXWoCGk18paRRJeNpkLRCtz86c0ADG48ONevYLPLWlYY9E3aJ5o/waI9i1Lb2v9fe6pEVOGf87ZVc2KOxDgVSjt49iCgk/cjyx7h/U7vp440KjpJj5xY1ncZXRZ0oXe93iy83/XQzLQ6z+/sUHohKTkpyzetX1z7IrP+mMVNZW5iQKMBWXqsz7J2geXS8MEM/fabHr6ZC+QKXWSLNZnb+67Pd9wYe4M9j+9xSOYfdPqAuiXr0qteL95s9yY96/WkZXnbeOD+Dfunbt9X577cDTwHnvrhKYImBDkNm7RP5ulZ9fcqbqTcoNYHtfhq91eUersUvxz7hcm/TE49p8sCPXrky91fOj1+bMxYpm5yLnm7/ODy1LLIQOqnCFcuXL3Ao8se5eLViw7t1v79jB6bkJTAvvh96R73OdYr9FwaPpihRo1yrY69XKELjwpSQdQpWcehbXjz4QxvPtzp3EU9F/HEyidoUb4FB0YcYG/8XrrW0hOD/PEGX8irjv/dMlpWcO72uVQoUoGk5CRKFyrNqxte1c8RFMKI5iNQSvH5js+dHmeflA3DYMGuBQz4dgDbhm5jxtYZzPxjJhHhEVy6domPtn7Ekj5LUsshW2vfbzu5jfpR9QkPsRX4uuOzO/jzvz9Tyzp4wrUb1xi8ZDDj24ynevHqHnnOsTFjeXXDq1wfc92pcJ0DM6/Qc5HUchF50qlLp7hy/QpVp1XN8Lw373qTF9a4vpG1qt8qOvxfB5fHfFlk/kg6Vu/IvL+cZ07uHLaTBqUa8MLqF5j8q+3qv22Vtqz9J52VeyymtJ/C3B1z2XlqJ8OaDqN68eqUK1KOPvX7pP6BTR6TnLpU4cxtMxkbM5bFvRbTsnxLJm6cyJiYMRx84iDVi1dn2m/TGL1uNBdevODy58X8E8Odn9/J7ZVuZ/2g9cReiCWqYBSbjm8iskBktur8FHqjEJevXyZhVILDSC0n9erpuu+7dmV5nc/YC7FUmFqB34f8TrNyzbIcY07ldE1RIbyuVCF9U+nXh35ly4ktbDu5jWLhxTh9+TSzu81m3T/rGPTdIIY3G07nmp15ZtUz/Pj3j5x74RzNZjXj0NlDVImoku6iHb7sTOIZl8kcoOGMhhQMLcjl65cd2jNL5gDjfxpPwlW9uv2f//3JjG0zAMer/pBXQzDGGSQlJ/Ho93p4Xus5rXm25bNM2awXux6ydAjL+i7jqR+eAnS5hZqRNZ1+3p2f22aGJiUnUWFqBfo37M8XO78AIPaZWMoVcR6BlZGw4DAuX7+c4Q3eNYfXUKJ1dRrv2QMREemeB7D8wHIqFK1Aw1INU9tWHVoFwPSt001J6Bnxr88bwu/cXOFmnmzxJJ91/4x3O77L/PvmEx4Szt017ub0/05TMKwgdUvWZVW/VRjjDCLCI1JHegQHBdO/UX+W9NE3assVLkfrirqb48d+P/rtDcC0ydxd1mQO8Nu/v6VuD14y2OG82X/OJv/r+R3arMkc4Mj5IxSZZLs6rvVBLYZ9P4xdp3cxfPlw1HjF7XNtKygZGKlDPZfuty1XXH5qefp9048hS4dw6dolZm6bSfyVeH468hNtP2/rckSPtWaR9fkGLxlMgdcLOJzT7ot23FR2qZ5xmnaJvzQ6L+hMoxmNUu8zLNu/jG0nnWeqrjq0ymGimlmky0X4nXEx45iwYULqx+6k5CTyv56fT7p8wsNNHnY4967P72LtP2uZ1WUW3Wt3p0SBEhiGwXf7vqPHVz1oV7Udb7d/m0YzGgEwsNFA+tbvy/mk8/RZ7DwbVWRdpaKVmNZpGt0Wdkv3nKiCUZy+fNqpvUSBEhx9+ihnE89y+9zbU+cFTGw7kZ2ndqYOLwVoU7kNxxOO8/c5PdS0Q7UOrHxwJUop+izqw45TO9g73LFUgLWraVaXWbQs39KhQuigxoOY020OKw6u4J759zChzQTG3D6GI+ePcO3GNZefSjxBulxEQHmlzSu80PoFCoTqK7PwkPB0b+blD9VXmlEFo1LLHiilKBimi08Vz1+chqUakjI2hQ+3fMjARgNTJ101Lt2Y2h/qqeUlCpRg/4j9RE6OBOD3Ib/TcV5HziZmYWEHi841OzsUK/N3RxOOZpjMAZfJHHSd/4JvOBcKe3Gtc82V9UfWO+yv+nsVs/+czY5TO1JHFhmGweRfJjNq7Shmdrat/LX9v+1OxeLmbp9Lz7o9+eOkLgM8dv1Y2lRukzrU9dwL57hy/Qr3f3U/m2I3sfyB5XSq3gnQv2O5Qa7QRUD7/d/f6Ty/M/tG7KN4/uKp7SlGCq/+9Cojmo8gskCky8cahpFaEnjsbWMZf8d4bqTcQCmVOpv25bUvk5icSGT+SEbHjKZLzS4s6rWII+ePUOuDWgCMvnU0tUvUpt+3uipgzMAY7vjsDt5p/47+o1GiNuWm6K6BrrW6OnRLCM8qX6Q8sRfSWXjDg049f8qp4qm7ZJFoIXLJ17u/pnrx6jQq3cihJEJa5xLPET0rmq97fk2TMk0AWHt4LSsPreTNu94kOCiY+CvxKBSRBSK5cPWCwygNNV7Rvlp7Vj64klFrRvHpn5/yfd/vaVy6Md2/7M75pPN83/d77v/6fjYc3eCx1xekglKLmQWrYG4Yjv3EeXlSWF72addPeeim7N2sl4QuhI+7dO0S+YLzOS1U4sqGoxu4fe7t/PDgD9SLqkehsEIkXk/kq91f8eLaF0lMTmT/iP2pfbzTfpuWOiJl5C0jaViqIf2+7ccbd77Bi7e+mNqPXCuyFvvPOK4be33MdUJfzTwm4Whax2k80eKJbD1W+tCF8HFZmc5/W6XbnO4ZRIRH8FTLp6hSrApv/PyGw0SeJ1s8SbOyzagUUYmyhfUMxgcb2kq/Wks6rHxwpcO8gIjwCEKCQrij8h0Os1VB9x+PXz+eDtU70KFaBz7a8hHda3en/NTyWXrdAOPbjGfc+nFZflxeltGs3JyQYYtCBJCutbqyechmp+6hmyvcnJrM01rUaxFXXrpClWJVGNZUrwZUtnBZ5nTTXS2v3/k6AMefOQ5A+2rtiQiPYGrHqXSs3hGlFMObD6dckXJsG7qN34f8TuLLibzb4V2nn/X7kN/556l/2D9iP0NuGkLhsMKMvd12M9JaGbRXvV4Oj2tX1XHBFntlC5elV71erBuwzmkUS1b0rtc7249Ny77+jidJl4sQwm3JKcmcTzqfOiIorUNnD1G2cNnUEUbu2npiKxuPbeTplk+7PP7V7q+IzB/JrZVuTS1VcCzhGA8vfZi1h9fqm4yWgmj/PPUPlSMq02leJ3449IPLEU43fXxT6spVO4ftpOEMPXGoQ7UOPBb9GKHBodz75b2UKFAitXSyMc5gb9xe6n5UN93XERYcluGkpnk95vHgNw8yqtUoJt410Y13xpl0uQghPCIkKCTdZA5kuyZLdNnodBctB+crcoCKRSuyur9tkZEjTx3hbOJZKkdUBuCbXt+kO9zReqN3Trc5NCjVgLDgMGpF1uKHfj+knnN1tF48fPqW6Xy3/zuA1DpF+YLz8dqdr1E/qj5tq7Sl96LelClUhmmdplFtWjWOJhzlyFNHKF2oNIOWDGLhroWpf2jOJp5NvTHuaXKFLoQIOA2nN+Sv03+xY9gOPc/ASEGh3BoffvT8UQqGFUz3D9veuL18vuNz3mj7Rq6MN5crdCGEsGMdfmm9l5DRkNO0KkVUyvB4nZJ1st2dklNyU1QIEXCsffzBysWKQz5MrtCFEAFnUc9FzN0+l9olapsdikdJQhdCBJxKEZUY18a/xraDdLkIIYTfkIQuhBB+wq2ErpTqqJTar5Q6pJQa5eJ4PqXUl5bjvymlKns6UCGEEBnLNKErpYKBD4FOQF2gr1Iq7VSph4FzhmFUB6YCb3o6UCGEEBlz5wq9OXDIMIzDhmFcAxYCaavRdwM+s2wvAtqq3KrgLoQQwiV3Eno54LjdfqylzeU5hmEkAwmA06oASqmhSqmtSqmtcXFx2YtYCCGES+4kdFdX2mnrBbhzDoZhzDQMI9owjOiSJUu6E58QQgg3uZPQY4EKdvvlgRPpnaOUCgGKAllfTFEIIUS2uTOxaAtQQylVBfgX6AM8kOacpcBAYBNwP7DOyKTq17Zt2+KVUkezHjIAJYD4bD7WXwT6exDorx/kPQjU159uMZlME7phGMlKqRHAKiAYmG0Yxm6l1ARgq2EYS4FPgS+UUofQV+Z93HjebPe5KKW2pldtLFAE+nsQ6K8f5D0I9NfviltT/w3DWAGsSNM21m47Cejp2dCEEEJkhcwUFUIIP+GrCX2m2QHkAYH+HgT66wd5DwL99TsxbcUiIYQQnuWrV+hCCCHSkIQuhBB+wucSemaVH/2FUuqIUuovpdR2pdRWS1txpdRqpdRBy/dilnallJpmeU92KqVyZ0nxXKaUmq2UOq2U2mXXluXXrJQaaDn/oFJqoBmvJTvSef2vKKX+tfwebFdK3W137EXL69+vlOpg1+6z/0eUUhWUUjFKqb1Kqd1Kqacs7QHze5AjhmH4zBd6HPzfQFUgDNgB1DU7rlx6rUeAEmnaJgOjLNujgDct23cDK9ElGFoCv5kdfzZf821AE2BXdl8zUBw4bPlezLJdzOzXloPX/wrwvItz61p+//MBVSz/L4J9/f8IUAZoYtkuDBywvNaA+T3IyZevXaG7U/nRn9lXtfwM6G7X/rmhbQYilFJlzAgwJwzD2IBzyYisvuYOwGrDMM4ahnEOWA10zP3ocy6d15+ebsBCwzCuGobxD3AI/f/Dp/+PGIZx0jCMPyzbF4G96OJ/AfN7kBO+ltDdqfzoLwzgR6XUNqXUUEtbKcMwToL+xQeiLO3+/L5k9TX743sxwtKdMNva1UAAvH7LQjk3Ab8hvwdu8bWE7lZVRz/RyjCMJuiFRYYrpW7L4NxAel+s0nvN/vZeTAeqAY2Bk8A7lna/fv1KqULAYuBpwzAuZHSqiza/eR+yytcSujuVH/2CYRgnLN9PA9+iP0qfsnalWL6ftpzuz+9LVl+zX70XhmGcMgzjhmEYKcAs9O8B+PHrV0qFopP5PMMwvrE0B/Tvgbt8LaGnVn5USoWhi4AtNTkmj1NKFVRKFbZuA+2BXdiqWmL5vsSyvRQYYLnj3xJIsH489QNZfc2rgPZKqWKW7on2ljaflOZeyL3o3wPQr7+P0uv5VgFqAL/j4/9HlFIKXexvr2EYU+wOBfTvgdvMviub1S/0Xe0D6Dv5L5sdTy69xqro0Qk7gN3W14leBWotcNDyvbilXaHXff0b+AuINvs1ZPN1L0B3K1xHX2E9nJ3XDDyEvkl4CBhs9uvK4ev/wvL6dqKTVxm781+2vP79QCe7dp/9PwK0RneN7AS2W77uDqTfg5x8ydR/IYTwE77W5SKEECIdktCFEMJPSEIXQgg/IQldCCH8RzJNzAAAABlJREFUhCR0IYTwE5LQhRDCT0hCF0IIP/H/B5Yi5g+KkUoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "loss_arr = []\n",
    "loss_bn_arr = []\n",
    "\n",
    "max_epochs = 20\n",
    "\n",
    "for epoch in range(max_epochs):\n",
    "    \n",
    "    running_loss = 0.0\n",
    "    running_loss_bn = 0.0\n",
    "    for i, data in enumerate(loader, 0):\n",
    "\n",
    "        inputs, labels = data\n",
    "\n",
    "        # training steps for normal model\n",
    "        opt.zero_grad()\n",
    "        outputs = net_without_batch_norm(inputs)\n",
    "        loss = loss_fn(outputs, labels)\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "        \n",
    "        # training steps for bn model\n",
    "        opt_bn.zero_grad()\n",
    "        outputs_bn = net_with_batch_norm(inputs)\n",
    "        loss_bn = loss_fn(outputs_bn, labels)\n",
    "        loss_bn.backward()\n",
    "        opt_bn.step()\n",
    "        \n",
    "        loss_arr.append(loss.item())\n",
    "        loss_bn_arr.append(loss_bn.item())\n",
    "        \n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        running_loss_bn += loss_bn.item()\n",
    "        if i % 40 == 0:    # print every 2000 mini-batches\n",
    "            print('[%d, %5d] loss: %.3f  loss_bn: %.3f'%\n",
    "                  (epoch + 1, (i + 1)*512, running_loss / 40, running_loss_bn / 40))\n",
    "            running_loss = 0.0\n",
    "            running_loss_bn = 0.0\n",
    "    print(\"=\"*40)\n",
    "            \n",
    "\n",
    "print(\"=\"*10 + \" Epoch {} \".format(epoch+1) + \"=\"*10)\n",
    "inputs = inputs.view(inputs.size(0), -1)\n",
    "\n",
    "net_without_batch_norm.eval()\n",
    "net_with_batch_norm.eval()\n",
    "\n",
    "a = net_without_batch_norm.classifier[0](inputs)\n",
    "a = a.detach().numpy().ravel()\n",
    "sns.distplot(a, kde=True, color='r', label='Normal') \n",
    "\n",
    "b = net_with_batch_norm.classifier[0](inputs)\n",
    "b = net_with_batch_norm.classifier[1](b)\n",
    "b = b.detach().numpy().ravel()\n",
    "sns.distplot(b, kde=True, color='g', label='BatchNorm') \n",
    "\n",
    "plt.title('%d: Loss = %0.2f, Loss with bn = %0.2f' % (epoch+1, loss.item(), loss_bn.item()))\n",
    "plt.legend()\n",
    "plt.savefig(\"notes/imgs/activation-distribution.png\", dpi=100)\n",
    "plt.show()\n",
    "plt.pause(0.5)\n",
    "\n",
    "net_without_batch_norm.train()\n",
    "net_with_batch_norm.train()\n",
    "print('----------------------')\n",
    "\n",
    "plt.plot(loss_arr, 'r', label='Normal')\n",
    "plt.plot(loss_bn_arr, 'g', label='BatchNorm')\n",
    "plt.savefig(\"notes/imgs/training-loss.png\", dpi=100)\n",
    "plt.legend()\n",
    "plt.show()\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}