tagi_complete.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tractable Gaussian Approximate Infernece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TAGI:\n",
    "    \"\"\"\n",
    "    Implementation of TAGI (Traceable Approximate Gaussian Inference) based on the paper:\n",
    "    Tractable approximate Gaussian inference for Bayesian neural networks (James-A. Goulet, Luong-Ha Nguyen, and Said Amiri. JMLR, 2021)\n",
    "    https://jmlr.csail.mit.edu/papers/volume22/20-1009/20-1009.pdf\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, layers):\n",
    "        \"\"\"\n",
    "        Initialize the TAGI model.\n",
    "\n",
    "        Input:\n",
    "        layers: list of integers, specifying the number of neurons in each layer.\n",
    "\n",
    "        Output:\n",
    "        Initializes the model parameters and hidden states.\n",
    "        \"\"\"\n",
    "        self.layers = layers\n",
    "        self.parameters = self.init_parameters(layers)\n",
    "        self.hidden_states = self.init_hidden_states(layers)\n",
    "        self.covariances = self.init_covariances(layers)\n",
    "        self.mu = 0\n",
    "        self.sigma = 1\n",
    "\n",
    "    def standardize_data(self, X):\n",
    "        \"\"\"\n",
    "        Standardize the input data by subtracting the mean and dividing by the standard deviation.\n",
    "\n",
    "        Input:\n",
    "        X: ndarray of shape (n_samples), the input data.\n",
    "\n",
    "        Output:\n",
    "        ndarray of shape (n_samples), the standardized data.\n",
    "        \"\"\"\n",
    "        self.mu = np.mean(X)\n",
    "        self.sigma = np.std(X)\n",
    "        return (X - self.mu) / self.sigma\n",
    "\n",
    "    def unstandardize_data(self, X):\n",
    "        \"\"\"\n",
    "        Convert standardized data back to its original scale.\n",
    "\n",
    "        Input:\n",
    "        X: ndarray of shape (n_samples), the standardized data.\n",
    "\n",
    "        Output:\n",
    "        ndarray of shape (n_samples), the unstandardized data.\n",
    "        \"\"\"\n",
    "        return X * self.sigma + self.mu\n",
    "\n",
    "    def init_parameters(self, layers):\n",
    "        \"\"\"\n",
    "        Initialize weights and biases for each layer.\n",
    "\n",
    "        Input:\n",
    "        layers: list of integers, specifying the number of neurons in each layer.\n",
    "\n",
    "        Output:\n",
    "        parameters: list of dictionaries, where each dictionary contains:\n",
    "            - mw: ndarray of shape (n_inputs, n_outputs), the mean values of the weights.\n",
    "            - Sw: ndarray of shape (n_inputs, n_outputs), the variances of the weights.\n",
    "            - mb: ndarray of shape (n_outputs,), the mean values of the biases.\n",
    "            - Sb: ndarray of shape (n_outputs,), the variances of the biases.\n",
    "        \"\"\"\n",
    "        parameters = []\n",
    "        for i in range(len(layers) - 1):\n",
    "            input_size = layers[i]\n",
    "            output_size = layers[i + 1]\n",
    "            parameters.append(\n",
    "                {\n",
    "                    \"mw\": np.random.randn(output_size, input_size)\n",
    "                    * (0.5 / input_size) ** 0.5,\n",
    "                    \"Sw\": np.ones((output_size, input_size)) / input_size,\n",
    "                    \"mb\": np.random.randn(output_size, 1) * 1 / input_size,\n",
    "                    \"Sb\": np.ones((output_size, 1)) * 0.5,\n",
    "                }\n",
    "            )\n",
    "        return parameters\n",
    "\n",
    "    def init_hidden_states(self, layers):\n",
    "        \"\"\"\n",
    "        Initialize the hidden states for each layer.\n",
    "\n",
    "        Input:\n",
    "        layers: list of integers, specifying the number of neurons in each layer.\n",
    "\n",
    "        Output:\n",
    "        hidden_states: list of dictionaries, where each dictionary contains:\n",
    "            - 'ma': ndarray of shape (n_neurons,1), the mean values of the activation units.\n",
    "            - 'Sa': ndarray of shape (n_neurons,n_neurons), the variances of the activation units.\n",
    "            - 'J': ndarray of shape (n_neurons,1), the Jacobian of the activation function.\n",
    "        \"\"\"\n",
    "        hidden_states = []\n",
    "        for layer_size in layers:\n",
    "            hidden_states.append(\n",
    "                {\n",
    "                    \"ma\": np.zeros((layer_size, 1)),\n",
    "                    \"Sa\": np.ones((layer_size, 1)),\n",
    "                    \"J\": np.ones((layer_size, 1)),\n",
    "                }\n",
    "            )\n",
    "        return hidden_states\n",
    "\n",
    "    def init_covariances(self, layers):\n",
    "        \"\"\"\n",
    "        Initialize the covariances for each layer.\n",
    "\n",
    "        Input:\n",
    "        layers: list of integers, specifying the number of neurons in each layer.\n",
    "\n",
    "        Output:\n",
    "        covariances: list of dictionaries, where each dictionary contains:\n",
    "            - 'covZw': ndarray of shape (n_inputs, n_outputs), the covariances of the hidden states and weights.\n",
    "            - 'covZb': ndarray of shape (n_outputs,), the covariances of the hidden states and biases.\n",
    "            - 'covZZ': ndarray of shape (n_neurons, n_neurons), the covariances of two consecutive hidden states.\n",
    "        \"\"\"\n",
    "        covariances = []\n",
    "        for i in range(len(layers) - 1):\n",
    "            input_size = layers[i]\n",
    "            output_size = layers[i + 1]\n",
    "            covariances.append(\n",
    "                {\n",
    "                    \"covZw\": np.zeros((output_size, input_size)),\n",
    "                    \"covZb\": np.zeros((output_size, 1)),\n",
    "                    \"covZZ\": np.zeros((output_size, input_size)),\n",
    "                }\n",
    "            )\n",
    "\n",
    "        return covariances\n",
    "\n",
    "    def GMA(self, ma, Sa, mw, Sw):\n",
    "        \"\"\"\n",
    "        Gaussian Multiplicative Approximation (GMA)\n",
    "\n",
    "        Input:\n",
    "        ma: ndarray of shape (n_inputs,), mean values of the activation units.\n",
    "        Sa: ndarray of shape (n_inputs,), variances of the activation units.\n",
    "        mw: ndarray of shape (n_inputs, n_outputs), mean values of the weights.\n",
    "        Sw: ndarray of shape (n_inputs, n_outputs), variances of the weights.\n",
    "\n",
    "        Output:\n",
    "        mu: ndarray of shape (n_outputs,), the means of the next hidden layer.\n",
    "        var: ndarray of shape (n_outputs,), the variances of the next hidden layer.\n",
    "\n",
    "        Remember that in GMA, given X1 and X2 random gaussian variables,\n",
    "        Expected value: E[X1 * X2] = μ1 * μ2 + cov(X1, X2)\n",
    "        Variance: var[X1 * X2] = var(X1) * var(X2) + cov(X1, X2)^2 +\n",
    "                  2cov(X1,X2)μ1μ2 + var(X1) * μ2^2 + var(X2) * μ1^2\n",
    "\n",
    "        Also remebmer that the activation units and the weights are independent in a neural network.\n",
    "        \"\"\"\n",
    "        # To-do: Implement GMA and compute mu and var.\n",
    "\n",
    "        # Expected value E[X1 * X2] = μ1 * μ2\n",
    "        mu = mw @ ma  # Matrix product of means\n",
    "        print(mu.shape)\n",
    "\n",
    "        # Variance calculation\n",
    "        # var[X1 * X2] = Sa * Sw + (Sa * mw^2) + (Sw * ma^2)\n",
    "        var = (Sw @ Sa) + (Sw @ ma**2) + (mw**2 @ Sa)\n",
    "\n",
    "        return mu, var\n",
    "\n",
    "    def ReLU(self, mz, Sz):\n",
    "        \"\"\"\n",
    "        Rectified Linear Unit (ReLU) activation function.\n",
    "\n",
    "        Input:\n",
    "        mz: ndarray of shape (n_samples,), the hidden states.\n",
    "        Sz: ndarray of shape (n_samples,), the variances of the hidden states.\n",
    "\n",
    "        Output:\n",
    "        ndarray of shape (n_samples,), the output values.\n",
    "        \"\"\"\n",
    "\n",
    "        # To-do: Implement ReLU activation function and compute ma, Sa and J\n",
    "        ma = np.maximum(0, mz)\n",
    "        Sa = Sz * (mz > 0)\n",
    "        J = (mz > 0).astype(int)\n",
    "\n",
    "        return ma, Sa, J\n",
    "\n",
    "    def forward(self, X):\n",
    "        \"\"\"\n",
    "        Perform a forward pass through the network using GMA.\n",
    "\n",
    "        Input:\n",
    "        X: ndarray of shape (n_inputs,), the input data.\n",
    "\n",
    "        Output:\n",
    "        Y: ndarray of shape (n_outputs,), the output values.\n",
    "\n",
    "        It is suggested to compute the coariances at the same time you perform the forward pass.\n",
    "        \"\"\"\n",
    "\n",
    "        # To-do: Implement forward pass using GMA and compute the covariances.\n",
    "        self.hidden_states[0][\"ma\"], self.hidden_states[0][\"Sa\"] = X, np.zeros_like(X)\n",
    "\n",
    "        for i in range(len(self.layers) - 1):\n",
    "            ma, Sa = self.hidden_states[i][\"ma\"], self.hidden_states[i][\"Sa\"]\n",
    "            mw, Sw = self.parameters[i][\"mw\"], self.parameters[i][\"Sw\"]\n",
    "            mb, Sb = self.parameters[i][\"mb\"], self.parameters[i][\"Sb\"]\n",
    "\n",
    "            # Apply GMA\n",
    "            mz, Sz = self.GMA(ma, Sa, mw, Sw)\n",
    "            mz += mb\n",
    "            Sz += Sb\n",
    "\n",
    "            # Apply ReLU if not the output layer\n",
    "            J = np.ones_like(mz)\n",
    "            if i < len(self.layers) - 2:\n",
    "                ma, Sa, J = self.ReLU(mz, Sz)\n",
    "            else:\n",
    "                ma, Sa = mz, Sz\n",
    "\n",
    "            self.hidden_states[i + 1] = {\"ma\": ma, \"Sa\": Sa, \"J\": J}\n",
    "\n",
    "            # Update covariances at the same time\n",
    "            # print (ma.shape,Sw.shape)\n",
    "            self.covariances[i][\"covZw\"] = Sw @ ma\n",
    "            self.covariances[i][\"covZb\"] = Sb\n",
    "            Sz = Sz[:, np.newaxis]\n",
    "            J = J[np.newaxis, :]\n",
    "            self.covariances[i][\"covZZ\"] = Sz @ J @ mw.T\n",
    "\n",
    "        # Output layer\n",
    "        return self.hidden_states[-1][\"ma\"]\n",
    "\n",
    "    def backward(self, Y, sigma_v):\n",
    "        \"\"\"\n",
    "        Perform a backward pass through the network using Bayesian inference.\n",
    "\n",
    "        Input:\n",
    "        Y: ndarray of shape (n_outputs,), the output values.\n",
    "        sigma_v: float, the observation noise.\n",
    "\n",
    "        We are just gonna update the parameters and not the hidden states.\n",
    "\n",
    "        Infer posterior mean and diagonal covariance for the output layer:\n",
    "        - f(z_out|y) = N(z_out; mu_{z_out|y}, Sigma_{z_out|y})\n",
    "        - mu_{z_out|y} = mu_{z_out} + cov_{Y, Z_out}^T * cov_{Y}^{-1} * (y - mu_Y)\n",
    "        - Sigma_{z_out|y} = Sigma_{z_out} - cov_{Y, Z_out}^T * cov_{Y}^{-1} * cov_{Y, Z_out}\n",
    "\n",
    "        RTS smoother for the parameters:\n",
    "        - f(\\theta|y) = N(\\theta; mu_{\\theta|y}, Sigma_{\\theta|y})\n",
    "        - mu_{\\theta|y} = mu_{\\theta} + J_{\\theta} * (mu_{Z+|y} - mu_{Z+})\n",
    "        - Sigma_{\\theta|y} = Sigma_{\\theta} + J_{\\theta} * (Sigma_{Z+|y} - Sigma_{Z+}) * J_{\\theta}^T\n",
    "\n",
    "            where:\n",
    "            - J_{\\theta} = cov_{\\theta, Z+} * cov_{Z+}^{-1}\n",
    "        \"\"\"\n",
    "\n",
    "        # Update output layer\n",
    "        mu_y = self.hidden_states[-1][\"ma\"]  # Predicted mean of the output\n",
    "        Sigma_y = (\n",
    "            self.hidden_states[-1][\"Sa\"] + sigma_v**2\n",
    "        )  # Total variance of the output (add noise)\n",
    "        cov_yZ = (\n",
    "            self.hidden_states[-1][\"Sa\"] / Sigma_y\n",
    "        )  # Covariance between output and last hidden state\n",
    "\n",
    "        # Infer posterior for the output layer\n",
    "        residual = Y - mu_y  # Error between predicted and true output\n",
    "        mu_z_out_y = mu_y + cov_yZ * residual  # Posterior mean of the last hidden state\n",
    "        Sigma_z_out_y = (\n",
    "            self.hidden_states[-1][\"Sa\"] - cov_yZ**2 * Sigma_y\n",
    "        )  # Posterior variance of the last hidden state\n",
    "\n",
    "        # Update the output layer's hidden state parameters\n",
    "        self.hidden_states[-1][\"ma\"] = mu_z_out_y\n",
    "        self.hidden_states[-1][\"Sa\"] = Sigma_z_out_y\n",
    "\n",
    "        # RTS smoother for parameters\n",
    "        for i in reversed(range(len(self.layers) - 1)):\n",
    "            # Retrieve forward pass values\n",
    "            J_theta = self.covariances[i][\"covZw\"] / (\n",
    "                self.hidden_states[i + 1][\"Sa\"] + 1e-8\n",
    "            )  # Avoid division by zero\n",
    "            mu_theta = self.parameters[i][\"mw\"]  # Mean of the parameters (weights)\n",
    "            Sigma_theta = self.parameters[i][\n",
    "                \"Sw\"\n",
    "            ]  # Variance of the parameters (weights)\n",
    "\n",
    "            # Retrieve posterior values from next layer\n",
    "            mu_z_next_y = self.hidden_states[i + 1][\"ma\"]\n",
    "            Sigma_z_next_y = self.hidden_states[i + 1][\"Sa\"]\n",
    "\n",
    "            # Update posterior for the current layer's parameters\n",
    "            delta_mu_z = mu_z_next_y - self.hidden_states[i + 1][\"ma\"]\n",
    "            delta_Sigma_z = Sigma_z_next_y - self.hidden_states[i + 1][\"Sa\"]\n",
    "\n",
    "            # Compute updated mean and variance for parameters\n",
    "            mu_theta_y = mu_theta + J_theta * delta_mu_z\n",
    "            Sigma_theta_y = Sigma_theta + J_theta * delta_Sigma_z * J_theta.T\n",
    "\n",
    "            # Save updated parameter posterior\n",
    "            self.parameters[i][\"mw\"] = mu_theta_y\n",
    "            self.parameters[i][\"Sw\"] = Sigma_theta_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x12fca30d0>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAGdCAYAAADT1TPdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAtgklEQVR4nO3df3Ac9X3/8ddxqQ9DbaW2iWX5zsjJeErAaUrslmJQkYbGHcpQpxdBsQilkxnPUONEspsfUDNBZmqrocU4xQ2M80dJp7Fxii8h/TXFoTaYISSOa7cZyDQwEVgIO8aEkYzTyuW83z/2u0J3utPt3u3efvZzz8eM5tBqtf4gnT773s/n/Xl/Uo7jOAIAADDUBXE3AAAAYCYEKwAAwGgEKwAAwGgEKwAAwGgEKwAAwGgEKwAAwGgEKwAAwGgEKwAAwGjvi7sBjTp//rzeeOMNzZkzR6lUKu7mAAAAHxzH0ZkzZ9TR0aELLph57CTxwcobb7yhXC4XdzMAAEAdRkZGlM1mZzwn8cHKnDlzJLn/s3Pnzo25NQAAwI/x8XHlcrnJ+/hMEh+seFM/c+fOJVgBACBh/KRwkGALAACMRrACAACMRrACAACMRrACAACMRrACAACMRrACAACMRrACAACMRrACAACMlviicIhPsSgdOiSdOCEtWiR1dUnpdNytAgDYhmAFdSkUpP5+6fXX3zuWzUpf+YqUz8fXLgCAfZgGQmCFgtTbWxqoSNLoqHu8UIinXQAAOxGsIJBi0R1RcZzpX/OODQy45wEAEAaCFQRy6ND0EZWpHEcaGXHPAwAgDAQrCOTEiXDPAwCgFoIVBLJoUbjnAQBQC8EKAunqclf9pFKVv55KSbmcex4AAGEgWEEg6bS7PFmaHrB4n+/YQb0VAEB4CFYQWD4vPfGEtHhx6fFs1j1OnRUAQJgoCoe65PPSmjVUsAUARI9gBXVLp6Xu7rhbAQCwHdNAAADAaAQrAADAaAQrAADAaAQrAADAaAQrAADAaKwGwjTFIkuSASQb/ZhdCFZQolCQ+vtLd1bOZqWHHpIWLOAPH4D5qvVjX/kKRSuTKuU4jhN3IxoxPj6utrY2jY2Nae7cuXE3J9EKBam3V/LzjuAPH4CJqvVj3nYgVNk2R5D7N8FKi6k2NFosSp2dpU8iM+EPH4BpavVjqZT7oDU8zMiwCYLcv0mwbSGFgvuH3NMj9fW5r52d7vFDh/wHKtJ7Ty0DA24HAQBxq9WPOY40MuKeh2QhWGkR3tBo+R/y6Kh7/Mkng1+TP3wAJjlxItzzYI5Ig5Vnn31WN910kzo6OpRKpfTtb3+75OuO42hwcFAdHR2aPXu2uru79eKLL0bZpJZULLrJZpUm/Lxj3/hG/dfnDx+ACRYtCvc8mCPSYOXs2bP66Ec/qp07d1b8+gMPPKDt27dr586dOnz4sNrb2/Xxj39cZ86cibJZLcfP0Oibb0qXXPJeLkoQ/OEDMEFXl5uTUq0fS6WkXM49D8kSabByww036M///M+Vr5CB6TiOduzYoc2bNyufz2v58uX6+te/rl/84hfavXt3lM1qOX5HPm67zX31G7Dwhw/AJOm0u0pRmt6PeZ/v2EFybRLFlrMyPDyskydPavXq1ZPHMpmMrrvuOj3//PNVv29iYkLj4+MlH5iZ35GPNWvc1T2LF9c+lz98ACbK5yv3Y9ksqxeTLLaicCdPnpQkLVy4sOT4woUL9dprr1X9vqGhIW3ZsiXSttnGGxodHa2ct+It5/OWMa9ZU7q8+c03pU2bphdY2rGj9A+fipEAPGH1B/VcJ5+f3o/RHyVb7BVsU2VjdY7jTDs21T333KNNmzZNfj4+Pq5cLhdZ+2zgDY329rqBydSApdIISTotdXeXXiOfn/kPn4qRADxh9QeNXKdSP4bkii1YaW9vl+SOsCyaMk9x6tSpaaMtU2UyGWUymcjbZxtvaLTSH375CEklM/3hV6sY6S2LZugVsMtMox1h9Qdh9yt+RmgYHTaY0ySSnG9961uTn58/f95pb293vvzlL08em5iYcNra2pxHH33U93XHxsYcSc7Y2FiYzbXWu+86zoEDjrN7t/v67ruNXy+bdRy3S5n+kUo5Ti7X+L8DwAz79k3/m89m3eNh9Qdh9ysztTnIOQhXkPt3pCMr77zzjl555ZXJz4eHh3Xs2DHNmzdPS5Ys0cDAgLZt26Zly5Zp2bJl2rZtmy666CL19fVF2ayWFvbQaJCKkQzJAslWa7RjcLCx/sAb2Xj66fD6FT8jNBKjw6aLNFj54Q9/qJ6ensnPvVyTO+64Q4899pi+8IUv6H/+53+0fv16vf3227rqqqv01FNPac6cOVE2CyGiYiTQGmoVl0ylpL/+a3/XqtQfVMpPqec6U/lps/f1mc4ZGHATdpkSik+kwUp3d7ecGfZJTKVSGhwc1ODgYJTNQISoGAm0Bj+jqG+95e9a5f1BkB3fZ7pOOT9trhUcMTpshthXAyHZgiyL9oskN8A8fkdH582T3n7bf38w0+hHNX77lTBHdBkdjhcbGaIhYVeMnGlnaADx8Ts62t/vvvrtD4Lu+B6kXwlzRJfR4XgRrKBhYVWMrLUzNAELEB+/++5s3hysPwg6YhGkX/HT5mzWbSv7CZkt5cyUVJIA4+Pjamtr09jYmObOnRt3c1paI9M3xaI7glLtCcvrVIaHmRIC4uI9UEiVi0tODSL81jV5+GFp48ba//a990rXXx98WthPmyX//18IT5D7N8EKjHDwoDvlU8uBAyS5AXGqtGonl6tdXLI8eDl92g1Sak0BhfGg4qfN9f5/oX5B7t8k2MIIfoeC9+1zX0m6BeJRz7479SxLlsLbMNVPm9lPyGyMrMAIfkdWPOw7BCRDvcuSJUY2bBfk/k2CLYxQKxGuHEm3gPnqWZbseeghd+qHQAUSwYpVikV3hGLPHve1WIy7Rf7NtAS6Eq/zGxhI1v8n0EqCLkueauFCpmDwHoIVSySlPslMAVW1JdDVTK0sCcA8jRRSC1LXJMkPavCHYMUCSalP4iegyuelV191V/1s2ODvulSWBMxUTyG1oHVNkvKghsYQrCRcrY26JDOmSoIEVN7O0J/8pL9rU1kSMFPQXLSgq3+S8qCGxhGsJJyfjbriniqpN6DyWzGTypKAmYLmogWpTpuUBzWEg2AlwYpF6emn/Z0b51RJvQFV2PsOAWi+arlouZz0zW+6U767d7uvQVb/JOFBDeGhKFxCBS2yFOdUid9AqdJ5XkdX/v+azVJ/AUiKKAquNdKvIHkIVhIoSJElv1upR8lvoFTtPCpLAsnn5aKFpdF+BclCBduEqbXh31SmbMLltXl0tHKAxSaFAIKiX0k+KthaLEiRpSDJalEi9wRA2OhXWgvBSsL4nX+9916zSlVXS7IzJaACkDz0K62DnJWE8Tv/ev315j1RkHsCIGzN7leKRfqwOJCzkjDM0wJAPCqtwmQH+PqRs2Ix5mkBoPmolhsvgpUEYp4WAJqHarnxI2clocj/AGAK2/M4glTLDbOWDN5DsJJgYRdZAoCgWiGPg2q58WMaCABQl1bJ46BabvwIVgAAgbVSHgc7wMePYAUAEFgr7XrMKsz4EawAAAJrtTwOVmHGiwRbAEBgrZjHwSrM+BCsJITtSwMBJIuXx1GrmrZteRyswowH00AJUCi4JfZ7eqS+Pve1s9OeTHsAyWNDHkexKB08KO3Z477akAxsK4IVw7XK0kAAyZPkPA4eApOFjQwN5m1aWC3jnk0LAZggadPU3kNg+d3PGxEyPdCyRZD7N8GKwQ4edKP9Wg4cYA4VAPzgIdAc7LpsiVZbGggAUWul+jA2IVgxWCsuDQSAKPEQmEwsXTZAtfneVl0aCABR4SEwmRhZidlMGek2LA2MCksOAdSDfX6SiWAlRn6WJSd5aWBUWHIIoF48BCYTq4FiEjQjPWlLA4MI8v/GkkMAYSgU3F2jp/bBuZwbqNCHNAdLlxOAZcmuSh1GNus++ZR3GCw5BMJl80OQH63+/x+3IPdvEmxj4jfTfN8+99XGP6JqoyTeNFj5KEmQJYc2B3hAGII8KNiKfX6Sg5yVmPjNNN+5086cjGLR7Sgrjet5xwYGShNnWXIIhINtPJA0BCsxqZWRXs62TqSewkwsOQQaV8+DAhA3gpWYzJSRXoltnUg9oyQsOQQaRwVXJBHBSoyqLUuuxqZOpJ5REpYcAo1jOhVJRLASs3xeevVVd9XPhg3+vseGTqTeURLqzgCNYToVSUSwYgAvI/2Tn/R3vg2dSCOjJFMDvN273dfhYQIVwA+mU5FEBCsGabVOpJFREi/AW7vWfWXqB/CH6VQkEUXhIha06JC3pFAqzda3uUIrhZmA5qOCa3jow+pDBVtD1Ft0iU4EQDNwk20cxfXqR7BigEb3sKETAQCzsVdZYwhWYsYeNgBgN/r5xgW5f5NgGwGKLgGA3ejnm4tgJQIUXQIAu9HPNxfBSgQougQAdqOfby6ClQi0Wr0UAGg1Qfv5YlE6eFDas8d9tWGPt2YiWAmR92b85jeldevcOUuKLgGAfYIU1ysU3GTcnh6pr8997ex0j8MfgpWQlL8Z77tPmj9fmjev9Dz2sAEAO/ipwu0tby5Pxh0ddY8TsPjD0uUQ1FprPzgoLVtGvRQAsFG1ulgsb54ZdVaaiDcjAKCSgwfd0fZaDhxw9zhrNYmqszI4OKhUKlXy0d7eHnezfGOtfbKQ5AagWVjeHJ73xd0ASbriiiv03e9+d/LzdIKGIHgzJgd7eABoJpY3h8eIYOV973tfokZTpuLNmAzV8oq8JDeSngGEzVvePDo6ve+R3ksToIxFbbFPA0nSyy+/rI6ODi1dulS33nqrfvrTn1Y9d2JiQuPj4yUfcaKmivmKRXdEpVJn4R0bGGBKCEC4gixvxsxiD1auuuoq/d3f/Z3+7d/+TV/72td08uRJrVq1Sm+99VbF84eGhtTW1jb5kcvlmtziUrwZzUdeEYC4+FnejNqMWw109uxZfehDH9IXvvAFbdq0adrXJyYmNDExMfn5+Pi4crlc7LsuV8qHyOXcQIU3Y7z27HFr39Sye7e0dm31ZYhAkvG+jhc//+mCrAYyImdlqosvvlgf+chH9PLLL1f8eiaTUSaTaXKrasvnpTVreDOaKEheEUm4sBHv6/il0625PDkssU8DlZuYmNCPf/xjLUpgRqr3Zly71n0lUDGD37yi06epNAn7UEEVNog9WPnc5z6nZ555RsPDw/r+97+v3t5ejY+P64477oi7abCEn7yiBx+UNm4kCRd2Ibkctog9WHn99de1du1a/eqv/qry+bxmzZqlF154QZdeemncTYMFvCJwExPutgfVktwuuYQkXNiH5HLYIvaclccffzzuJsBSlebpFy+WtmyZvlfTnj3+rklxPyQJRSthi9hHVoAoVJunf+MNd4QlkynNK6K4H2zE+xq2MG7pclBxb2QI89SzuaT3PbUqTbIhJZKE9zVMlqiNDIGw1TNPT3E/2Ij3dTzYMDV8BCuwTr3z9FSahI14XzdXoeCOZvX0uMUoe3rcz1ki3himgWCdgwfdDqKWAwcqF2mi0iRsxPs6etU2TPVGsQgOSwW5fxOsNBkdRvSCztPzOwHQqHpy5VodOSuGYniwOYLM0/M7ARAGatpEi2ClSSh53Vx+5un5nQAICzVtosU0UBMwPBifalM8/E4AhKnRXLlWlOhdl20UZHiQN3G4qu10GvR3Ql4LgJl4G6bWypXr6mp+22zANFATMDxoniC/E/JaANRCTZtoEaw0ASWvzeP3Z/3yy+S1APCHmjbRIWelCSh5bR4/v5PFi92vjY5Wvga/NwCVMG3sD0uXDcPwoHn8/E7WraseqEgsRQRQmZcrt3Zt6YapqB/BSpMwPGieWr+TZcv8XYdcIwCIFquBmiifl9asYXjQJDP9Tg4e9HcNco0AIFrkrABVkGsEANEhZwUIAblGAGAGgpUGFIvuVMGePe5rsRh3ixA2co0AIH7krNSpUJD6+0vrb2Sz7pM4NzC7kGsEIAoscfaPnJU6eBvglf/kvKkBnrgBADPhgZeclUgVi+4brFKI5x0bGGBKCABQGTu+B0ewElCQDfAAAJiKB976EKwExKaEAIB68cBbH4KVgNiUEABQLx5460OwElBXl5sEVV53w5NKSbmcex4AAFPxwFsfgpWAKBQGIG7UeEouHnjrQ7BSBwqFAYhLoeBuA9HTI/X1ua+dnawgSQoeeOtDnZUGUNAHQDNR48keleqs5HJuoNIqv8Mg92+CFQBIAG9jzWorSdhYM3la/YE3yP2bcvsAkAB+l7w+/LD0mc+01k0vqdJpqbs77lYkAzkrAJAAfpeybtxIDgvsQ7ACAAbzVv689JL/76FsO2zDNBAAGKpSEqYfjuPmsAwMuDuGMyWEpGNkBQAMVG2zO78o2w6bMLJSRatnaQOIz0yb3QVF2XbYgGClgkpDr9msW8inVda/A4hPrZU/QVC2HTZgGqhMtaFXEtYANIvf0ZB58yjbjtZAsDLFTEOv3rGBAfbhABAtv6Mh/f3uK2XbYTuClSn8Fl0iYQ1AlPxudrd5M/uUoTWQszKF36FXEtYARMnb7K631w1Mpo72lo+a5PPu8mQWBMBmBCtT+B16JWENQNS83d0rJfuXb3ZH2XbYjo0Mp/A2ChsdrZy3wkZhAJqNMgqYyqb3AxsZ1inI0Ctam00dBszGqAk8rVxWgwTbMt7QKwlrqKZQcEfgenqkvj73lY3jAESp1ctqMA1UBU/OqMTrMMr/aryRNwJaAGHzUhSqrVZNaopCkPs3wQrgk60dBgCzHTzojuDWcuBAsqYMg9y/mQYCfKIOD4A4UFaDYAXwjQ4DQBwoq0GwAvhGhwEgDn4rGtu8DxTBCuATHQaAOHhlNaTW3QeKYAXwiQ4DQFxavawGq4GAgCoVZsrlppdAB4Cw2VRWg6XLQMRs6jAAIA6U2wciRgl0AGgeclYAAIDRCFYAAIDRCFYAAIDRCFYAAIDRCFYAAIDRCFYAAIDRCFYAAIDRjAhWvvrVr2rp0qW68MILtWLFCh06dCjuJgEAAEPEHqzs3btXAwMD2rx5s44ePaquri7dcMMNOn78eNxNAwAABoi93P5VV12lj33sY3rkkUcmj334wx/WJz7xCQ0NDdX8fsrtAwCQPEHu37GOrJw7d05HjhzR6tWrS46vXr1azz//fMXvmZiY0Pj4eMkHAACwV6zByunTp1UsFrVw4cKS4wsXLtTJkycrfs/Q0JDa2tomP3K5XDOaCgCAkYpF6eBBac8e97VYjLtF4Ys9Z0WSUqlUyeeO40w75rnnnns0NjY2+TEyMtKMJgIAYJxCQerslHp6pL4+97Wz0z1uk1h3XV6wYIHS6fS0UZRTp05NG23xZDIZZTKZZjQPAABjFQpSb69Unnk6Ouoef+IJKZ+Pp21hi3VkZdasWVqxYoX2799fcnz//v1atWpVTK0CAMBsxaLU3z89UJHeOzYwYM+UUKwjK5K0adMm3X777Vq5cqWuvvpq7dq1S8ePH9edd94Zd9MAADDSoUPS669X/7rjSCMj7nnd3U1rVmRiD1b+8A//UG+99Zbuv/9+nThxQsuXL9e//Mu/6NJLL427aQAAGOnEiXDPM13swYokrV+/XuvXr4+7GQAAJMKiReGeZzojVgMBAAD/urqkbFaqsnBWqZSUy7nn2YBgBQCAhEmnpa98xf3v8oDF+3zHDvc8GxCsAIBBWqHAF8KRz7vLkxcvLj2ezdq1bFkyYG+gRrE3EICkKhbd1RonTri5BW++KW3aVLrKI5t1n6BtuvEgXOXvo1WrpOeff+/zri4zR1iC3L+NSLAFgFZTKLh1MmZafirZWeAL4Uqn31ueXChIH/qQfQEv00AA0GRe5dFagYpkZ4EvRKPa+8oLeJNcgp9gBQCaaKbKo9VMLfAFVGJ7RVuCFQBoolqVR2diS4EvhC9IRdskIlgBgCZqJOCwpcAXwmd7RVsSbAGgieoJOFIpN0nSlgJfCJ/tFW0ZWQGAJqpVebScjQW+ED7bK9oSrABAE3jF3r75TWndOjeHwE/AYmOBL4TP9oq2TAMBQMQq1VSZP999feut947lctL27dKCBeYX9IJ5vIq25e+1bNYNVJIc8BKsAECEvNoX5UtKf/5z93XLFmnZMgIThCOfl9asKa1oa8P7inL7ABCRYlHq7Ky+pNRLnB0eTv7NBAgqyP2bnBUAiIjttS+QXEnbMJNpIACIiO21L5BMlXKoTN8/iJEVAIiI7bUvkDxJ3T+IYAUAImJ77QskS5L3DyJYAYCI2F77AsmS5BwqghUAiJBX+2Lx4tLjFHtDsyU5h4oEWwCImK21L5AsSc6hIlgBgCZIp6Xu7rhbgVbm5VCNjlbOWzF5w0ymgQAAaAFJzqEiWAEAoEUkNYeKaSAAAFpIEnOoCFYAAGgxScuhIlgBAMByxWKyRlLKEawAAGCxJO4FVI4EWwAALJXUvYDKEawAQASKRengQWnPHvfVxP1WYLck7wVUjmAFAEJWKEidnVJPj9TX5752dibnKRZ2SPJeQOUIVgAgRLYMuyP5krwXUDmCFQAIiU3D7ki+JO8FVI5gBQBCYtOwO5LP2wuovLS+J5WScjkz9wIqR7ACxIxETHvYNOyO5EvyXkDlCFaAGJGIaRebht1hh6TuBVQu5TiVZleTY3x8XG1tbRobG9PcuXPjbg7gm5eIWf4X6D3xJKkjgatYdIPN0dHKeSuplHuTGB5OxtMs7GFiBdsg92+CFSAG3k2tWn4DN7Xk8oJQqTRgIQgFSgW5fzMNBMSAREx72TLsDpiEvYGAGARNxDRxCBfvKf/9rFnjfvA7A8JBsALEIEgipg2bkNmM3w8QPaaBgBjUqn8gSfPmuUuZP/lJqqGaimq1QHOQYAvEpFoipl8k4caLJGmgMSTYAglQLRHTL5Jw40WSNNA8BCtAE5VXq12zRnr1Vem733WnfepBNdR4UK0WaB4SbIEmmSkRc9486ec/r++6VEONB9VqgeZhZAVoglqJmE8+GfyaSdqEzEY2bRIHmI5gBYhYseiOqFRKovWOfeMbwa6ZtE3IbGTTJnGA6QhWgIj5ScR8803pkktmXso8FdVQzUC1WqA5yFkBIuY3wfK229wn9VSq8p4yg4PSsmVUQzVNPk+1WiBqBCtAxPwmWK5Z497kKiXh7tjBU7rJ0mmpuzvuVgD2IlgBIuYlYo6OVs5b8YqHeU/jPKUDQCmCFSBiXiJmb2/1KZ6piZh+ntLZ2BBA2EzuV0iwBZogzETMQsEt897TI/X1ua+dnexDA6B+pvcr7A0ENFGjTy5evZbyv1pvhIYVKACCiqtfCXL/JlgBEoKN8wCELc5+hY0MAQuxcV48yvdzKhbjbhEQnqT0KyTYAgnBxnnNV20/p+3b3SJ+JiYiAkEkpV8hWAESgo3zmqvaPP7rr0u33FJ6zNuQknwhJE1S+hVyVoCE8OaWa9VrIWelcbXm8cuR4IykirNfIWcFsBAb5zVPrXn8cl4nPzBATguSJSn9SqzBSmdnp1KpVMnH3XffHWeTAKOxcV5z1DM/b0oiIhDUTP3K3r3SvHnxJ5jHnrNy//33a926dZOf//Iv/3KMrQHMx8Z50Wtkfj7uRESgHpX6ldOnpY0bpyeYx5GfFXuwMmfOHLW3t8fdDCBR2DgvWrX2c5pJ3ImIQL2m9iuFgptIXv7+Hx11E8+bPZIba4JtZ2enJiYmdO7cOeVyOd188836/Oc/r1mzZlX9nomJCU1MTEx+Pj4+rlwuR4ItgFB5q4EkfwELCc6wRbMKxSUmwba/v1+PP/64Dhw4oA0bNmjHjh1av379jN8zNDSktra2yY9cLtek1gJoJdXm8SsxKRERaJSJheJCH1kZHBzUli1bZjzn8OHDWrly5bTj+/btU29vr06fPq358+dX/F5GVgA0U/l+TpXm8XM5N1AhwRk22LPH3cywlt27pbVr6/93goyshJ6zsmHDBt16660zntPZ2Vnx+G/91m9Jkl555ZWqwUomk1Emk2mojQDgV6X8oD/4AxKcYS8TC8WFHqwsWLBACxYsqOt7jx49KklaRIYaAIOR4Ayb1Uow93JWurqa16bYVgN973vf0wsvvKCenh61tbXp8OHD2rhxo37/939fS5YsiatZAAC0NK9QXG+vG5hMDVjiys+KLcE2k8lo79696u7u1uWXX64vfelLWrdunfbs2RNXkwAAgMwrQMneQAAAoKLyBPMw87NiTbAFAAB2MCU/i40MAQCA0QhWAACA0QhWAACA0QhWAACA0QhWAACA0VgNBKBlRbksE0B4CFYAtKRCQervL92QMJt1K3eyISFgFqaBALScQsEtJT41UJHcvVB6e92vAzAHwQqAllIsuiMqlWp3e8cGBtzzAJiBYAVASzl0aPqIylSOI42MuOcBMAPBCoCWcuJEuOcBiB7BCoCWsmhRuOcBiB7BCoCW0tXlrvpJpSp/PZWScjn3PABmIFgB0FLSaXd5sjQ9YPE+37GDeiuASQhWALScfF564glp8eLS49mse5w6K4BZKAoHoCXl89KaNVSwBZKAYAVAy0qnpe7uuFsBoBamgQAAgNEIVgAAgNEIVgAAgNEIVgAAgNEIVgAAgNFYDQRgRsUiy3sBxItgBbBUGEFGoSD195fuUpzNuhVgKZwGoFmYBgIsVChInZ1ST4/U1+e+dna6x4Nco7e3NFCRpNFR93iQawFAIwhWAMuEEWQUi+6IiuNM/5p3bGDAPQ8AokawAlgkrCDj0KHpwU75tUZG3PMAIGoEK4BFwgoyTpzw9+/5PQ8AGkGwAlgkrCBj0SJ/1/F7HgA0gmAFsEhYQUZXl7vqJ5Wq/PVUSsrl3PMAIGoEK4BFwgoy0ml3ebL3PeXXkKQdO6i3AqA5CFYAi4QZZOTz0hNPSIsXlx7PZt3j1FkB0Cwpx6m0biA5xsfH1dbWprGxMc2dOzfu5gBGqFTMLZdzA5WgQQYVbAFEIcj9m2AFsBRBBgCTBbl/U24fsFQ6LXV3x90KAGgcOSsAAMBoBCsAAMBoBCsAAMBoBCsAAMBoBCsAAMBoBCsAAMBoLF0GkDjUkAFaC8EKgESpVJ03m3W3GWALAMBOTAMBSIxCQertLQ1UJGl01D1eKMTTLgDRIlgBkAjFojuiUmmDEO/YwIB7HgC7EKwASIRDh6aPqEzlONLIiHseALsQrABIhBMnwj0PQHIQrABIhEWL/J33s58xFQTYJuU4lWaAkyPIFtMAkqtYlDo73WTaWr1WNitt3y5dcgnLmwFTBbl/s3QZQNM0Uh8lnXaXJ/f2SqnUzAHL669Lt9xSeozlzUByMQ0EoCkKBXdkpKdH6utzXzs7gy03zuelJ56QFi8O/u+zvBlILoIVAJELsz5KPi+9+qr00EPB2sDyZiC5CFYARCqK+ijptLRwYfC2sLwZSCaCFQCRiqo+it/VQZWwvBlIFoIVAJGKqj5KV5ebNJtKBW9TI4EOgOYjWAEQKb+BQdAAwlsdJPkPWFIpKZdzAx0AyUGwAiBStUZAGgkggqwO8v79HTuotwIkDcEKgEjNNAISRgDhrQ46cEDavdt9/Yd/cAOkqbJZN7ChzgqQPFSwBdAUhYK7Kmhqsm0u5wYqUQQQjRSgAxC9IPdvghUATeMngCDIAFoD5fYBGCmdlrq7q3+90ugLZfIBRJqzsnXrVq1atUoXXXSR3v/+91c85/jx47rpppt08cUXa8GCBfrsZz+rc+fORdksAAYKs8otALtEGqycO3dON998s/7kT/6k4teLxaJuvPFGnT17Vs8995wef/xx7du3T3/6p38aZbMAGCaKKrcA7BHpNNCWLVskSY899ljFrz/11FN66aWXNDIyoo6ODknSgw8+qD/+4z/W1q1byUEBWkSQKrczTSMBsFOsS5e/973vafny5ZOBiiT97u/+riYmJnTkyJGK3zMxMaHx8fGSDwDJFlWVWwB2iDVYOXnypBaW7Ub2K7/yK5o1a5ZOnjxZ8XuGhobU1tY2+ZHL5ZrRVAA1FIvSwYPSnj3ua5ApG7/Va3/2M6aCgFYUOFgZHBxUKpWa8eOHP/yh7+ulKpS1dByn4nFJuueeezQ2Njb5MTIyEvR/AUDICgWps1Pq6ZH6+tzXzk7/SbF+9/nZuDHYdQHYIXDOyoYNG3TrrbfOeE5nZ6eva7W3t+v73/9+ybG3335b//d//zdtxMWTyWSUyWR8XR9A9LxVPOXJsd4qHj9VY70qt729bsAyU/WnINcFYIfAwcqCBQu0YMGCUP7xq6++Wlu3btWJEye06P+PAz/11FPKZDJasWJFKP8GgOjUWsWTSrmreNasqV3Yzdvnp7zOSqPXBZB8keasHD9+XMeOHdPx48dVLBZ17NgxHTt2TO+8844kafXq1br88st1++236+jRo3r66af1uc99TuvWrWMlEBCTILknQVbx+OHt8/PQQzOfF/S6AJIt0qXLX/rSl/T1r3998vMrr7xSknTgwAF1d3crnU7rn//5n7V+/Xpdc801mj17tvr6+vRXf/VXUTYLQBVBK8hGsYonnZaqzAI3dF0AyRVpsPLYY49VrbHiWbJkif7pn/4pymYA8KGe3BO/q3j8nhf1dQEkExsZAlCx6K6yqTalk0q5IyzDw6U5It73jY5Wzlup9n1+2xP2dQGYI8j9O9Y6KwDMUG/uibeKR5q+7Nj7fMeO4AFFVNcFkEwEKwAayj3xVvEsXlx6PJttbHlxVNcFkDyR5qwASIZGc0TyeXcZ8aFDbkCzaJFb6K3RkY+orgsgWchZAUCOCICmI2cFQCDkiAAwGcEKAEnh54g0srEhAExFzgqASWHliAQtLgcAMyFnBUCoqhWX86aTWMkDQCJnBUBMam1sKLkbEDIlBCAIghUAoQlaXI68FgB+kLMCIDRBisuR1wLAL0ZWAITGb3G5l19281rKR2G8TRMLhfDbBiC5CFYAhKaryx0dKa/V4vGKy+3aRV4LAP8IVgCExk9xuXXr3BGUaqptmgigdRGsAAhVreJyy5b5u47f/BcA9iPBFkDoZioud/Cgv2v4zX8BYD+CFQCRSKel7u7px728llqbJnZ1Rd5EAAnBNBCApmLTRABBEawAaLqwN00EYDemgQDEIqxNEwHYj2AFQGyq5bUAwFRMAwEAAKMRrAAAAKMRrAAAAKMRrAAAAKMRrAAAAKMRrAAAAKMRrAAAAKMRrAAAAKMRrAAAAKMlvoKt8/+3bR0fH4+5JQAAwC/vvu1U2n69TOKDlTNnzkiScrlczC0BAABBnTlzRm1tbTOek3L8hDQGO3/+vN544w3NmTNHqfL95hs0Pj6uXC6nkZERzZ07N9Rr4z38nJuDn3Nz8HNuDn7OzRPVz9pxHJ05c0YdHR264IKZs1ISP7JywQUXKJvNRvpvzJ07lz+GJuDn3Bz8nJuDn3Nz8HNunih+1rVGVDwk2AIAAKMRrAAAAKMRrMwgk8novvvuUyaTibspVuPn3Bz8nJuDn3Nz8HNuHhN+1olPsAUAAHZjZAUAABiNYAUAABiNYAUAABiNYAUAABiNYKWKr371q1q6dKkuvPBCrVixQocOHYq7SVYZGhrSb/zGb2jOnDn6wAc+oE984hP67//+77ibZb2hoSGlUikNDAzE3RQrjY6O6lOf+pTmz5+viy66SL/+67+uI0eOxN0sq7z77ru69957tXTpUs2ePVsf/OAHdf/99+v8+fNxNy3Rnn32Wd10003q6OhQKpXSt7/97ZKvO46jwcFBdXR0aPbs2eru7taLL77YtPYRrFSwd+9eDQwMaPPmzTp69Ki6urp0ww036Pjx43E3zRrPPPOM7rrrLr3wwgvav3+/3n33Xa1evVpnz56Nu2nWOnz4sHbt2qVf+7Vfi7spVnr77bd1zTXX6Jd+6Zf0r//6r3rppZf04IMP6v3vf3/cTbPKl7/8ZT366KPauXOnfvzjH+uBBx7QX/7lX+rhhx+Ou2mJdvbsWX30ox/Vzp07K379gQce0Pbt27Vz504dPnxY7e3t+vjHPz65P1/kHEzzm7/5m86dd95Zcuyyyy5z7r777phaZL9Tp045kpxnnnkm7qZY6cyZM86yZcuc/fv3O9ddd53T398fd5Os88UvftG59tpr426G9W688Ubn05/+dMmxfD7vfOpTn4qpRfaR5HzrW9+a/Pz8+fNOe3u78xd/8ReTx/73f//XaWtrcx599NGmtImRlTLnzp3TkSNHtHr16pLjq1ev1vPPPx9Tq+w3NjYmSZo3b17MLbHTXXfdpRtvvFG/8zu/E3dTrPWd73xHK1eu1M0336wPfOADuvLKK/W1r30t7mZZ59prr9XTTz+tn/zkJ5Kk//zP/9Rzzz2n3/u934u5ZfYaHh7WyZMnS+6LmUxG1113XdPui4nfyDBsp0+fVrFY1MKFC0uOL1y4UCdPnoypVXZzHEebNm3Stddeq+XLl8fdHOs8/vjj+o//+A8dPnw47qZY7ac//akeeeQRbdq0SX/2Z3+mH/zgB/rsZz+rTCajP/qjP4q7edb44he/qLGxMV122WVKp9MqFovaunWr1q5dG3fTrOXd+yrdF1977bWmtIFgpYpUKlXyueM4044hHBs2bNB//dd/6bnnnou7KdYZGRlRf3+/nnrqKV144YVxN8dq58+f18qVK7Vt2zZJ0pVXXqkXX3xRjzzyCMFKiPbu3au///u/1+7du3XFFVfo2LFjGhgYUEdHh+644464m2e1OO+LBCtlFixYoHQ6PW0U5dSpU9OiSjTuM5/5jL7zne/o2WefVTabjbs51jly5IhOnTqlFStWTB4rFot69tlntXPnTk1MTCidTsfYQnssWrRIl19+ecmxD3/4w9q3b19MLbLT5z//ed1999269dZbJUkf+chH9Nprr2loaIhgJSLt7e2S3BGWRYsWTR5v5n2RnJUys2bN0ooVK7R///6S4/v379eqVatiapV9HMfRhg0bVCgU9O///u9aunRp3E2y0vXXX68f/ehHOnbs2OTHypUrddttt+nYsWMEKiG65pprpi2//8lPfqJLL700phbZ6Re/+IUuuKD01pVOp1m6HKGlS5eqvb295L547tw5PfPMM027LzKyUsGmTZt0++23a+XKlbr66qu1a9cuHT9+XHfeeWfcTbPGXXfdpd27d+vJJ5/UnDlzJkey2traNHv27JhbZ485c+ZMywO6+OKLNX/+fPKDQrZx40atWrVK27Zt0y233KIf/OAH2rVrl3bt2hV306xy0003aevWrVqyZImuuOIKHT16VNu3b9enP/3puJuWaO+8845eeeWVyc+Hh4d17NgxzZs3T0uWLNHAwIC2bdumZcuWadmyZdq2bZsuuugi9fX1NaeBTVlzlEB/8zd/41x66aXOrFmznI997GMsqQ2ZpIoff/u3fxt306zH0uXo/OM//qOzfPlyJ5PJOJdddpmza9euuJtknfHxcae/v99ZsmSJc+GFFzof/OAHnc2bNzsTExNxNy3RDhw4ULFPvuOOOxzHcZcv33fffU57e7uTyWSc3/7t33Z+9KMfNa19KcdxnOaERQAAAMGRswIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIxGsAIAAIz2/wAni/0vx03zBgAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Generate data\n",
    "X = np.linspace(0, 10, 100).reshape(-1, 1)  # Ensure X is 2D\n",
    "Y = (np.sin(X).flatten() + np.random.normal(0, 0.1, 100)) * 10\n",
    "\n",
    "# Shuffle data\n",
    "indices = np.arange(100)\n",
    "np.random.shuffle(indices)\n",
    "X = X[indices]\n",
    "Y = Y[indices].reshape(-1, 1)\n",
    "\n",
    "# Split data\n",
    "train_X, test_X = X[:80], X[80:]\n",
    "train_Y, test_Y = Y[:80], Y[80:]\n",
    "\n",
    "# Plot the training data\n",
    "plt.scatter(train_X, train_Y, marker=\"o\", color=\"b\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 1) (50, 1) (50, 1) (50, 1)\n",
      "(1, 1) (1, 1) (1, 1)\n",
      "(50, 1) (50, 1) (50, 1)\n",
      "-------------------\n",
      "(1, 50) (1, 50) (1, 1) (1, 1)\n",
      "(50, 1) (50, 1) (50, 1)\n",
      "(1, 50) (1, 1) (1, 50)\n",
      "-------------------\n",
      "(50,)\n",
      "(50, 1)\n",
      "[[ 1.16086536]\n",
      " [ 0.04063328]\n",
      " [ 1.35129743]\n",
      " [ 0.47088579]\n",
      " [ 0.69244952]\n",
      " [-1.11214724]\n",
      " [-0.14585151]\n",
      " [ 0.35183856]\n",
      " [-1.35210007]\n",
      " [ 1.71153226]\n",
      " [-0.48583218]\n",
      " [ 0.84340244]\n",
      " [ 0.57993642]\n",
      " [-0.8325415 ]\n",
      " [-0.99929367]\n",
      " [ 1.31858694]\n",
      " [-0.31663125]\n",
      " [-0.81267474]\n",
      " [ 0.28662022]\n",
      " [-0.47165138]\n",
      " [ 0.41006863]\n",
      " [-0.38878381]\n",
      " [-0.08083376]\n",
      " [-0.19521674]\n",
      " [ 0.68244962]\n",
      " [-0.64028493]\n",
      " [-0.07037773]\n",
      " [ 1.10855274]\n",
      " [ 1.41792343]\n",
      " [ 0.84868649]\n",
      " [ 1.31456219]\n",
      " [ 0.21532622]\n",
      " [-0.78184303]\n",
      " [-0.18647984]\n",
      " [-0.03499289]\n",
      " [ 0.77010087]\n",
      " [-0.3007951 ]\n",
      " [ 0.51033972]\n",
      " [-0.94175051]\n",
      " [ 1.49623762]\n",
      " [ 0.24770997]\n",
      " [ 0.22090454]\n",
      " [ 1.65499934]\n",
      " [ 0.1369016 ]\n",
      " [-0.29267989]\n",
      " [-0.1802707 ]\n",
      " [-0.07754098]\n",
      " [ 0.61889833]\n",
      " [-0.36762121]\n",
      " [-0.06722829]]\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "non-broadcastable output operand with shape (50,) doesn't match the broadcast shape (50,50)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[64], line 32\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(epochs):\n\u001b[1;32m     31\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m x, y \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(train_X, train_Y):\n\u001b[0;32m---> 32\u001b[0m         \u001b[43mnet\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mforward\u001b[49m\u001b[43m(\u001b[49m\u001b[43mx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     33\u001b[0m         net\u001b[38;5;241m.\u001b[39mbackward(y, \u001b[38;5;241m0.01\u001b[39m)\n",
      "Cell \u001b[0;32mIn[62], line 208\u001b[0m, in \u001b[0;36mTAGI.forward\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[38;5;28mprint\u001b[39m(mb\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m    207\u001b[0m \u001b[38;5;28mprint\u001b[39m(mz\u001b[38;5;241m.\u001b[39mreshape(\u001b[38;5;241m50\u001b[39m,\u001b[38;5;241m1\u001b[39m))\n\u001b[0;32m--> 208\u001b[0m mz \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m mb\n\u001b[1;32m    209\u001b[0m Sz \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m Sb\n\u001b[1;32m    211\u001b[0m \u001b[38;5;66;03m# Apply ReLU if not the output layer\u001b[39;00m\n",
      "\u001b[0;31mValueError\u001b[0m: non-broadcastable output operand with shape (50,) doesn't match the broadcast shape (50,50)"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "net = TAGI([1, 50, 1])\n",
    "train_X = net.standardize_data(train_X)\n",
    "train_Y = net.standardize_data(train_Y)\n",
    "\n",
    "\n",
    "for i in range(len(net.layers) - 1):\n",
    "    print(\n",
    "        net.parameters[i][\"mw\"].shape,\n",
    "        net.parameters[i][\"Sw\"].shape,\n",
    "        net.parameters[i][\"mb\"].shape,\n",
    "        net.parameters[i][\"Sb\"].shape,\n",
    "    )\n",
    "    print(\n",
    "        net.hidden_states[i][\"ma\"].shape,\n",
    "        net.hidden_states[i][\"Sa\"].shape,\n",
    "        net.hidden_states[i][\"J\"].shape,\n",
    "    )\n",
    "    print(\n",
    "        net.covariances[i][\"covZw\"].shape,\n",
    "        net.covariances[i][\"covZb\"].shape,\n",
    "        net.covariances[i][\"covZZ\"].shape,\n",
    "    )\n",
    "    print(\"-------------------\")\n",
    "\n",
    "# net.forward(train_X[0])\n",
    "# net.backward(train_Y[1], 0.01)\n",
    "\n",
    "epochs = 1\n",
    "for epoch in range(epochs):\n",
    "    for x, y in zip(train_X, train_Y):\n",
    "        net.forward(x)\n",
    "        net.backward(y, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009 -1.45961009\n",
      " -1.45961009 -1.45961009]\n",
      "[-1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364 -1.86225364\n",
      " -1.86225364 -1.86225364]\n",
      "[-1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807 -1.0583807\n",
      " -1.0583807 -1.0583807]\n",
      "[-0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158 -0.09504158\n",
      " -0.09504158 -0.09504158]\n",
      "[-0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471 -0.95145471\n",
      " -0.95145471 -0.95145471]\n",
      "[-0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674 -0.36413674\n",
      " -0.36413674 -0.36413674]\n",
      "[-1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066 -1.8478066\n",
      " -1.8478066 -1.8478066]\n",
      "[-1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062 -1.23020062\n",
      " -1.23020062 -1.23020062]\n",
      "[-0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754 -0.98749754\n",
      " -0.98749754 -0.98749754]\n",
      "[-0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382 -0.53347382\n",
      " -0.53347382 -0.53347382]\n",
      "[-1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194 -1.00523194\n",
      " -1.00523194 -1.00523194]\n",
      "[-1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215 -1.67444215\n",
      " -1.67444215 -1.67444215]\n",
      "[-0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439 -0.68129439\n",
      " -0.68129439 -0.68129439]\n",
      "[-1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438 -1.26549438\n",
      " -1.26549438 -1.26549438]\n",
      "[-1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032 -1.53235032\n",
      " -1.53235032 -1.53235032]\n",
      "[-0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538 -0.40798538\n",
      " -0.40798538 -0.40798538]\n",
      "[-0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267 -0.81979267\n",
      " -0.81979267 -0.81979267]\n",
      "[-1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438 -1.76112438\n",
      " -1.76112438 -1.76112438]\n",
      "[-1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559 -1.51402559\n",
      " -1.51402559 -1.51402559]\n",
      "[-0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631 -0.83961631\n",
      " -0.83961631 -0.83961631]\n",
      "[[  5.12036633]\n",
      " [  1.52294486]\n",
      " [-10.15282335]\n",
      " [  1.40854141]\n",
      " [ -6.80492819]\n",
      " [ 10.41619552]\n",
      " [  3.53004582]\n",
      " [ -7.39830981]\n",
      " [ -8.87442799]\n",
      " [ 10.00862719]\n",
      " [ -9.83376315]\n",
      " [  9.97712093]\n",
      " [  8.84365709]\n",
      " [ -5.7975826 ]\n",
      " [  7.89045139]\n",
      " [  9.79155124]\n",
      " [  0.48095593]\n",
      " [  9.33510326]\n",
      " [  7.81515854]\n",
      " [ -1.79384722]]\n"
     ]
    }
   ],
   "source": [
    "# Predict\n",
    "test_Y_pred = []\n",
    "for x in test_X:\n",
    "    prediction = net.forward(x)\n",
    "    test_Y_pred.append(prediction[0])\n",
    "    print(prediction)\n",
    "test_Y_pred = np.array(test_Y_pred).flatten()\n",
    "test_Y_pred = net.unstandardize_data(test_Y_pred)\n",
    "\n",
    "print(test_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x10782c6d0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAisAAAGdCAYAAADT1TPdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAs3ElEQVR4nO3df3RUZX7H8c8wQAJIxpUfIWEGEn/0iGKVHz0U3GjobrHWUugYV8FaOKu2trImoitaXAVXTLXVhq5CgV1ht3tAWpjdrj3urqwFjWVZWJbYXaGLKyAxJA14bAZYCTA8/eM6MZNMJjPJzNw7c9+vc+bkzJ2b5CtjZj7z3Of5Ph5jjBEAAIBDDbC7AAAAgEQIKwAAwNEIKwAAwNEIKwAAwNEIKwAAwNEIKwAAwNEIKwAAwNEIKwAAwNEG2l1Af124cEHHjh3T8OHD5fF47C4HAAAkwRijkydPqrS0VAMGJB47yfmwcuzYMQUCAbvLAAAAfdDY2Ci/35/wnJwPK8OHD5dk/ccWFRXZXA0AAEhGOBxWIBDoeB9PJOfDSvTST1FREWEFAIAck8wUDibYAgAARyOsAAAARyOsAAAARyOsAAAARyOsAAAARyOsAAAARyOsAAAARyOsAAAAR8v5pnDoWSQi1ddLzc1SSYlUUSF5vXZXBQBAaggreSoUkqqrpQ8//OyY3y+tXCkFg/bVBQBAqrgMlIdCIamqKjaoSFJTk3U8FLKnLgAA+oKwkmciEWtExZjuj0WP1dRY5wEAkAsIK3mmvr77iEpnxkiNjdZ5AADkAsJKnmluTu95AADYjbCSZ0pK0nseAAB2YzVQnqmosFb9NDXFn7fi8ViPV1Rkty6WUQPINbxuOUdGR1beeustzZ49W6WlpfJ4PPr+978f87gxRsuWLVNpaamGDBmiyspKvfvuu5ksKe95vdbyZMkKJp1F79fVZfcPLhSSysqkmTOl+fOtr2VlrEoC4Fy8bjlLRsPK6dOnde211+rFF1+M+/hzzz2nF154QS+++KL27NmjMWPG6A//8A918uTJTJaV94JBacsWaezY2ON+v3U8m31WWEYNINfwuuU8HmPiXSzIwC/yePS9731Pc+fOlWSNqpSWlqqmpkZLliyRJLW3t6u4uFjPPvus/uqv/iqpnxsOh+Xz+dTW1qaioqJMlZ+Tkh3CzNRQZyRifRLpaXVS9JLU4cMMrQJwBl63sieV92/bJtgePnxYLS0tmjVrVsexgoIC3Xjjjdq5c2eP39fe3q5wOBxzQ3xer1RZKc2bZ32N94eVyaFOllEDyDW8bjmTbWGlpaVFklRcXBxzvLi4uOOxeGpra+Xz+TpugUAgo3Xms0wPdbKMGkCu4XXLmWxfuuzpMgvUGNPtWGePPfaY2traOm6NjY2ZLjEvZaPTLcuoAeQaXrecybawMmbMGEnqNorS2trabbSls4KCAhUVFcXckLpsDHVGl1H3lD09HikQyP4yagDoCa9bzmRbWCkvL9eYMWO0bdu2jmNnz57Vm2++qRkzZthVlmtkY6jTicuoASARXrecKaNh5dSpU2poaFBDQ4Mka1JtQ0ODjh49Ko/Ho5qaGj3zzDP63ve+p1/96ldauHChhg4dqvnz52eyLCh7Q51OWkYNAMngdct5Mrp0eceOHZo5c2a34wsWLNCGDRtkjNHy5cu1Zs0affzxx5o2bZpeeuklTZw4MenfwdLlvokuz+ut0226lufRCRJAruF1K7NSef/OWp+VTCGs9F10NZAUG1iiQ518ggCA7HNLSMqJPiuwH0OdAOAstPmPj5EVuCbFA4CTRUe7u74r5+toN5eBAADIIW5s889lIAAAcght/hMjrAAAYDPa/CdGWAEAwGa0+U+MsAIAgM1o858YYQV5LRKRduyQNm2yvvZnY0YAyBTa/CdGWEHeol8BgFxC76uesXQZeclt/QoA5A+39L6izwpczY39CgAg19BnBa5GvwIAyC+EFeQd+hUAQH4hrCDv0K8AAPILYQV5h34FAJBfCCvIO/QrAID8QlhBXqJfAQDkj4F2FwBkSjAozZnjjn4FAJDPCCvIa16vVFlpdxUAgP7gMhAAAHA0wgoAAHA0wgoAAHA05qwAAGzhlg370H+EFQBAVnQOJ++9J61dKzU1ffa432/1SKK1ALoirAAAMi4UkqqrE28y2tQkVVXRCwndMWcFAJBRoZAVQhIFFcnaEV2SamqsURggirACAMiYSMQaUYkGkd4YIzU2WpeLgCjCCgAgY+rrex9Riae5Of21IHcRVgAAGdPX0FFSkt46kNuYYAsAyJhUQ4fHY60KqqjITD3ITYysAAAypqLCCh8eT+/nRs+pq6PfCmIRVgAAGeP1Wr1TpN4Di9/PsmXEx2UgAEBGBYNWCOnaZ8Xvl+69V7riCjrYIjHCCgAg44JBac4c2uujbwgrAICs8Hqlykq7q0AuYs4KAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwNMIKAABwtIF2FwAAyB2RiFRfLzU3SyUlUkWF5PXaXRXyHWEFAJCUUEiqrpY+/PCzY36/tHKlFAzaVxfyH5eBAAC9CoWkqqrYoCJJTU3W8VDInrrgDraHlWXLlsnj8cTcxowZY3dZAIBPRSLWiIox3R+LHqupsc4DMsERl4Guvvpq/eQnP+m47+UCKAA4Rn199xGVzoyRGhut8yors1YWXMQRYWXgwIGMpgCAQzU3p/c8IFW2XwaSpPfee0+lpaUqLy/XHXfcoUOHDvV4bnt7u8LhcMwNAJA5JSXpPQ9Ile1hZdq0afrOd76jH//4x1q3bp1aWlo0Y8YMffTRR3HPr62tlc/n67gFAoEsVwwA7lJRYa368XjiP+7xSIGAdR6QCR5j4k2Zss/p06d12WWX6ZFHHtHixYu7Pd7e3q729vaO++FwWIFAQG1tbSoqKspmqQDgGtHVQFLsRNtogNmyheXLSE04HJbP50vq/dv2kZWuhg0bpmuuuUbvvfde3McLCgpUVFQUcwMAZFYwaAWSsWNjj/v9BBVkniMm2HbW3t6uAwcOqILxRABwlGBQmjOHDrbIPtvDysMPP6zZs2dr3Lhxam1t1dNPP61wOKwFCxbYXRoAoAuvl+XJyD7bw8qHH36oefPm6cSJExo1apR+//d/X7t27dL48ePtLg0AADiA7WHllVdesbsEAADgYI6bYAsAANAZYQUAADgaYQUAADgaYQUAADgaYQUAADgaYQUAADgaYQUAADgaYQUAADgaYQUAADgaYQUAADgaYQUAADia7XsDAQAA+0UiUn291NwslZRIFRXWLttOQFgBAMDlQiGpulr68MPPjvn90sqVUjBoX11RXAYCAMDFQiGpqio2qEhSU5N1PBSyp67OCCsAALhUJGKNqBjT/bHosZoa6zw7EVYAAHCp+vruIyqdGSM1Nlrn2YmwAgCASzU3p/e8TCGsAADgUiUl6T0vUwgrAAC4VEWFterH44n/uMcjBQLWeXYirAAA4FJer7U8WeoeWKL36+rs77dCWAEAwMWCQWnLFmns2Njjfr913Al9VmgKBwCAywWD0pw5dLAFAAAO5vVKlZV2VxEfl4EAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjEVYAAICjDbS7AAAA4EyRiFRfLzU3SyUlUkWF5PVmvw7CCgAA6CYUkqqrpQ8//OyY3y+tXCkFg9mthctAPYhEpB07pE2brK+RiN0VAQCQHaGQVFUVG1QkqanJOh4KZbcewkocoZBUVibNnCnNn299LSvL/pMDAEC2RSLWiIox3R+LHqupye6HeMJKF05LkwAAZFN9fff3wM6MkRobrfOyhbDSSec0OUAR3agdukObdKN2yGOsCJntNAkAQDY1N6f3vHQgrHQSTZN/ppCOqEw7NFObNF87NFNHVKa5JpT1NAkAQDaVlKT3vHQgrHTS3GwFlS2q0ljFjoGNVZO2qEp/plBW0yQAANlUUWGt+vF44j/u8UiBgHVethBWOikZHdFKVUsy3f5hBsiaVVSnGpWM5joQACA/eb3W8mSpe2CJ3q+ry26/FcJKJxWqV0Af9viPMkBG49SoCnEdCACQv4JBacsWaezY2ON+v3U8231WaArXibc1ues7yZ4HAECuCgalOXPoYOs8TpxVBACATbxeqbLS7iq4DBTLibOKAABwOcJKZ06cVQQAgMsRVrpy2qwiAABcjjkr8ThpVhEAAC5HWOmJU2YVAQDgcoQVIAsiEQbqAKCvCCtAhoVC1gaZnXcx9futudxMgQKA3jHBFsigUEiqquq+3XpTk3U8FLKnLgDIJYQVIEMiEWtExZjuj0WP1dRY5wEAekZYATKkvr77iEpnxkiNjdZ5AICeEVaADGlOcgupZM8DALcirAAZwlZTAJAehBUgQ9hqCgDSwxFhZdWqVSovL1dhYaGmTJmiei7iIw+w1RQApIftYWXz5s2qqanR0qVLtW/fPlVUVOjmm2/W0aNH7S4N6De2mgKA/vMYE29hZfZMmzZNkydP1urVqzuOTZgwQXPnzlVtbW2v3x8Oh+Xz+dTW1qaioqJMlgr0GR1sASBWKu/ftnawPXv2rPbu3atHH3005visWbO0c+dOm6oC0o+tpgCg72wNKydOnFAkElFxcXHM8eLiYrW0tMT9nvb2drW3t3fcD4fDGa0RAADYy/Y5K5Lk6TL70BjT7VhUbW2tfD5fxy0QCGSjRAAAYBNbw8rIkSPl9Xq7jaK0trZ2G22Jeuyxx9TW1tZxa2xszEapAADAJraGlcGDB2vKlCnatm1bzPFt27ZpxowZcb+noKBARUVFMTcAAJC/bJ2zIkmLFy/WXXfdpalTp2r69Olau3atjh49qvvuu8/u0gAAgAPYHlZuv/12ffTRR3rqqafU3NysiRMn6rXXXtP48ePtLi33sV4WAJAHbO+z0l/0WelBKCRVV8du++v3Wy1V6UQGALBZKu/fjlgNhDQLhaSqqtigIklNTdbxUMieugAA6APCSr6JRKwRlXgDZtFjNTXWeQAA5ADCSr6pr+8+otKZMVJjo3VeVCQi7dghbdpkfSXIAAAcxPYJtkiz5ubUzmNuCwDA4RhZyTclJcmfx9wWAEAOIKzkm4oKa2Skh+0K5PFIgYA0YwZzWwAAOYGwkm+8XusSjtQ9sETv19VJO3emPrcFAAAbEFbyUTAobdkijR0be9zvt44Hg6nPbQEAwCZMsM1XwaA0Z07PHWxTmdsCAICNCCv5zOuVKivjPxad29LUFH/eisdjPV5RkdESAQDoDZeB3CrZuS3sJQQAsBlhxc2SmdsCAIDNuAzkdr3NbQEAwGaEFSSe2wIAgM24DAQAAByNsAIAAByNy0DIjkiEeTEAgD4hrCDz2Nk5LvIbACSHy0DILHZ2jisUksrKpJkzpfnzra9lZa795wCAhAgryJxIhJ2d4yC/AUBqCCvInPp6dnbugvwGAKkjrCBz2Nm5G/IbAKSOsILMYWfnbshvAJA6wgoyJ7qzc9eNEqM8HikQcNXOzuQ3AEgdYQWZw87O3ZDfACB1hBVkFjs7xyC/AUDqPMbEW5eQO8LhsHw+n9ra2lRUVGR3OegJHdBixOuTFwhYQcVl+Q2AS6Xy/k1YAWxCfgPgZqm8f9NuH7CJ1ytVVtpdBQA4H3NWAACAoxFWAACAoxFWAACAozFnBfmNWawAkPMIK8hf8dYH+/1WoxPWBwNAzuAyEPJTKCRVVXXfNbCpyToeCtlTFwAgZYQV5J9IxBpRiddCKHqspsY6DwDgeIQV5J/6+u4jKp0ZIzU2WucBAByPsIL809yc3vMAALYirCD/lJSk9zwAgK0IK8g/FRXWqp+u2xpHeTzWroEVFdmtCwDQJ4QV5B+v11qeLHUPLNH7dXX0WwGAHEFYQX4KBqUtW6SxY2OP+/3WcfqsAEDOoCkc8lcwKM2ZQwdbAMhxhBXkN69Xqqy0uwoAQD9wGQgAADgaYQUAADgaYQUAADgac1aAbIhEmOgLAH1EWAEyLRSyNlbsvF+R32/1gmEJNQD0irACZFIoJFVVdd8BuqlJuvVWafly6YorGG0BgAQ8xnR9Fc0t4XBYPp9PbW1tKioqsrsc4DORiFRWlngH6M4YbQHgIqm8fzPBFsiU+vrkg4pkjbZUVVmjMQCADoQVIFOam1M7PzrIWVNjjcoAACQRVoDMKSlJ/XuMkRobrVEZAIAkwgqQORUV1jyUrjs/JyPVURkAyGOEFSBTvF5rwqyUemDpy6gMAOQpwgqQScGgtGWLNHZscud7PFIgYI3KAAAkEVaAzAsGpSNHpO3bpY0brd4qHk/30Zbo/bo6+q0AQCc0hQOyweuVKis/uz9xYvyutnV19FkBgC4IK4AdgkFpzhz2CwKAJBBWALt0HW0BAMRFWAHchh2gAeQYwgrgJuwADSAHsRoIcIvoDtBd9ytiTyIADkdYAdwgErFGVOJtss6eRAAcjrACuEFvO0DH25MoEpF27JA2bbK+EmQA2IQ5K4AbJLvXUPQ85rYAcBDCCuAGye41VFLy2dyWrpeMmpqkW2+1OvBecQUriQBkjceYeBexc0c4HJbP51NbW5uKiorsLgdwpkhEKiuzAke8P3mPxxo5+c1vpMsuS3zJqDNGWwD0USrv37bOWSkrK5PH44m5Pfroo3aWBOSnRDtAd96TaOfO5IOKxEoiAFlh+wTbp556Ss3NzR23xx9/3O6SgPzU0w7Qfr91PBhMfm5LFCuJAGSB7XNWhg8frjFjxthdBuAOve1JlOzcls46ryRi+wAAGWDrnJWysjK1t7fr7NmzCgQCuu222/TVr35VgwcP7vF72tvb1d7e3nE/HA4rEAgwZwVIh97mtiSycaM0b17in02bfwCfypk5K9XV1XrllVe0fft2LVq0SHV1dfqbv/mbhN9TW1srn8/XcQsEAlmqFnCBRHNbepNoVCYUskLQzJnS/PnW17Iy5roASEraR1aWLVum5cuXJzxnz549mjp1arfjW7duVVVVlU6cOKERI0bE/V5GVoAsiNdnpSfRlUSHD8cfKelpKXQ0DEXnywBwlVRGVtIeVk6cOKETJ04kPKesrEyFhYXdjjc1Ncnv92vXrl2aNm1aUr+PpctAhnS+bPPee9KyZdbxzi8ZvQWO6GWlnkJPb0EHQN5K5f077RNsR44cqZEjR/bpe/ft2ydJKunLJD8A6eX1xk6YnTgxflfburqeR0ZSafPP5FwAPbBtNdBPf/pT7dq1SzNnzpTP59OePXv04IMP6k//9E81btw4u8oC0JPeVhLFk2qbfwCIw7awUlBQoM2bN2v58uVqb2/X+PHjde+99+qRRx6xqyQAvek62tKbVNr8A0APaLcPIHOSbfPPnBXAdXJm6TKAPJdsm3+CCoAECCsAMiuZNv95JhKRduyQNm2yvrITAdA/trfbB+ACfZmcm6Pitahhc2qgf5izAgBpQv87IHnMWQGALItErBGVeB//2Jwa6B/CCgCkQSr97wCkhrACAGlA/zsgcwgrAJAG9L8DMoewAgBpUFFhrfrp2k4myuORAgHrPACpIawAQBrQ/w7IHMIKAKSJC/vfAVlBUzgASFYk0mtjOxf1vwOyhrACAMlIoTVtqptTA0iMy0AA0Jtoa9qujVSamqzjoZA9dQEuQVgBgET60pqWnQyBtCKsAEAiqbamDYWksjJp5kxp/nzra1kZoy9APxBWACCRVFrTcrkIyAjCCuAyXKFIUbItZ0ePZidDIEMIK4CLcIWiD5JtTSuxkyGQIYQVwCW4QtFHybambW1N7uexkyGQMsIK4AJ9WdCCTpJpTctOhkDGeIyJ9/KVO8LhsHw+n9ra2lRUVGR3OYAj7dhhXfLpzfbtNDNLKFEH20jEuqbW1BQ/FXo8Vrg5fJh2toBSe/+mgy3gAqksaEECiVrTRi8XVVVZwaRzYGEnQ6BfuAwEuABXKLKEnQyBjOAyEOACXKHIsiQ2PATcjstAAGJwhSLL2MkQSCsuAwEuwRWK/qOhHmAPRlYAFwkGpTlzuELRF6GQtfy7c58av98asSLoAZnFnBUA6EW0oV7XV8voJTRGpoDUpfL+zWUgAEiAhnqA/QgrAJBAfT1b/gB2I6wAQAI01APsR1gBgARoqAfYj7ACAAlUVFirfrpuuBzl8UiBgHUegMwgrABAAtGGelL3wEJDPSA7CCsA0Asa6nVCZzzYgKZwAJAEGuqJzniwDU3hAAC9ozMe0oymcACA9KEzHmxGWAEAJEZnPNiMsAIASIzOeLAZE2wBAImlqzNeJOLyGcroK0ZWAACJpaMzXigklZVJM2dK8+dbX8vKrONALwgrAIDE+tsZL7qSqOu8l6Ym6dZbpQcfpGcLEiKsAAB619fOeMmsJKqrY6QFCdFnBQCQvFTnnezYYQWRZNCzxVVSef92zQTbSCSic+fO2V0G+mjQoEHyMhEPsJ/XK1VWJn9+KiuEjLECS02N1S6Yv3l8Ku/DijFGLS0t+r//+z+7S0E/XXzxxRozZow8PU3yA+A8ya4kiurcsyWVUIS8lvdhJRpURo8eraFDh/JGl4OMMfrtb3+r1tZWSVJJqi9+AOwTXUnU1BR/3kpP6NmCTvI6rEQikY6gMmLECLvLQT8MGTJEktTa2qrRo0dzSQjIFdGVRFVV1iWeZAMLH0rQSV6vBorOURk6dKjNlSAdos8jc4+yIxKx5kZu2sSqUvRTTyuJ4kmmZwtcJ6/DShSXfvIDz2P20L8LaRcMSkeOSNu3WxNo40mmZwtcyRVhBUDyEvXvqqoisKAfoiuJ/vEfpa1brbksnfXWswWulddzVgCkprf+XawqRdoEg9b/SOwVhCQwsuIwHo8n4W3hwoV2l4g8Vl/ffUSls86rSoF+i460zJtnfSWooAeMrCQhmxuFNndarrd582Y98cQT+vWvf91xLLoqJurcuXMaNGhQZoqB6yS7WpRVpQCyiZGVXmR7ouGYMWM6bj6fTx6Pp+P+mTNndPHFF+tf//VfVVlZqcLCQn33u9/VsmXLdN1118X8nLq6OpWVlcUcW79+vSZMmKDCwkJdeeWVWrVqVWb+I5Czkl0tyqpSANlEWEnAqRMNlyxZogceeEAHDhzQTTfdlNT3rFu3TkuXLtWKFSt04MABPfPMM/ra176mb3/72xmuFrkk2r+rp4VXrCoFYAcuA/XAyRMNa2pqFExxtvzXv/51Pf/88x3fV15erv3792vNmjVasGBBJspEDkrUv4tVpXCdbM4BQEKMrPTAyRMNp06dmtL5x48fV2Njo+6++25ddNFFHbenn35a77//foaqRK7qqX8Xq0rhKjQbchRGVnrg5ImGw4YNi7k/YMAAmS5DQJ27vF64cEGSdSlo2rRpMefRth7xsKoUrhadA9B1aD06B4DUnnWElR7k0kTDUaNGqaWlRcaYji6vDQ0NHY8XFxdr7NixOnTokO68806bqkSuia4qBVzFyXMAXIyw0oPeNgr1eKzHnTDRsLKyUsePH9dzzz2nqqoq/ehHP9IPf/hDFRUVdZyzbNkyPfDAAyoqKtLNN9+s9vZ2/fznP9fHH3+sxYsX21g9ADhIKnMASPNZw5yVHkQnGkrdV0Y4baLhhAkTtGrVKr300ku69tprtXv3bj388MMx59xzzz365je/qQ0bNuiaa67RjTfeqA0bNqi8vNymqgHAgZw8B8DFPKbrZIccEw6H5fP51NbWFjOSIElnzpzR4cOHVV5ersLCwj79/FDIGhHsHLQDASuocMkyu9LxfAJAQjt2WJNpe7N9OyMr/ZTo/bsrLgP1gomGwGdYyYm8l0tzAFyEsJIEJhoC8UcZ/X7rcimjjMgbNBtyJOasAOiVU7s5AxlBsyHHYWQFQEKs5IQr2TkHgOut3WR0ZGXFihWaMWOGhg4dqosvvjjuOUePHtXs2bM1bNgwjRw5Ug888IDOnj2bybIApMDJ3ZyBjIrOAZg3z/qajcBA59y4MhpWzp49q9tuu01//dd/HffxSCSiW265RadPn9bbb7+tV155RVu3btVDDz2UybIApCBdKzkjEWuhxaZN1tdIpL+VAXmG6609yuhloOXLl0uSNmzYEPfx119/Xfv371djY6NKS0slSc8//7wWLlyoFStW9LqUCUDmpaObM5NzgV5wvTUhWyfY/vSnP9XEiRM7gook3XTTTWpvb9fevXvjfk97e7vC4XDMDUDmRFdydm2OGOXxWL2HelrJyYdFIAlcb03I1rDS0tKi4uLimGOf+9znNHjwYLW0tMT9ntraWvl8vo5bIBDIRqmAa/Wnm3NvHxYl68Mil4TgenTOTSjlsLJs2TJ5PJ6Et5///OdJ/zxPnI9rnTfk6+qxxx5TW1tbx62xsTHV/wR0smzZMl133XUd9xcuXKi5c+dmvY4jR47I4/HEbMAI5+jrSk4+LAJJyqXdc22Q8pyVRYsW6Y477kh4TllZWVI/a8yYMfrZz34Wc+zjjz/WuXPnuo24RBUUFKigoCCpn5/LFi5cqG9/+9uSpIEDByoQCCgYDGr58uUaNmxYxn7vypUrlewODEeOHFF5ebn27dsXE3iQn/qykpMPi0CS6JybUMphZeTIkRo5cmRafvn06dO1YsUKNTc3q+TTtPj666+roKBAU6ZMScvvSAub1rz/0R/9kdavX69z586pvr5e99xzj06fPq3Vq1fHnHfu3DkNGjQoLb/T5/Ol5ecgP6XazZkPi0CS6JybUEbnrBw9elQNDQ06evSoIpGIGhoa1NDQoFOnTkmSZs2apauuukp33XWX9u3bpzfeeEMPP/yw7r33XuesBLJxzXtBQYHGjBmjQCCg+fPn684779T3v//9jks3L7/8si699FIVFBTIGKO2tjb95V/+pUaPHq2ioiL9wR/8gd55552Yn/l3f/d3Ki4u1vDhw3X33XfrzJkzMY93vQx04cIFPfvss7r88stVUFCgcePGacWKFZLUsWPzpEmT5PF4VNnpXWz9+vWaMGGCCgsLdeWVV2rVqlUxv2f37t2aNGmSCgsLNXXqVO3bty+N/3Jwiv5OzgVcxe7OuU7uL2AyaMGCBUZSt9v27ds7zvnggw/MLbfcYoYMGWIuueQSs2jRInPmzJmkf0dbW5uRZNra2ro99sknn5j9+/ebTz75pG//AVu3GuPxGGNl3M9uHo9127q1bz83CQsWLDBz5syJOfaVr3zFjBgxwjz55JNm2LBh5qabbjK/+MUvzDvvvGMuXLhgrr/+ejN79myzZ88ec/DgQfPQQw+ZESNGmI8++sgYY8zmzZvN4MGDzbp168z//M//mKVLl5rhw4eba6+9tsff+8gjj5jPfe5zZsOGDeY3v/mNqa+vN+vWrTPGGLN7924jyfzkJz8xzc3NHb9n7dq1pqSkxGzdutUcOnTIbN261VxyySVmw4YNxhhjTp06ZUaNGmVuv/1286tf/cq8+uqr5tJLLzWSzL59+3r8N+n38wlbRP+Muv4pZeHPCMhN588bs327MRs3Wl/Pn8/879y61Ri/P/aP1O835t/+LWO1JHr/7iqjYSUbMhZWzp/v/sR1faUNBDL2P1HX0PCzn/3MjBgxwnzpS18yTz75pBk0aJBpbW3tePyNN94wRUVF3YLeZZddZtasWWOMMWb69Onmvvvui3l82rRpPYaVcDhsCgoKOsJJV4cPH44bMAKBgNm4cWPMsa9//etm+vTpxhhj1qxZYy655BJz+vTpjsdXr15NWMlj8V4HAwGCCuAIPX0wj3fz+9P2h5tKWGEjw544YBnDf/zHf+iiiy5SYWGhpk+frhtuuEHf+MY3JEnjx4/XqFGjOs7du3evTp06pREjRuiiiy7quB0+fFjvv/++JOnAgQOaPn16zO/oer+zAwcOqL29XV/4wheSrvn48eNqbGzU3XffHVPH008/HVPHtddeq6FDhyZVB3JfMCgdOSJt3y5t3Gh9PXyYhnCA7RL1F4jHpgZJbGTYEwcsY5g5c6ZWr16tQYMGqbS0NGYSbdcVQRcuXFBJSYl27NjR7ef0tC9Tb4YMGZLy91y4cEGStG7dOk2bNi3mMe+nE8NMsn8UyCupTs4FkAW9fTDvyqZuuoys9MQByxiGDRumyy+/XOPHj+91tc/kyZPV0tKigQMH6vLLL4+5RVdvTZgwQbt27Yr5vq73O7viiis0ZMgQvfHGG3EfHzx4sCRrj6eo4uJijR07VocOHepWR3RC7lVXXaV33nlHn3zySVJ1AAAypC8fuG1okERY6UmOLWP44he/qOnTp2vu3Ln68Y9/rCNHjmjnzp16/PHHO5r0VVdX6+WXX9bLL7+sgwcP6sknn9S7777b488sLCzUkiVL9Mgjj+g73/mO3n//fe3atUvf+ta3JEmjR4/WkCFD9KMf/Uj/+7//q7a2NklWo7na2lqtXLlSBw8e1C9/+UutX79eL7zwgiRp/vz5GjBggO6++27t379fr732mv7hH/4hw/9CAIBu+vOBO4sNkggrPelPj3EbeDwevfbaa7rhhhv05S9/Wb/zO7+jO+64Q0eOHOlosHf77bfriSee0JIlSzRlyhR98MEHPe6IHfW1r31NDz30kJ544glNmDBBt99+u1pbWyVZzer+6Z/+SWvWrFFpaanmzJkjSbrnnnv0zW9+Uxs2bNA111yjG2+8URs2bOgYWbnooov06quvav/+/Zo0aZKWLl2qZ599NoP/OgCAuHr7YJ5IFhskeUyOTyAIh8Py+Xxqa2vr1pvlzJkzOnz4sMrLy1VYWNi3XxBvu9hAwAoqzA7MqrQ8nwCAWNHdRqXkJtpGu+kePtyvD+yJ3r+7YoJtb/rSYxwAgFwRbUbX9YN5PDZdWSCsJINlDACAfBbvg/nx49LixbEBxu+35coCYQUAAMT/YB4MOuLKAmEFAADE55ArC6wGAgAAjuaKsBLtqorcxvMIAO6U15eBBg8erAEDBujYsWMaNWqUBg8eLE9f1pLDVsYYnT17VsePH9eAAQM6OucCANwhr8PKgAEDVF5erubmZh07dszuctBPQ4cO1bhx4zRggCsGBAEAn8rrsCJZoyvjxo3T+fPnY/awQW7xer0aOHAgI2MA4EJ5H1YkqxX9oEGDet0MEAAAOA/j6QAAwNEIKwAAwNEIKwAAwNFyfs5KdNPocDhscyUAACBZ0fdtk8ROzzkfVk6ePClJCgQCNlcCAABSdfLkSfl8voTneEwykcbBLly4oGPHjmn48OH9WtYaDocVCATU2NiooqKiNFaIVPA8OAPPgzPwPDgDz0NmGGN08uRJlZaW9to/K+dHVgYMGCC/35+2n1dUVMT/jA7A8+AMPA/OwPPgDDwP6dfbiEoUE2wBAICjEVYAAICjEVY+VVBQoCeffFIFBQV2l+JqPA/OwPPgDDwPzsDzYL+cn2ALAADyGyMrAADA0QgrAADA0QgrAADA0QgrAADA0QgrklatWqXy8nIVFhZqypQpqq+vt7skV6mtrdXv/d7vafjw4Ro9erTmzp2rX//613aX5Xq1tbXyeDyqqamxuxRXampq0p//+Z9rxIgRGjp0qK677jrt3bvX7rJc5fz583r88cdVXl6uIUOG6NJLL9VTTz2lCxcu2F2a67g+rGzevFk1NTVaunSp9u3bp4qKCt188806evSo3aW5xptvvqn7779fu3bt0rZt23T+/HnNmjVLp0+ftrs019qzZ4/Wrl2r3/3d37W7FFf6+OOPdf3112vQoEH64Q9/qP379+v555/XxRdfbHdprvLss8/qn//5n/Xiiy/qwIEDeu655/T3f//3+sY3vmF3aa7j+qXL06ZN0+TJk7V69eqOYxMmTNDcuXNVW1trY2Xudfz4cY0ePVpvvvmmbrjhBrvLcZ1Tp05p8uTJWrVqlZ5++mldd911qqurs7ssV3n00Uf1X//1X4zy2uxP/uRPVFxcrG9961sdx2699VYNHTpU//Iv/2JjZe7j6pGVs2fPau/evZo1a1bM8VmzZmnnzp02VYW2tjZJ0iWXXGJzJe50//3365ZbbtEXv/hFu0txrR/84AeaOnWqbrvtNo0ePVqTJk3SunXr7C7LdT7/+c/rjTfe0MGDByVJ77zzjt5++2398R//sc2VuU/Ob2TYHydOnFAkElFxcXHM8eLiYrW0tNhUlbsZY7R48WJ9/vOf18SJE+0ux3VeeeUV/eIXv9CePXvsLsXVDh06pNWrV2vx4sX627/9W+3evVsPPPCACgoK9Bd/8Rd2l+caS5YsUVtbm6688kp5vV5FIhGtWLFC8+bNs7s013F1WInyeDwx940x3Y4hOxYtWqT//u//1ttvv213Ka7T2Nio6upqvf766yosLLS7HFe7cOGCpk6dqmeeeUaSNGnSJL377rtavXo1YSWLNm/erO9+97vauHGjrr76ajU0NKimpkalpaVasGCB3eW5iqvDysiRI+X1eruNorS2tnYbbUHmfeUrX9EPfvADvfXWW/L7/XaX4zp79+5Va2urpkyZ0nEsEonorbfe0osvvqj29nZ5vV4bK3SPkpISXXXVVTHHJkyYoK1bt9pUkTt99atf1aOPPqo77rhDknTNNdfogw8+UG1tLWEly1w9Z2Xw4MGaMmWKtm3bFnN827ZtmjFjhk1VuY8xRosWLVIoFNJ//ud/qry83O6SXOkLX/iCfvnLX6qhoaHjNnXqVN15551qaGggqGTR9ddf3235/sGDBzV+/HibKnKn3/72txowIPZt0uv1snTZBq4eWZGkxYsX66677tLUqVM1ffp0rV27VkePHtV9991nd2mucf/992vjxo3693//dw0fPrxjpMvn82nIkCE2V+cew4cP7zZPaNiwYRoxYgTzh7LswQcf1IwZM/TMM8/oS1/6knbv3q21a9dq7dq1dpfmKrNnz9aKFSs0btw4XX311dq3b59eeOEFffnLX7a7NPcxMC+99JIZP368GTx4sJk8ebJ588037S7JVSTFva1fv97u0lzvxhtvNNXV1XaX4UqvvvqqmThxoikoKDBXXnmlWbt2rd0luU44HDbV1dVm3LhxprCw0Fx66aVm6dKlpr293e7SXMf1fVYAAICzuXrOCgAAcD7CCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcDTCCgAAcLT/ByGJ178+VlLmAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot\n",
    "plt.scatter(test_X.flatten(), test_Y, color=\"blue\", label=\"True\")\n",
    "plt.scatter(test_X.flatten(), test_Y_pred, color=\"red\", label=\"Predicted\")\n",
    "plt.legend()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "IFT6269-PGM",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}