10_rmbs_TF2_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 10 - Recommender Systems Using Restricted Boltzmann Machines  (TensorFlow v 2.1.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Main'''\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, time, re\n",
    "import pickle, gzip, datetime\n",
    "from datetime import datetime\n",
    "\n",
    "'''Data Viz'''\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "color = sns.color_palette()\n",
    "import matplotlib as mpl\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "'''Data Prep and Model Evaluation'''\n",
    "from sklearn import preprocessing as pp\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.model_selection import StratifiedKFold \n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import precision_recall_curve, average_precision_score\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score, mean_squared_error\n",
    "\n",
    "'''Algos'''\n",
    "import lightgbm as lgb\n",
    "\n",
    "'''TensorFlow and Keras'''\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "K = keras.backend\n",
    "\n",
    "from tensorflow.keras.models import Sequential, Model\n",
    "from tensorflow.keras.layers import Activation, Dense, Dropout\n",
    "from tensorflow.keras.layers import BatchNormalization, Input, Lambda\n",
    "from tensorflow.keras.layers import Embedding, Flatten, dot\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.losses import mse, binary_crossentropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check library versions & set seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sklearn    0.20.3\n",
      "tensorflow 2.1.0\n",
      "keras      2.2.4-tf\n",
      "numpy      1.17.2\n"
     ]
    }
   ],
   "source": [
    "import sys, sklearn\n",
    "print(f'sklearn    {sklearn.__version__}')\n",
    "print(f'tensorflow {tf.__version__}')\n",
    "print(f'keras      {keras.__version__}')\n",
    "print(f'numpy      {np.__version__}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# To make the output stable across runs\n",
    "tf.random.set_seed(42)\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MovieLens Dataset "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|MovieLens 20M Dataset||\n",
    "|:------------|----------|\n",
    "|ratings     |20,000,263| \n",
    "|movies      |27,278| \n",
    "|users       |138,493|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_path = os.getcwd()\n",
    "file = os.path.sep.join(['', 'datasets', 'movielens_data', 'ratings.csv'])\n",
    "ratingDF = pd.read_csv(current_path + file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert fields into appropriate data types "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratingDF.userId = ratingDF.userId.astype(str).astype(int)\n",
    "ratingDF.movieId = ratingDF.movieId.astype(str).astype(int)\n",
    "ratingDF.rating = ratingDF.rating.astype(str).astype(float)\n",
    "ratingDF.timestamp = ratingDF.timestamp.apply(lambda x: datetime.utcfromtimestamp(x).strftime('%Y-%m-%d %H:%M:%S'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Store DataFrame as pickle for faster loading in the future"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_file = os.path.sep.join(['', 'datasets', 'movielens_data', 'ratingPickle'])\n",
    "ratingDF.to_pickle(current_path + pickle_file)\n",
    "ratingDF = pd.read_pickle(current_path + pickle_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preview data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2005-04-02 23:53:47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2005-04-02 23:31:16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>32</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2005-04-02 23:33:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2005-04-02 23:32:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>50</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2005-04-02 23:29:40</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating            timestamp\n",
       "0       1        2     3.5  2005-04-02 23:53:47\n",
       "1       1       29     3.5  2005-04-02 23:31:16\n",
       "2       1       32     3.5  2005-04-02 23:33:39\n",
       "3       1       47     3.5  2005-04-02 23:32:07\n",
       "4       1       50     3.5  2005-04-02 23:29:40"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratingDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate summary statistics on full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique users: 138493\n",
      "Number of unique movies: 26744\n",
      "Number of total ratings: 20000263\n",
      "Average number of ratings per user: 144.4\n"
     ]
    }
   ],
   "source": [
    "n_users = ratingDF.userId.unique().shape[0]\n",
    "n_movies = ratingDF.movieId.unique().shape[0]\n",
    "n_ratings = len(ratingDF)\n",
    "avg_ratings_per_user = n_ratings/n_users\n",
    "\n",
    "print(f'Number of unique users: {n_users}')\n",
    "print(f'Number of unique movies: {n_movies}')\n",
    "print(f'Number of total ratings: {n_ratings}')\n",
    "print(f'Average number of ratings per user: {round(avg_ratings_per_user,1)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reduce size of dataset by taking top 1000 movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId       12840344\n",
       "movieId      12840344\n",
       "rating       12840344\n",
       "timestamp    12840344\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movieIndex = ratingDF.groupby(\"movieId\").count().sort_values(by= \n",
    "                \"rating\",ascending=False)[0:1000].index\n",
    "ratingDFX2 = ratingDF[ratingDF.movieId.isin(movieIndex)]\n",
    "ratingDFX2.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reduce size of dataset by sampling 1000 users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "userId       90213\n",
       "movieId      90213\n",
       "rating       90213\n",
       "timestamp    90213\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "userIndex = ratingDFX2.groupby(\"userId\").count().sort_values(by= \n",
    "    \"rating\",ascending=False).sample(n=1000, random_state=2018).index\n",
    "ratingDFX3 = ratingDFX2[ratingDFX2.userId.isin(userIndex)]\n",
    "ratingDFX3.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reindex movie ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>originalMovieId</th>\n",
       "      <th>newMovieId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>163</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>216</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>296</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   originalMovieId  newMovieId\n",
       "0               50           1\n",
       "1              163           2\n",
       "2              216           3\n",
       "3              296           4\n",
       "4              333           5"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies = ratingDFX3.movieId.unique()\n",
    "moviesDF = pd.DataFrame(data=movies,columns=['originalMovieId'])\n",
    "moviesDF['newMovieId'] = moviesDF.index+1\n",
    "moviesDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reindex user ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>originalUserId</th>\n",
       "      <th>newUserId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>260</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>311</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>319</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>499</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   originalUserId  newUserId\n",
       "0              49          1\n",
       "1             260          2\n",
       "2             311          3\n",
       "3             319          4\n",
       "4             499          5"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users = ratingDFX3.userId.unique()\n",
    "usersDF = pd.DataFrame(data=users,columns=['originalUserId'])\n",
    "usersDF['newUserId'] = usersDF.index+1\n",
    "usersDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate newly merged DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>newMovieId</th>\n",
       "      <th>newUserId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>49</td>\n",
       "      <td>50</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013-05-03 02:50:26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "      <td>163</td>\n",
       "      <td>3.5</td>\n",
       "      <td>2013-05-03 02:43:37</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>216</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-05-03 02:45:58</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>49</td>\n",
       "      <td>296</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2013-05-03 02:50:13</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>49</td>\n",
       "      <td>333</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2013-05-03 02:44:38</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userId  movieId  rating            timestamp  newMovieId  newUserId\n",
       "0      49       50     5.0  2013-05-03 02:50:26           1          1\n",
       "1      49      163     3.5  2013-05-03 02:43:37           2          1\n",
       "2      49      216     3.0  2013-05-03 02:45:58           3          1\n",
       "3      49      296     5.0  2013-05-03 02:50:13           4          1\n",
       "4      49      333     3.0  2013-05-03 02:44:38           5          1"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratingDFX3 = ratingDFX3.merge(moviesDF,left_on='movieId',\n",
    "                              right_on='originalMovieId')\n",
    "ratingDFX3.drop(labels='originalMovieId', axis=1, inplace=True)\n",
    "ratingDFX3 = ratingDFX3.merge(usersDF,left_on='userId', \n",
    "                              right_on='originalUserId')\n",
    "ratingDFX3.drop(labels='originalUserId', axis=1, inplace=True)\n",
    "ratingDFX3.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save as pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle_file = os.path.sep.join(['', 'datasets', 'movielens_data', 'ratingReducedPickle'])\n",
    "ratingDFX3.to_pickle(current_path + pickle_file)\n",
    "ratingDFX3 = pd.read_pickle(current_path + pickle_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate summary statistics on reduced dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique users: 1000\n",
      "Number of unique movies: 1000\n",
      "Number of total ratings: 90213\n",
      "Average number of ratings per user: 90.2\n"
     ]
    }
   ],
   "source": [
    "n_users = ratingDFX3.userId.unique().shape[0]\n",
    "n_movies = ratingDFX3.movieId.unique().shape[0]\n",
    "n_ratings = len(ratingDFX3)\n",
    "avg_ratings_per_user = n_ratings/n_users\n",
    "\n",
    "print(f'Number of unique users: {n_users}')\n",
    "print(f'Number of unique movies: {n_movies}')\n",
    "print(f'Number of total ratings: {n_ratings}')\n",
    "print(f'Average number of ratings per user: {round(avg_ratings_per_user,1)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split into validation and test, such that each is 5% of the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = train_test_split(ratingDFX3, test_size=0.10, shuffle=True, random_state=2018)\n",
    "X_valid, X_test = train_test_split(X_test,     test_size=0.50, shuffle=True, random_state=2018)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confirm size of train, validation, and test datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of train set: 81191\n",
      "Size of validation set: 4511\n",
      "Size of test set: 4511\n"
     ]
    }
   ],
   "source": [
    "for (l,x) in [('train',X_train),('validation',X_valid),('test',X_test)]:\n",
    "    print(f'Size of {l} set: {len(x)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the cost Function: Mean Squared Error "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate ratings matrix for train, validation and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_train = np.zeros((n_users, n_movies))\n",
    "ratings_valid = np.zeros((n_users, n_movies))\n",
    "ratings_test  = np.zeros((n_users, n_movies))\n",
    "for (X,ratings) in [(X_train,ratings_train),(X_valid,ratings_valid),(X_test,ratings_test)]:\n",
    "    for row in X.itertuples():\n",
    "        ratings[row[6]-1, row[5]-1] = row[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1000, 1000), (1000, 1000), (1000, 1000))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_train.shape, ratings_valid.shape, ratings_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculation the sprasity of the train, validation & test rationg matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_sparsity(label, ratings):\n",
    "    sparsity = float(len(ratings.nonzero()[0]))\n",
    "    sparsity /= (ratings.shape[0] * ratings.shape[1])\n",
    "    sparsity *= 100\n",
    "    print('{:s} Sparsity: {:4.2f}%'.format(label, sparsity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Sparsity: 8.12%\n",
      "Validation Sparsity: 0.45%\n",
      "Test Sparsity: 0.45%\n"
     ]
    }
   ],
   "source": [
    "for (lbl, rtg) in [('Train',ratings_train),('Validation',ratings_valid),('Test', ratings_test)]:\n",
    "    calc_sparsity(lbl,rtg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform Baseline Experiments "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Flatten the sprace matrix with the rations for the validation set. This will be the vector of actual ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "actual_valid = ratings_valid[ratings_valid.nonzero()].flatten()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment One \n",
    "#### Assign naive 3.5 rating and calculate baseline MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_valid = np.zeros((len(X_valid),1))\n",
    "pred_valid[pred_valid==0] = 3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error using naive prediction: 1.06\n"
     ]
    }
   ],
   "source": [
    "naive_prediction = mean_squared_error(pred_valid, actual_valid)\n",
    "print(f'Mean squared error using naive prediction: {round(naive_prediction,2)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment Two\n",
    "#### Predict a user's rating based on user's average rating for all other movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error using user average: 0.909\n"
     ]
    }
   ],
   "source": [
    "ratings_valid_pred = np.zeros((n_users, n_movies))\n",
    "i = 0\n",
    "for row in ratings_train:\n",
    "    ratings_valid_pred[i][ratings_valid_pred[i]==0] = np.mean(row[row>0])\n",
    "    i += 1\n",
    "\n",
    "pred_valid = ratings_valid_pred[ratings_valid.nonzero()].flatten()\n",
    "user_average = mean_squared_error(pred_valid, actual_valid)\n",
    "print(f'Mean squared error using user average: {round(user_average,3)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment Three\n",
    "#### Predict a user's rating for a movie based on the average rating other users have given that movie "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error using movie average: 0.914\n"
     ]
    }
   ],
   "source": [
    "ratings_valid_pred = np.zeros((n_users, n_movies)).T\n",
    "i = 0\n",
    "for row in ratings_train.T:\n",
    "    ratings_valid_pred[i][ratings_valid_pred[i]==0] = np.mean(row[row>0])\n",
    "    i += 1\n",
    "\n",
    "ratings_valid_pred = ratings_valid_pred.T\n",
    "pred_valid = ratings_valid_pred[ratings_valid.nonzero()].flatten()\n",
    "movie_average = mean_squared_error(pred_valid, actual_valid)\n",
    "print(f'Mean squared error using movie average: {round(movie_average,3)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Matrix Factorization "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment Four - Recommender System using Matrix Factorization\n",
    "#### One Latent Factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_latent_factors = 1\n",
    "\n",
    "user_input = Input(shape=[1], name='user')\n",
    "user_embedding = Embedding(input_dim=n_users + 1, \n",
    "                           output_dim=n_latent_factors, \n",
    "                           name='user_embedding')(user_input)\n",
    "user_vec = Flatten(name='flatten_users')(user_embedding)\n",
    "\n",
    "movie_input = Input(shape=[1], name='movie')\n",
    "movie_embedding = Embedding(input_dim=n_movies + 1, \n",
    "                            output_dim=n_latent_factors,\n",
    "                            name='movie_embedding')(movie_input)\n",
    "movie_vec = Flatten(name='flatten_movies')(movie_embedding)\n",
    "\n",
    "product = dot([movie_vec, user_vec], axes=1)\n",
    "model = Model(inputs=[user_input, movie_input], outputs=product)\n",
    "model.compile('adam', 'mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 81191 samples, validate on 4511 samples\n",
      "Epoch 1/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 13.4436 - val_loss: 11.5197\n",
      "Epoch 2/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 8.8163 - val_loss: 6.2817\n",
      "Epoch 3/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 4.4879 - val_loss: 3.2602\n",
      "Epoch 4/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 2.5015 - val_loss: 2.0229\n",
      "Epoch 5/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 1.6251 - val_loss: 1.4304\n",
      "Epoch 6/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 1.1894 - val_loss: 1.1305\n",
      "Epoch 7/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.9683 - val_loss: 0.9748\n",
      "Epoch 8/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.8552 - val_loss: 0.8942\n",
      "Epoch 9/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7964 - val_loss: 0.8501\n",
      "Epoch 10/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7653 - val_loss: 0.8281\n",
      "Epoch 11/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7488 - val_loss: 0.8155\n",
      "Epoch 12/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7397 - val_loss: 0.8071\n",
      "Epoch 13/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7348 - val_loss: 0.8047\n",
      "Epoch 14/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7320 - val_loss: 0.8017\n",
      "Epoch 15/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7303 - val_loss: 0.7990\n",
      "Epoch 16/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7294 - val_loss: 0.7991\n",
      "Epoch 17/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7287 - val_loss: 0.7988\n",
      "Epoch 18/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7286 - val_loss: 0.7980\n",
      "Epoch 19/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7281 - val_loss: 0.7993\n",
      "Epoch 20/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7281 - val_loss: 0.7982\n",
      "Epoch 21/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7280 - val_loss: 0.7968\n",
      "Epoch 22/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7280 - val_loss: 0.7988\n",
      "Epoch 23/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7981\n",
      "Epoch 24/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7280 - val_loss: 0.8002\n",
      "Epoch 25/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7982\n",
      "Epoch 26/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7982\n",
      "Epoch 27/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7983\n",
      "Epoch 28/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7982\n",
      "Epoch 29/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7988\n",
      "Epoch 30/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7979\n",
      "Epoch 31/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7984\n",
      "Epoch 32/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7980\n",
      "Epoch 33/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7980\n",
      "Epoch 34/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7969\n",
      "Epoch 35/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7978\n",
      "Epoch 36/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7978\n",
      "Epoch 37/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7978\n",
      "Epoch 38/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7972\n",
      "Epoch 39/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7279 - val_loss: 0.7975\n",
      "Epoch 40/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7278 - val_loss: 0.7978\n",
      "Epoch 41/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7993\n",
      "Epoch 42/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7278 - val_loss: 0.7975\n",
      "Epoch 43/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7979\n",
      "Epoch 44/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7990\n",
      "Epoch 45/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7987\n",
      "Epoch 46/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7990\n",
      "Epoch 47/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7276 - val_loss: 0.7977\n",
      "Epoch 48/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7276 - val_loss: 0.7975\n",
      "Epoch 49/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7997\n",
      "Epoch 50/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7990\n",
      "Epoch 51/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7995\n",
      "Epoch 52/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7987\n",
      "Epoch 53/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7991\n",
      "Epoch 54/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7277 - val_loss: 0.7987\n",
      "Epoch 55/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7277 - val_loss: 0.7980\n",
      "Epoch 56/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7989\n",
      "Epoch 57/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7976\n",
      "Epoch 58/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7968\n",
      "Epoch 59/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7979\n",
      "Epoch 60/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7278 - val_loss: 0.7969\n",
      "Epoch 61/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7974\n",
      "Epoch 62/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7987\n",
      "Epoch 63/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7973\n",
      "Epoch 64/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7276 - val_loss: 0.7980\n",
      "Epoch 65/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7970\n",
      "Epoch 66/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7968\n",
      "Epoch 67/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7978\n",
      "Epoch 68/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7977\n",
      "Epoch 69/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7973\n",
      "Epoch 70/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7990\n",
      "Epoch 71/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7978\n",
      "Epoch 72/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7983\n",
      "Epoch 73/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7983\n",
      "Epoch 74/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7985\n",
      "Epoch 75/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7276 - val_loss: 0.7981\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7989\n",
      "Epoch 77/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7980\n",
      "Epoch 78/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7977\n",
      "Epoch 79/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7993\n",
      "Epoch 80/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7990\n",
      "Epoch 81/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7975\n",
      "Epoch 82/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7278 - val_loss: 0.7970\n",
      "Epoch 83/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7979\n",
      "Epoch 84/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7278 - val_loss: 0.7990\n",
      "Epoch 85/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7976\n",
      "Epoch 86/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7980\n",
      "Epoch 87/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7279 - val_loss: 0.7980\n",
      "Epoch 88/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7984\n",
      "Epoch 89/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7981\n",
      "Epoch 90/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7974\n",
      "Epoch 91/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7960\n",
      "Epoch 92/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7974\n",
      "Epoch 93/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7276 - val_loss: 0.7977\n",
      "Epoch 94/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7984\n",
      "Epoch 95/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7278 - val_loss: 0.7975\n",
      "Epoch 96/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7279 - val_loss: 0.7983\n",
      "Epoch 97/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7980\n",
      "Epoch 98/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7967\n",
      "Epoch 99/100\n",
      "81191/81191 [==============================] - 2s 20us/sample - loss: 0.7277 - val_loss: 0.7989\n",
      "Epoch 100/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7277 - val_loss: 0.7985\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x=[X_train.newUserId, X_train.newMovieId], \n",
    "                    y=X_train.rating, epochs=100, \n",
    "                    validation_data=([X_valid.newUserId, X_valid.newMovieId], X_valid.rating), \n",
    "                    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum MSE: 0.796\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXiU5dX48e+Z7CEJIRtLEiCEsARkMyyioLhiXVC7CLYu1NbyVq1dX7X9dbW+3VvborVq3VoVdwW1oOKuCISdQIAQthDIQoBskJDk/P6YJ3GSTJJJmCHBnM915WLm2eYkJHPmvs9934+oKsYYY4w/uLo7AGOMMZ8fllSMMcb4jSUVY4wxfmNJxRhjjN9YUjHGGOM3wd0dQHdKSEjQoUOHdncYxhhzWlmzZk2pqiZ629erk8rQoUPJzs7u7jCMMea0IiJ72tpn3V/GGGP8xpKKMcYYv7GkYowxxm8sqRhjjPEbSyrGGGP8xpKKMcYYv7GkYowxxm96dVIpKj/O8RP13R2GMcZ8bvTqpFJcUUPF8bruDsMYYz43enVSAThWay0VY4zxl16fVKpPWEvFGGP8JaBJRURmi8g2EckTkbu87O8rIktEZIOI5IjIfI99j4pIsYhsbnHOL0Rkv4isd76+4LHvbue1tonIJb7EWG0tFWOM8ZuAJRURCQLuBy4FMoF5IpLZ4rBbgS2qOh44D/iTiIQ6+x4HZrdx+b+o6gTn6w3n9TKBucAY57wHnBjaVV1jScUYY/wlkC2VKUCequarai2wCJjT4hgFokVEgCigDKgDUNUPnOe+mgMsUtUaVd0F5DkxtKu61rq/jDHGXwKZVJKBfR7PC5xtnhYCo4FCYBNwh6o2+HDt20Rko9NF1q8Tr4eI3CIi2SKSDXDMhhQbY4zfBDKpiJdt2uL5JcB6YBAwAVgoIjEdXPcfQLpz/AHgT514PVT1IVXNUtUssJqKMcb4UyCTSgGQ6vE8BXeLxNN84CV1ywN2AaPau6iqFqlqvdOieZjPurh8eb1WLKkYY4z/BDKprAYyRCTNKb7PBRa3OGYvcAGAiPQHRgL57V1URAZ6PL0aaBwdthiYKyJhIpIGZACrOgqyusZqKsYY4y8Bu52wqtaJyG3AMiAIeFRVc0RkgbP/QeAe4HER2YS7++pOVS0FEJFncI8ISxCRAuDnqvov4PciMgF319Zu4FvO9XJE5DlgC+5i/62q2m4zRIBqq6kYY4zfiGqrskOvETlohP7vgy/ziyvHdHcoxhhz2hCRNY116ZZ69Yx6l4gNKTbGGD/q3UnFBVVWqDfGGL/p1UlFRGxBSWOM8aNenVSCrPvLGGP8qlcnFZfY0vfGGONPvTupuMRqKsYY40e9OqmItVSMMcavenVSsSHFxhjjX706qbgL9dZSMcYYf+nVSUUEauoaqG/ovasKGGOMP/XqpOJyuVfLty4wY4zxj96dVMSdVKxYb4wx/tGrk0qQc1svq6sYY4x/9Oqk0thSsaRijDH+0auTiojVVIwxxp96dVJxOd+9tVSMMcY/AppURGS2iGwTkTwRucvL/r4iskRENohIjojM99j3qIgUi8jmFuf8QURyRWSjiLwsIrHO9qEickxE1jtfD3YUn3V/GWOMfwUsqYhIEHA/cCmQCcwTkcwWh90KbFHV8bhvHfwn5372AI8Ds71c+i1grKqOA7YDd3vs26mqE5yvBR3F2DT664R1fxljjD8EsqUyBchT1XxVrQUWAXNaHKNAtLiLG1FAGe77y6OqHzjPm5+g+qaqNmaBT4GUrgbYmFSqaqylYowx/hDIpJIM7PN4XuBs87QQGA0UApuAO1S1oROv8XXgvx7P00RknYi8LyIzvJ0gIreISLaIZB8uOwTYPBVjjPGXQCYV8bKt5XoolwDrgUHABGChiMT4dHGRn+Bu1TzlbDoADFbVicD3gae9XUtVH1LVLFXNSkxMAKymYowx/hLIpFIApHo8T8HdIvE0H3hJ3fKAXcCoji4sIjcClwNfVVUFUNUaVT3kPF4D7ARGtHsdIDTYRbXVVIwxxi8CmVRWAxkikuYU3+cCi1scsxe4AEBE+gMjgfz2Liois4E7gStVtdpje6IzOAARGQZkdHQtgMjQIKqtpmKMMX4RsKTiFNNvA5YBW4HnVDVHRBaISOPIrHuA6SKyCVgO3KmqpQAi8gywAhgpIgUicrNzzkIgGnirxdDhmcBGEdkAvAAsUNVWhf6WIkOCrPvLGGP8JDiQF1fVN4A3Wmx70ONxIXBxG+fOa2P78Da2vwi82NkYI0KDbEixMcb4Sa+eUQ/QJyzYWirGGOMnvT6pRIRYTcUYY/yl1yeVyNAgG/1ljDF+Ykkl1Lq/jDHGXyyphAbZjHpjjPETSyqhQVTVWPeXMcb4Q69PKhGhwRw7YS0VY4zxh16fVCJDgzhRr5yo78w6lsYYY7yxpBIaBNiiksYY4w+WVELdiwpYsd4YY06eJRWnpVJVa8V6Y4w5Wb0+qUQ4ScVaKsYYc/J6fVKxmooxxviPJRWnplJt3V/GGHPSLKlYS8UYY/zGkoolFWOM8ZuAJhURmS0i20QkT0Tu8rK/r4gsEZENIpIjIvM99j0qIsUisrnFOXEi8paI7HD+7eex727ntbaJyCW+xPhZod66v4wx5mQFLKk494u/H7gUyATmiUhmi8NuBbao6njgPOBPzv3sAR4HZnu59F3AclXNwH0L4ruc18sE5gJjnPMeaLxnfXv6NNVUrKVijDEnK5AtlSlAnqrmq2otsAiY0+IYBaJFRIAooAyoA1DVD5znLc0BnnAePwFc5bF9karWqOouIM+JoV0RIY3zVCypGGPMyQpkUkkG9nk8L3C2eVoIjAYKgU3AHara0SJc/VX1AIDzb1InXg8RuUVEskUku6SkBJdLCA9xWfeXMcb4QSCTinjZpi2eXwKsBwYBE4CFIhITwNdDVR9S1SxVzUpMTATsRl3GGOMvgUwqBUCqx/MU3C0ST/OBl9QtD9gFjOrgukUiMhDA+be4E6/nld2oyxhj/COQSWU1kCEiaU7xfS6wuMUxe4ELAESkPzASyO/guouBG53HNwKvemyfKyJhIpIGZACrfAk0MjTI1v4yxhg/CFhSUdU64DZgGbAVeE5Vc0RkgYgscA67B5guIptwj+S6U1VLAUTkGWAFMFJECkTkZuec3wIXicgO4CLnOaqaAzwHbAGWAreqqk/Njwjr/jLGGL8Ibm+nMyorRVX3tXdcW1T1DeCNFtse9HhcCFzcxrnz2th+CKd142XfvcC9nY0zMsS6v4wxxh/abamoqgKvnKJYuk1kaJC1VIwxxg986f76VEQmBzySbhQZZvepN8YYf2i3+8sxC/iWiOwBqnAP3VVVHRfQyE6hyJAgqmqsUG+MMSfLl6RyacCj6GYRNqTYGGP8osPuL1XdA8QCVzhfsc62z43I0CCqT9TjLiEZY4zpqg6TiojcATyFezmUJOA/InJ7oAM7lfqEBVPfoNTWd7RCjDHGmPb40v11MzBVVasAROR3uOeP/D2QgZ1KjYtKVtfUExbc4cLGxhhj2uDL6C8BPAsO9XhfZ+u01XSjLhsBZowxJ8WXlspjwEoRedl5fhXwr8CFdOrZjbqMMcY/OkwqqvpnEXkPOAd3C2W+qq4LdGCnkt2oyxhj/KOjZVpcwEZVHQusPTUhnXqN3V9VNZZUjDHmZHS0TEsDsEFEBp+ieLpFU/fXCev+MsaYk+FLTWUgkCMiq3DPqAdAVa8MWFSnWKR1fxljjF/4klR+GfAoulnT6C9LKsYYc1I6qqkEAT9V1QtPUTzdoimp2PpfxhhzUjqqqdQD1SLS9xTF0y2aur9snooxxpwUX7q/jgObROQtmtdUvhOwqE6x8BAXItiiksYYc5J8mVH/OvBT4ANgjcdXh0RktohsE5E8EbnLy/6+IrJERDaISI6IzO/oXBF5VkTWO1+7RWS9s32oiBzz2Pdgy9drJ04iQuxGXcYYc7LabKmISIyqlqvqE172dTjE2KnH3I/7PvIFwGoRWayqWzwOuxXYoqpXiEgisE1EnsK9FIzXc1X1Wo/X+BNw1ON6O1V1QkexeRMZGky1zag3xpiT0l5L5b3GByKyvMU+X24xPAXIU9V8Va0FFgFzWhyjQLSICBAFlAF1vpzrnPMV4BkfYumQ3VLYGGNOXntJxXPRyLh29rUlGdjn8bzA2eZpITAaKAQ2AXc4Ey59OXcGUKSqOzy2pYnIOhF5X0RmeAtKRG4RkWwRyS4pKWnabknFGGNOXntJRdt47O25N94ST8vzLgHWA4OACcBCEYnx8dx5NG+lHAAGq+pE4PvA0861ml9E9SFVzVLVrMTExKbtdvdHY4w5ee2N/koSke/jfoNvfIzzPLHt05oUAKkez1Nwt0g8zQd+q+5bLuaJyC5gVEfnikgwcA1wZuM2Va0BapzHa0RkJzACyPYhVvpYTcUYY05aey2Vh4Fo3LWOxseNzx/x4dqrgQwRSRORUGAusLjFMXuBCwBEpD8wEsj34dwLgVxVLWjcICKJzuAARGQYkOFcyycR1v1ljDEnrc2Wiqqe1PIsqlonIrcBy4Ag4FFVzRGRBc7+B4F7gMdFZBPuFtCdqloK4O1cj8vPpXWBfibwKxGpwz16bIGqlvkab/+YMFbmH0JVcY8BMMYY01m+TH7sMlV9A3ijxbYHPR4XAhf7eq7Hvpu8bHsReLGrsaYnRlF+vI7SyloSo8O6ehljjOnVfJn82CukJ0YBkF9S2c2RGGPM6cuSiiM9yZ1UdpZUdXCkMcaYtnTY/SUiYcAXgaGex6vqrwIX1qk3MCaciJAgdlpLxRhjusyXmsqruJdCWYMzZPfzyOUShiX2saRijDEnwZekkqKqswMeSQ8wLDGK9fsOd3cYxhhz2vKlpvKJiJwR8Eh6gPTEPhQcPsZxu6+KMcZ0iS9J5RxgjbMM/UYR2SQiGwMdWHdIT4xCFXaVWrHeGGO6wpfur0sDHkUP0TiseGdJJaMHtlo2zBhjTAc6bKmo6h4gFrjC+Yp1tn3upCX0QQR2FltLxRhjuqLDpCIidwBPAUnO139E5PZAB9YdIkKDSI6NsBFgxhjTRb50f90MTFXVKgAR+R2wAvh7IAPrLumJUZZUjDGmi3wp1AvuBRob1ePbTbpOS+mJUeSXVNHQ4MstY4wxxnjypaXyGLBSRF52nl8F/CtwIXWv9KQ+HDtRz8Hy4wyKjejucIwx5rTSYVJR1T+LyHu4hxYLMF9V1wU6sO7iOQLMkooxxnROm0lFRGJUtVxE4oDdzlfjvrjO3KvkdNKUVIormZHhyw0ujTHGNGqvpfI0cDnuNb88CwziPB8WwLi6TUJUKDHhwbZasTHGdEGbhXpVvdz5N01Vh3l8pamqTwlFRGY7M/HzROQuL/v7isgSEdkgIjkiMr+jc0XkFyKyX0TWO19f8Nh3t3P8NhG5xNcfQouYSE+yEWDGGNMVvsxTWe7LNi/HBAH3456RnwnME5HMFofdCmxR1fHAecCfRCTUh3P/oqoTnK83nNfLxH2b4THAbOCBxnvWd5YNKzbGmK5pM6mISLhTT0kQkX4iEud8DQUG+XDtKUCequarai2wCJjT4hgFosV9U/gooAyo8/HcluYAi1S1RlV3AXnOdTptWGIfisprqDh+oiunG2NMr9VeS+VbuOspo5x/G79exd2K6EgysM/jeYGzzdNCYDRQCGwC7lDVBh/Ovc1Z3PJREenXiddDRG4RkWwRyS4pKfEa+Ge3Fra6ijHGdEZ7NZW/qmoa8EOPWkqaqo5X1YU+XNvbBMmWMwovAdbjbvlMABaKSEwH5/4DSHeOPwD8qROvh6o+pKpZqpqVmOh9dJfnsGJjjDG+82Weyt9FZCzu2ka4x/YnOzi1AEj1eJ6Cu0XiaT7wW1VVIE9EduFuGbV5rqoWNW4UkYeB1zrxej4ZEh9JsEssqRhjTCf5Uqj/Oe51vv4OzAJ+D1zpw7VXAxkikiYiobiL6ItbHLMXuMB5nf7ASCC/vXNFZKDH+VcDm53Hi4G5IhImImlABrDKhzhbCQlykdIvgj2HqrtyujHG9Fq+LNPyJWA8sE5V5ztv/o90dJKq1onIbcAyIAh4VFVzRGSBs/9B4B7gcRHZhLv76k5VLQXwdq5z6d+LyATcXVu7cdd+cK79HLAFd7H/VlXt8i0cU/pFUnD4WFdPN8aYXsmXpHJMVRtEpM6pdxTj48RHZ7jvGy22PejxuBC42Ndzne3Xt/N69wL3+hJbR1L6RfD21mJ/XMoYY3oNX5JKtojEAg/jHv1VSRe7lU4nKf0iKK2s4fiJesJDujTdxRhjeh1fCvXfdh4+KCJLgRhV/Vzeo95TSr9IAAoOH2N4UlQ3R2OMMaeH9haUnNTePlVdG5iQeoaUfu4VigsOV1tSMcYYH7XXUmmc/xEOZAEbcBfTxwErcS+F/7nl2VIxxhjjm/YmP85S1VnAHmCSM2HwTGAi7iVQPteSosMICRJLKsYY0wm+3E54lKpuanyiqptxz2b/XHO5hOTYCAoO21wVY4zxlS+jv7aKyCPAf3DPDfkasDWgUfUQNlfFGGM6x5eWynwgB7gD+C7uyYXz2z3jcyKlX4QlFWOM6QRfhhQfB/7ifPUqNlfFGGM6p70hxc+p6lecJVS8rfY7LqCR9QA2V8UYYzqnvZbKHc6/l5+KQHqixrkq+49YUjHGGF+0mVRU9YDz755TF07PkuwxAdIYY0zH2uv+qsBLtxfuCZCqqjEBi6qHSIoOt7kqxhjTCe21VKJPZSA9UZBLGBRrI8CMMcZXvsxTAUBEkmh+58e9AYmoh3EPK7buL2OM8YUvd368UkR2ALuA93HfGOu/AY6rx0iJtQmQxhjjK18mP94DTAO2q2oa7tv/fuzLxUVktohsE5E8EbnLy/6+IrJERDaISI6IzO/oXBH5g4jkishGEXnZudcLIjJURI6JyHrn68GWr9cVKf0iKKlwz1UxxhjTPl+SyglVPQS4RMSlqu/iw9pfIhIE3A9cCmQC80Qks8VhtwJbVHU8cB7wJxEJ7eDct4CxzjyZ7cDdHtfbqaoTnK8FPnxvHUqJ+2xYsTHGmPb5klSOiEgU8AHwlIj8Ffc94DsyBchT1XxVrQUWAXNaHKNAtIgIEAWUOddu81xVfVNVG1//UyDFh1i6zJbAN8YY3/mSVOYAx4DvAUuBncAVPpyXDOzzeF7gbPO0EBgNFAKbgDtUtcHHcwG+TvP6TpqIrBOR90VkhregROQWEckWkeySkpIOv4kUm6tijDE+a2+eykLgaVX9xGPzE524tnjZ1nLeyyXAeuB8IB14S0Q+9OVcEfkJ7lbNU86mA8BgVT0kImcCr4jIGFUtb3YR1YeAhwCysrK8zcNpxuaqGGOM79prqezAXePYLSK/E5HO3kOlAEj1eJ6Cu0XiaT7wkrrl4R5hNqqjc0XkRtzLx3xVVRVAVWuc2g+qugZ3i2pEJ2NuxeaqGGOM79q78+NfVfUs4FzctY7HRGSriPxMRHx5s14NZIhImoiEAnOBxS2O2Yt7NBki0h8YCeS3d66IzAbuBK5U1aY+KRFJdAr8iMgwIMO51kmzuSrGGOObDmsqqrpHVX+nqhOB64Cr8eEmXU4x/TZgmXP8c6qaIyILRKRxZNY9wHRnJeTlwJ2qWtrWuc45C4Fo3F1lnkOHZwIbRWQD8AKwQFXLfPkhdMTmqhhjjG86nFEvIiHAbNythQtwT4D8pS8XV9U3gDdabHvQ43EhcLGv5zrbh7dx/IvAi77E1Vmec1XsvirGGNO29gr1FwHzgMuAVbiH9d6iqlWnKLYeI9ljCfz0RFsC3xhj2tJe99ePgRXAaFW9QlWf6o0JBWyuijHG+Kq9VYpnncpAerJUZ1b9vjIr1htjTHt8mfzY6/WPDic02MVeSyrGGNMuSyo+cLmEwXGR7C7tlb1/xhjjM0sqPhoSF2ktFWOM6YAlFR8NjncnFWcCvzHGGC8sqfhoSFwk1bX1lFTWdHcoxhjTY1lS8dGQhD4A7D1kXWDGGNMWSyo+GhLnnquy25KKMca0yZKKj1L6ReIS2HvIRoAZY0xbLKn4KDTYxcC+EeyxEWDGGNMmSyqdMCQ+kj3W/WWMMW2ypNIJQ+L72FwVY4xphyWVThgSH0lZVS3lx090dyjGGNMjWVLphMYRYDas2BhjvLOk0gmD491JxeoqxhjjXUCTiojMFpFtIpInInd52d9XRJaIyAYRyRGR+R2dKyJxIvKWiOxw/u3nse9u5/htInKJv7+fIfHuCZB7ynrGsOLXNhZyw6OrbOkYY0yPEbCkIiJBwP3ApUAmME9EMlscdiuwRVXHA+cBfxKR0A7OvQtYrqoZuO9rf5fzepm4b3k8Bvftjx9wruM3UWHBxPcJ7THdX//dfJAPtpdw4Ojx7g7FGGOAwLZUpgB5qpqvqrW4b0c8p8UxCkSLiABRQBlQ18G5c4AnnMdPAFd5bF+kqjWqugvIc67jVz1pWPG2gxUAbD1Q3s2RGGOMWyCTSjKwz+N5gbPN00JgNFAIbALuUNWGDs7tr6oHAJx/kzrxeojILSKSLSLZJSUlnf6mhsT3YU8PmFV//EQ9u5z7u+Q6ycUYY7pbIJOKeNnWsvP/EmA9MAiYACwUkRgfz+3K66GqD6lqlqpmJSYmdnDJ1gbHRXKg/Dg1dfWdPtef8oorqW9wf3tbrKVyyp2ob+juEIzpkQKZVAqAVI/nKbhbJJ7mAy+pWx6wCxjVwblFIjIQwPm3uBOvd9KGxEeiCvvKjvn70p3S2PWVntjHur9OsUWr9jL1/5ZTUmG3QTCmpUAmldVAhoikiUgo7iL64hbH7AUuABCR/sBIIL+DcxcDNzqPbwRe9dg+V0TCRCQNyABW+fubGuIMK97bzSPAcg+WExrs4tKxA9ldWsWx2u5tOfUmq3aVUVZVy9+W7+juUIzpcQKWVFS1DrgNWAZsBZ5T1RwRWSAiC5zD7gGmi8gm3CO57lTV0rbOdc75LXCRiOwALnKe4+x/DtgCLAVuVVW/v9M2DSvu5mJ97sEKMpKiGJscQ4PC9iKrq5wq24vdP+tnVu1tqmt5yj1YTlVN3akOy5geITiQF1fVN4A3Wmx70ONxIXCxr+c62w/htG687LsXuPckQu5QfJ9Q+oQGdXtS2XawghkZiYweGAO4R4CNT43t1ph6g4YGJa+4kivHD+LtrUX8cdk27v/qpKb9b+YcZMF/1nBRZn/+eX1WN0ZqTPewGfWdJCIMPokRYIcqa/jzW9tPqtBfVlVLcUUNowZEk9ovkj6hQb2yrrKlsJyX1hac0tfcd7ia4ycamJ4ezzdmDOP1TQdYv+8IANm7y7j9mXVEhASxLKeIzfuPntLYjO+W5Rzk569utnX8AsCSShcMiYvs8n1VFq3ex9+W72Dp5oNdfv3cg+4EMmpgNC6XMGpgDFt74bDi+97ezo9e2MjxE6eunrS9qBKAEQOiuWXmMBKiQvnNG1vZUVTBzU9kMyg2gjfumEHfiBDue3v7KYvLdM7DH+TzxIo9XPH3jyz5+5kllS4YEh9JQdkxXlxTwG//m8vNj6/mrhc3+tSP/k6ue7Daq+u7PjAt94A7gYwcEA3AqAHRbD1Q3quWa2loUFbtLqO+QckpPHWttMbaVUZSFFFhwXznggxW7irji//4hNBgF09+fQpD4vtwy8xhvL21mA1OK6bRkg2FfPWRTykqt1UQusvxE/VsLDjKeSMTqa1r4Jp/fMJ/Pt3Tq/5+AsmSShekJ0VRW9/AD57fwL8+ymdvWTXPZe/j2odWUFzR9ptFWVUta/ceJjo8mPe3l3CosmtDUrcdrCC+TyiJUWEAjB4YQ8XxOvYf6d5hzqfStqIKjlS7uy42Fhzp4GiorKlj4Ts7KPTyM9p7qJov/eMTfvLypg6vs6OogkF9w4kODwFg7uTBDI2PpEHhsZsmk+qsZH3j9KH0iwzhLx6tlaWbD/DdZ9fzcd4hbn5ida8s5ucVV/DAe3kBeQNvaFCfrrt5/1Fq6xuYN2Uwr39nBtPT4/l/r2zmgfd2dul1N+8/6nXARm9lSaUL5kwYxJNfn8Lb3z+XLb+azVvfP5eHb8hiZ3EVV9//CXnF3rui3t9ejCr89LJM6huU1zYe6NLr5x4sZ+SAaNyr2+BRrO96F9ix2vrT6k3u0/xDAESGBrVqDXizaNVe/vjmdi7+ywc8vXJv05vP21uKuPzvH5K95zAvrd3fYa1re1ElGf2jm56HBrt45pZpvP6dcxib3Ldpe1RYMN86N533tpWwZs9h3s0t5vZn1jEhNZb7r5vE1gMV3Pb0WuramUTZ0KA8l72PytPo/6UjT3yyh98v3XZSv6venKhv4MK/vM9vl+Z2eGz2nsMAZA3pR1yfUB69cTJT0+J4df3+Tr9uXX0DNz22ijtf2Njpc08Xew5V8bNXN3O02rf6kyWVLggLDmLmiESGJ0UREuT+EV4wuj/PfmsaNXUNXPPAJ6zZU9bqvHdyS0iICuNLZ6YwakA0r/jwS7ylsLzZG11Dg7K9qJJRA2Kato1yusFyu1isV1VueHQl1z2y8rTpAvg0/xAp/SI4e3gCGws67hNfsvEAw5OiGJfSlx+/vInrHl7JPa9t4RtPZjM4PpJfXjmGYyfqWbWr9f9bo/oGZWdJJSP6RzXbPrBvRNNQc083nDWE+D6h/PilTSz4zxpGDojmsfmTuWzcQH41Zwzvbivh54tz2vyZv7mliP99YSP/XrGnw+/vdNE4qGHp5q59oGrLaxsLyS+p4qlP93b44Sh7dxnDEvoQ77T0XS5h5ohEthdVUlZV26nX/XBHKaWV7h6Iis9p0f+1jQd4csUern90JUePdfw9WlLxo3Epsbz87enERobyoxc2Ni2jAu5PNO9vK2bWyERcLuHqicms23uE3e00mw9X1XLlwo/4xeKcpm17y6o5dqK+KZEA9AkLZkh8JFsPdi2pvLHpIKt3H2bDviNsOg2Klg0NyqpdZUwbFs/4lL7kl1a1+8u+r6yaDfuO8MVJKTz1jan85poz2LT/KP/6aBfzpgzmhQXT+UpWKqHBLt7NbXs9uL1l1dTUNTRrqbQnMiiYy2EAACAASURBVDSY/zkvnW1FFQyJj+TJr08lxuk2++rUISw4N52nVu7l4Q/zvZ7/5IrdACzfWuTT63XGmj1lPPrRLr9ftz3HT9Q3jVL870kMVGlJVfnn+/n0iwyhsqau3XplQ4OSvecwWUP7Nds+bVgcAKt2HerUa7+8bj8iUNegfJrf9geS09m2gxVEhQWz9UA5Nz66qsPkaUnFz1LjIrnr0lHkl1Tx+qbPPo2t2XOY8uN1nD/Kvf7llRMGIUK7rZWcwnLqGpRFq/eRU+h+s/cc+eVp9ICYLnUp1NTV89ulWxmeFEV4iItFq/d1fFI3215cweHqE0wbFs+4FPfcnPZG8DT+P1w+biAiwrwpg3n7++fy9DfdCSY8JIiI0CDOGhbPe9uL27xOY5F+hI9JBeBr04bw08sz+c83phLXJ7TZvv+9ZCSzxwzgj8u2s6/FaMIdRRV8svMQSdFhrN17uMv1t7bc9/YO7nl9i89dGv6QU3iUugbl7OHx7CiuJK+40i/X/SivlNyDFdz9hdGMGhDdbtE9v7SSI9UnyBoS12z7GcmxhIe4OpUYKmvqeHPLQb58ZgoRIUF8uKPzC9SeDrYXVTA1LY6F101i8/6j3PTY6naPt6QSALPHDCAjKYq/L99Bg9NaeSe3mJAg4ZyMBMDdZTItLZ5X1xe2+Qew5YD7jTI6LJhfLdmCqpJ7sAIRyEhqkVQGxrD7UBXVtZ3rf//3ij3sKzvGzy7P5LIzBrF4fWGPr618utP9aXJqWhzjUtx1jPXt1FVe21jI+NTYpiI6wIC+4UxPT2h23HkjE8kvqWpzDtIOj5FfvgoPCeLmc9JIig5vtc/lEn5+ZSYi8Oe3mg8/fmLFbkKDXfz+S+NoUHh3m//esI7V1rNyVxmq8GknP5mfjHV73f9Hd84eBXS+C2zbwQqeWbW3WQ8AwEMf5JMUHcacCYP42rQhbDlQzro2fh9W73bqKS1aKqHBLrKGxDXV6nzx300HOH6igWsnpzJtWBwf7ijt1PdzOjhR3+Du8h0QzSVjBvD3eRPb/VsDSyoB4XIJt1+QwY7iSpbmuJv57+QWMyUtrmnUEMDVE5PZVVrFhjZqAjmF5QzsG86PZo9i5a4yluUcJPdABWnxfYgIbX7/sVEDo1H9bKFJXxypdq9fNXNEIjNHJDJvSiqVNXXNWljgLtTNe+hTPs7rGX80K3eVkRwbQWpcJLGRoQyNj2xzBNju0io27y/n8jMGdnjdWSPdrcj32ngD315USXJsBH3C/LcQxcC+Ecw/O41X1u9vao2WHz/BS2v3c8W4QZw7IpH+MWF+7QJbkV9KbZ17gMCKnacuqazfd4RBfcMZlxLLxMGxPneBFR45xg+f38Dsv37A3S9t4ofPb2ga4JBTeJQPd5Ry09lDCQsO4qqJyfQJDeKpT/d6vVb27sPE9wklLaF1DWxqWpwzqtC3usor6/czJD6SSYP7MSMjkV2lVa1anB1ZmX+ITwLwd1VccZybH1/dbve6L3aVVnGiXhnptM4vPWMgf507od1zLKkEyGVnDGRYYh/+tnwHew9Vs6O4sulNq9HsMwYQGuzilXXeu8ByCssZMyiGeZNTGdk/mnvf2MrmwqNN81M8ZbYzAuxwVS3PrNrL959dz9Mr93LYKUb+/Z08Kmvq+MkXRgNw5pB+DE+KYtGqz/4gT9Q38J1F61mRf4hvPJHN6t3d22/c0KCsdOopjcalxLZZrH9to7t//bJxHSeVoQl9GBofyXvbvHeBbS+qaFWk94f/OS+dvhEh/Pa/7pFLL64poLq2npumD0VEOH9Ufz7YXuK32y28m1tCZGgQ04bF8cnOU/dBYf2+I0wY7O6uvHTsAHIKy5vdRbW+Qfn90lxue3otP355E79bmsvPXt3MrD++x+L1hXzjnDS+c/5wXl63n+89504sj3y4iz6hQXx16hDAPeru6knJvLax0GtyyN5TxplD+jWNnPQ0dVg8qrQ7WKPRgaPH+GTnIa6akIyIMHOEu9X7UScSxPPZ+5j38Kdc98hKbnt6bbvTETrrb8t3sDy3mIfaqNcdq633qeje+CHV8z3n8nGD2j3HkkqABLmE288fTu7BCn7yinv+Q2M9pVFMeAgXjk7itY0HmrrJGh2rrSe/pJLMQX0JDnLx08sz2Vd2jILDx7wmlZR+EUSHBfP21iJeWbef51bv49GPdnHjo6uYfO/b3P3SJt7aWsSPX97E5Hvf5oZHV/Hkit18JSu16XoiwtzJqazde6SpfvC35TvYsO8Iv5ozhoF9w5n/2GqfhvAGyo5i9widqcM+6xMfl9KXA0ePU+xlQuFrGw9w5pB+DIqN8On6541M4pOdh1rN0q+rbyC/pKpT9RRf9Y0I4bZZw/lwRykfbC/hyRV7mDg4ljOcrr2LMpOoqq33SyFYVXl3WzHT0+M5b2QS24sqT8kS/qWVNRQcPsaE1Mak4k7yS3M+axX/8c1tPPDeTjbtP8qbOQd55MN8/v3pHi47YyDv/PBcfnJZJt+/eCR3XTqKJRsK+eaT2SzZUMjcKYPpG/FZD8DXpg2hpq6BF9Y0X8KnuOI4ew5VM3lo83pKo/GpfQkLdrHSh6Ti7raGqya67wOYnhjFwL7hPtdVHvkwnx+9sJGzhyfwvQtH8GZOERf86X2eWrmn1XtBZ+05VMWiVfsIcz6weluK5pZ/Z3PO797h3dy2a4jg/iAV5BKGJbZu2bXFkkoAXTFuEEPjI/lwRylpCX0Yltj6U+75o/pTWlnTtPJto60Hy2lQGDPI3QI5JyOBC0f3B2g2nLiRiDBhcCzv5Bbz3WfX878vbuRXr21hZ0klN89I47Xbz2Hjzy/mtdvP4RszhrGzuJLI0GC+f9GIZte5emIyIUHColX7WLWrjPvfzePLZ6Zww1lDeeqbU+nXJ4QbHl3VbWuNNfZ5n+XRUml8o2rZjZhXXEHuwQou96GV0mjWqCRq6hpY0aJvfU9ZNbX1vo/86qyvTRtCcmwE3312PbtKq7hp+tCmfdPTEwgPcfmlC2xnSRUFh49x7sgkpqe7f4Ytv9dAaPwgMiHVXctIjYtkzKCYpi6w1zYW8o/3dnLd1MG8/6NZZP+/i9j+60vZ/utL+fO1E0jp91k9bMG56fy/y0bz7rYSFJh/9tBmrzVqQAxZQ/rx1Mq9zd6g1zj1lDNb1FMahQUHMXFwrE91lVfW7Wfi4NimbjQR4ZzhCXy0o7RVzceTqvLHZdv49etb+cIZA3jkxizuuDCDpd+dwZhBMfzk5c0sfDevw9dvz31v7yA4SLj/uklU19bzUovkumLnIT7cUUqQS/j6E6vbnYyae7CCtIQ+hAUHed3vTUBXKe7tgoNc3DprOD96YWOrrq9GjX/YH+cdapYstjhLjzQmFYCfX5FJWIiLs9Lj8eYfXzuTgsPVhAa5CA12fyVGhTVr6o9N7svY5L7cOXskNXUNhIc0/2WJjwrj4jEDeGldActyDjI4LpJfXDkGcPf/P/2NaXz5wRVcufAjwkOCENx/UOeNTOSvcyd2/ofUSSt3HSI5NoKUfp+1PMYM6kuQS9hYcISLMvs3bV+y4QAi8AUf6imNpqbFER7i4v1tJc3+z3Y0jfzyf/cXuAv6P7h4BN9/bgMJUWFNn+Qb950zPJHlW4v55ZXqtevGV41de+eNSGRQbATR4cGs2FnKleM/69JQVZ74ZDcXZvZv9mZ+MtbvO0KQSzjDY4LopWMH8Mc3t/NubjE/en4jWUP68YsrxjTtFxFCgrx/r9+YMYyEqDAqa+q8xvjVaYP53rMbeC57H3OnDAbcRfqwYBdjB/VtdXyjacPi+evyHRw9dqJZ68fTlsJycg9WcM+cMc22zxiRyPNrCthYcISJg92Jq/DIMe57ezv7jxyjpKKGkooaDlef4NqsVP7vmjMIcrm/v2GJUTzzzWl888k1PPJhPvPPHtqs/uqr3IPlvLJ+P7fMHMaFmf0ZnxrLvz/dw41OV6qq8pe3t9M/Joxl353JT1/N4fdLt5Gzv5w/fHkckaHNU8L2oopmk3p9YS2VALtqYjK3zkpv9snT06DYCNIS+rCiRd92TmE5fSNCSPbotkmNi+T+6ya1+cseFRbMqAExDEuMIqVfJEnR4W2+AYlIq4TSaO7kVI5Un+Bg+XHumzuxWWE6NS6SZ781jflnp/HFSSlcMymF0QOjWbyh0Gv3kz+puucCTB0W1+z7iggNIiMpqllLpaFBeW1jIZOHxtE/pvXIq7aEhwQxPT2Bd3KLm316a1xIcngnRn511lUTkrlkTH/uuDCD0ODmf5oXZSax/8ixdoeNP/JhPm9tab818/72EoYnRZEaF0mQS5g2LJ6P85p/Ml++tZhfLNnC35ef3CdmT+v3HWFk/+hmA0xmO4nzG09mExMRzANfm9Tq+27PVROT+dq0IV73XXbGIM4aFs9dL23i/nfdn8TX7CljfGpsu68xNc1dV8lup3b40toCgl3CZS1qC+cMT0CEplFgJRU1fO2RlSzZ4B4lNjS+D5eeMZD/u/oMfvvFzxJKIxF3l3n58TqeWul9oEFH/rhsO1GhwfzPuekA3DBtCDtLqpoGZKzYeYhVu8r49nnDiY0M5W9zJ3D3paP47+YDTTW9RtW1dewtq24q0vvKWioBFhLk4keXjGr3mLPS41myvpC6+gaCnRn6WwqPkjkw5qQ+lXbV2ekJXJTZn5kjEpu6ljwNie/Dj53iPriLeZfc9wFvbilq9Ud+or6BP7+1nZ3FlRRV1FBcfpyyqlo8W9uT0/rx+PwpTasTtCX3YAVlVbVMS2vdUhufEsuyLQebEsE9r29hZ0kVC5w/rs6YNTKRd3KL2VVa1dRlub2ogtS4iFaf5PzJ5ZI278Eyy6nHLd9aROag1t2fZVW1/Oa/uaT2i+DC0Ulef2+qaupYmV/GDWd99n80PT2et7YUsa+smtS4SFSV+5a7hze/sekAv5wzps0PH41+/doWsvcc5qz0eM4ZnsCZQ/o1O6ehQVm/7whXjG/+Jjw8KYqMpCj2HKrmn9dneR123VWhwS4e//pk7nxhI39Yts09CrCwnAXnDmv3vImDYwkNctdVLhjdv9X+mrp6XlxbwEWZ/VvNO4rrE8rYQX35cEcJN04fyg2PruLA0eP8++YpZLVRx2lpfGosMzISeOTDXdw0fWiHP3tPa/ce5u2tRfzgohHERrpju2zcQH79+haeXLGHs9Lj+fNb2xkQE861k913XhcRvnVuOhsLjvLfzQf5xRVjcDnJLq+4EtXOzcuCALdURGS2iGwTkTwRucvL/h+JyHrna7OI1ItInLPvDmdbjoh81+OcZz3O2S0i653tQ0XkmMe+B1u+Xk81PT2eipo6NjtdXnX1DeQerGjW9XUquVzCwzdkcX0bnwJbGtE/irSEPizLaT1EdFnOQf7x3k7ySiqJCQ9menoCN00fys0z0rh5RhpfPDOFj/MO8djH7c/uXr/vCDc/vprwEBczRiS02j8+NZYj1SfYW1bNX97ewWMf72b+2UP50pkpvn3THs5zur1+v3Qbpc6kwx1FlYxICkw9xRdJ0eGMT43l7TYKq29sOkB9g7L7UHWbczRW7DxEbX1DU4ICmubqNNZV3tpSxOb95VwzMZmKmjqWb22/kHu4qpYnVuzmwNFjPPxBPl99ZCXjfvkmT638bGmZ/NIqKo7Xef2AsvC6STz7rWle952ssOAg/nLtBG6bNZzn1xRQ36CtJj22FB4SxITUtusqy3KKOFx9gnlOl1pLMzISWLv3CDc+uoq84gr+ef2ZPieURt8+bzillTU8n+37ROSaunp+88ZWEqJC+fo5ac2+n69MTuWtrUU8v6aA7D2HufX84a2S1UWZ/SmpqGGDx9D8XC8jv3wRsI9dIhIE3I/7lr8FwGoRWayqWxqPUdU/AH9wjr8C+J6qlonIWOCbwBSgFlgqIq+r6g5VvdbjNf4EeFZnd6pq+4Ooe6DGovPHeaVMSI1lZ0kVNXUNjEnunqTSWSLCJWMG8MiH+Ryprm36lATuW+4mx0bw1vfObdXcb1RScZz73t7B5eMGeR2l9cyqvfz81RwSo8N4YcF0BvZtfUzjJMifvLyZj/JK+UpWCj+9LLNLLb3UuEjuuCCD+9/N4+OdpdxxQQb5pZWcP9p7XexUuWh0En98czsFh6tb1REWbyhkcFwkReXHeXntfiYNbl2Mfm97MZGhQc0m/o3oH0VCVCgrdh7iy2emcN/bOxgaH8lvvngGH+8s5eV1Be0Ox35tYyEn6pXHbprC4PhIVu8q4+EP8/n5qzlkJEUzJS2uabLcRC+Jo7NvWJ0lIvzwkpGk9Itg8YZCJqd1/AY/dVgc97+bR8XxE63qGs+s3EtKvwjOGd76gw24B9Q88N5ONhYc4f7rJjFzRGKnY542LI5Jg2N58P185k4Z3GELvqyqlm/9O5vVuw/zhy+NazWP6mtTh/DQB/nc/dImBvUN5ytZrT9ozRqZRJBLeGtLUVM9aPvBCsJDXAyO61xdLZAtlSlAnqrmq2otsAiY087x84BnnMejgU9Vtdq5X/37wNWeB4v73eIrHuectuKjwhg1ILqp37NxJv2YdgqKPc3ssQOoa9Bmn2x3l1bxcd4h5k5ObTOhAPz8ijE0qPLLJTnNth+rrefOFzZy90ubmJYez2u3n9Nm0XDkgGjCgl18lFfKZWcM5DfXjGtqxnfF9y4awdLvzmRCaiy/fn0rJ+o1YEV6X10zKYVgl/DYx7ubbT9w9Bird5fxxUkpXDxmAEs2FjZNbmykqrybW8L09IRmI3lEhLPSE/hkZynLcorYcqCc28/PcE8knJDMe9vav0XDi2v3M2pANJmDYogKC2bWqCQevP5MUvpFcOvTaykuP876fYeJCgsm3cvox1Nl7pTBPP3NaUT5MHF1alo8DUqrIdz5JZWsyD/EvCmD2/zdyhoSx8WZ/fnzVyZwaScGiHgSEW6dNZz9R451eN+lvOJKrn7gYzYUHGXhdRP5clZqq2NS4yKZNTKJ+gbl1vOHex3J1TcyhClD45rV5LYVVZCRFN3u3643gUwqyYBn+63A2daKiEQCs4EXnU2bgZkiEu/s+wLQ8qc1AyhS1R0e29JEZJ2IvC8iM9p4rVtEJFtEsktKes5aPdPTE1i9u4yaunpy9pcTFuximJdZvz3VuOS+DOwb3rSCALjvchnkEr4yufUvuqfUuEi+c0EGy3KKeCfX/UudU3iUKxZ+xLPZ+7ht1nAeu2ky/Vr0YXsKCXJx4ej+XDKmP3+5dkKn/xC8GZ4UxZNfn8JD15/J7DEDmJHR+U+d/jQoNoIrxg9i0aq9zSauvb7xAKpwxfiBXDMxmSPVJ1pN4NxZUsn+I8eYNar19zA9PZ6i8hp+sTiHtIQ+zJngrn1cNTGZugZttcJCo/ySStbvO8LVE5v/WceEh/DP67OoPF7Ht59aS/buw4xL6XtSSf5Uyhraj4F9w/n161uaLZ74rPP7/OV2ulRDg108dENW0/yVrjp/VBKjBkTzwHt5bQ5RXr27jGse+JjK43UsumVau5MSv3/RCK7NSuXLZ7b9t3hRZn92FFc2zcLfdrCiS/OyAplUvP0GtTWA+wrgY1UtA1DVrcDvgLeApcAGoOWCVJ4tG4ADwGBVnQh8H3haRFr1H6nqQ6qapapZiYnd+ybhaXp6PDV1Dazdc4ScwnJGDYhuKtqfDlwudxfYB9tLqK6to7augRfW7OOCUUk+jb76xjnDyEiK4mev5vDP93dy9f2fUHH8BP+5eSo/vGSkT0ni/q9O4p/XZ3VqBFFHRISLxwzgwevPJMFZKr07fXPGMKpq65vVLJZsKGRssnvU34yMBBKiQnnZY5WGhgbl169vJTTYxQWjWhefG4e1Hyw/zu3nD2/6vRs9MIZRA6J5aa33FR9ecVbonTOh9RvoyAHR/PaLZ5C95zC5BysCUjMJlPCQIP46dyL7yqr5f69sRlWpqavn+TUFXDg6iaROjCbsKhHhf85LJ7+kqs3lkX65JIe+kSG8cuvZXrs7PY1N7svvvjSu3b+NxuH4b28t4nBVLcUVNc1WQ/dVIN+1CmjeukgB2mrLzaVFN5aq/ktVJ6nqTKAMaGqRiEgwcA3wrMfxNap6yHm8BtgJNJ/Z14NNGRaHS+CTnaXkFB4l8zTq+mp0yZgB1NQ18P62Et7eWkRpZS3zpnovaLYUGuzinqvGUnD4GL/5by7njUxk6R0zmxbgNG6Zg2KYkZHAYx/vpqaunt3O2nGN80yCg1xcMX4Qy7cWN61A/OjHu3hvWwk/vWw0A/q2fkMcHBdJcmwEwxL6NJuvAnDNpGTW7zvS6s6GDQ3KS+v2c87wBK/XBHeyaZyY2NGbXk8zJS2O7104glfXF/J8dgFvbSmirKqW66b6NnjFHy7OdC/j5G0tupKKGjbvL2fu5MHNFko9GalxkYwaEM2bW4o+W5G7hyWV1UCGiKSJSCjuxLG45UEi0hc4F3i1xfYk59/BuBOIZ9K5EMhV1QKP4xOdwQGIyDAgA/C+8E0PFBMewriUWF5au5/y43XdNvLrZEwe6r6T3tKcg00F+pmd6DKaNiyeX145hj98aRz/vP7Mdru7erNvzUynpKKGV9cVsmRD49pmnyWDayamUFvfwOubDrCp4Ci/W5rLJWP6tzmnQ8Q92u/Rmya3ah3PmZCMCM1aPuC+e2LB4WOtur5a+skXRvOvG7OajTg7XXx71nCmp8fzs8WbWfhOHsmxEcxoo0AfCBGhQUxNi+N9L7djaGy9dObvyxcXZfYne3dZUz2ps3NUIIBJxSmw3wYsA7YCz6lqjogsEJEFHodeDbypqi2X03xRRLYAS4BbVfWwx75WLRtgJrBRRDYALwALGrvTThfT0+Ob7jN/OiaV4CAXF45O4s2cIj7cUcq1HRTovblx+lC+nJXaLfNzThdnD48nc2AM//xgp3tE09B+zSbJjk2OISMpikWr93L7M2tJjArjd18c1+7PNHNQDEO91PD6x4RzdnoCr6zb32wy6MvrCogICeKSMQPajTU4yMUFo/v7pcZ1qgW5hPuunUCf0GByD1Ywb0rqKa8LnTsikZ0lrVc//mB7CXF9Qv3+PnFRZn8a1H3rhZjwYPrHdL7LN6Cd9qr6hqqOUNV0Vb3X2fagqj7occzjqjrXy7kzVDVTVcer6vIW+27yvIaz7UVVHeMcP0lVlwTq+wqUxjkDLvG+vtfpYPbYARw7UY9L4CteRqKYk+eesDaMnSVV7CiubDWpUES4elIyGwuOsresmvvmTmw2zLuzvnhmMnvLqplz/8e8uKaA8uMneG3jAWaPHeDX2wD0REkx4fxt3kSmpMV1OOAkEM4b6W6JfOCxUGVDg/LBjlLOGZ7g9yQ3dlBf+seEUVZVy6gBXZt8ffpUgnuBM4f0IzTIxbDEqFb3SzldnD08gejwYC4Y3b/NvnZz8r5wxkCSYyNwtbG22dUTk4kJD+YHF49kig9zM9ozZ3wy9149luraen7w/Aam3ruciuN1XDPp5EY4nS7OHp7Ac986y68z/n2VnhhFcmwE73vUVbYeLKe0sqZLc2A64nJJ08K1IwZ0bQj45/tjxmkmIjSIayenntZvxmHBQbz0P9OJ7wEjpT7PQoJc/N81Z5BXXOl1VNrAvhGs+smFnVrmoy0ul/DVqUO4bspgPtl5iMc/2U3F8ROt7pxp/M99r5ZElmwo5ER9AyFBLj7Y7q6nzAjQIJaLMvvz1Mq9XaqnAEhbSx73BllZWZqdnd3dYRhjTJuWbj7Igv+s4dlbpjF1WDzXPfwpZVW1LP3uzIC8Xn2D8uSK3VwzKaXNxWtFZI2qel2ozrq/jDGmB5s+PJ5gl/C+Mwcse/fhgHR9NQpyCfPPTmszoXTEur+MMaYHiwkPYdKQfry/vYSsof2orW/w+1Bif7KWijHG9HDnjkgkp7CcF9fuJzzE1WxR0J7GkooxxvRw5zrdXa9vPMDUtHi/DMAIFEsqxhjTw2UOjGka5RfIeoo/WFIxxpgezuUSZjo3p5vZw9fDs0K9McacBm6ZOYzUfpEMT+re+/p0xJKKMcacBkYNiDktlm+y7i9jjDF+Y0nFGGOM31hSMcYY4zeWVIwxxviNJRVjjDF+Y0nFGGOM31hSMcYY4zeWVIwxxvhNr75Jl4hUANu6Ow4vEoDS7g7Ci54YV0+MCXpmXD0xJrC4OqOnxDREVb0uQtbbZ9Rva+vuZd1JRLItLt/0xJigZ8bVE2MCi6szemJMLVn3lzHGGL+xpGKMMcZventSeai7A2iDxeW7nhgT9My4emJMYHF1Rk+MqZleXag3xhjjX729pWKMMcaPLKkYY4zxm16bVERktohsE5E8EbmrG+N4VESKRWSzx7Y4EXlLRHY4//Y7xTGlisi7IrJVRHJE5I4eEle4iKwSkQ1OXL/sCXE5MQSJyDoRea0HxbRbRDaJyHoRye4JcYlIrIi8ICK5zu/XWT0gppHOz6jxq1xEvtvdcTmxfc/5Xd8sIs84fwPdHld7emVSEZEg4H7gUiATmCcimd0UzuPA7Bbb7gKWq2oGsNx5firVAT9Q1dHANOBW5+fT3XHVAOer6nhgAjBbRKb1gLgA7gC2ejzvCTEBzFLVCR5zG7o7rr8CS1V1FDAe98+sW2NS1W3Oz2gCcCZQDbzc3XGJSDLwHSBLVccCQcDc7o6rQ6ra676As4BlHs/vBu7uxniGAps9nm8DBjqPB+KepNmdP69XgYt6UlxAJLAWmNrdcQEpuP+4zwde6yn/h8BuIKHFtm6LC4gBduEMEOoJMXmJ8WLg454QF5AM7APicE9Uf82Jr8f8vLx99cqWCp/9ZzUqcLb1FP1V9QCA829SdwUiIkOBicDKnhCX0820HigG3lLVnhDXfcD/Ag0e27o7JgAF3hSRNSJySw+IaxhQAjzmdBU+IiJ9ujmmluYCzziPuzUuLJmsPgAAA8pJREFUVd0P/BHYCxwAjqrqm90dV0d6a1IRL9tsbHULIhIFvAh8V1XLuzseAFWtV3c3RQowRUTGdmc8InI5UKyqa7ozjjacraqTcHfz3ioiM7s5nmBgEvAPVZ0IVNGDum5EJBS4Eni+u2MBcGolc4A0YBDQR0S+1r1Rday3JpUCINXjeQpQ2E2xeFMkIgMBnH+LT3UAIhKCO6E8paov9ZS4GqnqEeA93PWo7ozrbOBKEdkNLALOF5H/dHNMAKhqofNvMe4awZRujqsAKHBalwAv4E4y3f6zclwKrFXVIud5d8d1IbBLVUtU9QTwEjC9B8TVrt6aVFYDGSKS5nw6mQss7uaYPC0GbnQe34i7pnHKiIgA/wK2quqfe1BciSIS6zyOwP1Hl9udcanq3aqaoqpDcf8evaP/v727CbExCgM4/n9C8p2PshESWShJkmSh7GZrIbGR1WxYyUIpZWM7saEsSJYspSyUiBSmYUEkFqZQkpKkx+Kc4TZmZPFe58b/V2/vuWdut+femel5zzn3PU/m/pYxAUTEvIhYMNGmzMWPtYwrM8eB1xGxvnbtAp60jGmSvfyc+oL2cb0CtkXE3Po/uYvyxYbWcf1e60WdVgcwBDwFngPHGsZxmTJf+pVyJXcQWEpZ+H1Wz0v+ckw7KNOBo8DDegwNQFwbgQc1rjHgeO1vGldPfDv5uVDf+rNaAzyqx+OJv/EBiGsTcL/+Dq8Ci1vHVOOaC7wHFvX0DUJcJygXTmPARWD2IMT1u8NtWiRJnflfp78kSX1gUpEkdcakIknqjElFktQZk4okqTMmFanPIuLbpF1wO7uLPCJWR88O11JrM1sHIP0HPmfZWkb65zlSkRqp9U5O1Rox9yJibe1fFRE3ImK0nlfW/uURcSVKPZlHEbG9vtSMiDhX625cr7sNSE2YVKT+mzNp+mtPz88+ZuZW4DRlt2Nq+0JmbgQuASO1fwS4maWezGbKnfIA64AzmbkB+ADs7vP7kablHfVSn0XEp8ycP0X/S0rRsRd1A8/xzFwaEe8o9TK+1v43mbksIt4CKzLzS89rrKaUAFhXHx8FZmXmyf6/M+lXjlSktnKa9nTPmcqXnvY3XCtVQyYVqa09Pec7tX2bsuMxwD7gVm3fAIbhR7GyhX8rSOlPeUUj9d+cWq1ywrXMnPha8eyIuEu5wNtb+w4B5yPiCKVS4oHafxg4GxEHKSOSYcoO19LAcE1FaqSuqWzJzHetY5G64vSXJKkzjlQkSZ1xpCJJ6oxJRZLUGZOKJKkzJhVJUmdMKpKkznwHlZ6UqQCpFYEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(history.history['val_loss'][10:]).plot(logy=False)\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Validation Error\")\n",
    "print(f\"Minimum MSE: {round(min(history.history['val_loss']),3)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment Five - Recommender System using Matrix Factorization\n",
    "#### Three Latent Factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_latent_factors = 3\n",
    "\n",
    "user_input = Input(shape=[1], name='user')\n",
    "user_embedding = Embedding(input_dim=n_users + 1, \n",
    "                           output_dim=n_latent_factors, \n",
    "                           embeddings_regularizer=regularizers.l1(10e-7), \n",
    "                           name='user_embedding')(user_input)\n",
    "user_vec = Flatten(name='flatten_users')(user_embedding)\n",
    "\n",
    "movie_input = Input(shape=[1], name='movie')\n",
    "movie_embedding = Embedding(input_dim=n_movies + 1, \n",
    "                            output_dim=n_latent_factors, \n",
    "                            embeddings_regularizer=regularizers.l1(10e-7), \n",
    "                            name='movie_embedding')(movie_input)\n",
    "movie_vec = Flatten(name='flatten_movies')(movie_embedding)\n",
    "\n",
    "product = dot([movie_vec, user_vec], axes=1)\n",
    "model = Model(inputs=[user_input, movie_input], outputs=product)\n",
    "model.compile('adam', 'mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 81191 samples, validate on 4511 samples\n",
      "Epoch 1/100\n",
      "   32/81191 [..............................] - ETA: 9:07 - loss: 13.9572"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/faisito/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n",
      "/home/faisito/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81191/81191 [==============================] - 2s 24us/sample - loss: 11.9497 - val_loss: 7.4388\n",
      "Epoch 2/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 4.2506 - val_loss: 2.4705\n",
      "Epoch 3/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 1.7421 - val_loss: 1.3478\n",
      "Epoch 4/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 1.0710 - val_loss: 0.9941\n",
      "Epoch 5/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.8546 - val_loss: 0.8717\n",
      "Epoch 6/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7824 - val_loss: 0.8287\n",
      "Epoch 7/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7571 - val_loss: 0.8119\n",
      "Epoch 8/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7483 - val_loss: 0.8112\n",
      "Epoch 9/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7452 - val_loss: 0.8082\n",
      "Epoch 10/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7436 - val_loss: 0.8102\n",
      "Epoch 11/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7429 - val_loss: 0.8104\n",
      "Epoch 12/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7424 - val_loss: 0.8065\n",
      "Epoch 13/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7424 - val_loss: 0.8095\n",
      "Epoch 14/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7423 - val_loss: 0.8064\n",
      "Epoch 15/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7420 - val_loss: 0.8042\n",
      "Epoch 16/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7420 - val_loss: 0.8059\n",
      "Epoch 17/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7417 - val_loss: 0.8068\n",
      "Epoch 18/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7418 - val_loss: 0.8055\n",
      "Epoch 19/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7410 - val_loss: 0.8085\n",
      "Epoch 20/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7410 - val_loss: 0.8058\n",
      "Epoch 21/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7407 - val_loss: 0.8032\n",
      "Epoch 22/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7403 - val_loss: 0.8078\n",
      "Epoch 23/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7398 - val_loss: 0.8048\n",
      "Epoch 24/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7395 - val_loss: 0.8095\n",
      "Epoch 25/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7387 - val_loss: 0.8043\n",
      "Epoch 26/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7377 - val_loss: 0.8040\n",
      "Epoch 27/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7369 - val_loss: 0.8046\n",
      "Epoch 28/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7358 - val_loss: 0.8041\n",
      "Epoch 29/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7340 - val_loss: 0.8042\n",
      "Epoch 30/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7326 - val_loss: 0.8008\n",
      "Epoch 31/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.7307 - val_loss: 0.8014\n",
      "Epoch 32/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7283 - val_loss: 0.7991\n",
      "Epoch 33/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7260 - val_loss: 0.7973\n",
      "Epoch 34/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7229 - val_loss: 0.7936\n",
      "Epoch 35/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7195 - val_loss: 0.7934\n",
      "Epoch 36/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7158 - val_loss: 0.7916\n",
      "Epoch 37/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7123 - val_loss: 0.7897\n",
      "Epoch 38/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7081 - val_loss: 0.7866\n",
      "Epoch 39/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.7042 - val_loss: 0.7840\n",
      "Epoch 40/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6999 - val_loss: 0.7832\n",
      "Epoch 41/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6958 - val_loss: 0.7842\n",
      "Epoch 42/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6920 - val_loss: 0.7781\n",
      "Epoch 43/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6881 - val_loss: 0.7785\n",
      "Epoch 44/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6848 - val_loss: 0.7789\n",
      "Epoch 45/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6813 - val_loss: 0.7774\n",
      "Epoch 46/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6782 - val_loss: 0.7760\n",
      "Epoch 47/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6751 - val_loss: 0.7723\n",
      "Epoch 48/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6723 - val_loss: 0.7709\n",
      "Epoch 49/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6697 - val_loss: 0.7749\n",
      "Epoch 50/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6674 - val_loss: 0.7726\n",
      "Epoch 51/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6650 - val_loss: 0.7716\n",
      "Epoch 52/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6629 - val_loss: 0.7694\n",
      "Epoch 53/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6608 - val_loss: 0.7692\n",
      "Epoch 54/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6588 - val_loss: 0.7675\n",
      "Epoch 55/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6569 - val_loss: 0.7677\n",
      "Epoch 56/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6552 - val_loss: 0.7688\n",
      "Epoch 57/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6538 - val_loss: 0.7669\n",
      "Epoch 58/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6521 - val_loss: 0.7653\n",
      "Epoch 59/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6509 - val_loss: 0.7674\n",
      "Epoch 60/100\n",
      "81191/81191 [==============================] - 2s 25us/sample - loss: 0.6493 - val_loss: 0.7642\n",
      "Epoch 61/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6481 - val_loss: 0.7646\n",
      "Epoch 62/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.6470 - val_loss: 0.7670\n",
      "Epoch 63/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6457 - val_loss: 0.7633\n",
      "Epoch 64/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6445 - val_loss: 0.7651\n",
      "Epoch 65/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6437 - val_loss: 0.7625\n",
      "Epoch 66/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6426 - val_loss: 0.7629\n",
      "Epoch 67/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6417 - val_loss: 0.7650\n",
      "Epoch 68/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6410 - val_loss: 0.7655\n",
      "Epoch 69/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6401 - val_loss: 0.7643\n",
      "Epoch 70/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6396 - val_loss: 0.7667\n",
      "Epoch 71/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6386 - val_loss: 0.7646\n",
      "Epoch 72/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6380 - val_loss: 0.7651\n",
      "Epoch 73/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6374 - val_loss: 0.7649\n",
      "Epoch 74/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6368 - val_loss: 0.7662\n",
      "Epoch 75/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6359 - val_loss: 0.7669\n",
      "Epoch 76/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6354 - val_loss: 0.7676\n",
      "Epoch 77/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6353 - val_loss: 0.7649\n",
      "Epoch 78/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6344 - val_loss: 0.7653\n",
      "Epoch 79/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6339 - val_loss: 0.7664\n",
      "Epoch 80/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6335 - val_loss: 0.7678\n",
      "Epoch 81/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6330 - val_loss: 0.7652\n",
      "Epoch 82/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6328 - val_loss: 0.7654\n",
      "Epoch 83/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6323 - val_loss: 0.7651\n",
      "Epoch 84/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6319 - val_loss: 0.7667\n",
      "Epoch 85/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6316 - val_loss: 0.7672\n",
      "Epoch 86/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6309 - val_loss: 0.7683\n",
      "Epoch 87/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6309 - val_loss: 0.7682\n",
      "Epoch 88/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6303 - val_loss: 0.7674\n",
      "Epoch 89/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6301 - val_loss: 0.7678\n",
      "Epoch 90/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6298 - val_loss: 0.7669\n",
      "Epoch 91/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6296 - val_loss: 0.7651\n",
      "Epoch 92/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6293 - val_loss: 0.7691\n",
      "Epoch 93/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6286 - val_loss: 0.7681\n",
      "Epoch 94/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6287 - val_loss: 0.7697\n",
      "Epoch 95/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6282 - val_loss: 0.7682\n",
      "Epoch 96/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6282 - val_loss: 0.7707\n",
      "Epoch 97/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6276 - val_loss: 0.7695\n",
      "Epoch 98/100\n",
      "81191/81191 [==============================] - 2s 21us/sample - loss: 0.6275 - val_loss: 0.7679\n",
      "Epoch 99/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6274 - val_loss: 0.7714\n",
      "Epoch 100/100\n",
      "81191/81191 [==============================] - 2s 22us/sample - loss: 0.6270 - val_loss: 0.7723\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x=[X_train.newUserId, X_train.newMovieId], \n",
    "                    y=X_train.rating, epochs=100, \n",
    "                    validation_data=([X_valid.newUserId, X_valid.newMovieId], X_valid.rating), \n",
    "                    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum MSE: 0.763\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3zV9fX48de5mWSTSQaQhBH2DNsBIooD0FoHta21/Wr9Vltrbb/VDju0v9ZabWu1tdaB1t0qQtWiiIDKDnsGsgkQkpCQQELmff/+uDcx4ya5gdzcm5vzfDzuI7mfce9JlJz7fp/3EGMMSimlVFsWdweglFLKM2mCUEop5ZAmCKWUUg5pglBKKeWQJgillFIO+bo7gJ4UHR1tkpOT3R2GUkr1Gdu3by81xsQ4OudVCSI5OZmMjAx3h6GUUn2GiOR3dE67mJRSSjmkCUIppZRDmiCUUko5pAlCKaWUQ5oglFJKOaQJQimllEOaIJRSSjnk0gQhIgtFJFNEskTkAQfnw0XkPyKyW0T2i8jtLc69ICLFIrLP2fera7D2VOhKKdXvuSxBiIgP8DRwFTAGWCoiY9pcdjdwwBgzEZgLPC4i/vZzy4CF3XnPs7UNFxKyUkqpFlzZgpgOZBljcowxdcAbwJI21xggVEQECAHKgAYAY8yn9udOq6rTBKGUUj3FlQkiETja4nmh/VhLTwGjgePAXuBeY0y3+olE5E4RyRCRjDPVdRcSr1JKqRZcmSDEwbG2+5teCewCEoBJwFMiEtadNzHGPGuMSTfGpDciFFXUtLvmpY15/GKF06UMt8gtrWLBE+s5fvqcu0NRSinAtQmiEBjc4nkStpZCS7cD7xibLCAXGHUhb5qR37pXyhjDs5/m8M/N+ZRXeW4L49PDJRwpPsuW3FPuDkUppQDXJohtwAgRSbEXnm8BVra5pgCYDyAicUAakHO+b2gRISOvvNWxzJNnOHb6HFYDnx4pOd+Xdrl9xyoAOFR0xs2RKKWUjcsShDGmAbgH+BA4CLxljNkvIneJyF32yx4GZovIXmAN8GNjTCmAiLwObALSRKRQRL7V1XsG+fuwLa91C2LNwWIAQgN8WZfpwQnieCUAhzVBKKU8hEv3gzDGfAB80ObYMy2+Pw5c0cG9S7v7fkH+Phw8UcnZ2gZCAmw/2scHTzIhKZxhMSGsP1xCo9XgY3FUHuk+q9Xw/Oe5XDsxnvjwAef9OrUNjRw5aUsMmZoglFIewqtmUgf7+2I1sLPA1s1UeraWXUdPM39UHPNGxVJWVcfuwtM99n47j57mNx8c5Jl12Rf0OoeLztJgNYxPDOd4RQ0V5+p7KEKllDp/XpUgggJ8sAhss9chPjlUjDEwf3Qsl4yIxiKw7lBxj73fxwdPAvDBviIarW0HaDlv/3Fb/eFLU2yjgJtaE0op5U5elSAsIowaFMZ2+0imNQdPEh8eyNiEMCKC/JkyZCBre7AOsfrASYL9fSg5U8vW3K7n9D2x+jDPfda+Br/veAWhgb5cPjoO0EK1UsozeFWCAJiWPJCdBaepqm3gsyOlXDYqFttEbZg3Kpa9xyooPtN+rkR35ZZWkVV8lrsvG84APx/e29N2BG9rxhhe2pjH39Zlt2tt7DtWyZj4MJIGDiAkwJfD2oJQSnkAr0sQ6cmRVNc18uKGXKrrGps/lQPMS4sFYH0PtCJWHygCYPHEBC4bHcuqfUU0NHY8Cfxo2TkqztVzqqqOXUe/qIM0NFo5VFTJuMRwRISRcSHaglBKeQQvTBADAfj7+hwG+Pkwa1hU87nR8aHEhQWwNvPC6xCrD5xkdHwYSQODWDQhnlNVdWzO6bibaa99nkPTvU1ySquoqbcyLtE2gTxtUBiHT57BmPOvaSilVE/wugQRHz6AxIgBnKlt4KIR0QT6+TSfExHmpcXy2eFS6jv5tN/S5pxTvLmtoNWxU2dr2Z5fzoIxttbJ3LRYgv0772bae6wCPx9hWvLA5uI2fFGgHpsQDkBaXAinq+spPlPr3A+slFIu4nUJAmx1CIDLR8e2OzdvVCxnahvYnl/e7lxbaw6e5OvPb+XHb+9l1b4Tzcc/OVSM1cAV9gQR6OfD5WPiWLW/qMPEs+9YBWmDQrl6fDxZxWfJLa2yH68k0M9CanQwYGtBgM6HUEq5n1cmiHmjbJ/oLxsV1+7cnOHR+PkIH7fo5nFkzcGT3PXKdkbFhzI+MZwfv723eSG91Qe+GB3V5Jrx8ZyurmdDVmm71zLGsPdYBeMTw5trIk3vv+9YBaPjw/D1sf2nSBsUCmiCUEq5n1cmiMUTE9j2s8uJCQ1ody4kwJfLRsXy9o5CauobHd7flBxGx4fxz2/N4Mmlk6lvtHLfm7uaR0ddPjqueXQUwKVpMYQG+PL+nhPtXq+pQD0uMZzBkUGMGhTK6oMnsVoNB45Xtko0kcH+xIQGkKkjmZRSbuaVCUJECPLveBWR22YnU15dz8rd7WsGGXllrZJD+AA/UqKD+dXisWzJLeOOlzM4V9/YXH9oEuDrw4KxcXy4v6jd1qdNBeoJiREALBgTR0ZeGXuOVXCmtoFx9vpDk7S4UG1BKKXczisTRFdmpUaRFhfKSxvzWo0WsloNv1i5n5iQgObk0OTLU5NYNDGBjdmnCA3wZWZqVLvXXTQhgcqaBta1GSXVVKAeOSgEsCUIq4GnPjkCfFGgbpI2KJQjxWcuaHa2UkpdqH6ZIESEr88eyv7jla2K1e/uOsb+45X838JRrZJD0z2PXDeOIZFBXDluEP6+7X91F42IJirYnxW7WrdMmgrUAb62EVXjEsKJCwvg44PF+Fq+SBxN0uJCqam3UlBW3VM/slJKdVu/TBAA109OJCzQl2Ub8wCoqW/ksQ8zmZAUzuKJCQ7vCR/gx0f3XcLvvjTe4Xk/HwuLJiaw+uBJKmtsC+61LFA3sViE+fZi9ci4LxJHEy1UK6U8Qb9NEEH+vtw8bTCr9hVRVFHD85/ncqKihp9cPRpLJ8uBB/r5NI84cuS6yYnUNVj5715bsbplgbqlphpGywJ1kxFxIYhoglBKuVe/TRAAX5uZTKMx/HnNEf62LpsFY+Ic1ha6Y2JSOCnRwSzfeQz4okA9vk2CmJUaxcSk8HbFbrAlryGRQR2uyXSi4hy//s8BXt2S3+nyHkopdSFcumGQpxsSFcT8UbG8vrUAX4vw4FUXtB02YKtVXD85kSdWH+bY6XPNBeqmbqMmgX4+rLjnog5fJy0ulP3HKzhTU09ooK0e0mi1Lfj3+EeZnKtvxGrgxQ15/OTqUcxL+2JRwnN1jdRbrYQF+nX4+kop1ZV+nSAAvjE7hY8PFnPrjCGkxoR0fYMTrptkSxArdx1n37EKh3WGrkwcHMFHB04y6dermZAUzszUKD4/UsreYxVcOjKGh5eM42BRJb/77yG+uSyDKUMi8PWxkH+qipOVtYQE+LLpwcuak4tSSnVXv08Qc4ZH8ezXpnLRiOgee80hUUFMHTqQ5TsLOVlZy9XjB3X7Ne66dBiTB0ewMfsUG7NLefbTHCKD/XnqK5O5Znw8IsKQqCDmpcXy6pZ8XttSwMAgfy4eEYO/r4XXthSwJaeMyx10YSmllDP6fYIQEa4Y2/0/4F25bnIiP393H0C7ArUzfCzC7OHRzB4eDaRRVduAn4+l3fBaf18Lt89J4fY5Kc3HahsaeWdHIZ9nlWqCUEqdt35dpHala8fH4+djqwm0LVCfj+AAX4dzLxwJ8PVhekoUnx3pud3zlFL9jyYIFxkY7M/ctFiHBerecPHwaLJLqjhRca7X31sp5R00QbjQQ9eO4R9fT+92gbonNNVUPj/SfnVZpZRyhiYIFxocGcTctPZ7UvSGtLhQokP8+dzB8uNKKeUMTRBeymIR5gyPZkNWKVZd9E8pdR40QXixi4ZHU3q2TveWUEqdF00QXuziETGA1iGUUudHE4QXGxQeyPDYED7TOoRS6jxogvByFw2PZmvuqebtVa1Ww8cHTlJUUePmyJRSnk4ThJe7aHg0NfVWdhSUc6iokhv/von/eTmDn9lneSulVEf6/VIb3m7msCh8LcIvV+4nu6SKsEBfZqVGsTazmOLKGmLDAt0dolLKQ2kLwsuFBPgydehADp88y41Tk/jk/rk8cv04Gq2Gt3ccc3d4SikP5tIEISILRSRTRLJE5AEH58NF5D8isltE9ovI7c7eq5z3xM2TWPX9i/ndDRMYGOzPsJgQpidH8q+MoxijcySUUo65LEGIiA/wNHAVMAZYKiJj2lx2N3DAGDMRmAs8LiL+Tt6rnJQYMYBRg1pvbXrTtMHklFaxLa/cTVEppTydK1sQ04EsY0yOMaYOeANY0uYaA4SKbSu0EKAMaHDyXnUBrh4/iJAAX97cdtTdoSilPJQrE0Qi0PKvT6H9WEtPAaOB48Be4F5jjNXJewEQkTtFJENEMkpKdHlrZwX5+7JoYgIf7D1BZU29u8NRSnkgVyYIcXCsbYf3lcAuIAGYBDwlImFO3ms7aMyzxph0Y0x6TEzMhcTb79w8bTDn6hv5z+7j7g5FKeWBXJkgCoHBLZ4nYWsptHQ78I6xyQJygVFO3qsu0MSkcNLiQnlLu5mUUg64MkFsA0aISIqI+AO3ACvbXFMAzAcQkTggDchx8l51gUSEm6YNZndhBQeOV7o7HKWUh3FZgjDGNAD3AB8CB4G3jDH7ReQuEbnLftnDwGwR2QusAX5sjCnt6F5XxdqffWlyIsH+Pjy9LsvdoSilPIx40zj49PR0k5GR4e4w+pw/fJjJU2uz+O+9FzM6PqzrG5RSXkNEthtj0h2d05nUijsuTiU00JcnVh92dyhKKQ+iCUIRHuTHHRensvrASfYUnnZ3OEopD6EJQgFw+5xkIoL8tBWhlGqmCUIBEBrox12XDmNdZgnb88vcHY5SygNoglDNvj5rKNEh/jz+kbYilFKaIFQLQf6+3HFxKhuzT5FXWuXucJRSbqYJQrUyf3QsAFtyT7k5EqWUu2mCUK0MiwkhKtifLTlah1Cqv9MEoVoREaanRLIlVxOEUv2dJgjVzoyUSI6dPkdhebW7Q1FKuZEmCNXOjNQoAO1mUqqf0wSh2kmLCyV8gB9btZtJqX6t0wQhNoM7u0Z5H4tFmJYcqSOZlOrnOk0QxrbU67u9FIvyIDNTI8k7Vc3Jyhp3h6KUchNnupg2i8g0l0eiPMr0lEgAHc2kVD/mTIKYB2wSkWwR2SMie0Vkj6sDU+41Jj6MkABftuRoN5NS/ZWvE9dc5fIolMfx9bEwdehAbUEo1Y912YIwxuQDEcAi+yPCfkx5uRmpkWQVn6X0bK27Q1FKuUGXCUJE7gVeBWLtj1dE5LuuDky534wU23yIbdqKUKpfcqYG8S1ghjHmIWPMQ8BM4A7XhqU8wfjEcAL9LNrNpFQ/5UwNQoDGFs8b7ceUl/P3tdUhXt9aQFbxWaYMHUj60IHMGhaFn4/OsVTK2zmTIF4EtojIcvvz64DnXReS8iQ/u2YMr20pICO/nKc+OYLVwJJJCfz5lsnuDk0p5WJimwvXxUUiU4CLsLUcPjXG7HR1YOcjPT3dZGRkuDsMr3Wmpp4nVh/mxQ15vPfdixiXGO7ukJRSF0hEthtj0h2d62qpDYuI7DPG7DDGPGmM+bOnJgfleqGBfty3YCQRQX489mGmu8NRSrlYV0ttWIHdIjKkl+JRHi4s0I//vXQY6w+X6CQ6pbycM5XGeGC/iKwRkZVND1cHpjzXbbOTiQsL4PcfZuJMF6VSqm9ypkj9K5dHofqUQD8fvjd/BD9dvo+1mcVcNirO3SEppVygqxqED/BzY8z6to9eik95qJvSBzM0KojHPjyM1aqtCKW8UVc1iEagWkR0uIpqxc/Hwg8WjOTgiUo+2HfC3eEopVzAmRpEDbBXRJ4XkSebHq4OTHm+RRMSSI0J5q9rs7UWoZQXciZBvA/8HPgU2N7iofo5i0W465JhHDhRyWdHSt0djlKqh3VYpBaRMGNMpTHmJQfndNirAmDJ5ASeWH2Yv63L5pKRMe4ORynVgzprQaxr+kZE1rQ559Q2pCKyUEQyRSRLRB5wcP5HIrLL/tgnIo0iEmk/d6/92H4R+b4z76d6X4CvD9+6KIVNOafYdfS0u8NRSvWgzhJEywX5Ijs55/hm2wiop7FtODQGWCoiY1peY4x5zBgzyRgzCXgQWG+MKRORcdhWjJ0OTASuFZERXf40yi2WzhhCWKAvz6zLdncoSqke1FmCMB187+i5I9OBLGNMjjGmDngDWNLJ9UuB1+3fjwY2G2OqjTENwHrgeifeU7lBSIAvt81O5sMDRWQVn3V3OEqpHtJZgogVkR+IyP0tvm967kxncyJwtMXzQvuxdkQkCFgIvG0/tA+4RESi7OeuBgZ3cO+dIpIhIhklJSVOhKVc4bbZyfj7WHj2U21FKOUtOksQ/wBCgZAW3zc9f86J13bUDdVRy2MRsMEYUwZgjDkIPAqsBlYBu4EGRzcaY541xqQbY9JjYrRI6i7RIQHcPG0wy3ce0y1KlfISHY5iMsZc6BIbhbT+1J8EHO/g2lv4onup6f2fx77vhIj8P/vrKQ92U/pgXt6Uz9pDxdyY7rDBp5TqQ1y5Ldg2YISIpIiIP7Yk0G6RP/ss7UuBFW2Ox9q/DgG+RJsEojzP2IQw4sICWJepXX1KeQNnFus7L8aYBhG5B/gQ8AFeMMbsF5G77OefsV96PfCRMaaqzUu8LSJRQD1wtzGm3FWxqp4hIsxLi+X9PSeob7TqtqRK9XEuSxAAxpgPgA/aHHumzfNlwDIH917sytiUa8xNi+WNbUfJyCtn1rAod4ejlLoAXSYIEQkAbgCSW15vjPm168JSfdVFI6Lx8xHWZhZrglCqj3OmD2AFtvkLDUBVi4dS7YQE+DIjJYpPDhW7OxSl1AVypospyRiz0OWRKK8xb1QsD793gKNl1QyODHJ3OEqp8+RMC2KjiIx3eSTKa1w2KhZAWxFK9XHOJIiLgO32Rff2iMheEdnj6sBU35USHUxKdLAmCKX6OGe6mK5yeRTK68xLi+WVLflU1zUQ5O/SwXJKKRfpsgVhjMkHIrAth7EIiLAfU6pD80bFUNdgZWPWKXeHopQ6T10mCBG5F3gViLU/XhGR77o6MNW3TU+JJMjfh7WZ2s2kVF/lTA3iW8AMY8xDxpiHgJnY9mpQqkMBvj5cNDyaNQeLsVp1v2ql+iJnEoQAjS2eN+LEhkFKXT0+nqLKGjLydZUUpfoiZ6qHLwJbRGS5/fl12FdZVaozC8bEEehnYcWuY0xPabspoVLK0zlTpH4CuB0oA8qB240xf3J1YKrvCw7wZcGYQXyw17Z4n1Kqb+kwQYhImP1rJJAHvAL8E8i3H1OqS0smJlBeXc/nR0rdHYpSqps662J6DbgW2E7rneDE/jzVhXEpL3HJyBjCB/ixYtcx5tlnWCul+obOdpS71v41pffCUd7G39fC1eMHsWLXcc7VNTLA38fdISmlnOTMPIg1zhxTqiOLJyZSXdfIxwdPujsUpVQ3dFaDCLTXGqJFZKCIRNofyUBCbwWo+r7pKZEMCgtkxa6OtiRXSnmizloQ38ZWfxhl/9r0WAE87frQlLfwsQiLJsaz/nAxp6vrHF5T29DIxqxSjNFJdUp5ig4ThDHmz/b6ww+NManGmBT7Y6Ix5qlejFF5gcUTE6lvNPx3X1G7c/WNVu5+dSdfeW4LB05UuiE6pZQjXU6UM8b8RUTGAWOAwBbHX3ZlYMq7jEsMY9SgUB557wCRwf5cOXYQAFar4Uf/2t1cn8gqPsvYhHB3hqqUsnOmSP0L4C/2xzzg98BiF8elvIyI8OLt0xgeF8q3/7mdJ1YfptFqeGjlPt7ddZzvzR+BCOSU6G62SnkKZ9Zi+jIwHygyxtwOTAQCXBqV8krx4QN4886Z3Dg1iSfXHGHBE+t5ZXMB3740lfsuH0FixABySzVBKOUpnEkQ54wxVqDBPru6GJ0kp85ToJ8Pv//yBH69ZCwFZdV8ZcYQHlg4ChEhJTpYE4RSHsSZxfoyRCQC+Ae2UUxnga0ujUp5NRHh67OSuW5yIqEBvojYFgdOjQ7m7R3HMMY0H1NKuY8zRerv2L99RkRWAWHGGN2TWl2wsEC/Vs9TY0I4W9tAydlaYkMDO7hLKdVbOkwQIjKls3PGmB2uCUn1VynRwQDkllRpglDKA3TWgnjc/jUQSAd2Y1uobwKwBbjItaGp/qY5QZRWMSM1ys3RKKU6myg3zxgzD8gHphhj0o0xU4HJQFZvBaj6j4SIAfj7WrRQrZSHcGYU0yhjzN6mJ8aYfcAk14Wk+isfi5AcFUSOJgilPIIzo5gOishz2DYMMsBXgYMujUr1WynRwWTrZDmlPIIzLYjbgf3AvcD3gQP2Y0r1uJToEPJPVdFo1UX7lHI3Z4a51gB/tD+UcqnU6GDqGw3Hys8xJCrI3eEo1a91Nsz1LWPMTSKyl9ZbjgJgjJng0shUv5QSYxvJlFN6VhOEUm7WWQviXvvXa8/3xUVkIfBnwAd4zhjzuzbnfwTc2iKW0UCMMaZMRO4D/gdbctoL3G5vzSgv1jTUNaekirlpbg5GqX6us2GuJ+xf8x09unphEfHBtrHQVdiWCl8qImPavMdjxphJxphJwIPAentySAS+B6QbY8ZhSzC3nO8PqfqOqGB/QgN9dairUh6gsy6mMzjoWsI2Wc4YY8K6eO3pQJYxJsf+em8AS7AVuR1ZCrzeJrYBIlIPBAG6X2U/ICKk6qJ9SnmEzloQocaYMAePUCeSA0AicLTF80L7sXZEJAhYCLxtf+9jwB+AAuAEUGGM+aiDe+8UkQwRySgpKXEiLOXpdFVXpTyDM8NcARCRWBEZ0vRw5hYHxzoau7gI2GCMKbO/10BsrY0UIAEIFpGvOrrRGPOsfZZ3ekxMjBNhKU+XEh3CsdPnqKlvdHcoSvVrzuwot1hEjgC5wHogD/ivE69dCAxu8TyJjruJbqF199LlQK4xpsQYUw+8A8x24j2VF2gayZR3SlsRSrmTMy2Ih4GZwGFjTAq23eU2OHHfNmCEiKSIiD+2JLCy7UUiEg5cCqxocbgAmCkiQWLbGGA+Onu730htsaqrIzX1jXy4v4iK6vreDEupfseZpTbqjTGnRMQiIhZjzFoRebSrm4wxDSJyD/AhtlFILxhj9ovIXfbzz9gvvR74yBhT1eLeLSLyb2AH0ADsBJ7t3o+m+qrkpqGubeoQR8uqeWVLPm9uO8rp6nquHBvH37+W7o4QleoXnEkQp0UkBPgUeFVEirH90e6SMeYD4IM2x55p83wZsMzBvb8AfuHM+yjvEhLgS2xoQHOhekdBOX9bl83HB09iEeGKMXFEBPnx+tajrMssZm5arJsjVso7OZMglgA1wH3YJrWFA792ZVBKpUQHsy2vjFue3cTmnDIigvy4e+5wbp05hPjwAdQ2NLI5p4xfrtzPh/dFEeDr4+6QlfI6nc2DeAp4zRizscXhl1wfklIwLDaELbll1NZb+dk1o1k6fQjBAV/87xrg68MvF4/lthe28txnudw9b7gbo1XKO3XWgjgCPC4i8cCbwOvGmF29E5bq77572XAuHh7NZaNjO2wdXDoyhoVjB/GXT46wZFICSQN17SalelJnE+X+bIyZhW2EURnwoogcFJGHRGRkr0Wo+qX48AFcNT6+y66jny+yrd7yyHs6yE2pntblMFf72kuPGmMmA1/BNupI/zUqj5AYMYDvXjaCVfuLWH3gpLvDUcqrODNRzk9EFonIq9gmyB0GbnB5ZEo56Y6LUxkdH8aD7+ylvKrO3eEo5TU6TBAiskBEXsA2I/pObMNVhxljbjbGvNtbASrVFX9fC4/fOJGKc3X8fMU+d4ejlNforAXxE2ATMNoYs8gY82rLyWxKeZIxCWHcO38E7+05wX9268K/SvWEzorU84wx/2haQE8pT3fXpcOYmBTOz1fso/iM7i2l1IVyejVXpTydr4+Fx2+axLm6Rn7yjnY1KXWhNEEorzI8NoRvXzqMjw+e1FaEUhdIE4TyOvPSbPuCbMstd3MkSvVtmiCU1xmXGM4APx+25p5ydyhK9WmaIJTX8fOxMGVoBFvztAWh1IXQBKG80vTkKA4VVVJxrvNNhQpOVTPnd5+w/3hFL0WmVN+hCUJ5pWkpAzEGtud3Pkp7/eFijp0+x7INeb0TmFJ9iCYI5ZUmDx6In4+wJbfzBLE939YN9Z89x6ms0S1MlWpJE4TySgP8fZiQFMHWLhJERn45qTHB1NRbWbHzWC9Fp1TfoAlCea1pyZHsLazgXF2jw/MnK2soLD/HV6YPYVxiGK9uKcAY08tRKuW5NEEorzUjJZIGq2FngePRTDvs3UtThw5k6fQhHCo6w+5CLVYr1UQThPJaU5MHIgJb8xx3M2XklxPga2FsQjiLJyYQ5O/D61sKejlKpTyXJgjltcIC/Rg9KKzDOsT2/HImJkXg72shNNCPxRMTWLn7OGe0WK0UoAlCebnpKZHsKCinrsHa6nhNfSP7j1cwZejA5mNLpw/hXH0jK3bpcuFKgSYI5eVmpERSU29lX5uJcHsKK6hvNKS3SBATksIZEx/Ga1sKsFq1WK2UJgjl1dKTIwHadTM1zX9o2YIQEW6fk8yBE5X8/sPM3gtSKTfp6oOQJgjl1WJCA0iNCWbNwZOthrBuzy8nNTqYyGD/Vtd/eWoSX505hGfWZ/PSxrxejlap3mO1Gn7WxRa9miCU1/vG7GS25ZXzxrajABhj2FFQ3qr10ERE+NXicSwYE8cv/7OfVftO9Ha4Srmc1Wr46bt7ea2LUXuaIJTX++qMocweFsUj7x3gaFk1uaVVlFXVtao/tORjEZ68ZTKTB0fwvTd2sa2DYbJK9UVWq+HBd/by+taj3D1vWKfXaoJQXs9iER67cSIWEe5/a3fzH/ypHSQIsC3V8fxt00iMGMAP3tqlRWvlFYwxPPDOHt7MOMr3LhvOD69I6/R6TRCqX0iMGMBDi8awNa+M36/KJCzQl2ExIZ3eMzDYn/sWjORo2Tk+yyrtpUiVcp3dhRW8lVHIXZcO4wdXpCEinV6vCUL1G1+emsTlo+M4VSLtjhAAABjoSURBVFXH1KEDsVg6/8cBcOXYOKKC/Xl1c34vRKiUa22wf9C54+IUp67XBKH6DRHht18aT2LEAC4fE+fUPQG+PtyYPpg1h4opqqhxcYRKdV9lTT3v7Tnu1EKTG7JKGTUolKiQAKde26UJQkQWikimiGSJyAMOzv9IRHbZH/tEpFFEIkUkrcXxXSJSKSLfd2Wsqn+ICQ3g8x/P49YZQ52+Z+n0wTRaDW/aR0Ep5Un+tPoI97y2k7+uy+70upr6RjLyy5kzPNrp13ZZghARH+Bp4CpgDLBURMa0vMYY85gxZpIxZhLwILDeGFNmjMlscXwqUA0sd1Wsqn/pqt+1raFRwVw8Ipo3thXQ0Gjt+galeklDo5WVu4/j5yP84aNM1mUWd3jt9nzbkjNzhkc5/fqubEFMB7KMMTnGmDrgDWBJJ9cvBV53cHw+kG2M0U5g5Ta3zhjKiYoa1maWuDsUpZptzD5F6dlaHr1hAmlxodz7xi4KTlU7vHZDVim+FmF6imckiESgZZu80H6sHREJAhYCbzs4fQuOE0fTvXeKSIaIZJSU6D9e5RrzR8cSGxrAa1v0c4rqfQeOV1Jxrv0qw+/uPEZYoC9Xj4/n2a+lA3DnPzOormtod+2G7FNMHBxBSICv0+/rygThqB3fURVlEbDBGNNqRpKI+AOLgX919CbGmGeNMenGmPSYmJjzDlapzvj5WLhl2mDWHS7haJnjT2hKuUJW8VkWP/U53/5nRqtCdHVdA6v2F3HNhHgC/XwYEhXEn2+ZRObJM/xseeslNCrO1bO38HS36g/g2gRRCAxu8TwJ6Ggd5Y5aCVcBO4wxJ3s4NqW67ebpQxDgpr9v4gdv7uLVLflkFZ91d1jKS3Q0CumR9w/QaAybc8p4f+8XS7+sPnCS6rpGlkz6omNmblos98wbzjs7j7VaoHJzzimsBuYMc757CVybILYBI0Qkxd4SuAVY2fYiEQkHLgVWOHiNjuoSSvW6xIgB/PXWKUxMiuDTI6X8dPk+Ln9iPSt2HXN3aKqPW76zkJm/XUNW8ZlWx9dmFrMus4QfLxzF2IQwfvP+webuo+U7j5EQHsh0+4rFTb4zdziDwgJ55P0DzSsAbMwqZYCfD5OHdLx6gCMuSxDGmAbgHuBD4CDwljFmv4jcJSJ3tbj0euAjY0xVy/vtdYkFwDuuilGp7lo4Lp5nvjaVbT+dz7ofzmV8Yji//eAQ5+oaO73PajU8sfow6w9rnUy1t/rASU5W1nLbC9s4WWmbb1PfaOWR9w6QEh3MN+ek8KvFYzlRUcNf12ZTeraWz46UsmRyYrsJnwP8ffi/hWnsKaxgxW7bh5cN2aeYlhKJv2/3/uS7dB6EMeYDY8xIY8wwY8xv7MeeMcY80+KaZcaYWxzcW22MiTLG6C7yyuOICMnRwfzsmtEUVdbw/Oc5nV7/+OpMnlxzhAff3kO9DpVVLRhjyMgrZ+LgCE5X13HbC1uprKnnlc35ZJdU8dOrR+PvayE9OZLrJyfy7Kc5PPVJFo1Ww/WTHY774bpJiUxICuf3qzLJK60iq/hst7uXQGdSK3VBZqRGccWYOP62LpviM45nWr+zo5Cn12YzaXAExytqWKlbmvZJ1XUNfPf1ndz/1m6Wbchle35Zly1HZxSWn6P4TC1fnpLIM1+bSlbxWf5nWQZ/+vgIF4+IZv7o2OZrH7xqFH4+wrKNeYyJD2NkXKjD17RYhJ9dM4YTFTV859UdAN0uUIMmCKUu2ANXjaK2wcofVx9pdy4jr4wH3t7LrNQo3vr2LNLiQvn7p9m6OmwfY4xtiez39hxn/eESfvmfA9zwt01M+vVH/Ht74QW9dtPuhlOHRnLxiBj+cONEtuaVcba2gZ9fO6bVxM7YsEDuvXwEANdNTuj0daenRHL1+EEcOFFJRJAfY+LDuh2b8wNilVIOpcaE8NWZQ3l5Ux7fmJ1M2qBQjDFknjzDt/+5nYSIQP721Sn4+1q4a24q9725m08OFTu9HpTqnq25ZeSVVnHTtMFdX+yklzfls2LXcX54xUjunjeck5W17D1WwYsbcvnhv3ZTcKqK+xaMbP5jboxhS24ZoYG+jE0I7/S1t+eXExLgS9ogW2vgusmJGAx1DVaHLYTb56QQGujHkkmdJwiABxaO5uMDxcweFuXU4pRtiTMLPPUV6enpJiMjw91hqH6ovKqOSx9by9CoYBIiAtlRcJqSM7WEBfqy/O45zUuL1zdamfvYOuLDA/n3/852c9SeqaHRyh8/PkyD1fDgVaO7dW/FuXrmP76e8uo6tv5kvtOL0nVme345tzy7iUtGxPCPr6e3+kNb32jlp8v38lZGIUsmJfDrJeP4YO8Jlm3II/PkGfx9LTz9lSks6OTDwFV//ozoEH/++a0ZFxxrR/EnRAQSHz7A4XkR2W6MSXd0TruYlOoBA4P9+cGCkew9VsGhojNcNDyah68bxwf3Xtxq3wk/Hwt3XJxCRn55pzvV/enjwyx9dnNvhO5RztTU8z8vZ/D02mz+8WkOJWdqu3X/71cd4lRVLY1Wwwf7ii44ntKztdz96g7iwwfwxE2T2n0K9/Ox8OgNE/jRlWms2HWcKQ+v5sF39mKx2FYOHh0fxl2vbOdfGY4XejxTU09mUSVTujn8tDumDh3YYXLoinYxKdVDbpudzI3pgwnuYimDm6cN4clPsnhmXTbTvhHZ7vyZmnqe+yyXs7UN5J+qYmhUcJfvbYzp9iKEnuZoWTXfemkbOSVVfPuSVP7+aQ6r9hfxtZnOrby7o6Cc17YWcPvsFD47UsLKXcecvrctYwzrMkt4dNUhyqvreOc7swkP8nN4rYhw97zhJEcFs+bgSW6aNpgZKZGICIsnJnDXK9v50b/3UF5dx52XtN7ic9fR01gNpCe7LkFcCG1BKNVDRKTL5AC2ceq3zUpmzaFiDhVVtjv/9vZCztbaJkM5M2/isyMljP3Fh316CZCtuWVc9/QGiipqeOmb03ngqlEMiwnm/T3Ojfiqb7Tyk3f2MigskB9cMZIlkxLYlldOYXn3fifGGD45dJIlT2/g9mXbOFvbwF9vndJlHQHgmgnxPHHzJGamRjUn6+AAX567LZ1rxsfz/z44xCttNp7KyCvHIjBpcES34uwtmiCUcoPbZg8lNNCX37x/sNUSC1ar4aVN+UwZEsHQqCDWObF67LINeVTXNfLpkd6dhHfgeCVjH1rVPArnfFithr+ty2bpPzYTNsCPd74zhznDoxERrhkfz9bcMqe6mV7ckMuhojP8YtFYQgJ8WTzRNj/gP7tPdHFna4+uyuSbyzIor67j0RvGs/aHc5k/+sIGEwT4+vDk0snMTI3kTx8fbrWQ3o6CctIGhREa6Lh14m6aIJRyg4ggW83isyOlfHTgi6XG1h8uIbe0im/MSWHuyBg2ZpdSU9/xWPuiihrW2vcA2JR9yuVxt/Tqlnyq6hr5x6edTxLsyOnqOu54OYNHVx1i4dhBrLxnDsNjv6jXXDMhAauBVfs7ryUcLavmj6uPcPnoWK4ca/tjPiQqiMlDIrq1DEpeaRXPf57D9ZMT+eT+udw8bQh+Pj3zJ9LHIvzoyjRKz9bx8iZbK6LRathZcJqpQz2z9QCaIJRym6/NHMrIuBAefu9AcxJ4cWMecWEBXDVuEHPTYqmpt7ZadK2tt3cUYjW2QuTmnDKntp3sCTX1jazcfRx/XwsfHSjqdlfOiYpzXPPk53x6pIRfLR7LU1+Z3O5T9Mi4kC67mSqq6/nmsm34WoRfLh7bqg6zZGICh4rOkFl0psP7W/r9h4fw87HYJ6P1/J/GqUMjuXRkDH9fn83Z2gYOFVVytraB9KHt61CeQhOEUm7i62Phl4vGUlh+jmc/zSGr+CyfHi7hqzOG4udjYWZqFP6+lg67maz2bVBnpUZxU3oSpWdryS7pndVlP9xfxJmaBn5z3TgAXtlc4PS9DY1W7n19F+XVdbz17VncNjvZYYG9q26m2oZG7vhnBnmnqvj716eSNDCo1flrJiRgEVi5u+tWxPb8Mj7YW8Sdl6QSGxbo9M/SXT9YMJLy6nqWbchlR/MEOc8sUIMmCKXcavbwaK4eP4i/rsvi96sO4e9jYemMIYCtmD0zNYp1hx1vI7k55xQFZdXcPG0wM1Nt6+xsyum4tdGT/pVRSGLEAG6YksQVYwbxxraCTrvCWnrykyy25pXxm+vHdbm6aEfdTFar4Qdv7WZrbhl/uHEis4e1X0YiJjSAOcOjWbHreKctK2MMv3n/ILGhAdx5SapTP8P5mjg4gstHx/HspzmszSwhNjSApIHnNwS1N2iCUMrNfnK1bTLYRwdOsmhiAtEtJnfNHRlDTkmVwxFKb2YcJSzQl4XjBjEkMoiE8EA291AdwhjD2kPFXPPkZ/zPS9ta7cVdWF7NhuxSvjw1CYtF+MacZE5X1zvV378p+xRPfXKEG6Ykcf3kpC6vd9TN1Gg1PPL+Qd7fc4IHrxrVaj+EtpZMSqSw/BzrD5fw2ZES/rLmCHe/toPnPsuhrKoOgP/uK2JHwWnuv2IkQf6uH/l/34IRVNY08MmhYqYOHejRw5M1QSjlZkkDg/jO3OGIwO1zkludm5tm2yWx7Wb0FdX1/HdfEddPTiTQzwcRYWZqFJtzTrX7tFxRXd88bNYZOwvKueXZzdy+bBslZ2r5+GAxj68+3Hz+7e3HMAa+PNX2B35GSiSjBoWybGN+83sbY1i17wR/X5/NtrwyauobKauq4/tv7iQ5KphfLxnrVCwtu5mOnT7Ha1sKuPyJ9bywIZfbZg3t8hP/lWPj8Pe18I0Xt/G157fy+OrD7Mgv55H3DzLj/33Md17dzm//e5C0uFC+PLXnlubozNiEcK4aNwjw7O4l0IlySnmEe+YN59oJ8aS2mHUNkBIdzJBI23DXr81Kbj7+7q5j1DVYuXnakOZjM4dF8c7OYxwpPtu8hk9dg5Xr/7qBwZFBvPTN6V3G8dqWAn6yfC/RIf48vGQsN08bwi9W7udv67KZbO8e+feOo8weFsXgSFufv4jwjdnJPPDOXrbmlhEdGsBDK/axIeuL1oy/j4WIID9OV9fzwjemOTVfpMk1ExJ48pMs5v1hHXUNVsYnhvP0V6Zw1bhBXX76Dg3047EvT+BoWTWTBg9kwuBwwgL9OHzyDG9uO8rynccoq6rjpW9Ox+c81io6X/dfkUZuadUFD6F1NU0QSnkAi0XaJQew/fGdmxbDvzIKqW1oJMDXh+q6Bl7fWsD4xHDGJHyxQuespjpE9qnmBPFmxlFySqvIPVXFiYpznS65UN9o5S+fHGHq0IG8/M3pzX/Ef7FoDPuPV3D/W7t58OrRHC07x/0L0lrdu2RSIr/97yF+/PYejp+uIcDPwsNLxrJwXDw7C8rZnl/OzqOnuXFqklOTzloaGRfC/FGx1FsNd12SyqxhUd3qlnHUBTUyLpSfXzuGHy8cRUFZdavhtb1heGwIq75/Sa++5/nQBKGUh5ubFsPLm/J5d+cxsorP8ua2o1TWNPDETRNbXTc4MojEiAFszjnFbbOTqa5r4Mk1RxgeG0JW8VlW7DrOXZcO6+BdYNW+Ik5U1PDwknGtPuEH+vnw11unsOgvn/OT5XsJDfDlyrGDWt07wN+HW2cM4a/rsrl+ciIPXj2K2FDbaKArxg7iijbXd4eI8Pw3pp33/Z3x97X0enLoS7QGoZSHm5Uajb+vhR+/vZcXNuRxycgY/n3XLL40pX2Rt6kOYbUaXtyQR8mZWh69YTyTh0Tw7s7Oi8gvbMglOSqIy0bFtjuXNDCIP98yGRFYPCmBAf4+7a65b8FI1v5wLn+8eVJzclB9m7YglPJwA/x9eGDhKMqr6/jKjCGddhPNTI3k7R2FbM0r45n12Vw+Oo6pQyP50uREfr5iPweOV7bqlmqyo6CcnQWn+dXisR3uG3DJyBhW3XsJQ6OCHJ7387GQEt31woKq79AWhFJ9wDcvSuH+K9K6XLa5aT7ED97cxdnaBn50pa1WcM2EBHwtwvKdjnc/e+HzXEIDfZtHJnUkbVAogX7tWw/KO2mCUMqLDI4MImngAI5X1HD95MTmXcoig/2ZmxbLil3HaWyz3emx0+f4774ilk4f0q3RRcr7aYJQysvMGRaNn49w3+UjWx3/0pREis/UsjG7tNXxlzflYYzh67POb+8E5b00QSjlZf5vYRrLvzOneZ5Ck8tGxRIa6MvyHV8Uq4vP1PD6lgIWjhvUbi0jpbQ9qZSXiQoJcLgXc6CfD9dOiGfFruPckFXKvzKO8sHeIhqN4Y6LXbsGkeqbNEEo1Y9cPzmJ17ce5dbnthAa4MtXZgzhqzOH6lwA5ZAmCKX6kfShA7ln3nDiIwK5blKiFqVVp/T/DqX6EYtF+OGVaV1fqBRapFZKKdUBTRBKKaUc0gShlFLKIU0QSimlHNIEoZRSyiFNEEoppRzSBKGUUsohTRBKKaUcEmNM11f1ESJyBsh0dxxtRAOlXV7V+zwxLk+MCTSu7vDEmEDj6sxQY0yMoxPeNpM60xiT7u4gWhKRDE+LCTwzLk+MCTSu7vDEmEDjOl/axaSUUsohTRBKKaUc8rYE8ay7A3DAE2MCz4zLE2MCjas7PDEm0LjOi1cVqZVSSvUcb2tBKKWU6iGaIJRSSjnkFQlCRBaKSKaIZInIA26M4wURKRaRfS2ORYrIahE5Yv86sJdjGiwia0XkoIjsF5F7PSSuQBHZKiK77XH9yhPissfgIyI7ReQ9D4opT0T2isguEcnwoLgiROTfInLI/v/YLHfGJSJp9t9R06NSRL7vIb+r++z/r+8Tkdft/wbcHldn+nyCEBEf4GngKmAMsFRExrgpnGXAwjbHHgDWGGNGAGvsz3tTA3C/MWY0MBO42/77cXdctcBlxpiJwCRgoYjM9IC4AO4FDrZ47gkxAcwzxkxqMW7eE+L6M7DKGDMKmIjt9+a2uIwxmfbf0SRgKlANLHdnTAAikgh8D0g3xowDfIBb3B1Xl4wxffoBzAI+bPH8QeBBN8aTDOxr8TwTiLd/H49tMp87f18rgAWeFBcQBOwAZrg7LiAJ2z/Uy4D3POW/IZAHRLc55u7fVRiQi32wi6fE1SKOK4ANnhATkAgcBSKxTVB+zx6fR/yuOnr0+RYEX/zimxTaj3mKOGPMCQD711h3BSIiycBkYIsnxGXvytkFFAOrjTGeENefgP8DrC2OuTsmAAN8JCLbReROD4krFSgBXrR3yT0nIsEeEFeTW4DX7d+7NSZjzDHgD0ABcAKoMMZ85O64uuINCUIcHNOxu22ISAjwNvB9Y0ylu+MBMMY0GltXQBIwXUTGuTMeEbkWKDbGbHdnHB2YY4yZgq0r9W4RucTdAWH7JDwF+JsxZjJQhYd0kYiIP7AY+Je7YwGw1xaWAClAAhAsIl91b1Rd84YEUQgMbvE8CTjuplgcOSki8QD2r8W9HYCI+GFLDq8aY97xlLiaGGNOA+uw1W/cGdccYLGI5AFvAJeJyCtujgkAY8xx+9dibH3q0z0grkKg0N7yA/g3toTh7rjAlkh3GGNO2p+7O6bLgVxjTIkxph54B5jtAXF1yhsSxDZghIik2D813AKsdHNMLa0EbrN/fxu2GkCvEREBngcOGmOe8KC4YkQkwv79AGz/gA65My5jzIPGmCRjTDK2/48+McZ81Z0xAYhIsIiENn2Pre96n7vjMsYUAUdFJM1+aD5wwN1x2S3li+4lcH9MBcBMEQmy/5ucj62g7+64OufuIkgPFYCuBg4D2cBP3RjH69j6F+uxfbr6FhCFreh5xP41spdjughbl9seYJf9cbUHxDUB2GmPax/wkP24W+NqEd9cvihSu/t3lQrstj/2N/0/7u647DFMAjLs/x3fBQa6Oy5sgx5OAeEtjnnC7+pX2D4E7QP+CQR4QlydPXSpDaWUUg55QxeTUkopF9AEoZRSyiFNEEoppRzSBKGUUsohTRBKKaUc0gShVDeISGOb1UJ7bOawiCRLi5WAlXI3X3cHoFQfc87YlgdRyutpC0KpHmDfr+FR+x4XW0VkuP34UBFZIyJ77F+H2I/Hichyse2HsVtEZttfykdE/mHfN+Aj+yxzpdxCE4RS3TOgTRfTzS3OVRpjpgNPYVsVFvv3LxtjJgCvAk/ajz8JrDe2/TCmYJshDTACeNoYMxY4Ddzg4p9HqQ7pTGqlukFEzhpjQhwcz8O2AVKOfXHEImNMlIiUYlvvv95+/IQxJlpESoAkY0xti9dIxrbs+Qj78x8DfsaYR1z/kynVnrYglOo5poPvO7rGkdoW3zeidULlRpoglOo5N7f4usn+/UZsK8MC3Ap8bv9+DfC/0LxxUlhvBamUs/TTiVLdM8C+C16TVcaYpqGuASKyBdsHr6X2Y98DXhCRH2Hbfe12+/F7gWdF5FvYWgr/i20lYKU8htYglOoB9hpEujGm1N2xKNVTtItJKaWUQ9qCUEop5ZC2IJRSSjmkCUIppZRDmiCUUko5pAlCKaWUQ5oglFJKOfT/AXoS5CjxzqscAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(history.history['val_loss'][10:]).plot(logy=False)\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Validation Error\")\n",
    "print(f\"Minimum MSE: {round(min(history.history['val_loss']),3)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment Six - Recommender System using Matrix Factorization\n",
    "#### Five Latent Factors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_latent_factors = 5\n",
    "\n",
    "user_input = Input(shape=[1], name='user')\n",
    "user_embedding = Embedding(input_dim=n_users + 1, \n",
    "                           output_dim=n_latent_factors, \n",
    "                           embeddings_regularizer=regularizers.l1(1e-6), \n",
    "                           name='user_embedding')(user_input)\n",
    "user_vec = Flatten(name='flatten_users')(user_embedding)\n",
    "\n",
    "movie_input = Input(shape=[1], name='movie')\n",
    "movie_embedding = Embedding(input_dim=n_movies + 1, \n",
    "                            output_dim=n_latent_factors, \n",
    "                            embeddings_regularizer=regularizers.l1(1e-6), \n",
    "                            name='movie_embedding')(movie_input)\n",
    "movie_vec = Flatten(name='flatten_movies')(movie_embedding)\n",
    "\n",
    "product = dot([movie_vec, user_vec], axes=1)\n",
    "model = Model(inputs=[user_input, movie_input], outputs=product)\n",
    "model.compile('adam', 'mean_squared_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 81191 samples, validate on 4511 samples\n",
      "Epoch 1/100\n",
      "   32/81191 [..............................] - ETA: 9:33 - loss: 13.9529"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/faisito/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n",
      "/home/faisito/anaconda3/envs/tf2/lib/python3.7/site-packages/tensorflow_core/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81191/81191 [==============================] - 2s 26us/sample - loss: 10.9793 - val_loss: 5.2609\n",
      "Epoch 2/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 2.8429 - val_loss: 1.6763\n",
      "Epoch 3/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 1.2204 - val_loss: 1.0329\n",
      "Epoch 4/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.8697 - val_loss: 0.8709\n",
      "Epoch 5/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7818 - val_loss: 0.8257\n",
      "Epoch 6/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7590 - val_loss: 0.8135\n",
      "Epoch 7/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7520 - val_loss: 0.8076\n",
      "Epoch 8/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7500 - val_loss: 0.8123\n",
      "Epoch 9/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7494 - val_loss: 0.8104\n",
      "Epoch 10/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7485 - val_loss: 0.8139\n",
      "Epoch 11/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7479 - val_loss: 0.8144\n",
      "Epoch 12/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7469 - val_loss: 0.8089\n",
      "Epoch 13/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7462 - val_loss: 0.8128\n",
      "Epoch 14/100\n",
      "81191/81191 [==============================] - 2s 25us/sample - loss: 0.7452 - val_loss: 0.8075\n",
      "Epoch 15/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7435 - val_loss: 0.8042\n",
      "Epoch 16/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7417 - val_loss: 0.8050\n",
      "Epoch 17/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7390 - val_loss: 0.8049\n",
      "Epoch 18/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7363 - val_loss: 0.8011\n",
      "Epoch 19/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7318 - val_loss: 0.8025\n",
      "Epoch 20/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7275 - val_loss: 0.7961\n",
      "Epoch 21/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7226 - val_loss: 0.7902\n",
      "Epoch 22/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7167 - val_loss: 0.7934\n",
      "Epoch 23/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.7107 - val_loss: 0.7858\n",
      "Epoch 24/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.7048 - val_loss: 0.7905\n",
      "Epoch 25/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6987 - val_loss: 0.7813\n",
      "Epoch 26/100\n",
      "81191/81191 [==============================] - 2s 25us/sample - loss: 0.6925 - val_loss: 0.7785\n",
      "Epoch 27/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6867 - val_loss: 0.7789\n",
      "Epoch 28/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6812 - val_loss: 0.7773\n",
      "Epoch 29/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6749 - val_loss: 0.7768\n",
      "Epoch 30/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.6697 - val_loss: 0.7711\n",
      "Epoch 31/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6641 - val_loss: 0.7726\n",
      "Epoch 32/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6581 - val_loss: 0.7708\n",
      "Epoch 33/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6531 - val_loss: 0.7699\n",
      "Epoch 34/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6471 - val_loss: 0.7687\n",
      "Epoch 35/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6417 - val_loss: 0.7695\n",
      "Epoch 36/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6364 - val_loss: 0.7703\n",
      "Epoch 37/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6318 - val_loss: 0.7702\n",
      "Epoch 38/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6269 - val_loss: 0.7691\n",
      "Epoch 39/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6226 - val_loss: 0.7696\n",
      "Epoch 40/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6183 - val_loss: 0.7719\n",
      "Epoch 41/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6144 - val_loss: 0.7753\n",
      "Epoch 42/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6109 - val_loss: 0.7700\n",
      "Epoch 43/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6075 - val_loss: 0.7749\n",
      "Epoch 44/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6046 - val_loss: 0.7775\n",
      "Epoch 45/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.6016 - val_loss: 0.7781\n",
      "Epoch 46/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5992 - val_loss: 0.7783\n",
      "Epoch 47/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5963 - val_loss: 0.7749\n",
      "Epoch 48/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5940 - val_loss: 0.7766\n",
      "Epoch 49/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5919 - val_loss: 0.7834\n",
      "Epoch 50/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5900 - val_loss: 0.7813\n",
      "Epoch 51/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5881 - val_loss: 0.7813\n",
      "Epoch 52/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5865 - val_loss: 0.7814\n",
      "Epoch 53/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5847 - val_loss: 0.7819\n",
      "Epoch 54/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5831 - val_loss: 0.7814\n",
      "Epoch 55/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5815 - val_loss: 0.7831\n",
      "Epoch 56/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5800 - val_loss: 0.7842\n",
      "Epoch 57/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5790 - val_loss: 0.7844\n",
      "Epoch 58/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5775 - val_loss: 0.7835\n",
      "Epoch 59/100\n",
      "81191/81191 [==============================] - 2s 25us/sample - loss: 0.5765 - val_loss: 0.7853\n",
      "Epoch 60/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5751 - val_loss: 0.7840\n",
      "Epoch 61/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5743 - val_loss: 0.7848\n",
      "Epoch 62/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5732 - val_loss: 0.7885\n",
      "Epoch 63/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5719 - val_loss: 0.7852\n",
      "Epoch 64/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5708 - val_loss: 0.7876\n",
      "Epoch 65/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5704 - val_loss: 0.7854\n",
      "Epoch 66/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5694 - val_loss: 0.7861\n",
      "Epoch 67/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5684 - val_loss: 0.7901\n",
      "Epoch 68/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5679 - val_loss: 0.7896\n",
      "Epoch 69/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5671 - val_loss: 0.7884\n",
      "Epoch 70/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5667 - val_loss: 0.7918\n",
      "Epoch 71/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5657 - val_loss: 0.7914\n",
      "Epoch 72/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5651 - val_loss: 0.7910\n",
      "Epoch 73/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5645 - val_loss: 0.7915\n",
      "Epoch 74/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5641 - val_loss: 0.7937\n",
      "Epoch 75/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5631 - val_loss: 0.7949\n",
      "Epoch 76/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5627 - val_loss: 0.7959\n",
      "Epoch 77/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5626 - val_loss: 0.7934\n",
      "Epoch 78/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5616 - val_loss: 0.7944\n",
      "Epoch 79/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5612 - val_loss: 0.7978\n",
      "Epoch 80/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5608 - val_loss: 0.7973\n",
      "Epoch 81/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5603 - val_loss: 0.7945\n",
      "Epoch 82/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5600 - val_loss: 0.7947\n",
      "Epoch 83/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5596 - val_loss: 0.7953\n",
      "Epoch 84/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5591 - val_loss: 0.7980\n",
      "Epoch 85/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5589 - val_loss: 0.7975\n",
      "Epoch 86/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5583 - val_loss: 0.7975\n",
      "Epoch 87/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5582 - val_loss: 0.7986\n",
      "Epoch 88/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5577 - val_loss: 0.7978\n",
      "Epoch 89/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5573 - val_loss: 0.7991\n",
      "Epoch 90/100\n",
      "81191/81191 [==============================] - 2s 24us/sample - loss: 0.5571 - val_loss: 0.7994\n",
      "Epoch 91/100\n",
      "81191/81191 [==============================] - 2s 25us/sample - loss: 0.5568 - val_loss: 0.7977\n",
      "Epoch 92/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5564 - val_loss: 0.8001\n",
      "Epoch 93/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5558 - val_loss: 0.8006\n",
      "Epoch 94/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5561 - val_loss: 0.8009\n",
      "Epoch 95/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5554 - val_loss: 0.7978\n",
      "Epoch 96/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5554 - val_loss: 0.8014\n",
      "Epoch 97/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5546 - val_loss: 0.8025\n",
      "Epoch 98/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5546 - val_loss: 0.8009\n",
      "Epoch 99/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5547 - val_loss: 0.8039\n",
      "Epoch 100/100\n",
      "81191/81191 [==============================] - 2s 23us/sample - loss: 0.5539 - val_loss: 0.8054\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(x=[X_train.newUserId, X_train.newMovieId], \n",
    "                    y=X_train.rating, epochs=100, \n",
    "                    validation_data=([X_valid.newUserId,X_valid.newMovieId], X_valid.rating), \n",
    "                    verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Minimum MSE: 0.769\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3ib5bn48e/tvfdIHMcrdjZk7zDDCJRdCgToAAqF0hZKf5zS0x5OT2lPS1sopdABbaEUyjisAIWEHUams3c8knjGe29Zz+8PSY6HbMvBsmz5/lyXLkvv+0q67di59az7EWMMSimlVG8+ng5AKaXU6KQJQimllFOaIJRSSjmlCUIppZRTmiCUUko55efpAIZTXFycSUtL83QYSik1Zmzfvr3SGBPv7JxXJYi0tDSys7M9HYZSSo0ZInK8v3PaxaSUUsopTRBKKaWc0gShlFLKKU0QSimlnNIEoZRSyilNEEoppZzSBKGUUsopr0oQbRarp0NQSimv4VUJornd4ukQlFLKa3hVgmjr0BaEUkoNF+9KENrFpJRSw8arEkSrpdPTISillNfwqgTRbrHSpklCKaWGhVclCIDjVc2eDkEppbyC1yWI3PLGPsfW7TvBXz/N90A0Sik1dnldgshzkiD+vCGP33+QgzHGAxEppdTY5FUJwt/Xh9yKngmitaOT/SV1NLRaKG9o81BkSik19nhVggj08+nTxbS/pI6OTlvLIaesb+tCKaWUc16VIIL8fcmvaMJqPdmVtLOgtut+TnmDJ8JSSqkxyasSRKCfDy0dnZTUtXQd21FQQ3J0MFEh/uQ4GZ9QSinlnNclCOg5k2nH8Vrmp0STlRBGrnYxKaWUy7wrQfj7ApBX0QRASW0LJ+pbmZ8SRWZCOEfKG3Qmk1JKucirEoSfjxAV4t/VgthRUAPA/FRbC6K2uYOqpnZPhqiUUmOGn6cDGG6Z8WFdayF2HK8l0M+H6RMiqGvpAGwzmeLCAj0ZolJKjQlubUGIyGoROSwiuSJyn5PzkSLypojsFpH9InJTt3N/F5FyEdk3lPfMTAgjz74WYmdhDacnRxLg50NWQjgAuTqTSSmlXOK2BCEivsDjwEXATGCNiMzsddmdwAFjzBzgbOAhEQmwn3saWD3U950SH0ZVUztl9a3sL65nfko0AIkRgYQH+ulMJqWUcpE7WxCLgVxjTL4xph14Abi81zUGCBcRAcKAasACYIz5xP54SDITwgBYu6uY9k4r8+wJQkTITAzTxXJKKeUidyaISUBht8dF9mPdPQbMAEqAvcBdxpgh7fojIreJSLaIZFdUVHQliP/LLgJgfmpU17VZCWHaglBKKRe5M0GIk2O955heCOwCkoC5wGMiEjGUNzHGPGGMWWiMWRgfH09SVDCBfj7klDeSHB1MQnhQ17VZCeFUNrZRozOZlFJqUO5MEEXA5G6Pk7G1FLq7CXjV2OQCR4HpX+RNfX2EjHhbK8Ix/uCQmWg73rugn1JKqb7cmSC2AVkikm4feL4OeKPXNQXAKgARSQSmAV944wZHN9O8lKgex7Psx3UcQimlBue2BGGMsQDfAdYDB4GXjDH7ReR2EbndftkDwHIR2Qt8APzQGFMJICLPA5uAaSJSJCK3uPreU+JDgb4tiElRwYQG+GrRPqWUcoFbF8oZY94G3u517M/d7pcAF/Tz3DWn+r5XzJ1EY6uFWUk9hzNEhMwEncmklFKu8KpSGw5pcaH85JKZ+Pn2/fYyE8K1BaGUUi7wygQxkKzEMMrq27pKbyillHJu/CUI+0B1753nlFJK9TQOE4TWZFJKKVeMuwQxKTqYIH8fjuhAtVJKDWjcJQhfH+G0SZFkHxtymSellBpXxl2CAFg+JY69xXXUNetAtVJK9WdcJoiVWXFYDWzKr/J0KEopNWqNywQxJzmKkABfNuZVejoUpZQatcZlggjw82FJegyf5WqCUEqp/ozLBAGwIjOO/IomSutaPB2KUkqNSuM2QSyfEgfA57k6DqGUUs6M2wQxfUI4saEBbNRuJqWUcmrcJggfH2HZlFg+y63EmN4b3SmllPfbcKRiwPPjNkEArMyMo7yhjTzdYU4pNc50dFp54K0DA17j1v0gRrsVmbZxiM9yKsm012j65+bjPP5hLnHhAaTGhpIWG8KKzLiuMQullPIG/9pSMGjR0nHdgpgcE0JKTAif51VhjOG36w/zX6/vIzk6mNjQQPYX1/GXDfl865ntdFq1G0op5R1qm9v53ftHWJEZO+B147oFAbAiM5a3dpdy3yt7eTG7kOsWTebnV8zu2mzo9Z3F3P3iLg6faGBmrx3qlFJqLHrk/RzqWzr4r0tm8q8BrhvXLQiwTXdtaLPwYnYh3zs3k19edVqPnegWpNr2td5eUOOpEJVSatjklDXwz83HWbM4hekTBv7QO+4TxJlZ8cxJjuTnV8zmngumISI9zidHBxMfHsiO45oglFJjmzGGB/59kJAAX+45f+qg14/7LqbIEH/Wfmdlv+dFhIWp0WQf1/LgSqmx7WBpA58cqeA/L55ObFjgoNeP+xaEKxakRlNY3UJ5faunQ1FKqVP20eFyAK6YN8ml6zVBuGC+YxxCu5mUUmPYhsMVzEqKICE8yKXrNUG4YHZSJAF+PpoglFJjVl1LB9sLajh7WrzLz9EE4YIAPx/mJEfqTCal1Jj1eW4lnVbDWVMTXH6OJggXzU+NZl9xHa0dnT2OF9U0Y+m0eigqpZTqqanNQkVDW5/jHx8uJzzIj/kpUS6/liYIFy1MjaGj07C3uK7r2K7CWs789Ue8sqPIg5EppZRNS3snV/7xc7706Kc0t1u6jhtj2HCkgjOy4nqs8xqMJggXObKuYxzCajXcv3YfVgOHTjR4MjSllALgZ2/t50hZI+UNbTz1+bGu44dONFBW38bZQ+heAk0QLosNCyQjLpTsY7YE8WJ2IXuK6vD3FQqqmj0cnVJqvHtzdwnPby3kjrOnsGp6An/ZkEddcwcAHx+2lfU+awgD1KAJYkjmp0azo6CGmqZ2fr3uEIvTYjh7WgIF1ZoglFKec7yqiR+9upf5KVHcc/5UfnDBNOpbLTzxaR5gG3+YMTGCxAjXprc6aIIYggWp0VQ3tXPXi7uob7XwP5fPIjUmhILqZqxa7VUp5QHtFivffX4nPgKPrpmHv68PM5MiuHROEn//7BhHK5vYfnxo01sdNEEMwUL7grlPjlTw1aWpzJgYQWpsCG0WK+VOZg0opZS7Obq7f3316SRHh3Qdv+f8qbR3Wrn1mWwsVsNZUzVBuNWU+DAigvyICwvg+/ZCVymxoYCtiaeUUu4wUA/F5zmVJEcHs3r2xB7H0+NCuWZhMrnljYQH+nVVph4KTRBD4OMj/PrqOfzpxgVEBvsDkBpjy9jHdRxCKTWMmtstvLWnhDue3c6M+9fx+Ee5fa4xxrD1WDVL0p1v/PO9VVkE+PmwMisO/yFMb3VwazVXEVkN/B7wBf5qjPlVr/ORwLNAij2W3xpjnnLluZ6yevaEHo+TooLxESjUBKGUGgZFNc08+kEOb+wuobXDSnx4IFEh/ry5u4Q7z8nscW1OeSPVTe0syYhx+loTI4N56VvLmDDEwWkHtyUIEfEFHgfOB4qAbSLyhjGm+y7ZdwIHjDGXikg8cFhEngM6XXjuqBDg50NSVDDHdaqrUuoLKG9o5fEPc/nX1gJEhKsXJHPZnCQWpcXwxCf5PLjuECfqWpkQefI/+y35VQAsSXeeIADmTnZ95XRvAyYIse2ek2yMKTyF114M5Bpj8u2v9QJwOdD9P3kDhNvfJwyoBizAEheeO2qkxoZoF5NS6pR9mlPBbc9sp6PTyjWLJvPdczOZGBncdf7safE8uO4QG46Uc+2ilK7jm49WMyEiiJSYEGcv+4UN2ClljDHA66f42pOA7omlyH6su8eAGUAJsBe4yxhjdfG5AIjIbSKSLSLZFRUVpxjqF5MSE0qBDlIrpU7RE5/kExMawPv3nMX/Xnlaj+QAMH1COBMigroWvIFt/GFLfjVLMmL67IQ5XFwZtdgsIotO4bWdRdx7KP5CYBeQBMwFHhORCBefaztozBPGmIXGmIXx8UOfxjUcUmNDqGnuoL61wyPvr5Qau8rrW/k8t5Ivz59EWlyo02tEhLOmxvNZTiUd9uKg+ZVNVDa29TtAPRxcSRDnAJtEJE9E9ojIXhHZ48LzioDJ3R4nY2spdHcT8KqxyQWOAtNdfO6o4WjeackNpVR3xhgOlNTzq3cOcf7DG3jVSWHPN3aXYDVw+SC7vJ09LZ6GNgs77PXgth61bYPc3wD1cHBlkPqiU3ztbUCWiKQDxcB1wPW9rikAVgGfikgiMA3IB2pdeO6o0ZUgqpuZPSnSw9EopUaDN3eX8PsPcsgtb8TXR4gI8uPBdYe4+LSJBPn7dl23dlcJp02KZEp82ICvtyIrDj8f4eMjFSzJiGVLfhVx9hpx7jJoC8IYcxyIAi6136LsxwZ7ngX4DrAeOAi8ZIzZLyK3i8jt9sseAJaLyF7gA+CHxpjK/p479G9vZKTG2tdCaAtCKQXkljfwg5d24+cjPHDFbLb+5yoev34+ZfVt/F92YbfrGtlbXMflc5MGfc2IIH/mp0bz8eEK2/jDUfeOP4ALLQgRuQu4FXjVfuhZEXnCGPOHwZ5rjHkbeLvXsT93u18CXODqc0er8CB/YkIDKKjWgWqlxpNOq8HXp+d/0Far4Uev7iU4wJd/3rKE+PBAAJZNCWBhajR//DiPaxZNJtDPl7W7ivERuGzO4AkCbN1Mv153mO3Hayita2XpANNbh4MrYxC3AEuMMfcbY+4HlmJLGKqbyfaifUqp8eHN3SWc/tP1PLelZ4fK89sK2Hashh9fPKMrOYBtoPl7q7IorWvlle3FGGNYu6uE5VPiSHBxIZtjP4dfrzsMwJIM9w1Qg2sJQrAtXHPoxPkso3EtNSZEu5iUGif2Fddx78u7McCPX9vHw+8exhhDWX0rv3r7EMsyYvnKwuQ+zzsjK465k6N4/KNcth6tpqC62aXuJYcZE8NJjAhk67FqYkIDyEoYeNzii3IlQTwFbBGRn4rIT4HNwN/cGtUYlBobQkltC+0W3Z9aqbFqY24l96/dR0ltS7/XVDW28a1/bic6JIAPf3A21yxM5tEPc7nvlb3cv3YfbZ1W/veq05yODYgId63Kori2hXte2k2gn0+f8j0DcUx3BVic5t7xB3BtkPphbNNRq4Ea4CZjzCNujWoMSokJwWqgeIBfLKXU8Nt+vIbzH97A/2UXYlvbe+oe+SCHZzYdZ9VDG3jswxxaOzp7nO/otPKdf+2ksrGNv3x1ARMig3jwy6fzvXMzeTG7kPX7y7hrVRbpA8wsOntaPKcnR1Jc28J5MxMJD/IfUoxnT7N1M7lzeqvDYKU2fIA9xpjZwA63RzOGpXYr+z3QL4dSanit3VVMTnkj9768h1d3FPOLK2eTMciUUWcqGtrYdqyaNYsnU9PUwW/fPcJL2UVcNX8SQf6+BPr5sKuwlk35VTx8zRxOT7bVOBIR7rlgGsnRIWw+WsVtZ2YM+D4iwt3nZXHz09lcvaBvN9Rgzp2ewJ3nTOGKuQOvmxgOAyYIY4xVRHaLSIoxpsDt0YxhjrUQWtVVqZG1Ob+KM7LiuGj2RH75zkFWP/Ip/3nxdL6xIn1Ir/PegTKMga8tS2PGxAg+zanggbcO8Mj7OT2u+9aZGVw1v+9/7Ncsmsw1iyb3Oe7MudMT+eyH5/TY4MdVQf6+3Hvh9CE/71S4slBuIrBfRLYCXfM4jTGXuS2qMSghPJBAPx8dqFZqBFU2tnGkrJEr5k3i+iUpnDczgR+8tJtfvH2QqxYkEzGE7pt39pWSFhvC9AnhAJyRFc+73z8LS6eV9k4rbR1WrMYQGxY4yCu55lSSw0hzJUH8j9uj8AI+PkJKjFZ1VWokbbaXu15mn+6ZEB7EXauy+DSnko8OlXO5i90wdc0dbMqr4pYz0vsM/Pr5+uDn60NIwPDGPhYMNgbhC/yXMea8EYpnTEuNDdF6TEqNoE15VYQG+PYocTM/JZq4sEDePVDmcoL44FAZFqvhol7bdo53g5X77gSa7Tu/qUE4Fst90ZkUSinXbM6vYlF6TI/tNH18hPNnJvLxofI+s5D6886+E0yMDOJ0raXWgyvrIFqBvSLyNxF51HFzd2BjUWpMCC0dnVQ0tnk6FKW8Xnl9K3kVTV3dS91dMCuRpvZONuVVDfo6TW0WPjlSwYWzJuDjo2uAu3NlDOLf9psaRGqcY6prMwnhp7YHrFIKjpQ10NDawYLU/uf6b3KMP0zpmyCWT4klLNCP9ftPcM70hAHf6+PDFbRZrENasDZe9JsgRCTCGFNvjPmHk3Mpzp4z3qXb10IcrWxiUZr7F7Eo5Y32Fddx3RObEWDbT87rURq7u835VYQH+TErqW+3UKCfL+dMT+C9A2X84sq+BfW6W7f/BLGhAfo368RAXUwfO+6IyAe9zp3qNqReLTk6GD8f4WilVnVV6lQcr2riG09twxhDQ5uFjw+X93vtprwqlqTH9Puf/4WzEqlqamdHQU2/r9Ha0cmHB8u4YFbigElkvBooQXT/afVOrfqTdMLP14fJMSEc0wSh1JCVN7Ty1b9txWK18uq3VxAXFsDaXc43kiyta+FYVTNLB6hmetbUeAJ8fVi/74TT8+0WK7965xBN7Z1cOEu7l5wZKEGYfu47e6zs0mJDtAWh1BA1tHbwjb9vo6Khjae+sYhpE8K55PQkPjhU7nSvd8f6h4ESRHiQPysyY1l/4ESfmYXHq5r4yp838vTGY9y4NIUzszyzn/1oN1CCSBCRe0TkB93uOx7rT7Mf6XFhHK9qxmrVHKqUK4wx3PfKXg6XNfDHG+czLyUagMvnJtFusbLOSQtgU14VkcH+zJwYMeBrXzhrAoXVLRw60YAxhhN1rbywtYAvPfoZRyub+NMN8/n5Fafp7KV+DDSL6Ukg3Ml9gL+6LaIxLj3ONtW1rKGViZHBng5HqVHvxW2F/HtvKT9cPZ1zpp2ccTR3chQpMSG8sauEaxb2rHG0Kd82/jDYf+znzUxEXtvLN/+RTX1rBw2tFgAWpkbz+zXzmBSlf6MD6TdBGGO0xMYpSIs7OZNJE4RSA8stb+Cnb+5nZWYc3+pVBVVEuHxuEo9/lEt5fWvXrmuf5lRQWN3CzS4U44sLC+RrS1M5WNrAORPimZoYzrTEcBakRuPn68oysPHNlXUQagjS7FNdj1U2s3yKh4NRahRr7ejkO//aSUiAHw9fM8dpa+DyuUn84cNc3txTyi0r0zlQUs8dz+5gWmK4y6Wy/+fy2cMd+rihKXSYJUUFE+Dnw7EqHahWaiC/fPsgh0408NBX5vS7J3NmQjizkiJ4Y1cxJbUt3PT0VsIC/Xj65kVD3mhHDZ0miGHm6yOkxuhMJqUGsreojn9sOs5NK9IGXel8+dwkdhfVsebJzTS3dfL0zYu0+3aEDNrFJCKBwJeBtO7XG2N+5r6wxra0uFCnCeLxj3KJCvHnhiWpHohKqdHjxewCAv18+P75Uwe99tI5SfzynUOU1Lbwj5sWM33CwDOX1PBxZQxiLVAHbAe0Cp0L0uNC2XC4gk7rySX+7RYrj3+Uy/QJ4Zog1LjW2tHJ2l0lXHzaRJc29JkYGcz9l8wkNTaE5ZlxIxChcnAlQSQbY1a7PRIvkh4XSnunlZLaFibbtyLdVVhLc3snpXWtHo5OKc9av/8EDa0WvrLQ9f2Ybxri9qFqeLgyBrFRRE5zeyRepGsmU7eB6s9yKgAoq2/F0mn1SFxKjQYvZRcyOSaYpen9r4JWo4MrCWIlsF1EDovIHhHZKyJ73B3YWJbebS2Ew2e5lQBYDZQ1aE+dGp8Kq5v5PLeKryyYrKuXxwBXupgucnsUXiYxIpBgf9+uBFHf2sHuojpOT45kT1EdpbUtuoJTjUsvby9CBL7s4hoG5VmDtiCMMceBKOBS+y3Kfkz1Q0RIiwvtquq6Jb+aTqvhK/Y/ihIdh1DjkNVqeHl7ESsz4/QD0hgxaIIQkbuA54AE++1ZEfmuuwMb69LjQjhW1QzA57mVBPv78qXTkwAoqW3xZGhKDYvSuhYue+wzfmWfgjqYjXlVFNe29KmrpEYvV7qYbgGWGGOaAETkQWAT8Ad3BjbWpcWGsn5/GR2dVj7NqWBxegwxoQGEB/lRqglCeYF395exp6iOfcV1PPlpPqtnT+CWlenMt1dj7c4Yw/PbCogM9uf8mYkeiFadClcGqQXo7Pa4E90waFBpcaF0Wg3Zx2rIq2hipX3+dlJksHYxKa+wMa+S5OhgPvmPc/jmynQ+PVLBVX/cyJonNrMpr6prD4bN+VVc+5fN/HtPKVcvSO53C1E1+rjSgngK2CIir9kfXwH8zX0heYcM+0ymZ7fYhmtWZtkTRFSQdjGpMc9qNWzOr+aCmYkkR4fwo4tn8L1VWbywrZA/b8hjzZObWZQWTYCfD5/nVpEYEcgDl8/i2kW6nf1YMmiCMMY8LCIfY5vuKsBNxpid7g5srHOU/V6/7wRxYQFMS7RtpzExKpjdRXWeDE2pL+xAaT11LR0szzy5liE00I9bVqZzw5IUXrQnio5OK/91yUxuWJKiLYcxqN8EISIRxph6EYkBjtlvjnMxxphq94c3dsWGBhAe6EdDm4XlU+K65nwnRQZR3dROa0en/sGoMcux5eeyjL6lL4L8ffn68jS+ujQVEdusPjU2DTQG8S/71+1Adreb4/GgRGS1fYFdrojc5+T8vSKyy37bJyKd9oSEiNxlP7ZfRO4e0nc1CjimugJd4w9gKwcOfWcyHSlrYOHP36OwunnkglTqFG3MqyIjLpQJkc7LdAP4+IgmhzGu3wRhjLnE/jXdGJPR7ZZujMno73kOIuILPI5tod1MYI2IzOz1Hr8xxsw1xswFfgRsMMZUi8hs4FZgMTAHuEREsk71m/QUx4rqFVknE4SjTHHvmkyf51ZS2djOoRMNIxegUr0crWziyU/yaWyz9HuNpdPK1qPVLJuipTK8nSvrID5w5ZgTi4FcY0y+MaYdeAG4fIDr1wDP2+/PADYbY5qNMRZgA3ClC+85qlwxL4mvLUvtsSgoKcr2iat3C+JAST0A5Q06w0l5zl825PGLtw9y3kMbeGdvaddMpO72FtfR2GbRBDEO9JsgRCTI3t0TJyLRIhJjv6UBSS689iSgsNvjIvsxZ+8VAqwGXrEf2gecKSKx9nMXA05X14jIbSKSLSLZFRUVLoQ1cs6dnsjPem136GiSl9T2TAT77QmiQus0KQ/aWVDLjIkRRIcGcMdzO7jp6W19uj035tnGH5ZmaILwdgO1IL6Fbbxhuv2r47YWW9fRYJx1Pvb9OGJzKfC5Y+DbGHMQeBB4D1gH7AactnmNMU8YYxYaYxbGx8e7EJZnBfr5EhcWSGndyRZEu8VKTrmta0kThPKUxjYLR8obuHBWIm9+ZwX/dclMth2t5ronNlPb3N513eb8KqYlhhMXFujBaNVIGGgM4vfGmHTg/3Ube0g3xswxxjzmwmsX0fNTfzJQ0s+113Gye8nx/n8zxsw3xpwJVAM5LrznmJAUFdRjsVxueSMdnbbcWa4JQnnInsJajIG5k6Pw8/XhlpXp/OvWpZQ3tHLvy3swxtBm6WTbMR1/GC9cWQfxB/ug8UwgqNvxZwZ56jYgS0TSgWJsSeD63heJSCRwFnBjr+MJxphyEUkBrgKWDRbrWJEUGUxeRWPX4wOl9fbjQdqCUB6zs7AWsCUIhzmTo7jvohk88NYBnt54jFlJkbR2WDVBjBOu7En938DZ2BLE29hmJX0GDJggjDEWEfkOsB7wBf5ujNkvIrfbz//ZfumVwLuOWk/dvCIisUAHcKcxpsbl72qUmxgVxKc5FRhjEBEOlNQT5O/DovQYso95zbepxpidBbVkxIUSFRLQ4/jNK9LYlFfF/759kHOmJSCCbvYzTrhSi+lqYBVwwhhzE7Zppy51Phpj3jbGTDXGTDHG/MJ+7M/dkgPGmKeNMdc5ee4ZxpiZ9i4tV2ZNjRlJkcE0tXdS32obVjlQWsf0CRFMiAiiorHN6cwRpfrTbrGybt+JL/R7Y4xhV2Etc1Oi+pwTEX77ldOJDwvk3QNlzE6KJDJk8L2k1djnSoJoMcZYAYuIRADlwKDrIFT/HIvlSutaMMZwoKSemUkRxIcH0m6xUt/S/xx0pXp7fWcxtz+7nS1HT724QVFNC5WNbcyb3DdBAESFBPCH6+fh6yNddcWU93OlWF+2iEQBT2KbxdQIbHVrVF5uYre1EGGBftS3Wpg5MYLwINs/R0Vjq35CUy7besyWGD7PrTzlqae77OMP85yU6nZYkBrD+rvP7FrLo7yfKzvKfdsYU2vvFjof+Lq9q0mdoqRIR7mN1q4Fco4WBOhMJjU02d0SxKnaVVhLoJ8P0yaED3hdZkIYIQGufK5U3mCgYn3zBzpnjNnhnpC8X3x4IH4+QmmdrVkvAtMnhHetrtaZTMpV5Q2tHKtqJirEn91FdTS0dhAe1LP1+fL2IsKD/Lhw1oR+X2dnQQ2nJ0fi7+tKr7MaLwb6KPCQ/WsQsBDbYjUBTge2YCv/rU6Br4+QGBFESW0rTW0W0uNCCQnwIz7M1nTXBKFctd0+6+3WMzL4zfrDbD1azaoZJ3dsa263cP/afYQG+nHu9ASnCaDdYmVfST1fX5Y6YnGrsWGghXLnGGPOAY4D8+2rlRcA84DckQrQWzk2DjpQWs/MiREARAT7EeDnowlCuWzbsRoC/Xz42rJUAu2b83T33oEymts7qWho48ND5U5f42BpPe0W64DjD2p8cqU9Od0Ys9fxwBizD5jrvpDGh6SoYHLKGymqaWFmki1BiAjxYYE6BqFcln28mrmTowgP8mdhWjQb83qOQ7y+s5iJkUEkRgTywtYCp6+xy8kCOaXAtQRxUET+KiJni8hZIvIkcNDdgXm7iZHBVDfZ6ts4WhBgG5/QFoRyRXO7hf0l9SxKiwFgRWYch040UNlo+/2pamzjk5xKLpubxFcWTGbDkQqn293uLKghMSKQiQPs7aDGJ1cSxE3AfuAu4G7ggIpmeLoAABwHSURBVP2Y+gK6TxV0tCAAEjRBKBftKqil02pYmGbrGloxxbY+wVFt9e29pXRaDVfMncS1iyZjNfBSdmHf1ymsZe7kKN3cR/XhyjTXVmPM74wxV9pvvzPG6KYFX5BjqmtcWCAJ4SeTRXx4oO4JoVyy7VgNIjA/1ZYgZk+KJCLIj4326a6v7yphWmI4MyZGMDkmhDOy4nhpWyGd1pMrrqub2jlW1azjD8qpgfaDeMn+da+I7Ol9G7kQvZNjsVz31gPYEkRNcwftFqsnwlJjSPbxaqZPiCDCPq3V10dYmhHL53mVFFY3s/14DZfPO7l1y3WLUiipa+WTHNu+Ka0dnfz3G/sBWJSmCUL1NdA017vsXy8ZiUDGG8cuc93HH4Cu1kRVU1vX9qRK9WbptLLjeA1XzU/ucXxFZhzvHijjsQ9tEw0vm3MyQZw/M5HY0ABe2FrAzIkR3PZMNnuK67j3wmnM1xaEcqLfBGGMKbV/PT5y4YwfUSEBPHLt3D5lk7tWU9drglD9O3Sigab2zq7xB4cVmbbfpxezC1mcFkNydEjXuQA/H768IJm/f3aUnQWf0dRm4YmvLuT8mYko5cxAXUwNIlLv5NYgIvUjGaS3umLeJBIjes4ccSQIHage33LLG1i//0S/5x3lNRwzmBymxIeRYP8d6t695HDdoslYrIYAPx9e+fZyTQ5qQAO1IAYuyqLcwvHHXdGoCWK8sloN335uB0fKGnnum0tYkdm3euq24zVMigruqgzsICKszIzjzT0lXDx7Yp/nZcSH8eq3lzvd90Gp3lwuvCIiCSKS4ri5M6jxLDbM9kerLYjx672DZRwpayQs0I8fvLSbuuaOHueNMWQfq+7TveTwH6un8+wtS4gOdZ4A5qdEa3JQLhk0QYjIZSKSAxwFNgDHgHfcHNe4FejnS1SIv051HaeMMTz+US4pMSH885bFVDa28ZO1+7rOd3Ra+fHr+yirb3PasgCYEBnEklMs+61Ud67U7X0AWAq8b4yZJyLnAGvcG9b4Fh+mi+W8UbvFyvGqJgprmimoaqa6uYPrFk3u0U30aU4le4rq+OVVpzEvJZq7VmXx0HtHOG9GAudMT+DO53bwaU4ld5w9hat7zWBSari5kiA6jDFVIuIjIj7GmI9E5EG3RzaOJURogvA2dc0dXPHHzzla2XPr9dd2FvHSt5Z1zVh77KNcJkQEcdX8SQDccfYUPjpczk9e38eEiCCOVjbx66tP55qFk0f8e1DjjytjELUiEgZ8AjwnIr8HdE9MN9KCfd7FGMMPX9lDYXUz/3vlabxyx3K2/fg8Xvv2cmqaOrj+yS2U17ey7Vg1W49Wc9uZGQT6+QLg5+vD766dS6fVUN7QxjO3LNbkoEaMKy2Iy4FW4PvADUAk8DN3BjXeOQr2GWO0Po4XeHZLAev2n+BHF03n+iUn53fEhwfyj5sX8dW/bWXNk5uJDQskNjSANYt7zgFJjQ1l7Z0rCA306zNrSSl3GmgdxGMistwY02SM6TTGWIwx/zDGPGqMqerveeqLSwgPos1ipaFNG2ru1mk1rN1V7LbSJgdL63ngrQOcOTWeW8/I6HN+QWoMT31jEcW1LWw9Ws3NK9MJDvDtc11WYrgmBzXiBupiygEeEpFjIvKgiOgeECOk+2pq5V5bjlZx1wu7+NeW4S8Y0Nxu4bvP7yQiyJ+HvjIHHx/nrcElGbE89Y3FXDYnia/prm5qFBloR7nfG2OWAWcB1cBTInJQRO4XkakjFuE4pKupR05BVTMAz2w+jjFmkKuH5tfrDpNX0cgj187t+jftz7IpsTy6Zl6f/aSV8iRXyn0fN8Y8aIyZB1wPXIluGORWupp65BTV2DbQya9o6rNd5xdRUNXMs5uPs2ZxCiuznK9XUGq0c2WhnL+IXCoiz2FbIHcE+LLbIxvHTnYx6WI5dyusaWZCRBAxoQE8s+nYsL3uI+8fwddHuGtV1rC9plIjrd9ZTCJyPrYFcV8CtgIvALcZY5r6e44aHpHB/gT4+mgLYgQU1bSQHhfK3JQo/rIhj+Lalq5S7KfqSFkDr+0q5rYzMvoUY1RqLBmoBfGfwCZghjHmUmPMc5ocRoaI6N7UI6Swupnk6GBusE8/fW7zFx+sfujdw4QF+HH7WVO+8Gsp5UkDDVKfY4x50hhTPZIBKZs4TRBu19rRSXlDG5NjQkiODuG8GYm8sK2Q1o7OQZ/b0t7Jox/ksPLBD/n5WwdoaLUV1NtdWMv6/WXcemZGv8XylBorXK7mqkaW1mNyv+Ja2wB1crStS+lry9Kobmrn7b2l/T7HajW8trOIcx/6mIffO0JsWCB/+/wo5z60gVd3FPGb9YeJCQ3g5pXpI/I9KOVOrqykVh6QEBHIzoIaT4fh1RwzmCbH2HZdW5EZS0Z8KH/6OI/WDivx4YHEhQXQ0t7JgdJ6DpY2sLOwhvyKJk6bFMkj185lSUYse4pquX/tfu55aTcAP/nSDMIC9U9LjX36WzxKJUcHU9XUTkVD26Bz6NWpKaqxrYFwtCBEhO+fN5W7X9zFf762t8/1CeGBzJgYwZ1nZ3LlvEldC99OT47i1TuW8/L2IrYeq+bGpbrYTXkHTRCj1NlTE/j1usN8cLCM6xbr/kzuUFjdgr+vkBB+cqbRpXOSuHDWBKqa2qhosN38fX2YMTFiwETt4yNcs2gy1yzSQnrKe2iCGKVmTAxnUlQw7x3QBOEuRTXNTIoKxrdXCYwAPx8mRgZ3leBWarxy6yC1iKwWkcMikisi9zk5f6+I7LLf9olIp4jE2M99X0T2248/LyLjakK5iHDBrEQ+za2kSYv2uUVhTQvJ0SGeDkOpUcttCUJEfIHHgYuAmcAaEZnZ/RpjzG+MMXONMXOBHwEbjDHVIjIJ+B6w0BgzG/AFrnNXrKPV+TMTabdY+TSnwtOheKXimmYmx2grQan+uLMFsRjINcbkG2Pasa3EvnyA69cAz3d77AcEi4gfEAKUuC3SUWpxWgyRwf68e6DM06F4neZ2C5WN7dqCUGoA7kwQk4DCbo+L7Mf6EJEQYDXwCoAxphj4LVAAlAJ1xph3+3nubSKSLSLZFRXe9Unbz9eHVdMT+PBQOZZO9+xXMF4V1/RcA6GU6sudCcJZ8fv+6ilfCnzuWLUtItHYWhvpQBIQKiI3OnuiMeYJY8xCY8zC+Pj4YQh7dDl/ZiK1zR1sO6ZrIoZTUVeC0BaEUv1xZ4IoArrP+Uum/26i6+jZvXQecNQYU2GM6QBeBZa7JcpR7syp8QT4+fDugROeDsWrFNrXQOgYhFL9c2eC2AZkiUi6iARgSwJv9L5IRCKxbUq0ttvhAmCpiISIbVPmVYzTPShCA/1YmRnHewfKemxoY4wZ9g1uxpOimhYC/XyID9NFiEr1x20JwhhjAb4DrMf2n/tLxpj9InK7iNze7dIrgXe7V4o1xmwBXgZ2AHvtcT7hrlhHuwtmJlJU08LB0gZO1LXyi38f4LSfvssfP87zdGhjlqOKq+3zh1LKGfGmT6ELFy402dnZng5j2FU0tLH4f98nKyGMo5VNWA1MiAiiprmdDfeeo6U4TsGlf/iMmNAA/nHzYk+HopRHich2Y8xCZ+e0musYEB8eyLKMWI5XNbNmcQof/7+zefabS2izWHn8o1xPhzeqNLVZeHf/Cdbt678iK9hWUesMJqUGpqU2xognvraQTqshMvjkpvbXLprMc1uOc/OKdFJix+9snNaOTl7YWsAHh8rZkl9Nu31K8PafnEeskzGGxjYLNc0dXVVclVLOaQtijAgL9OuRHADuWpWFr4/wu/ePeCgqzztS1sBlj33GT988QGldK19fnsp/X2pbsL/1qPO9rnpXcVVKOacJYgxLjAjiphXpvL6rmIOl9Z4OZ0QZY3huy3Eu/cNnVDe184+bF/P+PWfx4y/N5MalqYQE+LIpv8rpcwur7ftA6BoIpQakCWKMu/3MKYQH+vGb9Yc9HcqIMcZw94u7+PFr+1icHsM7d53JWVNPLpL09/VhYVoMm/KcJwhtQSjlGk0QY1xkiD93nJ3Jh4fK+dU7h2hpH3w/5bFuZ2Eta3eV8K2zMvjHTYudzuJalhFLTnmj021bC6tbCAnwJUb3jFZqQJogvMBNK9K4ekEyf96Qx3kPb+B9Ly/u9+buEgJ8fbjznMyuXd16WzYlFoDNTrqZHDOYdA2EUgPTBOEFgvx9+e1X5vDibUsJCfDlm89kc/cLO71ypXWn1fDvPaWcPS2eiCD/fq+bnRRBWKCf03GIopoWHX9QygWaILzIkoxY/v29M7hhSQqv7yrhRH2rp0MadluOVlHe0MZlc5MGvM7P14fF6TFs7jUO0Wk1XauolVID0wThZQL8fPjSaRMByCtvGuTqsefN3aWEBPhy7vSEQa9dlhFLfmUTZd0S5es7i2los3R1QSml+qcJwgtNSQgDIL+y0cORDK+OTivv7CvlvBmJhAQMvsbTkQQcs5naLJ08/N4RTpsUyQUzJ7g1VqW8gSYIL5QQHkhogC/5Fd7Vgvgst5La5g4umzNw95LDjIkRRAT5dQ1UP7+lgOLaFu69cFq/g9tKqZM0QXghESEjPoy8Cu9qQby5q4SIID/OmBrn0vW+PsKSjFg25VfR1GbhsY9yWZoRwxlZrj1fqfFOE4SXyogPHXMtiLqWjn7PtXZ08u6BMlbPnkCgn6/Lr+kocvjzfx+ksrGd/1g9Xae3KuUiTRBeakp8GMW1LWNm4dyGIxUseOA9CqqanZ7/+HA5jW0WLpvjdFvzfi3NsI1DPL+1gPNmJDI/JfoLx6rUeKEJwktlxIcCcLRybLQi3j9QhsVq2FNc6/T8m7tLiQsLYGlGzJBed/qEcKJD/BGBey+cNhyhKjVuaLlvL5URZ5vJlFfRyMykCA9HMzjHgracMufjJtnHqzkzKx4/36F9pvHxEW5ZmU67xcq0CeFfOE6lxhNNEF4qPS4UEcbEOER5fSu55bbE4PjaXW1zO2X1bUyfeGr/wX/n3KwvFJ9S45V2MXmp4ABfkiKDx8RaCEfrYVJUMDnlDX3OHzphOzZtwuhvCSnlTTRBeLGxMpNpY24VEUF+XDJnIkcrm+iw7wjncNieIKZrF5FSI0oThBebEh9GfkXjqC/atzG/kqUZsUyfEE5Hp+F4Vc+kduhEA1Eh/iQ4KeutlHIfTRBebEp8KE3tnZTV990TYbQorG6msLqF5VNiyUqwtRB6D1QfPlHPtMRwXb+g1AjTBOHFMuLtNZlG8YpqR52kZVPimBIfhgjkdBuoNsZwpKxRu5eU8gBNEF7MsRYibxSvhdiYV0lsaABTE8MIDvAlOTq4R4Ioqmmhsc2iA9RKeYAmCC82ISKIkABf8pxMHR0NjDFsyq9i2ZTYru6jrIRwcspOzmQ63DWDKcwjMSo1nmmC8GK2on2h5PdqQby7/wSF1c5LWowk214NbSyfcrJ4XlZCGPmVTVjsM5kO25PF1ETtYlJqpGmC8HIZcWE9xiC2H6/mtn9u565RsCXpRvv4w/Jum/dkJoTRbrFSYE9gh040MCkqmPABthdVSrmHJggvlxEfSnFtC60dnXRaDfev3Y+vj7CjoNbpfs0jaVNeJUmRQaTGntwf2tFScIxDHD5RrwPUSnmIJggvlxEfhjG2on0vbCtgf0k9D375dBLCA3nsw1yPxWW1GjblVbG02/gDnNwNL7e8kXaLlfyKJq2hpJSHaILwclPsM5l2FtTy2/WHWZIew5fnT+K2MzPYmFfF9uM1Holr27Fqapo7ODMrvsfxsEA/W8mNsgbyKhqxWI0mCKU8RBOEl0uPsyWIX75zkLqWDn562SxEhOuXpBAd4s/jH3mmFfHy9iJCA3y5YFZin3OZCWHklDd2K7GhU1yV8gRNEF4uJMCPpMggGlotfHVpKjMmRnQd/+YZGXx4qJx9xXUjGlNzu4W395Zy8WkTCQnoW1A4KyGM3PJGDpbW4+8rXes5lFIjSxPEODDVvmnOPef33DDnq8tSCQ/yG/FWxLp9J2hq7+TqBclOz2clhtFmsfL+wTKmxIfhP8Q9IJRSw0P/8saBX1x5Gi/fsZzIkJ5TRSOC/PnG8jTW7T/B7kLnO7m5w8vbi0iJCWFRmvPd4TLtNZnydIBaKY9ya4IQkdUiclhEckXkPifn7xWRXfbbPhHpFJEYEZnW7fguEakXkbvdGas3mxQVzJR45yuRb16RzsSIIG56eltXn787FdU0szGvii/PT8bHx3nxvcyEk7FqglDKc9yWIETEF3gcuAiYCawRkZndrzHG/MYYM9cYMxf4EbDBGFNtjDnc7fgCoBl4zV2xjmfRoQH869al+PkIN/x1C3lDKOzX2tHJXzbk8fhHuby0rZAPD5VxbJC6T6/tKAbgqvmT+r0mMtifxAhbaW9dA6GU57hzy9HFQK4xJh9ARF4ALgcO9HP9GuB5J8dXAXnGmONuiVKRFhfKv25dynVPbOL6Jzfz0reWkRo78MBwXUsHt/4jm63HqnscF4E/3bCA1bMn9HmOMYaXdxSxNCOGyTEhfc53l5UQTll9m5bYUMqD3NnFNAko7Pa4yH6sDxEJAVYDrzg5fR3OE4fjubeJSLaIZFdUVHyBcMe3zIQwnv3mEtosVq5/cgtVjf3vIVFe38q1f9nEzsIaHrt+Hgd/tppP/+McXvv2cuYkR/GDl3ZxpKxvd1X28RqOVzVz9YLJg8Yzd3IUEyKCmBQV/IW+L6XUqXNngnDWwdxf8Z9Lgc+NMT0+jopIAHAZ8H/9vYkx5gljzEJjzML4+Pj+LlMumD4hgmduXkxFYxvff2k3Vmvff66jlU1c9aeNFFY389Q3FnPJ6UkEB/gyOSaEeSnR/OWrCwgJ9OPWZ7KpbW7vel6n1fDPTccJCfDlIieti96+uyqTdXefoZsEKeVB7kwQRUD3j4rJQEk/1/bXSrgI2GGMKRvm2FQ/Tk+O4r8vncknRyr404a8Hud2FtRw9Z820tLeyfO3LWVlVlyf5ydGBPHnGxdQWtvKd5/fSbvFyhu7S7jgdxt4Y3cJ1yycTGjg4D2bgX6+RIUEDNv3pZQaOneOQWwDskQkHSjGlgSu732RiEQCZwE3OnmN/sYllBtdvziFTXlVPPzeERalxbA4PYZ1+05w94s7SYwI4qlvLOrarc6ZBanR/OzyWdz36l6W/fIDqpramZoYxp9umM+FswZvPSilRge3JQhjjEVEvgOsB3yBvxtj9ovI7fbzf7ZfeiXwrjGmx/QX+7jE+cC33BWjck5E+OVVp7GvuI7vPb+TG5ak8PD7R5g7OYq/fm0hsWGBg77GdYtTOFrZxCc5lfz3ZbO45LSJ/U5rVUqNTuLpPQGG08KFC012dranw/Aa+4rruOpPG2m3WFk9awKPXDeXIH9fT4ellBpGIrLdGLPQ2Tl3djGpMW72pEgeWzOPvIombjszA19tASg1rmiCUAO6QMcMlBq3tBaTUkoppzRBKKWUckoThFJKKac0QSillHJKE4RSSimnNEEopZRyShOEUkoppzRBKKWUcsqrSm2ISANw2NNx9BIHVHo6CCdGY1yjMSbQuIZiNMYEGtdAUo0xTvdK8LaV1If7qyniKSKSPdpigtEZ12iMCTSuoRiNMYHGdaq0i0kppZRTmiCUUko55W0J4glPB+DEaIwJRmdcozEm0LiGYjTGBBrXKfGqQWqllFLDx9taEEoppYaJJgillFJOeUWCEJHVInJYRHJF5D4PxvF3ESkXkX3djsWIyHsikmP/Gj3CMU0WkY9E5KCI7BeRu0ZJXEEislVEdtvj+p/REJc9Bl8R2Skib42imI6JyF4R2SUi2aMorigReVlEDtl/x5Z5Mi4RmWb/GTlu9SJy9yj5WX3f/ru+T0Set/8NeDyugYz5BCEivsDjwEXATGCNiMz0UDhPA6t7HbsP+MAYkwV8YH88kizAD4wxM4ClwJ32n4+n42oDzjXGzAHmAqtFZOkoiAvgLuBgt8ejISaAc4wxc7vNmx8Ncf0eWGeMmQ7MwfZz81hcxpjD9p/RXGAB0Ay85smYAERkEvA9YKExZjbgC1zn6bgGZYwZ0zdgGbC+2+MfAT/yYDxpwL5ujw8DE+33J2JbzOfJn9da4PzRFBcQAuwAlng6LiAZ2x/qucBbo+XfEDgGxPU65umfVQRwFPtkl9ESV7c4LgA+Hw0xAZOAQiAG2wLlt+zxjYqfVX+3Md+C4OQP3qHIfmy0SDTGlALYvyZ4KhARSQPmAVtGQ1z2rpxdQDnwnjFmNMT1CPAfgLXbMU/HBGCAd0Vku4jcNkriygAqgKfsXXJ/FZHQURCXw3XA8/b7Ho3JGFMM/BYoAEqBOmPMu56OazDekCDEyTGdu9uLiIQBrwB3G2PqPR0PgDGm09i6ApKBxSIy25PxiMglQLkxZrsn4+jHCmPMfGxdqXeKyJmeDgjbJ+H5wJ+MMfOAJkZJF4mIBACXAf/n6VgA7GMLlwPpQBIQKiI3ejaqwXlDgigCJnd7nAyUeCgWZ8pEZCKA/Wv5SAcgIv7YksNzxphXR0tcDsaYWuBjbOM3noxrBXCZiBwDXgDOFZFnPRwTAMaYEvvXcmx96otHQVxFQJG95QfwMraE4em4wJZIdxhjyuyPPR3TecBRY0yFMaYDeBVYPgriGpA3JIhtQJaIpNs/NVwHvOHhmLp7A/i6/f7XsY0BjBgREeBvwEFjzMOjKK54EYmy3w/G9gd0yJNxGWN+ZIxJNsakYfs9+tAYc6MnYwIQkVARCXfcx9Z3vc/TcRljTgCFIjLNfmgVcMDTcdmt4WT3Eng+pgJgqYiE2P8mV2Eb0Pd0XAPz9CDIMA0AXQwcAfKAH3swjuex9S92YPt0dQsQi23QM8f+NWaEY1qJrcttD7DLfrt4FMR1OrDTHtc+4H77cY/G1S2+szk5SO3pn1UGsNt+2+/4Hfd0XPYY5gLZ9n/H14FoT8eFbdJDFRDZ7dho+Fn9D7YPQfuAfwKBoyGugW5aakMppZRT3tDFpJRSyg00QSillHJKE4RSSimnNEEopZRyShOEUkoppzRBKDUEItLZq1rosK0cFpE06VYJWClP8/N0AEqNMS3GVh5EKa+nLQilhoF9v4YH7XtcbBWRTPvxVBH5QET22L+m2I8nishrYtsPY7eILLe/lK+IPGnfN+Bd+ypzpTxCE4RSQxPcq4vp2m7n6o0xi4HHsFWFxX7/GWPM6cBzwKP2448CG4xtP4z52FZIA2QBjxtjZgG1wJfd/P0o1S9dSa3UEIhIozEmzMnxY9g2QMq3F0c8YYyJFZFKbPX+O+zHS40xcSJSASQbY9q6vUYatrLnWfbHPwT8jTE/d/93plRf2oJQaviYfu73d40zbd3ud6LjhMqDNEEoNXyu7fZ1k/3+RmyVYQFuAD6z3/8AuAO6Nk6KGKkglXKVfjpRamiC7bvgOawzxjimugaKyBZsH7zW2I99D/i7iNyLbfe1m+zH7wKeEJFbsLUU7sBWCVipUUPHIJQaBvYxiIXGmEpPx6LUcNEuJqWUUk5pC0IppZRT2oJQSinllCYIpZRSTmmCUEop5ZQmCKWUUk5pglBKKeXU/wc64z1gj1t7JQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(history.history['val_loss'][10:]).plot(logy=False)\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Validation Error\")\n",
    "print(f\"Minimum MSE: {round(min(history.history['val_loss']),3)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment Seven - Recommender System using RBMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#make code compatible with v1 of TF\n",
    "tf.compat.v1.disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define RBM class\n",
    "class RBM(object):\n",
    "    \n",
    "    def __init__(self, input_size, output_size, \n",
    "                 learning_rate, epochs, batchsize):\n",
    "        # Define hyperparameters\n",
    "        self._input_size = input_size\n",
    "        self._output_size = output_size\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epochs = epochs\n",
    "        self.batchsize = batchsize\n",
    "        \n",
    "        # Initialize weights and biases using zero matrices\n",
    "        self.w = np.zeros([input_size, output_size], dtype=np.float32)\n",
    "        self.hb = np.zeros([output_size], dtype=np.float32)\n",
    "        self.vb = np.zeros([input_size], dtype=np.float32)\n",
    "\n",
    "    def prob_h_given_v(self, visible, w, hb):\n",
    "        return tf.nn.sigmoid(tf.matmul(visible, w) + hb)\n",
    "\n",
    "    def prob_v_given_h(self, hidden, w, vb):\n",
    "        return tf.nn.sigmoid(tf.matmul(hidden, tf.transpose(w)) + vb)\n",
    "    \n",
    "    def sample_prob(self, probs):\n",
    "        return tf.nn.relu(tf.sign(probs - tf.random.uniform(tf.shape(probs))))\n",
    "\n",
    "    def train(self, X):\n",
    "        _w = tf.compat.v1.placeholder(tf.float32, [self._input_size, self._output_size])\n",
    "        _hb = tf.compat.v1.placeholder(tf.float32, [self._output_size])\n",
    "        _vb = tf.compat.v1.placeholder(tf.float32, [self._input_size])\n",
    "        \n",
    "        prv_w = np.zeros([self._input_size, self._output_size], dtype=np.float32)\n",
    "        prv_hb = np.zeros([self._output_size], dtype=np.float32)\n",
    "        prv_vb = np.zeros([self._input_size], dtype=np.float32)\n",
    "        \n",
    "        cur_w = np.zeros([self._input_size, self._output_size], dtype=np.float32)\n",
    "        cur_hb = np.zeros([self._output_size], dtype=np.float32)\n",
    "        cur_vb = np.zeros([self._input_size], dtype=np.float32)\n",
    "        \n",
    "        v0 = tf.compat.v1.placeholder(tf.float32, [None, self._input_size])\n",
    "        h0 = self.sample_prob(self.prob_h_given_v(v0, _w, _hb))\n",
    "        v1 = self.sample_prob(self.prob_v_given_h(h0, _w, _vb))\n",
    "        h1 = self.prob_h_given_v(v1, _w, _hb)\n",
    "        \n",
    "        positive_grad = tf.matmul(tf.transpose(v0), h0)\n",
    "        negative_grad = tf.matmul(tf.transpose(v1), h1)\n",
    "        \n",
    "        update_w = _w + self.learning_rate * \\\n",
    "            (positive_grad - negative_grad) / tf.cast(tf.shape(v0)[0], tf.float32)\n",
    "        update_vb = _vb +  self.learning_rate * tf.reduce_mean(v0 - v1, 0)\n",
    "        update_hb = _hb +  self.learning_rate * tf.reduce_mean(h0 - h1, 0)\n",
    "        \n",
    "        err = tf.reduce_mean(tf.square(v0 - v1))\n",
    "        \n",
    "        error_list = []\n",
    "        \n",
    "        with tf.compat.v1.Session() as sess:\n",
    "            sess.run(tf.compat.v1.global_variables_initializer())\n",
    "            \n",
    "            for epoch in range(self.epochs):\n",
    "                for start, end in zip(range(0, len(X), \\\n",
    "                        self.batchsize),range(self.batchsize,len(X), \\\n",
    "                                              self.batchsize)):\n",
    "                    batch = X[start:end]\n",
    "                    cur_w = sess.run(update_w, feed_dict={v0: batch, \\\n",
    "                                    _w: prv_w, _hb: prv_hb, _vb: prv_vb})\n",
    "                    cur_hb = sess.run(update_hb, feed_dict={v0: batch, \\\n",
    "                                    _w: prv_w, _hb: prv_hb, _vb: prv_vb})\n",
    "                    cur_vb = sess.run(update_vb, feed_dict={v0: batch, \\\n",
    "                                    _w: prv_w, _hb: prv_hb, _vb: prv_vb})\n",
    "                    prv_w = cur_w\n",
    "                    prv_hb = cur_hb\n",
    "                    prv_vb = cur_vb\n",
    "                error = sess.run(err, feed_dict={v0: X, \\\n",
    "                                _w: cur_w, _vb: cur_vb, _hb: cur_hb})\n",
    "                print ('Epoch: %d' % epoch,'reconstruction error: %f' % error)\n",
    "                error_list.append(error)\n",
    "            self.w = prv_w\n",
    "            self.hb = prv_hb\n",
    "            self.vb = prv_vb\n",
    "            return error_list\n",
    "\n",
    "    def rbm_output(self, X):\n",
    "        \n",
    "        input_X = tf.constant(X)\n",
    "        _w = tf.constant(self.w)\n",
    "        _hb = tf.constant(self.hb)\n",
    "        _vb = tf.constant(self.vb)\n",
    "        out = tf.nn.sigmoid(tf.matmul(input_X, _w) + _hb)\n",
    "        hiddenGen = self.sample_prob(self.prob_h_given_v(input_X, _w, _hb))\n",
    "        visibleGen = self.sample_prob(self.prob_v_given_h(hiddenGen, _w, _vb))\n",
    "        with tf.compat.v1.Session() as sess:\n",
    "            sess.run(tf.compat.v1.global_variables_initializer())\n",
    "            return sess.run(out), sess.run(visibleGen), sess.run(hiddenGen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Begin the training cycle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert inputX into float32\n",
    "inputX = ratings_train\n",
    "inputX = inputX.astype(np.float32)\n",
    "\n",
    "# Define the parameters of the RBMs we will train\n",
    "rbm=RBM(1000,1000,1,1000,200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0 reconstruction error: 1.111619\n",
      "Epoch: 1 reconstruction error: 1.077608\n",
      "Epoch: 2 reconstruction error: 1.090634\n",
      "Epoch: 3 reconstruction error: 1.092777\n",
      "Epoch: 4 reconstruction error: 1.094247\n",
      "Epoch: 5 reconstruction error: 1.094252\n",
      "Epoch: 6 reconstruction error: 1.090279\n",
      "Epoch: 7 reconstruction error: 1.084947\n",
      "Epoch: 8 reconstruction error: 1.086759\n",
      "Epoch: 9 reconstruction error: 1.089347\n",
      "Epoch: 10 reconstruction error: 1.089152\n",
      "Epoch: 11 reconstruction error: 1.092489\n",
      "Epoch: 12 reconstruction error: 1.089138\n",
      "Epoch: 13 reconstruction error: 1.096748\n",
      "Epoch: 14 reconstruction error: 1.090379\n",
      "Epoch: 15 reconstruction error: 1.092049\n",
      "Epoch: 16 reconstruction error: 1.096488\n",
      "Epoch: 17 reconstruction error: 1.094620\n",
      "Epoch: 18 reconstruction error: 1.093840\n",
      "Epoch: 19 reconstruction error: 1.094844\n",
      "Epoch: 20 reconstruction error: 1.096987\n",
      "Epoch: 21 reconstruction error: 1.093371\n",
      "Epoch: 22 reconstruction error: 1.095827\n",
      "Epoch: 23 reconstruction error: 1.097365\n",
      "Epoch: 24 reconstruction error: 1.094983\n",
      "Epoch: 25 reconstruction error: 1.097685\n",
      "Epoch: 26 reconstruction error: 1.098858\n",
      "Epoch: 27 reconstruction error: 1.099048\n",
      "Epoch: 28 reconstruction error: 1.098292\n",
      "Epoch: 29 reconstruction error: 1.099416\n",
      "Epoch: 30 reconstruction error: 1.096866\n",
      "Epoch: 31 reconstruction error: 1.099463\n",
      "Epoch: 32 reconstruction error: 1.099581\n",
      "Epoch: 33 reconstruction error: 1.099614\n",
      "Epoch: 34 reconstruction error: 1.099498\n",
      "Epoch: 35 reconstruction error: 1.098447\n",
      "Epoch: 36 reconstruction error: 1.096521\n",
      "Epoch: 37 reconstruction error: 1.098922\n",
      "Epoch: 38 reconstruction error: 1.096617\n",
      "Epoch: 39 reconstruction error: 1.099167\n",
      "Epoch: 40 reconstruction error: 1.097492\n",
      "Epoch: 41 reconstruction error: 1.096475\n",
      "Epoch: 42 reconstruction error: 1.096676\n",
      "Epoch: 43 reconstruction error: 1.097460\n",
      "Epoch: 44 reconstruction error: 1.097276\n",
      "Epoch: 45 reconstruction error: 1.096038\n",
      "Epoch: 46 reconstruction error: 1.095927\n",
      "Epoch: 47 reconstruction error: 1.099882\n",
      "Epoch: 48 reconstruction error: 1.096874\n",
      "Epoch: 49 reconstruction error: 1.098382\n",
      "Epoch: 50 reconstruction error: 1.095273\n",
      "Epoch: 51 reconstruction error: 1.094012\n",
      "Epoch: 52 reconstruction error: 1.094841\n",
      "Epoch: 53 reconstruction error: 1.097198\n",
      "Epoch: 54 reconstruction error: 1.094653\n",
      "Epoch: 55 reconstruction error: 1.094597\n",
      "Epoch: 56 reconstruction error: 1.097319\n",
      "Epoch: 57 reconstruction error: 1.094568\n",
      "Epoch: 58 reconstruction error: 1.095319\n",
      "Epoch: 59 reconstruction error: 1.094970\n",
      "Epoch: 60 reconstruction error: 1.095592\n",
      "Epoch: 61 reconstruction error: 1.095934\n",
      "Epoch: 62 reconstruction error: 1.096143\n",
      "Epoch: 63 reconstruction error: 1.095281\n",
      "Epoch: 64 reconstruction error: 1.094828\n",
      "Epoch: 65 reconstruction error: 1.094566\n",
      "Epoch: 66 reconstruction error: 1.094376\n",
      "Epoch: 67 reconstruction error: 1.093920\n",
      "Epoch: 68 reconstruction error: 1.096303\n",
      "Epoch: 69 reconstruction error: 1.094059\n",
      "Epoch: 70 reconstruction error: 1.093658\n",
      "Epoch: 71 reconstruction error: 1.094090\n",
      "Epoch: 72 reconstruction error: 1.092595\n",
      "Epoch: 73 reconstruction error: 1.095554\n",
      "Epoch: 74 reconstruction error: 1.093719\n",
      "Epoch: 75 reconstruction error: 1.094411\n",
      "Epoch: 76 reconstruction error: 1.093439\n",
      "Epoch: 77 reconstruction error: 1.096120\n",
      "Epoch: 78 reconstruction error: 1.094599\n",
      "Epoch: 79 reconstruction error: 1.095620\n",
      "Epoch: 80 reconstruction error: 1.094705\n",
      "Epoch: 81 reconstruction error: 1.094103\n",
      "Epoch: 82 reconstruction error: 1.093544\n",
      "Epoch: 83 reconstruction error: 1.094764\n",
      "Epoch: 84 reconstruction error: 1.094851\n",
      "Epoch: 85 reconstruction error: 1.095412\n",
      "Epoch: 86 reconstruction error: 1.092505\n",
      "Epoch: 87 reconstruction error: 1.092955\n",
      "Epoch: 88 reconstruction error: 1.093624\n",
      "Epoch: 89 reconstruction error: 1.093331\n",
      "Epoch: 90 reconstruction error: 1.092582\n",
      "Epoch: 91 reconstruction error: 1.093052\n",
      "Epoch: 92 reconstruction error: 1.092831\n",
      "Epoch: 93 reconstruction error: 1.092214\n",
      "Epoch: 94 reconstruction error: 1.093647\n",
      "Epoch: 95 reconstruction error: 1.093734\n",
      "Epoch: 96 reconstruction error: 1.092280\n",
      "Epoch: 97 reconstruction error: 1.092160\n",
      "Epoch: 98 reconstruction error: 1.093050\n",
      "Epoch: 99 reconstruction error: 1.094720\n",
      "Epoch: 100 reconstruction error: 1.090343\n",
      "Epoch: 101 reconstruction error: 1.092571\n",
      "Epoch: 102 reconstruction error: 1.091486\n",
      "Epoch: 103 reconstruction error: 1.092816\n",
      "Epoch: 104 reconstruction error: 1.090893\n",
      "Epoch: 105 reconstruction error: 1.091668\n",
      "Epoch: 106 reconstruction error: 1.090894\n",
      "Epoch: 107 reconstruction error: 1.092499\n",
      "Epoch: 108 reconstruction error: 1.090189\n",
      "Epoch: 109 reconstruction error: 1.092276\n",
      "Epoch: 110 reconstruction error: 1.090032\n",
      "Epoch: 111 reconstruction error: 1.091431\n",
      "Epoch: 112 reconstruction error: 1.090124\n",
      "Epoch: 113 reconstruction error: 1.090884\n",
      "Epoch: 114 reconstruction error: 1.091494\n",
      "Epoch: 115 reconstruction error: 1.091328\n",
      "Epoch: 116 reconstruction error: 1.088989\n",
      "Epoch: 117 reconstruction error: 1.090045\n",
      "Epoch: 118 reconstruction error: 1.090193\n",
      "Epoch: 119 reconstruction error: 1.089466\n",
      "Epoch: 120 reconstruction error: 1.088796\n",
      "Epoch: 121 reconstruction error: 1.089375\n",
      "Epoch: 122 reconstruction error: 1.088413\n",
      "Epoch: 123 reconstruction error: 1.089268\n",
      "Epoch: 124 reconstruction error: 1.088645\n",
      "Epoch: 125 reconstruction error: 1.088426\n",
      "Epoch: 126 reconstruction error: 1.087984\n",
      "Epoch: 127 reconstruction error: 1.087760\n",
      "Epoch: 128 reconstruction error: 1.088873\n",
      "Epoch: 129 reconstruction error: 1.086825\n",
      "Epoch: 130 reconstruction error: 1.087891\n",
      "Epoch: 131 reconstruction error: 1.088374\n",
      "Epoch: 132 reconstruction error: 1.086540\n",
      "Epoch: 133 reconstruction error: 1.088457\n",
      "Epoch: 134 reconstruction error: 1.086056\n",
      "Epoch: 135 reconstruction error: 1.087278\n",
      "Epoch: 136 reconstruction error: 1.086054\n",
      "Epoch: 137 reconstruction error: 1.087731\n",
      "Epoch: 138 reconstruction error: 1.086730\n",
      "Epoch: 139 reconstruction error: 1.086707\n",
      "Epoch: 140 reconstruction error: 1.087093\n",
      "Epoch: 141 reconstruction error: 1.086709\n",
      "Epoch: 142 reconstruction error: 1.087172\n",
      "Epoch: 143 reconstruction error: 1.086933\n",
      "Epoch: 144 reconstruction error: 1.085430\n",
      "Epoch: 145 reconstruction error: 1.086198\n",
      "Epoch: 146 reconstruction error: 1.085312\n",
      "Epoch: 147 reconstruction error: 1.085917\n",
      "Epoch: 148 reconstruction error: 1.085217\n",
      "Epoch: 149 reconstruction error: 1.086977\n",
      "Epoch: 150 reconstruction error: 1.085487\n",
      "Epoch: 151 reconstruction error: 1.084999\n",
      "Epoch: 152 reconstruction error: 1.084725\n",
      "Epoch: 153 reconstruction error: 1.084795\n",
      "Epoch: 154 reconstruction error: 1.085214\n",
      "Epoch: 155 reconstruction error: 1.085827\n",
      "Epoch: 156 reconstruction error: 1.084136\n",
      "Epoch: 157 reconstruction error: 1.084368\n",
      "Epoch: 158 reconstruction error: 1.083972\n",
      "Epoch: 159 reconstruction error: 1.084191\n",
      "Epoch: 160 reconstruction error: 1.083841\n",
      "Epoch: 161 reconstruction error: 1.085238\n",
      "Epoch: 162 reconstruction error: 1.084242\n",
      "Epoch: 163 reconstruction error: 1.084414\n",
      "Epoch: 164 reconstruction error: 1.084131\n",
      "Epoch: 165 reconstruction error: 1.085593\n",
      "Epoch: 166 reconstruction error: 1.083099\n",
      "Epoch: 167 reconstruction error: 1.084324\n",
      "Epoch: 168 reconstruction error: 1.083076\n",
      "Epoch: 169 reconstruction error: 1.083920\n",
      "Epoch: 170 reconstruction error: 1.083418\n",
      "Epoch: 171 reconstruction error: 1.083513\n",
      "Epoch: 172 reconstruction error: 1.083341\n",
      "Epoch: 173 reconstruction error: 1.083011\n",
      "Epoch: 174 reconstruction error: 1.083110\n",
      "Epoch: 175 reconstruction error: 1.082206\n",
      "Epoch: 176 reconstruction error: 1.083694\n",
      "Epoch: 177 reconstruction error: 1.082664\n",
      "Epoch: 178 reconstruction error: 1.083066\n",
      "Epoch: 179 reconstruction error: 1.083272\n",
      "Epoch: 180 reconstruction error: 1.082615\n",
      "Epoch: 181 reconstruction error: 1.082320\n",
      "Epoch: 182 reconstruction error: 1.082140\n",
      "Epoch: 183 reconstruction error: 1.082386\n",
      "Epoch: 184 reconstruction error: 1.082234\n",
      "Epoch: 185 reconstruction error: 1.082968\n",
      "Epoch: 186 reconstruction error: 1.081908\n",
      "Epoch: 187 reconstruction error: 1.082592\n",
      "Epoch: 188 reconstruction error: 1.082685\n",
      "Epoch: 189 reconstruction error: 1.081511\n",
      "Epoch: 190 reconstruction error: 1.082402\n",
      "Epoch: 191 reconstruction error: 1.082422\n",
      "Epoch: 192 reconstruction error: 1.082147\n",
      "Epoch: 193 reconstruction error: 1.081287\n",
      "Epoch: 194 reconstruction error: 1.082844\n",
      "Epoch: 195 reconstruction error: 1.081213\n",
      "Epoch: 196 reconstruction error: 1.080904\n",
      "Epoch: 197 reconstruction error: 1.081262\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 198 reconstruction error: 1.081575\n",
      "Epoch: 199 reconstruction error: 1.080888\n",
      "Epoch: 200 reconstruction error: 1.082314\n",
      "Epoch: 201 reconstruction error: 1.081298\n",
      "Epoch: 202 reconstruction error: 1.081427\n",
      "Epoch: 203 reconstruction error: 1.080536\n",
      "Epoch: 204 reconstruction error: 1.081583\n",
      "Epoch: 205 reconstruction error: 1.081099\n",
      "Epoch: 206 reconstruction error: 1.081509\n",
      "Epoch: 207 reconstruction error: 1.080428\n",
      "Epoch: 208 reconstruction error: 1.081316\n",
      "Epoch: 209 reconstruction error: 1.081023\n",
      "Epoch: 210 reconstruction error: 1.080940\n",
      "Epoch: 211 reconstruction error: 1.080698\n",
      "Epoch: 212 reconstruction error: 1.080750\n",
      "Epoch: 213 reconstruction error: 1.080737\n",
      "Epoch: 214 reconstruction error: 1.081004\n",
      "Epoch: 215 reconstruction error: 1.080549\n",
      "Epoch: 216 reconstruction error: 1.080842\n",
      "Epoch: 217 reconstruction error: 1.080484\n",
      "Epoch: 218 reconstruction error: 1.080767\n",
      "Epoch: 219 reconstruction error: 1.080176\n",
      "Epoch: 220 reconstruction error: 1.080750\n",
      "Epoch: 221 reconstruction error: 1.081082\n",
      "Epoch: 222 reconstruction error: 1.080982\n",
      "Epoch: 223 reconstruction error: 1.081189\n",
      "Epoch: 224 reconstruction error: 1.080449\n",
      "Epoch: 225 reconstruction error: 1.080518\n",
      "Epoch: 226 reconstruction error: 1.080281\n",
      "Epoch: 227 reconstruction error: 1.080559\n",
      "Epoch: 228 reconstruction error: 1.080124\n",
      "Epoch: 229 reconstruction error: 1.080271\n",
      "Epoch: 230 reconstruction error: 1.081195\n",
      "Epoch: 231 reconstruction error: 1.080248\n",
      "Epoch: 232 reconstruction error: 1.080484\n",
      "Epoch: 233 reconstruction error: 1.080703\n",
      "Epoch: 234 reconstruction error: 1.080634\n",
      "Epoch: 235 reconstruction error: 1.080428\n",
      "Epoch: 236 reconstruction error: 1.080781\n",
      "Epoch: 237 reconstruction error: 1.080176\n",
      "Epoch: 238 reconstruction error: 1.080665\n",
      "Epoch: 239 reconstruction error: 1.080672\n",
      "Epoch: 240 reconstruction error: 1.079538\n",
      "Epoch: 241 reconstruction error: 1.079710\n",
      "Epoch: 242 reconstruction error: 1.080044\n",
      "Epoch: 243 reconstruction error: 1.079602\n",
      "Epoch: 244 reconstruction error: 1.079445\n",
      "Epoch: 245 reconstruction error: 1.079744\n",
      "Epoch: 246 reconstruction error: 1.079411\n",
      "Epoch: 247 reconstruction error: 1.079694\n",
      "Epoch: 248 reconstruction error: 1.079380\n",
      "Epoch: 249 reconstruction error: 1.079134\n",
      "Epoch: 250 reconstruction error: 1.079350\n",
      "Epoch: 251 reconstruction error: 1.079895\n",
      "Epoch: 252 reconstruction error: 1.079620\n",
      "Epoch: 253 reconstruction error: 1.079846\n",
      "Epoch: 254 reconstruction error: 1.079142\n",
      "Epoch: 255 reconstruction error: 1.078902\n",
      "Epoch: 256 reconstruction error: 1.079458\n",
      "Epoch: 257 reconstruction error: 1.078560\n",
      "Epoch: 258 reconstruction error: 1.079151\n",
      "Epoch: 259 reconstruction error: 1.078665\n",
      "Epoch: 260 reconstruction error: 1.079247\n",
      "Epoch: 261 reconstruction error: 1.078655\n",
      "Epoch: 262 reconstruction error: 1.079052\n",
      "Epoch: 263 reconstruction error: 1.078313\n",
      "Epoch: 264 reconstruction error: 1.079005\n",
      "Epoch: 265 reconstruction error: 1.078096\n",
      "Epoch: 266 reconstruction error: 1.078318\n",
      "Epoch: 267 reconstruction error: 1.078264\n",
      "Epoch: 268 reconstruction error: 1.079602\n",
      "Epoch: 269 reconstruction error: 1.078419\n",
      "Epoch: 270 reconstruction error: 1.078967\n",
      "Epoch: 271 reconstruction error: 1.078784\n",
      "Epoch: 272 reconstruction error: 1.078856\n",
      "Epoch: 273 reconstruction error: 1.079352\n",
      "Epoch: 274 reconstruction error: 1.078464\n",
      "Epoch: 275 reconstruction error: 1.078179\n",
      "Epoch: 276 reconstruction error: 1.078308\n",
      "Epoch: 277 reconstruction error: 1.078944\n",
      "Epoch: 278 reconstruction error: 1.077963\n",
      "Epoch: 279 reconstruction error: 1.078189\n",
      "Epoch: 280 reconstruction error: 1.078015\n",
      "Epoch: 281 reconstruction error: 1.077772\n",
      "Epoch: 282 reconstruction error: 1.077785\n",
      "Epoch: 283 reconstruction error: 1.078478\n",
      "Epoch: 284 reconstruction error: 1.077873\n",
      "Epoch: 285 reconstruction error: 1.077881\n",
      "Epoch: 286 reconstruction error: 1.077752\n",
      "Epoch: 287 reconstruction error: 1.078084\n",
      "Epoch: 288 reconstruction error: 1.077818\n",
      "Epoch: 289 reconstruction error: 1.078176\n",
      "Epoch: 290 reconstruction error: 1.077623\n",
      "Epoch: 291 reconstruction error: 1.077821\n",
      "Epoch: 292 reconstruction error: 1.077783\n",
      "Epoch: 293 reconstruction error: 1.077507\n",
      "Epoch: 294 reconstruction error: 1.077253\n",
      "Epoch: 295 reconstruction error: 1.077760\n",
      "Epoch: 296 reconstruction error: 1.077749\n",
      "Epoch: 297 reconstruction error: 1.077780\n",
      "Epoch: 298 reconstruction error: 1.077644\n",
      "Epoch: 299 reconstruction error: 1.077850\n",
      "Epoch: 300 reconstruction error: 1.077358\n",
      "Epoch: 301 reconstruction error: 1.077955\n",
      "Epoch: 302 reconstruction error: 1.077474\n",
      "Epoch: 303 reconstruction error: 1.077855\n",
      "Epoch: 304 reconstruction error: 1.077642\n",
      "Epoch: 305 reconstruction error: 1.077113\n",
      "Epoch: 306 reconstruction error: 1.076813\n",
      "Epoch: 307 reconstruction error: 1.077163\n",
      "Epoch: 308 reconstruction error: 1.077474\n",
      "Epoch: 309 reconstruction error: 1.077467\n",
      "Epoch: 310 reconstruction error: 1.077520\n",
      "Epoch: 311 reconstruction error: 1.077214\n",
      "Epoch: 312 reconstruction error: 1.077214\n",
      "Epoch: 313 reconstruction error: 1.077353\n",
      "Epoch: 314 reconstruction error: 1.076823\n",
      "Epoch: 315 reconstruction error: 1.077628\n",
      "Epoch: 316 reconstruction error: 1.076941\n",
      "Epoch: 317 reconstruction error: 1.077680\n",
      "Epoch: 318 reconstruction error: 1.076488\n",
      "Epoch: 319 reconstruction error: 1.077613\n",
      "Epoch: 320 reconstruction error: 1.076480\n",
      "Epoch: 321 reconstruction error: 1.077750\n",
      "Epoch: 322 reconstruction error: 1.077332\n",
      "Epoch: 323 reconstruction error: 1.077085\n",
      "Epoch: 324 reconstruction error: 1.077137\n",
      "Epoch: 325 reconstruction error: 1.077325\n",
      "Epoch: 326 reconstruction error: 1.076740\n",
      "Epoch: 327 reconstruction error: 1.077299\n",
      "Epoch: 328 reconstruction error: 1.076646\n",
      "Epoch: 329 reconstruction error: 1.077206\n",
      "Epoch: 330 reconstruction error: 1.077258\n",
      "Epoch: 331 reconstruction error: 1.077428\n",
      "Epoch: 332 reconstruction error: 1.076695\n",
      "Epoch: 333 reconstruction error: 1.077180\n",
      "Epoch: 334 reconstruction error: 1.076406\n",
      "Epoch: 335 reconstruction error: 1.077018\n",
      "Epoch: 336 reconstruction error: 1.076782\n",
      "Epoch: 337 reconstruction error: 1.076915\n",
      "Epoch: 338 reconstruction error: 1.076288\n",
      "Epoch: 339 reconstruction error: 1.076766\n",
      "Epoch: 340 reconstruction error: 1.076463\n",
      "Epoch: 341 reconstruction error: 1.076377\n",
      "Epoch: 342 reconstruction error: 1.076460\n",
      "Epoch: 343 reconstruction error: 1.076493\n",
      "Epoch: 344 reconstruction error: 1.076635\n",
      "Epoch: 345 reconstruction error: 1.076635\n",
      "Epoch: 346 reconstruction error: 1.076041\n",
      "Epoch: 347 reconstruction error: 1.076684\n",
      "Epoch: 348 reconstruction error: 1.076450\n",
      "Epoch: 349 reconstruction error: 1.075925\n",
      "Epoch: 350 reconstruction error: 1.076115\n",
      "Epoch: 351 reconstruction error: 1.076144\n",
      "Epoch: 352 reconstruction error: 1.076414\n",
      "Epoch: 353 reconstruction error: 1.076206\n",
      "Epoch: 354 reconstruction error: 1.075084\n",
      "Epoch: 355 reconstruction error: 1.075729\n",
      "Epoch: 356 reconstruction error: 1.076133\n",
      "Epoch: 357 reconstruction error: 1.076084\n",
      "Epoch: 358 reconstruction error: 1.075976\n",
      "Epoch: 359 reconstruction error: 1.076041\n",
      "Epoch: 360 reconstruction error: 1.076182\n",
      "Epoch: 361 reconstruction error: 1.076162\n",
      "Epoch: 362 reconstruction error: 1.076177\n",
      "Epoch: 363 reconstruction error: 1.076316\n",
      "Epoch: 364 reconstruction error: 1.076323\n",
      "Epoch: 365 reconstruction error: 1.076205\n",
      "Epoch: 366 reconstruction error: 1.076184\n",
      "Epoch: 367 reconstruction error: 1.076723\n",
      "Epoch: 368 reconstruction error: 1.076061\n",
      "Epoch: 369 reconstruction error: 1.076473\n",
      "Epoch: 370 reconstruction error: 1.076102\n",
      "Epoch: 371 reconstruction error: 1.076329\n",
      "Epoch: 372 reconstruction error: 1.075876\n",
      "Epoch: 373 reconstruction error: 1.076077\n",
      "Epoch: 374 reconstruction error: 1.076161\n",
      "Epoch: 375 reconstruction error: 1.075662\n",
      "Epoch: 376 reconstruction error: 1.075955\n",
      "Epoch: 377 reconstruction error: 1.076048\n",
      "Epoch: 378 reconstruction error: 1.075789\n",
      "Epoch: 379 reconstruction error: 1.076131\n",
      "Epoch: 380 reconstruction error: 1.075655\n",
      "Epoch: 381 reconstruction error: 1.075855\n",
      "Epoch: 382 reconstruction error: 1.075930\n",
      "Epoch: 383 reconstruction error: 1.076059\n",
      "Epoch: 384 reconstruction error: 1.075783\n",
      "Epoch: 385 reconstruction error: 1.076272\n",
      "Epoch: 386 reconstruction error: 1.075626\n",
      "Epoch: 387 reconstruction error: 1.075747\n",
      "Epoch: 388 reconstruction error: 1.076154\n",
      "Epoch: 389 reconstruction error: 1.075729\n",
      "Epoch: 390 reconstruction error: 1.075632\n",
      "Epoch: 391 reconstruction error: 1.075603\n",
      "Epoch: 392 reconstruction error: 1.075433\n",
      "Epoch: 393 reconstruction error: 1.075613\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 394 reconstruction error: 1.075943\n",
      "Epoch: 395 reconstruction error: 1.075513\n",
      "Epoch: 396 reconstruction error: 1.075611\n",
      "Epoch: 397 reconstruction error: 1.075719\n",
      "Epoch: 398 reconstruction error: 1.075621\n",
      "Epoch: 399 reconstruction error: 1.075215\n",
      "Epoch: 400 reconstruction error: 1.075688\n",
      "Epoch: 401 reconstruction error: 1.075560\n",
      "Epoch: 402 reconstruction error: 1.075245\n",
      "Epoch: 403 reconstruction error: 1.075241\n",
      "Epoch: 404 reconstruction error: 1.075372\n",
      "Epoch: 405 reconstruction error: 1.075228\n",
      "Epoch: 406 reconstruction error: 1.075359\n",
      "Epoch: 407 reconstruction error: 1.075374\n",
      "Epoch: 408 reconstruction error: 1.075281\n",
      "Epoch: 409 reconstruction error: 1.075446\n",
      "Epoch: 410 reconstruction error: 1.075416\n",
      "Epoch: 411 reconstruction error: 1.075279\n",
      "Epoch: 412 reconstruction error: 1.075541\n",
      "Epoch: 413 reconstruction error: 1.075837\n",
      "Epoch: 414 reconstruction error: 1.075328\n",
      "Epoch: 415 reconstruction error: 1.075719\n",
      "Epoch: 416 reconstruction error: 1.075711\n",
      "Epoch: 417 reconstruction error: 1.075117\n",
      "Epoch: 418 reconstruction error: 1.075712\n",
      "Epoch: 419 reconstruction error: 1.075104\n",
      "Epoch: 420 reconstruction error: 1.075107\n",
      "Epoch: 421 reconstruction error: 1.075377\n",
      "Epoch: 422 reconstruction error: 1.075017\n",
      "Epoch: 423 reconstruction error: 1.075686\n",
      "Epoch: 424 reconstruction error: 1.075263\n",
      "Epoch: 425 reconstruction error: 1.075163\n",
      "Epoch: 426 reconstruction error: 1.075207\n",
      "Epoch: 427 reconstruction error: 1.075305\n",
      "Epoch: 428 reconstruction error: 1.074793\n",
      "Epoch: 429 reconstruction error: 1.075014\n",
      "Epoch: 430 reconstruction error: 1.074733\n",
      "Epoch: 431 reconstruction error: 1.075305\n",
      "Epoch: 432 reconstruction error: 1.075019\n",
      "Epoch: 433 reconstruction error: 1.075122\n",
      "Epoch: 434 reconstruction error: 1.074764\n",
      "Epoch: 435 reconstruction error: 1.075336\n",
      "Epoch: 436 reconstruction error: 1.074463\n",
      "Epoch: 437 reconstruction error: 1.075114\n",
      "Epoch: 438 reconstruction error: 1.075184\n",
      "Epoch: 439 reconstruction error: 1.074801\n",
      "Epoch: 440 reconstruction error: 1.074949\n",
      "Epoch: 441 reconstruction error: 1.075091\n",
      "Epoch: 442 reconstruction error: 1.074427\n",
      "Epoch: 443 reconstruction error: 1.075420\n",
      "Epoch: 444 reconstruction error: 1.074659\n",
      "Epoch: 445 reconstruction error: 1.074872\n",
      "Epoch: 446 reconstruction error: 1.074918\n",
      "Epoch: 447 reconstruction error: 1.074836\n",
      "Epoch: 448 reconstruction error: 1.074821\n",
      "Epoch: 449 reconstruction error: 1.075140\n",
      "Epoch: 450 reconstruction error: 1.074738\n",
      "Epoch: 451 reconstruction error: 1.074587\n",
      "Epoch: 452 reconstruction error: 1.074744\n",
      "Epoch: 453 reconstruction error: 1.075168\n",
      "Epoch: 454 reconstruction error: 1.074335\n",
      "Epoch: 455 reconstruction error: 1.074868\n",
      "Epoch: 456 reconstruction error: 1.074559\n",
      "Epoch: 457 reconstruction error: 1.075066\n",
      "Epoch: 458 reconstruction error: 1.074924\n",
      "Epoch: 459 reconstruction error: 1.075132\n",
      "Epoch: 460 reconstruction error: 1.075194\n",
      "Epoch: 461 reconstruction error: 1.074978\n",
      "Epoch: 462 reconstruction error: 1.074772\n",
      "Epoch: 463 reconstruction error: 1.075003\n",
      "Epoch: 464 reconstruction error: 1.074281\n",
      "Epoch: 465 reconstruction error: 1.074872\n",
      "Epoch: 466 reconstruction error: 1.074463\n",
      "Epoch: 467 reconstruction error: 1.074636\n",
      "Epoch: 468 reconstruction error: 1.074240\n",
      "Epoch: 469 reconstruction error: 1.074590\n",
      "Epoch: 470 reconstruction error: 1.074841\n",
      "Epoch: 471 reconstruction error: 1.074757\n",
      "Epoch: 472 reconstruction error: 1.074396\n",
      "Epoch: 473 reconstruction error: 1.074407\n",
      "Epoch: 474 reconstruction error: 1.074155\n",
      "Epoch: 475 reconstruction error: 1.074535\n",
      "Epoch: 476 reconstruction error: 1.074260\n",
      "Epoch: 477 reconstruction error: 1.074103\n",
      "Epoch: 478 reconstruction error: 1.074119\n",
      "Epoch: 479 reconstruction error: 1.074281\n",
      "Epoch: 480 reconstruction error: 1.073707\n",
      "Epoch: 481 reconstruction error: 1.074870\n",
      "Epoch: 482 reconstruction error: 1.073460\n",
      "Epoch: 483 reconstruction error: 1.074615\n",
      "Epoch: 484 reconstruction error: 1.073869\n",
      "Epoch: 485 reconstruction error: 1.074453\n",
      "Epoch: 486 reconstruction error: 1.073761\n",
      "Epoch: 487 reconstruction error: 1.074396\n",
      "Epoch: 488 reconstruction error: 1.073946\n",
      "Epoch: 489 reconstruction error: 1.074200\n",
      "Epoch: 490 reconstruction error: 1.073879\n",
      "Epoch: 491 reconstruction error: 1.074142\n",
      "Epoch: 492 reconstruction error: 1.073460\n",
      "Epoch: 493 reconstruction error: 1.074098\n",
      "Epoch: 494 reconstruction error: 1.073395\n",
      "Epoch: 495 reconstruction error: 1.074440\n",
      "Epoch: 496 reconstruction error: 1.073810\n",
      "Epoch: 497 reconstruction error: 1.073953\n",
      "Epoch: 498 reconstruction error: 1.073738\n",
      "Epoch: 499 reconstruction error: 1.073684\n",
      "Epoch: 500 reconstruction error: 1.073702\n",
      "Epoch: 501 reconstruction error: 1.074247\n",
      "Epoch: 502 reconstruction error: 1.074026\n",
      "Epoch: 503 reconstruction error: 1.074039\n",
      "Epoch: 504 reconstruction error: 1.073632\n",
      "Epoch: 505 reconstruction error: 1.074018\n",
      "Epoch: 506 reconstruction error: 1.073895\n",
      "Epoch: 507 reconstruction error: 1.074108\n",
      "Epoch: 508 reconstruction error: 1.073604\n",
      "Epoch: 509 reconstruction error: 1.074257\n",
      "Epoch: 510 reconstruction error: 1.073228\n",
      "Epoch: 511 reconstruction error: 1.073441\n",
      "Epoch: 512 reconstruction error: 1.073670\n",
      "Epoch: 513 reconstruction error: 1.073539\n",
      "Epoch: 514 reconstruction error: 1.073707\n",
      "Epoch: 515 reconstruction error: 1.073545\n",
      "Epoch: 516 reconstruction error: 1.073864\n",
      "Epoch: 517 reconstruction error: 1.073611\n",
      "Epoch: 518 reconstruction error: 1.073650\n",
      "Epoch: 519 reconstruction error: 1.073665\n",
      "Epoch: 520 reconstruction error: 1.073514\n",
      "Epoch: 521 reconstruction error: 1.073807\n",
      "Epoch: 522 reconstruction error: 1.073061\n",
      "Epoch: 523 reconstruction error: 1.073846\n",
      "Epoch: 524 reconstruction error: 1.073737\n",
      "Epoch: 525 reconstruction error: 1.073676\n",
      "Epoch: 526 reconstruction error: 1.073784\n",
      "Epoch: 527 reconstruction error: 1.073151\n",
      "Epoch: 528 reconstruction error: 1.073637\n",
      "Epoch: 529 reconstruction error: 1.073499\n",
      "Epoch: 530 reconstruction error: 1.073856\n",
      "Epoch: 531 reconstruction error: 1.073671\n",
      "Epoch: 532 reconstruction error: 1.073253\n",
      "Epoch: 533 reconstruction error: 1.073593\n",
      "Epoch: 534 reconstruction error: 1.073405\n",
      "Epoch: 535 reconstruction error: 1.073668\n",
      "Epoch: 536 reconstruction error: 1.073928\n",
      "Epoch: 537 reconstruction error: 1.073068\n",
      "Epoch: 538 reconstruction error: 1.073647\n",
      "Epoch: 539 reconstruction error: 1.073735\n",
      "Epoch: 540 reconstruction error: 1.073380\n",
      "Epoch: 541 reconstruction error: 1.073071\n",
      "Epoch: 542 reconstruction error: 1.073581\n",
      "Epoch: 543 reconstruction error: 1.073372\n",
      "Epoch: 544 reconstruction error: 1.073503\n",
      "Epoch: 545 reconstruction error: 1.073347\n",
      "Epoch: 546 reconstruction error: 1.073061\n",
      "Epoch: 547 reconstruction error: 1.073701\n",
      "Epoch: 548 reconstruction error: 1.072994\n",
      "Epoch: 549 reconstruction error: 1.073122\n",
      "Epoch: 550 reconstruction error: 1.073611\n",
      "Epoch: 551 reconstruction error: 1.073161\n",
      "Epoch: 552 reconstruction error: 1.073006\n",
      "Epoch: 553 reconstruction error: 1.073053\n",
      "Epoch: 554 reconstruction error: 1.073354\n",
      "Epoch: 555 reconstruction error: 1.072860\n",
      "Epoch: 556 reconstruction error: 1.073171\n",
      "Epoch: 557 reconstruction error: 1.073704\n",
      "Epoch: 558 reconstruction error: 1.072943\n",
      "Epoch: 559 reconstruction error: 1.073267\n",
      "Epoch: 560 reconstruction error: 1.073544\n",
      "Epoch: 561 reconstruction error: 1.072814\n",
      "Epoch: 562 reconstruction error: 1.073470\n",
      "Epoch: 563 reconstruction error: 1.073506\n",
      "Epoch: 564 reconstruction error: 1.073184\n",
      "Epoch: 565 reconstruction error: 1.073117\n",
      "Epoch: 566 reconstruction error: 1.073228\n",
      "Epoch: 567 reconstruction error: 1.072773\n",
      "Epoch: 568 reconstruction error: 1.073354\n",
      "Epoch: 569 reconstruction error: 1.072922\n",
      "Epoch: 570 reconstruction error: 1.072898\n",
      "Epoch: 571 reconstruction error: 1.073331\n",
      "Epoch: 572 reconstruction error: 1.073277\n",
      "Epoch: 573 reconstruction error: 1.072983\n",
      "Epoch: 574 reconstruction error: 1.073136\n",
      "Epoch: 575 reconstruction error: 1.073277\n",
      "Epoch: 576 reconstruction error: 1.073145\n",
      "Epoch: 577 reconstruction error: 1.073040\n",
      "Epoch: 578 reconstruction error: 1.072976\n",
      "Epoch: 579 reconstruction error: 1.072834\n",
      "Epoch: 580 reconstruction error: 1.073046\n",
      "Epoch: 581 reconstruction error: 1.072966\n",
      "Epoch: 582 reconstruction error: 1.072605\n",
      "Epoch: 583 reconstruction error: 1.073025\n",
      "Epoch: 584 reconstruction error: 1.072490\n",
      "Epoch: 585 reconstruction error: 1.072881\n",
      "Epoch: 586 reconstruction error: 1.072590\n",
      "Epoch: 587 reconstruction error: 1.072996\n",
      "Epoch: 588 reconstruction error: 1.072520\n",
      "Epoch: 589 reconstruction error: 1.072888\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 590 reconstruction error: 1.072567\n",
      "Epoch: 591 reconstruction error: 1.072875\n",
      "Epoch: 592 reconstruction error: 1.071919\n",
      "Epoch: 593 reconstruction error: 1.072741\n",
      "Epoch: 594 reconstruction error: 1.072590\n",
      "Epoch: 595 reconstruction error: 1.072708\n",
      "Epoch: 596 reconstruction error: 1.072204\n",
      "Epoch: 597 reconstruction error: 1.072448\n",
      "Epoch: 598 reconstruction error: 1.072508\n",
      "Epoch: 599 reconstruction error: 1.072505\n",
      "Epoch: 600 reconstruction error: 1.072227\n",
      "Epoch: 601 reconstruction error: 1.072860\n",
      "Epoch: 602 reconstruction error: 1.072490\n",
      "Epoch: 603 reconstruction error: 1.072320\n",
      "Epoch: 604 reconstruction error: 1.072657\n",
      "Epoch: 605 reconstruction error: 1.072494\n",
      "Epoch: 606 reconstruction error: 1.072194\n",
      "Epoch: 607 reconstruction error: 1.072358\n",
      "Epoch: 608 reconstruction error: 1.072512\n",
      "Epoch: 609 reconstruction error: 1.072263\n",
      "Epoch: 610 reconstruction error: 1.072106\n",
      "Epoch: 611 reconstruction error: 1.073311\n",
      "Epoch: 612 reconstruction error: 1.072098\n",
      "Epoch: 613 reconstruction error: 1.072899\n",
      "Epoch: 614 reconstruction error: 1.072335\n",
      "Epoch: 615 reconstruction error: 1.072278\n",
      "Epoch: 616 reconstruction error: 1.072047\n",
      "Epoch: 617 reconstruction error: 1.072747\n",
      "Epoch: 618 reconstruction error: 1.072351\n",
      "Epoch: 619 reconstruction error: 1.072227\n",
      "Epoch: 620 reconstruction error: 1.072356\n",
      "Epoch: 621 reconstruction error: 1.072538\n",
      "Epoch: 622 reconstruction error: 1.072266\n",
      "Epoch: 623 reconstruction error: 1.072387\n",
      "Epoch: 624 reconstruction error: 1.072037\n",
      "Epoch: 625 reconstruction error: 1.072274\n",
      "Epoch: 626 reconstruction error: 1.072227\n",
      "Epoch: 627 reconstruction error: 1.072410\n",
      "Epoch: 628 reconstruction error: 1.071780\n",
      "Epoch: 629 reconstruction error: 1.072407\n",
      "Epoch: 630 reconstruction error: 1.072381\n",
      "Epoch: 631 reconstruction error: 1.072706\n",
      "Epoch: 632 reconstruction error: 1.072256\n",
      "Epoch: 633 reconstruction error: 1.072245\n",
      "Epoch: 634 reconstruction error: 1.072528\n",
      "Epoch: 635 reconstruction error: 1.072657\n",
      "Epoch: 636 reconstruction error: 1.072070\n",
      "Epoch: 637 reconstruction error: 1.072245\n",
      "Epoch: 638 reconstruction error: 1.072505\n",
      "Epoch: 639 reconstruction error: 1.072554\n",
      "Epoch: 640 reconstruction error: 1.072160\n",
      "Epoch: 641 reconstruction error: 1.072397\n",
      "Epoch: 642 reconstruction error: 1.072608\n",
      "Epoch: 643 reconstruction error: 1.072305\n",
      "Epoch: 644 reconstruction error: 1.072400\n",
      "Epoch: 645 reconstruction error: 1.072286\n",
      "Epoch: 646 reconstruction error: 1.072463\n",
      "Epoch: 647 reconstruction error: 1.072309\n",
      "Epoch: 648 reconstruction error: 1.072119\n",
      "Epoch: 649 reconstruction error: 1.072183\n",
      "Epoch: 650 reconstruction error: 1.072292\n",
      "Epoch: 651 reconstruction error: 1.072222\n",
      "Epoch: 652 reconstruction error: 1.072078\n",
      "Epoch: 653 reconstruction error: 1.072240\n",
      "Epoch: 654 reconstruction error: 1.072099\n",
      "Epoch: 655 reconstruction error: 1.072067\n",
      "Epoch: 656 reconstruction error: 1.072109\n",
      "Epoch: 657 reconstruction error: 1.072047\n",
      "Epoch: 658 reconstruction error: 1.071856\n",
      "Epoch: 659 reconstruction error: 1.072374\n",
      "Epoch: 660 reconstruction error: 1.072011\n",
      "Epoch: 661 reconstruction error: 1.072122\n",
      "Epoch: 662 reconstruction error: 1.072163\n",
      "Epoch: 663 reconstruction error: 1.072407\n",
      "Epoch: 664 reconstruction error: 1.071859\n",
      "Epoch: 665 reconstruction error: 1.071926\n",
      "Epoch: 666 reconstruction error: 1.071988\n",
      "Epoch: 667 reconstruction error: 1.071823\n",
      "Epoch: 668 reconstruction error: 1.072176\n",
      "Epoch: 669 reconstruction error: 1.071697\n",
      "Epoch: 670 reconstruction error: 1.071993\n",
      "Epoch: 671 reconstruction error: 1.071838\n",
      "Epoch: 672 reconstruction error: 1.071996\n",
      "Epoch: 673 reconstruction error: 1.071617\n",
      "Epoch: 674 reconstruction error: 1.072201\n",
      "Epoch: 675 reconstruction error: 1.071658\n",
      "Epoch: 676 reconstruction error: 1.071787\n",
      "Epoch: 677 reconstruction error: 1.071643\n",
      "Epoch: 678 reconstruction error: 1.072247\n",
      "Epoch: 679 reconstruction error: 1.071795\n",
      "Epoch: 680 reconstruction error: 1.071733\n",
      "Epoch: 681 reconstruction error: 1.072114\n",
      "Epoch: 682 reconstruction error: 1.071803\n",
      "Epoch: 683 reconstruction error: 1.071744\n",
      "Epoch: 684 reconstruction error: 1.071859\n",
      "Epoch: 685 reconstruction error: 1.072060\n",
      "Epoch: 686 reconstruction error: 1.071558\n",
      "Epoch: 687 reconstruction error: 1.071715\n",
      "Epoch: 688 reconstruction error: 1.071946\n",
      "Epoch: 689 reconstruction error: 1.071656\n",
      "Epoch: 690 reconstruction error: 1.071893\n",
      "Epoch: 691 reconstruction error: 1.071884\n",
      "Epoch: 692 reconstruction error: 1.071246\n",
      "Epoch: 693 reconstruction error: 1.071808\n",
      "Epoch: 694 reconstruction error: 1.071468\n",
      "Epoch: 695 reconstruction error: 1.071916\n",
      "Epoch: 696 reconstruction error: 1.071442\n",
      "Epoch: 697 reconstruction error: 1.071628\n",
      "Epoch: 698 reconstruction error: 1.071336\n",
      "Epoch: 699 reconstruction error: 1.071808\n",
      "Epoch: 700 reconstruction error: 1.071774\n",
      "Epoch: 701 reconstruction error: 1.071326\n",
      "Epoch: 702 reconstruction error: 1.071826\n",
      "Epoch: 703 reconstruction error: 1.071604\n",
      "Epoch: 704 reconstruction error: 1.071977\n",
      "Epoch: 705 reconstruction error: 1.071507\n",
      "Epoch: 706 reconstruction error: 1.072145\n",
      "Epoch: 707 reconstruction error: 1.071360\n",
      "Epoch: 708 reconstruction error: 1.071887\n",
      "Epoch: 709 reconstruction error: 1.071301\n",
      "Epoch: 710 reconstruction error: 1.071844\n",
      "Epoch: 711 reconstruction error: 1.071499\n",
      "Epoch: 712 reconstruction error: 1.071540\n",
      "Epoch: 713 reconstruction error: 1.071623\n",
      "Epoch: 714 reconstruction error: 1.071502\n",
      "Epoch: 715 reconstruction error: 1.071663\n",
      "Epoch: 716 reconstruction error: 1.071635\n",
      "Epoch: 717 reconstruction error: 1.071512\n",
      "Epoch: 718 reconstruction error: 1.071651\n",
      "Epoch: 719 reconstruction error: 1.071797\n",
      "Epoch: 720 reconstruction error: 1.071519\n",
      "Epoch: 721 reconstruction error: 1.071282\n",
      "Epoch: 722 reconstruction error: 1.071936\n",
      "Epoch: 723 reconstruction error: 1.071342\n",
      "Epoch: 724 reconstruction error: 1.071504\n",
      "Epoch: 725 reconstruction error: 1.071489\n",
      "Epoch: 726 reconstruction error: 1.071795\n",
      "Epoch: 727 reconstruction error: 1.070981\n",
      "Epoch: 728 reconstruction error: 1.071408\n",
      "Epoch: 729 reconstruction error: 1.071354\n",
      "Epoch: 730 reconstruction error: 1.071296\n",
      "Epoch: 731 reconstruction error: 1.071200\n",
      "Epoch: 732 reconstruction error: 1.070956\n",
      "Epoch: 733 reconstruction error: 1.070819\n",
      "Epoch: 734 reconstruction error: 1.071089\n",
      "Epoch: 735 reconstruction error: 1.071350\n",
      "Epoch: 736 reconstruction error: 1.070729\n",
      "Epoch: 737 reconstruction error: 1.071627\n",
      "Epoch: 738 reconstruction error: 1.070923\n",
      "Epoch: 739 reconstruction error: 1.071012\n",
      "Epoch: 740 reconstruction error: 1.070892\n",
      "Epoch: 741 reconstruction error: 1.071450\n",
      "Epoch: 742 reconstruction error: 1.070851\n",
      "Epoch: 743 reconstruction error: 1.071427\n",
      "Epoch: 744 reconstruction error: 1.070750\n",
      "Epoch: 745 reconstruction error: 1.071017\n",
      "Epoch: 746 reconstruction error: 1.070992\n",
      "Epoch: 747 reconstruction error: 1.071023\n",
      "Epoch: 748 reconstruction error: 1.070704\n",
      "Epoch: 749 reconstruction error: 1.071066\n",
      "Epoch: 750 reconstruction error: 1.070861\n",
      "Epoch: 751 reconstruction error: 1.070807\n",
      "Epoch: 752 reconstruction error: 1.070740\n",
      "Epoch: 753 reconstruction error: 1.070791\n",
      "Epoch: 754 reconstruction error: 1.071098\n",
      "Epoch: 755 reconstruction error: 1.070501\n",
      "Epoch: 756 reconstruction error: 1.070534\n",
      "Epoch: 757 reconstruction error: 1.070748\n",
      "Epoch: 758 reconstruction error: 1.070799\n",
      "Epoch: 759 reconstruction error: 1.070392\n",
      "Epoch: 760 reconstruction error: 1.070531\n",
      "Epoch: 761 reconstruction error: 1.070346\n",
      "Epoch: 762 reconstruction error: 1.070549\n",
      "Epoch: 763 reconstruction error: 1.070465\n",
      "Epoch: 764 reconstruction error: 1.070755\n",
      "Epoch: 765 reconstruction error: 1.070475\n",
      "Epoch: 766 reconstruction error: 1.071010\n",
      "Epoch: 767 reconstruction error: 1.070681\n",
      "Epoch: 768 reconstruction error: 1.070493\n",
      "Epoch: 769 reconstruction error: 1.070711\n",
      "Epoch: 770 reconstruction error: 1.070765\n",
      "Epoch: 771 reconstruction error: 1.070722\n",
      "Epoch: 772 reconstruction error: 1.070948\n",
      "Epoch: 773 reconstruction error: 1.070711\n",
      "Epoch: 774 reconstruction error: 1.070793\n",
      "Epoch: 775 reconstruction error: 1.070711\n",
      "Epoch: 776 reconstruction error: 1.070827\n",
      "Epoch: 777 reconstruction error: 1.070981\n",
      "Epoch: 778 reconstruction error: 1.070320\n",
      "Epoch: 779 reconstruction error: 1.070640\n",
      "Epoch: 780 reconstruction error: 1.070585\n",
      "Epoch: 781 reconstruction error: 1.070981\n",
      "Epoch: 782 reconstruction error: 1.070204\n",
      "Epoch: 783 reconstruction error: 1.070660\n",
      "Epoch: 784 reconstruction error: 1.070436\n",
      "Epoch: 785 reconstruction error: 1.071000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 786 reconstruction error: 1.070451\n",
      "Epoch: 787 reconstruction error: 1.070573\n",
      "Epoch: 788 reconstruction error: 1.070289\n",
      "Epoch: 789 reconstruction error: 1.070712\n",
      "Epoch: 790 reconstruction error: 1.070081\n",
      "Epoch: 791 reconstruction error: 1.070691\n",
      "Epoch: 792 reconstruction error: 1.070153\n",
      "Epoch: 793 reconstruction error: 1.070598\n",
      "Epoch: 794 reconstruction error: 1.070354\n",
      "Epoch: 795 reconstruction error: 1.070444\n",
      "Epoch: 796 reconstruction error: 1.070575\n",
      "Epoch: 797 reconstruction error: 1.070601\n",
      "Epoch: 798 reconstruction error: 1.070485\n",
      "Epoch: 799 reconstruction error: 1.070390\n",
      "Epoch: 800 reconstruction error: 1.070776\n",
      "Epoch: 801 reconstruction error: 1.070904\n",
      "Epoch: 802 reconstruction error: 1.070390\n",
      "Epoch: 803 reconstruction error: 1.070922\n",
      "Epoch: 804 reconstruction error: 1.070719\n",
      "Epoch: 805 reconstruction error: 1.070490\n",
      "Epoch: 806 reconstruction error: 1.070546\n",
      "Epoch: 807 reconstruction error: 1.070500\n",
      "Epoch: 808 reconstruction error: 1.070359\n",
      "Epoch: 809 reconstruction error: 1.070393\n",
      "Epoch: 810 reconstruction error: 1.070114\n",
      "Epoch: 811 reconstruction error: 1.070330\n",
      "Epoch: 812 reconstruction error: 1.070462\n",
      "Epoch: 813 reconstruction error: 1.070356\n",
      "Epoch: 814 reconstruction error: 1.070390\n",
      "Epoch: 815 reconstruction error: 1.070456\n",
      "Epoch: 816 reconstruction error: 1.070246\n",
      "Epoch: 817 reconstruction error: 1.070676\n",
      "Epoch: 818 reconstruction error: 1.069811\n",
      "Epoch: 819 reconstruction error: 1.070753\n",
      "Epoch: 820 reconstruction error: 1.070189\n",
      "Epoch: 821 reconstruction error: 1.070335\n",
      "Epoch: 822 reconstruction error: 1.069896\n",
      "Epoch: 823 reconstruction error: 1.070603\n",
      "Epoch: 824 reconstruction error: 1.069778\n",
      "Epoch: 825 reconstruction error: 1.070385\n",
      "Epoch: 826 reconstruction error: 1.069934\n",
      "Epoch: 827 reconstruction error: 1.070444\n",
      "Epoch: 828 reconstruction error: 1.070114\n",
      "Epoch: 829 reconstruction error: 1.070621\n",
      "Epoch: 830 reconstruction error: 1.070079\n",
      "Epoch: 831 reconstruction error: 1.070619\n",
      "Epoch: 832 reconstruction error: 1.070079\n",
      "Epoch: 833 reconstruction error: 1.070392\n",
      "Epoch: 834 reconstruction error: 1.070179\n",
      "Epoch: 835 reconstruction error: 1.071048\n",
      "Epoch: 836 reconstruction error: 1.070366\n",
      "Epoch: 837 reconstruction error: 1.070567\n",
      "Epoch: 838 reconstruction error: 1.070521\n",
      "Epoch: 839 reconstruction error: 1.070773\n",
      "Epoch: 840 reconstruction error: 1.070114\n",
      "Epoch: 841 reconstruction error: 1.070514\n",
      "Epoch: 842 reconstruction error: 1.070304\n",
      "Epoch: 843 reconstruction error: 1.070500\n",
      "Epoch: 844 reconstruction error: 1.070431\n",
      "Epoch: 845 reconstruction error: 1.070536\n",
      "Epoch: 846 reconstruction error: 1.070307\n",
      "Epoch: 847 reconstruction error: 1.070462\n",
      "Epoch: 848 reconstruction error: 1.070174\n",
      "Epoch: 849 reconstruction error: 1.070508\n",
      "Epoch: 850 reconstruction error: 1.070554\n",
      "Epoch: 851 reconstruction error: 1.070676\n",
      "Epoch: 852 reconstruction error: 1.070186\n",
      "Epoch: 853 reconstruction error: 1.071105\n",
      "Epoch: 854 reconstruction error: 1.070222\n",
      "Epoch: 855 reconstruction error: 1.070415\n",
      "Epoch: 856 reconstruction error: 1.070567\n",
      "Epoch: 857 reconstruction error: 1.070138\n",
      "Epoch: 858 reconstruction error: 1.070773\n",
      "Epoch: 859 reconstruction error: 1.070253\n",
      "Epoch: 860 reconstruction error: 1.070680\n",
      "Epoch: 861 reconstruction error: 1.069847\n",
      "Epoch: 862 reconstruction error: 1.070658\n",
      "Epoch: 863 reconstruction error: 1.070410\n",
      "Epoch: 864 reconstruction error: 1.070369\n",
      "Epoch: 865 reconstruction error: 1.070557\n",
      "Epoch: 866 reconstruction error: 1.070550\n",
      "Epoch: 867 reconstruction error: 1.070420\n",
      "Epoch: 868 reconstruction error: 1.070842\n",
      "Epoch: 869 reconstruction error: 1.070397\n",
      "Epoch: 870 reconstruction error: 1.070660\n",
      "Epoch: 871 reconstruction error: 1.070232\n",
      "Epoch: 872 reconstruction error: 1.070539\n",
      "Epoch: 873 reconstruction error: 1.070176\n",
      "Epoch: 874 reconstruction error: 1.070385\n",
      "Epoch: 875 reconstruction error: 1.070449\n",
      "Epoch: 876 reconstruction error: 1.070125\n",
      "Epoch: 877 reconstruction error: 1.070397\n",
      "Epoch: 878 reconstruction error: 1.070395\n",
      "Epoch: 879 reconstruction error: 1.070356\n",
      "Epoch: 880 reconstruction error: 1.070138\n",
      "Epoch: 881 reconstruction error: 1.070454\n",
      "Epoch: 882 reconstruction error: 1.069957\n",
      "Epoch: 883 reconstruction error: 1.070443\n",
      "Epoch: 884 reconstruction error: 1.070200\n",
      "Epoch: 885 reconstruction error: 1.069991\n",
      "Epoch: 886 reconstruction error: 1.070196\n",
      "Epoch: 887 reconstruction error: 1.070212\n",
      "Epoch: 888 reconstruction error: 1.070138\n",
      "Epoch: 889 reconstruction error: 1.070438\n",
      "Epoch: 890 reconstruction error: 1.070158\n",
      "Epoch: 891 reconstruction error: 1.070438\n",
      "Epoch: 892 reconstruction error: 1.070236\n",
      "Epoch: 893 reconstruction error: 1.070099\n",
      "Epoch: 894 reconstruction error: 1.070186\n",
      "Epoch: 895 reconstruction error: 1.070459\n",
      "Epoch: 896 reconstruction error: 1.070305\n",
      "Epoch: 897 reconstruction error: 1.070235\n",
      "Epoch: 898 reconstruction error: 1.070024\n",
      "Epoch: 899 reconstruction error: 1.070312\n",
      "Epoch: 900 reconstruction error: 1.070153\n",
      "Epoch: 901 reconstruction error: 1.070282\n",
      "Epoch: 902 reconstruction error: 1.070130\n",
      "Epoch: 903 reconstruction error: 1.070192\n",
      "Epoch: 904 reconstruction error: 1.070274\n",
      "Epoch: 905 reconstruction error: 1.069917\n",
      "Epoch: 906 reconstruction error: 1.070014\n",
      "Epoch: 907 reconstruction error: 1.070153\n",
      "Epoch: 908 reconstruction error: 1.070089\n",
      "Epoch: 909 reconstruction error: 1.069940\n",
      "Epoch: 910 reconstruction error: 1.070143\n",
      "Epoch: 911 reconstruction error: 1.070392\n",
      "Epoch: 912 reconstruction error: 1.070009\n",
      "Epoch: 913 reconstruction error: 1.069965\n",
      "Epoch: 914 reconstruction error: 1.070107\n",
      "Epoch: 915 reconstruction error: 1.070189\n",
      "Epoch: 916 reconstruction error: 1.070321\n",
      "Epoch: 917 reconstruction error: 1.070076\n",
      "Epoch: 918 reconstruction error: 1.070297\n",
      "Epoch: 919 reconstruction error: 1.070122\n",
      "Epoch: 920 reconstruction error: 1.070341\n",
      "Epoch: 921 reconstruction error: 1.070204\n",
      "Epoch: 922 reconstruction error: 1.070101\n",
      "Epoch: 923 reconstruction error: 1.070138\n",
      "Epoch: 924 reconstruction error: 1.070163\n",
      "Epoch: 925 reconstruction error: 1.070027\n",
      "Epoch: 926 reconstruction error: 1.070168\n",
      "Epoch: 927 reconstruction error: 1.070166\n",
      "Epoch: 928 reconstruction error: 1.070173\n",
      "Epoch: 929 reconstruction error: 1.070431\n",
      "Epoch: 930 reconstruction error: 1.069819\n",
      "Epoch: 931 reconstruction error: 1.070694\n",
      "Epoch: 932 reconstruction error: 1.069836\n",
      "Epoch: 933 reconstruction error: 1.070356\n",
      "Epoch: 934 reconstruction error: 1.070127\n",
      "Epoch: 935 reconstruction error: 1.070173\n",
      "Epoch: 936 reconstruction error: 1.069878\n",
      "Epoch: 937 reconstruction error: 1.070189\n",
      "Epoch: 938 reconstruction error: 1.070091\n",
      "Epoch: 939 reconstruction error: 1.069968\n",
      "Epoch: 940 reconstruction error: 1.070207\n",
      "Epoch: 941 reconstruction error: 1.069811\n",
      "Epoch: 942 reconstruction error: 1.070210\n",
      "Epoch: 943 reconstruction error: 1.069880\n",
      "Epoch: 944 reconstruction error: 1.070335\n",
      "Epoch: 945 reconstruction error: 1.069671\n",
      "Epoch: 946 reconstruction error: 1.070052\n",
      "Epoch: 947 reconstruction error: 1.070380\n",
      "Epoch: 948 reconstruction error: 1.070151\n",
      "Epoch: 949 reconstruction error: 1.069973\n",
      "Epoch: 950 reconstruction error: 1.070304\n",
      "Epoch: 951 reconstruction error: 1.069882\n",
      "Epoch: 952 reconstruction error: 1.070487\n",
      "Epoch: 953 reconstruction error: 1.069865\n",
      "Epoch: 954 reconstruction error: 1.070258\n",
      "Epoch: 955 reconstruction error: 1.069687\n",
      "Epoch: 956 reconstruction error: 1.070290\n",
      "Epoch: 957 reconstruction error: 1.069720\n",
      "Epoch: 958 reconstruction error: 1.070027\n",
      "Epoch: 959 reconstruction error: 1.069916\n",
      "Epoch: 960 reconstruction error: 1.069849\n",
      "Epoch: 961 reconstruction error: 1.070032\n",
      "Epoch: 962 reconstruction error: 1.070061\n",
      "Epoch: 963 reconstruction error: 1.069792\n",
      "Epoch: 964 reconstruction error: 1.070210\n",
      "Epoch: 965 reconstruction error: 1.069994\n",
      "Epoch: 966 reconstruction error: 1.070017\n",
      "Epoch: 967 reconstruction error: 1.069998\n",
      "Epoch: 968 reconstruction error: 1.069731\n",
      "Epoch: 969 reconstruction error: 1.070238\n",
      "Epoch: 970 reconstruction error: 1.069875\n",
      "Epoch: 971 reconstruction error: 1.069942\n",
      "Epoch: 972 reconstruction error: 1.069909\n",
      "Epoch: 973 reconstruction error: 1.070171\n",
      "Epoch: 974 reconstruction error: 1.069983\n",
      "Epoch: 975 reconstruction error: 1.070158\n",
      "Epoch: 976 reconstruction error: 1.069878\n",
      "Epoch: 977 reconstruction error: 1.070220\n",
      "Epoch: 978 reconstruction error: 1.069919\n",
      "Epoch: 979 reconstruction error: 1.070456\n",
      "Epoch: 980 reconstruction error: 1.070104\n",
      "Epoch: 981 reconstruction error: 1.069999\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 982 reconstruction error: 1.069870\n",
      "Epoch: 983 reconstruction error: 1.070230\n",
      "Epoch: 984 reconstruction error: 1.069927\n",
      "Epoch: 985 reconstruction error: 1.069474\n",
      "Epoch: 986 reconstruction error: 1.069867\n",
      "Epoch: 987 reconstruction error: 1.069734\n",
      "Epoch: 988 reconstruction error: 1.069898\n",
      "Epoch: 989 reconstruction error: 1.069607\n",
      "Epoch: 990 reconstruction error: 1.070014\n",
      "Epoch: 991 reconstruction error: 1.069523\n",
      "Epoch: 992 reconstruction error: 1.069991\n",
      "Epoch: 993 reconstruction error: 1.069834\n",
      "Epoch: 994 reconstruction error: 1.069958\n",
      "Epoch: 995 reconstruction error: 1.069993\n",
      "Epoch: 996 reconstruction error: 1.069689\n",
      "Epoch: 997 reconstruction error: 1.069693\n",
      "Epoch: 998 reconstruction error: 1.069870\n",
      "Epoch: 999 reconstruction error: 1.070186\n"
     ]
    }
   ],
   "source": [
    "# Train RBM model\n",
    "err = rbm.train(inputX)\n",
    "outputX, reconstructedX, hiddenX = rbm.rbm_output(inputX)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot reconstruction errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd5xU5dn/8c+1jd2l916VJtgQsQuKotggpukvGqNPHqPRWJKYaDSaRE3I88SWxNixxdhRecQYrIgFkSogvfelb+/X748ZhlmYnR3YmZ0t3/frNS/m3OfMOdecxLn2Pnczd0dERGR/KckOQERE6iclCBERiUgJQkREIlKCEBGRiJQgREQkorRkBxBPHTp08D59+iQ7DBGRBmP27Nnb3b1jpH2NKkH06dOHWbNmJTsMEZEGw8zWVrdPj5hERCQiJQgREYlICUJERCJSghARkYiUIEREJCIlCBERiUgJQkREImpUCaK4rDLZIYiINBqNKkGUlFckOwQRkUajUSUIERGJHyUIERGJSAlCREQiUoIQEZGIEpYgzGyimeWY2cJq9g8ysy/MrMTMfnkwnxURkcRLZA3iGeDcKPt3AjcAfzmEz4qISIIlLEG4+ycEkkB1+3Pc/Sug7GA/KyIiidfg2yDM7Gozm2Vms3Jzc5MdjohIo9HgE4S7P+7uw919eMtWrZIdjohIo9HgE0QVnuwAREQaj8aVIEREJG7SEnViM3sRGAV0MLMNwF1AOoC7P2pmXYBZQCug0sxuAo5w99xIn3X3pxIVq4iIHChhCcLdL61h/xagx6F8VkREEk+PmEREJCIlCBERiaiRJQh1YxIRiZdGliBERCRelCBERCSiRpUg9IBJRCR+GlWCEBGR+FGCEBGRiJQgREQkIiUIERGJSAlCREQiUoIQEZGIlCBERCSixpUgNBBCRCRuGleCEBGRuFGCEBGRiJQgREQkokaVINQEISISP40qQYiISPwoQYiISERKECIiEpEShIiIRKQEISIiESlBiIhIREoQIiISkRKEiIhE1KgShAbKiYjET6NKECIiEj9KECIiEpEShIiIRKQEISIiESlBiIhIRI0rQagbk4hI3DSuBKEMISISN40sQYiISLxETRBmlmpm79dVMLWl+oOISPxETRDuXgEUmlnrOopHRETqibQYjikGFpjZe0DB3kJ3vyFhUYmISNLF0gYxBfgt8AkwO+wVlZlNNLMcM1tYzf5BZvaFmZWY2S/323eumS01sxVmdmsMMYqISJzVWINw92fNLAMYECxa6u5lMZz7GeDvwHPV7N8J3ACMDy80s1TgYeBsYAPwlZlNdvdvYrimiIjESY01CDMbBSwn8KP9D2CZmZ1e0+fc/RMCSaC6/Tnu/hWwf7IZAaxw91XuXgq8BIyr6Xq1kVtcxodLtibyEiIiDU4sj5juA8a4+0h3Px04B3gggTF1B9aHbW8IlkVkZleb2Swzm1WQX1DdYVVs2l2E+74+Tz9/eR5XPTOLTbuLDjFkEZHGJ5YEke7uS/duuPsyID1xIWERyqrtweruj7v7cHcf3rxF86gnLi2v5O63v+HkCR/y6LRVofLV2wOJpbC04tAiFhFphGJJELPM7CkzGxV8PUEMjdS1sAHoGbbdA9gUjxPf+vrXPPXpagA+XppDZaXj7pgFclJ4rUJEpKmLJUFcCywi0KB8I/ANcE0CY/oK6G9mfYON45cAk2t70opKZ9LcjaHtVdsL6Pebd/jnjLWhKovSg4jIPlF7MQV7FD3l7pcB9x/Mic3sRWAU0MHMNgB3EXw05e6PmlkXYBbQCqg0s5uAI9w918yuB/4DpAIT3X3RwX2tA739ddVKyLa8EgBenrWelGANolI1CBGRkKgJwt0rzKyjmWUEexTFzN0vrWH/FgKPjyLtewd452CuB9FrACtz8iOWl1c4wfxAeYUShIjIXrGMpF4DfGZmk6k6kvqgahTJVl0DdEWlk5oSyBBLtuQxtLtmFRERgdjaIDYBbwePbRn2qn+iVAByiyOP7auo9NAjpl++Oj8RUYmINEixtEG0cPdb6iiehMktKo9YXuH7HjGJiMg+sczmOqyOYkmovJLINYjwNggREdknljaIecH2h1ep2gYxKWFRJUBecTmZ6SkUl1VWKS+vrESdl0REDhRLgmgH7ADODCtzoMEkiNXbC/h6wx7aZKcfkCC25pawNbcktF1QUs7stbs4fUDHug5TRKReiWU21yvrIpBE+sETM4DAVBs1GXLXfwCYdssoerePPnWHiEhjVm0bhJm9Evb+z/vtm5rIoOLpvW+2smlPMRB5kqfqFJRoXiYRadqiNVL3D3t/9n776v3zl2c/X8PbX2/iZy/OCZX1aJsNwJgjOicrLBGRBiPaI6ZoTbf1slk3PKi7Jh84O0e3Npm8/JMTWZ6Tz9Rvoq//4PXzK4qI1JloNYhsMzvWzI4DsoLvh+3drqP44q5NdgZH92jDaf07RK1JVAabK7bmFnPf1KVUViphiEjTEq0GsZl9E/RtoepkfVsSFlEC7f2Jz0hL4fn/OgGAVdvyOfO+aQcc+8GSrdz+5gLSU1OYvXYXZw7qRMvMdP70zmIe/sEwMtNT6zByEZG6V22CcPcz6jKQuhBpvEPHls0iHvvg+8sBaNkscIsq3blr8kI+W7GDr9bs5LT+9b4ZRkSkVmKZi6nRiPSQqGVmOo9ffhwn9G0X8TN5JYEpOkrLPfTYae663QmKUESk/mhaCaKaIdNjhnRh9OBOUT976RMz+GLVDgDuf28ZS7bkMnvtrrjHKCJSX8QykrpJSE89uFx57oPTAVgz4fxEhCMiknQxJQgz6w70Dj/e3T9JVFCJEm3OpSw1OouIVFFjggiOov4+gbWo9w4vdqDBJYjLT+pd7b6WmekHlHVtncnm4CjsWGzNLaZNdjrN0pRsRKThi+W5ynhgoLuf5+4XBl8XJTqwQxKlhnB0zzacM6RLtftbZFbNlR1aZDDyICbsq6x0TvjjB9z88ryYPyMiUp/FkiBWAQf+eV0POU55ReQJ+VJrmIip5X4J4q4LhzBmSPQpOVLCzllcHqhcTV0UfYS2iEhDEUsbRCGBNSE+AELzYrv7DQmL6hDl5JVw8oQP+fI3ow/Yl1LDqkAtguMderTN4tNfB2Y2Ly6LPmFf+KOkvWteZ6Q1qY5hItKIxZIgJgdfDUJOXgkVEabFSEmJniAygr2YurfZN4tITaOlK8NavYuCCeJge0OJiNRXsawH8ayZZQADgkVL3T3y+p31RHmkBFHDI6Y+HZpz9/ihjB1atZ3izEGd+HBJTsTPlFZUUl5Ryda8Eu6Z8g2gGoSINB6x9GIaBTwLrCGwpEJPM7uiPndzDU8QqSlGRaWTWlOGAC4/8cBeThN/dDx9bp0S8Xh3OOYP79E6K52Nu4uAfTUREZGGLpZfs/uAMe4+0t1PB84BHkhsWLVTUbEvQYwK9kRqm51xyOeb+KPhVbYf/P4xdGmVCUB+SXkoOYBqECLSeMTSBpHu7kv3brj7MjOr172ayir39WQ66bD2nHRYe747vOchn69zMBl0atmMf994Gu1bNGNHQSl3v/3NAceqBiEijUUsCWKWmT0FPB/c/gEwO3Eh1V55WA0iPTWFK07uU6vzZWcEblNWRirtWwRmf63uiVVRWQUbdxdVaewWEWmIYvlz91pgEXADcCOBEdXXJDKo2ioPq0HE0vZQk/TgIIqy8n3n3ZpbEvHYdTsLOWXCh7W+pohIstWYINy9xN3vd/eL3f1b7v6Au0f+dawnvlqzM/Q+LQ4Jok2w/WJM2Ejs/JLoHbm+99gX5BbX685eIiJRVfuIycxecffvmdkCIkxi4e5HJTSyWrj55fmh9/GoQbRolsZXt59F2+x9TS83nzWAds2bcVK/9lz6xIwDPjNz9U6+88jnTL15ZK2vLyKSDNHaIG4M/ntBXQSSKGk1zbERo/1Xnmvfohk/P3tAxEF5ey3bmh+Xa4uIJEO1j5jcfXPw7U/dfW34C/hp3YRXexmpiZ1ZNTXFWDPhfObfNSah1xERqWuxNFKfHaFsbLwDSZTsjLqZert1VuSevyXlFdWuZCciUp9VmyDM7Npg+8MgM/s67LUaWFB3IdZOVh0liOoMvONdfvnq10mNQUTkUESrQfwLuBB4K/jv3tdx7v6DOogtLuqqBhHN63M2JDsEEZGDFq0NYo+7rwEeAnaGtT+UmdkJdRVgbdWHBCEi0hDF0gbxCBDeHacgWNYgZGXEtOy2iIjsJ5YEYR7WyurulcQ2C+xEM8sxs4XV7Dcz+6uZrQi2bQwL23ejmS00s0VmdlMsX6Q62TWs6ZAI/77xtDq/pohIvMW05KiZ3WBm6cHXjQSWIa3JM8C5UfaPBfoHX1cTrJWY2VDgv4ERwNHABWbWP4brRVTToj+JcFjHFgeUqSeTiDQ0sSSIa4CTgY3ABuAEAj/oUQXXi9gZ5ZBxwHMeMANoY2ZdgcHADHcvdPdyYBrwrRjijKguezH97dJj+dW5A0NzN4X7wZNf1lkcIiLxEMuKcjnAJQm4dndgfdj2hmDZQuBeM2sPFAHnAbMO5QLjj+lW2xgPyoVH77veuUO68O6iLaHtz1fuwN2xGtbGFhGpL2JpS3iayHMxXVXLa0f6pXR3X2xmfwbeI9A4Ph8ojxLf1QRrNBldDq+yryKJT3UeuWwYG3cX8fbXm5nw7yVAYHGhlpn1eikNEZGQWB4xvQ1MCb4+AFpRtVfTodoAhK/i0wPYBODuT7n7sOAKdjuB5dWdxN0fd/fh7j58/30VYdN+1zUzo0fbbIpKK0Jlq7cXJC0eEZGDFct036+HvV4AvgcMjcO1JwM/DPZmOhHYs3f+JzPrFPy3F3Ax8OKhXCDaRHp1pbh8X4K46O+fMX/97iRGIyISu0MZJNAf6FXTQWb2IjAK6GBmG4C7gHQAd38UeIdA+8IKoBC4MuzjrwfbIMqA69x91yHESc+22YfysbgqDqtBAIx7+DPWTDg/SdGIiMQuljaIPKq2QWwBfl3T59z90hr2O3BdNftqPZDgrMGduOXcgbU9Ta2d1r8jz36xtkrZ5U99yd8uPTa0EJGISH0U9RGTBbrcDHH3VmGvAe7+eh3Fd8iuPKUvzdKSP83GWUd0ZsndVYeDTF++nTfmbkxSRCIisYmaIIJ/5b9RR7HEVUZaLO3vdSMzPZUfntS7Stnv/+8brv3n7CRFJCJSs1h+RWeY2fEJjyTOMlLrT4IAuGbkYWTtN6r73wu3VHO0iEjyxfIregbwhZmtDM6ZtMDM6v0CB/WpBgHQrU0WX94+OtlhiIjELJZeTA1m9bhw9S1BALSKMEhu8vxNXHBkV1JSNMJaROqXWH5F74mwJvU9iQ6sturbI6a9rh11WJXtG16cy8uz1ldztIhI8sTyKzokfMPMUoHjEhNO/DSrhzUIgOvOOPyAsk27i5IQiYhIdNHWpL4tOAbiKDPLDb7ygBwCy5DWa/XxERNAZoS4/vbhCm59vd4364hIExNtydE/uXtL4H/DxkC0dPf27n5bHcZ4SOprgkir5tHXS1+t5/OV2xly57us2haPqa5ERGonpsn6zKw5gJldZmb3m1nvmj6UbPW1DSKaB99fTkFpBa/P2ZDsUEREYl6TutDMjgZ+BawFnktoVHFQ3V/q9cGnvz4jYnlxWWDepnowx6CISEwJojw4onoc8JC7PwS0TGxYjVuPttn0aJsV2p52yygAVm8LTAdeqeVJRaQeiCVB5JnZbcBlwJRgLyatelNL48JWu8vOCAxHySsJrItUqSqEiNQDsSSI7wMlwH+5+xYCy4L+b0KjagIuOT4wY/qgLi1p3qzqFBxlFc7zM9ZSWp68BY9ERGJZk3oLcH/Y9joaQBtEfdezXXZoXQjf75HSP2espbzSKa+o5MpT+jJ//W7at8igRz1Y30JEmo4aaxBmdrGZLTezPXvHQphZbl0E11QEZlXfpzz4iKk8uKj2uIc/49Q/f1TncYlI0xbLI6b/AS5y99ZhYyFaJTqwpqZN9oHNOikpxo0vzU1CNCIisSWIre6+OOGRNHFzf3s2l51YdSXXl2au4615m5IUkYg0dbHM5jrLzF4G3iTQWA2Au09KWFS1kJGawtDuDa+CY2bcM/5IOrbI5IH3lwFQokZqEUmiWBJEK6AQGBNW5kC9TBDL7h17QKNvQzJ6cKdQgiirUIIQkeSJpRfTlXURSDzt3+jbkAzt3pold5/LoN++y+Y9xVX2lVVUkl6PR4iLSOMSSy+mHmb2hpnlmNlWM3vdzHrURXBNVeZ+S5Pu9ejHK+s4EhFpymL5c/RpYDLQjcAguf8LlkkCpUZYYe6RaUoQIlJ3YkkQHd39aXcvD76eATomOK4mb8HvxhxQVlhaQXFZBf/6cl1oYj8RkUSJJUFsD07znRp8XQbsSHRgTV12Rhrf/OGcA2Z+vfLpr/jNGwt44L1lSYpMRJqKWBLEVcD3gC3AZuA7wTJJsOyMtAOm1/hiVSA3P/bJKgqCk/uJiCRCjQnC3de5+0Xu3tHdO7n7eHdfWxfBSXT/+nKdJvQTkYSJpRfTs2bWJmy7rZlNTGxYEu6lq08E4IcnVV3Ib/GWXAbc8W/eXbgZ98AMsKpViEi8WE2DysxsrrsfW1NZfdCsa38v2bw82WEkVJ9bp4TeZ2ekUlgaaKy+Z/xQ7nhzIf/vhF788VtHJis8EWlgzGy2uw+PtC+WNogUM2sbdrJ2xDYCWxJsb3IAuOPNhQDsyC+p7nARkYMSS4K4D/jczO42sz8AnxOY4VWS6NIRvSKWb80toc+tU3h34eY6jkhEGptYGqmfA74NbAW2ARe7+/OJDkwi+92FR9CzXRbnH9k14v6lW/IAeOHLdXUZlog0QrFO7NMOKHD3vwHbzKxvAmOSKH50Sl+m/+pMTu3fgQ9+MfKA/WmpgRHYexcbEhE5VLH0YroL+DVwW7AoHfhnIoOS2BzWsQXTbhlVpSyvONCL6YtVO3juizVUVCpRiMihiaUG8S3gIqAAwN03AS0TGZTErnf75tXuu/OtRdzwolakE5FDE0uCKPVAX1gHMLPqf5EkKb7+3RhGDYw8PdaUBZvJyd03bfhrszdw88vzKCmv4IYX54baLERE9hdLgnjFzB4D2pjZfwPvA08mNiw5GK0y03ns8uOq3f+LV+cz/uHPWL41j1++Op835m5k2tJtTJ6/iZtenleHkYpIQxJLL6a/AK8BrwMDgTvd/a+JDkwOTrO0VG4bOyjivunLtzNv/W7OfuCTUNnG3UUALN6cy6j//Yj7py6lqFQzxIrIPjWOpD7gA2apwCXu/kINx00ELgBy3H1ohP0GPAScR2BJ0x+5+5zgvpuBHxN4rLUAuNLdi/c/x/6awkjqmlRWOv9ZtIVrX5hz0J/9/UVDuOLkPvEPSkTqrUMaSW1mrczsNjP7u5mNsYDrgVUEZnetyTPAuVH2jwX6B19XA48Er9sduAEYHkwsqcAlMVxPgJQU44xBnRh/TDcgsPDQ8nvHklXNKnXhdhWWJjo8EWlAoj1iep7AI6UFBP6anwp8Fxjn7uNqOrG7fwLsjHLIOOA5D5hBoI1j7+ivNCDLzNKAbGBTjd9EQjLTU3nwkmOZeftoZt1+FumpKSy+e1+uvmf8vgpdi2b7Zk35z6KtmuxPREKizanUz92PBDCzJ4HtQC93j1e3l+7A+rDtDUB3d59lZn8B1gFFwFR3nxqnazYpnVpmVtn+v+tPJT3NGNSlFRmpKVS48+1hPViRk895f53O4s25/PathfzlO0dz+5sL6dgig5+PGZik6EUk2aIliLK9b9y9wsxWxzE5ABy46DJ4cGLAcUBfYDfwqpld5u4RB+eZ2dUEHlGR0eXwOIbX+BzZo3Xo/feO7xl6f0S3VqH3k+ZsZNKcjaHti4f1oE+HQM/m8opKFm/Oq3IeEWm8oj1iOtrMcoOvPOCove/NLDcO194A9Azb7kHgUdJZwGp33+buZcAk4OTqTuLuj7v78OoaWaR2fvXa11QGR2M/9MFyLvz7p8xbv1trYos0AdUmCHdPdfdWwVdLd08Le9+qus8dhMnAD4ON3ycCe9x9M4FHSyeaWXawp9NoYHEcridRvHPDafzmvAO7yc5cs5NLnpjBym35zN+wB4DxD3/GEXe+W9chikgdO+hurjGf2OxFYBTQgcBMsHcRmMcJd380+OP/dwI9nQoJdGWdFfzs74HvA+XAXODH7l7jQgfq5hof4YsShUtLMcrD5nb65JYz6NU+O+KxItIwROvmmrAEkQxKEPGxYVchFZXOyP/9uMZj/3PT6Qzsoqm5RBqq2q4oJ01Mj7bZ9G7fnGeuPL7GYzfvKaqDiEQkGZQgpFon9mtf4zH/WbSVPrdOYd2OwjqISETqkhKEVCszPZU1E85n8R+qDoi//3tHh96/ODOwct1LX63j3infUFpeWacxikjiKEFIjbIyUrnlnMCAud9fNITxx3TntP4dqhzzj49X8sT01Uxbtg2AHfkl3Dd1KbnFZQecT0QaBjVSS0wqK50tucV0a5MFgLsz+v5prNpWcMCxx/Vuy+y1u0Lbq/90HoFOa7B6ewHrdhaycVcR/++EXnUTvIhUS72YJCHcndvfXMjqbQXMWbeLkmoeL909fiiFJeV8vXEPU77eHCpfce9Y0lJTKCwtJzsj2qB+EUmUJpMghh49zBfOP/hpriU+pi3bxhUTZ8Z8fO/22Qzt1popCzbzf9efqik8RJKgyXRzzUxvVF+nwRk5oCNDuwcG2f/zv07g52cPiHr82h2FTFkQqFFc+PdPNX2HSD2jX1SJq9euOZmFvz+HU/t34IbR/UPl937rgDWjDjDot++ys0BrUojUF0oQEleZ6alV1pjYW6P4fyNia5Aedvd7fLFyBze+NJcZq3YcsD+3uIwfPT2T9Ts17kIk0RpVG8Tw4cN91qxZyQ5DwhSUlFNYWkHHls3YU1jG0X/Yt7THT0b247Fpq6J+/tmrRnDq4R1ITQn0gnpp5jpunbQAgDUTzk9c4CJNRJNpg5D6p3mzNDq2bAZA6+x0Jv4o8P/Dp64Yzm1jB7Nmwvm88pOTqv38FRNn8tgnK8kvKee+qUtDyQHg+n/NCbVblJRXcMebC1i2NY/G9EePSDKpBiH1Ql5xGS/NXE/LzDQ27i7ibx+uOORz/fnbRzLmiC60yU4Pjb8QkchUg5B6r2VmOv99ej8uGdGLH57UJ+IxZw3uzOw7zuKqU/pGPddfP1jBsXe/xzOfrzlg31vzNvLqrPWUVVSqpiFSA9UgpF5ydz5cksNNL80jr6ScCRcfySVhDd1LtuRy7oPTYzrXP34wjPOO7Iq70/e2d0LlZw3uxMzVO/n8ttFVGtZFmpJoNQj9VyH1kpkxenBnCoNtDEO6VR1EN6hLK6befDofL82hTVYGbZtnsGjTHmav3cX05durHPvTF+bw3eN68OrsDVXK31+cA8Cny7dz+oAOlFc663YUMrR7a95ZsJn84vIqa3eLNDVKEFKvVQRXsOvaJvOAfQM6t2RA532LFZ19RGcAnv18DXuKyrj/vWWhffsnh3DX/HN2le2Lh3Vn0pyNAJzQrx3b80tYvDmPi4d15x8freSiY7pVua5IY6VHTFKvfesfnzF33W5W/fE8UlIOrsF54cY93Dd1KQUlFaSlGpeO6EVGWgofL90Wmqb8UD131QhOH9CxSllxWQUZqSkHHadIMjWZuZiUIBqfPUVlbN5TxKAureJ2zspKZ8W2fP69YAsDOrfg2hcObf6uswZ3ZlteMS//5CQy01Ppc+sULjm+JxO+fVTcYhVJNPVikgardVZ6XJMDQEqKMaBzS248qz9jj+zK3eMD04C0yU6v8bN3nD849P79xVuZv2EPt01awJItuQC89NV6VuTkhY4pKdf8UtJwKUFIk3f5ib1ZM+F8pt1yBgDnHdkFCCSMVX88j+8e1yN0bKQ1LN6Yu7FKj6qz7v+EV2etZ/rybQy8413mr9/Npt2BtbsLSsrZmlucyK8jEjdqpBYJap2VzvRfnUGX1plcOmIHR/VoQ0qK8edvH8WOglKuOLkP2RlpHN+nLV+t2RX1XLe89nXo/b3vLGbm6p3cfNYAHng/0HCuaUKkIVAbhMhBqqh0KiqdZVvz6NiyGVdMnMmSLXk1fzDMqj+ex/uLtzJqYCcWb86lZ7ts3J0Nu4ooKa/k85Xbuems6NOli8SDGqlFEuynL8zmnQVb4nrOi4/tzv985yjenLeJ7IxURg7oyNodhXy8LIefjjo8rteSpksJQiTBNuwq5NQ/fwQEGrJXbsvnxZnrARjUpSW7C8vYUsu2h34dm7MxWMMAuPKUPtx5wRGab0pqRSOpRRKsR9tsrjqlL7PW7uTHp/Vj4cY9oQTx7k2nA4EuuxM/Xc1DHwTWTb/ujMN4+KOVMV9j1baCKttPf7aGpz9bA8AxPdswamBHfnZm/9DU6NEs2LCH+99bymOXDycjTX1VJDIlCJE4ufPCI0Lvh3ZvzYLfjaGgZF8319ZZ6Zx3ZFce+mA5L199Iif0a09mWir3hY34PmNgRz5auu2grz1v/W7mrd/N7LW7GNGnHScf3oGM1JRq1/n+5avzWbo1j5Xb8hncNb7diKXx0CMmkTrm7qHHQtvzS/jZv+Zy+/mDKSmv5LjebcktLuMXr8ynTVY6R/VozW/fWlSr67153Skc07MNr8/eQGqKMf7Y7pzzwCcs3ZrHiL7tePSy42jXPCMeX00aILVBiDRgFZVOYWk5lZXwmzcW0L9zCx58fzl/GDeEO99aRPvmGTx71Qg6t8rk+Hvfj3iO0YM68cGSwOSEv79oCM/PWMuKnPzQ/snXn8KQbq1JTTF+8cp8Zq3dybBebfnJyH7MX7+b0/p3pFubrDr5vlK3lCBEGqnt+SW0b54RqpHsLChl2N3vHdK5erXL5vyjuvLIx5HbRX54Um9+d+EQUlKM0vJKPl6aw/b8Uk7o147DOrY45O8gyaUEIdKE5OQV88XKHXRqmcmlT8yI+/kz01MYPagzUxZsBqBP+2yuHXUYY4/sypPTV7NhVyEn9mvPuGO60SwtNeI5dhWUkpNXwsAumhU32ZQgRJqoPiFi3WgAAAyaSURBVLdOAeDUwzvQsWUzTuzXjmc/X8s3m3OrHNe/UwuWhz1yGju0C/9eGBjXkZZiXDysO6/Mqn7K9OoM7tqKW84ZwPF92rFpdzEDu7SkpLyCgXe8C8CKe8eSlhroReXufLFyByf2a68ZceuQEoRIE9Xn1im0zkpn/l1jqpRv2VPMvPW7SDEjPS2FUQM68rcPV4TW0Fgz4XymfL2Z4/u2pWOLZpgZG3cXsauglPcXb+XB95cfUjwDOrdg2db8KmX/uel02mSnM/ah6ewsKKV7myzu+dZQzhjY6dC+tBwUJQiRJqqgpBwzyM6IrUf7k9NXkVdczs1nR5/mY8mWXHYVlNG+RQb/+nJdxPW/a2vNhPPZllfCmAem8exVIziye2sNCkwAJQgRSagte4pZvb2AV2at5/BOLbhm5GFs2l3EtS/MZvHmvNDKgAfjge8fzaQ5G6ssIfvKT05iRN928Qy9yVOCEJGkeW32Bn756nx6tsti/c7AtOcv/PgEfvDklwDMu/NsmqWl8rMX5/L+4q1Rz3XukC5cf+bhdG2dycJNuYzcb1W/RHB3tuWV0KlV1WVvS8sreWfBZsYd061B12yUIEQkadydmat3MqJvuyo/pO5OpROaGqSsopLCkgpG3z+N7fklMZ170e/PoXmzNF6auY5bJy3gtWtOYtLcjbjDypx8dhWWsjwnn0cvO45zh3YJfW7Gqh384+OV3Dt+KD3bZUe9xr++XMdv3ljAv288rcqo8wffX8aD7y/nscuP45wh+869bkchvdpHP2d9ogQhIg3Gipw8Zq/dxeOfrGLlfvNPRfL0j47nyme+qvG4MUd05u7xQ0lNMYbfExhQ2LV1Jl/cNhqAwtJyfvzsLEb0bcc1Iw+jWVoKa3cUctfkRUxbFpj+JDwZDL/nPbbnl9K7fTbrdxYy4eKjyMoI1IR+c94grj79sEO9BVVUVjrPfbGG7w7vSfNm8Z8dKSkJwswmAhcAOe4+NMJ+Ax4CzgMKgR+5+xwzGwi8HHZoP+BOd3+wpmsqQYg0HnuKynjlq/Vs3F3E7ecP5snpq1m/q5CZq3dSXFbBhl1FcbnObWMH0bdDc65+fnaV8mgLQzVLSwnNqluda0YexgVHdSW/pJyNu4r41rHdmfrNVnq0zWLltnzGDu1aZaLE4rIK0lIs1O13r+dnrOW3by7k8hN7h5bHhX01sImfruY7x/WgbZTpUj5aksPxfdvRIkKCSVaCOB3IB56rJkGcB/yMQII4AXjI3U/Y75hUYCNwgruvremaShAiTUNBSTnX/HN2lQZsgF+cPYCfje7Pdx/9vMZV/+pay8w08orLq5S1b57BRcd04+azB3DU76YyvHdbVm7LZ1dhGb84ewBXj+wXGjMC8NmtZ7KnsIz731t2QHvNU1cMZ/TgzizfmkenVpks2LCH4/u2ZeHGXL79yOcM6tKSRy87jj4dmoc+s7uwlLbNmyXnEZOZ9QHeriZBPAZ87O4vBreXAqPcfXPYMWOAu9z9lFiupwQh0rQs35rH8zPWcsPo/nRo0SxUviInn398tIJJczfy/eE9eXnW+tC+D34xkg27imiekUq/ji0OemqSUw/vwKcrtkfc99jlx7F4c+4hjxOpratO6cvEz1ZHPWbpPecyfdl2erTL4qMl2/jpGYfXywTxNjDB3T8Nbn8A/NrdZ4UdMxGY4+5/j3KNq4GrAXr16nXc2rU1VjREpInYsKuQbq2z+GZzLj99YQ4PXXIMx/ZqW+WYdxduobC0nJMOa89Jf/ow4nkmX38Ke4rK2LKnmO8O78m89btplZlGUVkFK3LyufGlecC+tcY37i5i+rJtlFVUVpmN97ITe1FaXkm75s14dNpKOrVsRk7egQ3yqSlG7/bZB6wBkghr/3xBvVwwKFK/sFC2MrMM4CLgtmgncffHgcchUIOIZ4Ai0rD1aBvoTTS0e2s++dUZEY/Z27uppDywdsfgrq1YHJyKZPWfzovYhfWYnm1C74d0a82m3cXMWrMzVNa9TRaXjOgFwDlDurB6ewF9OjSnc1hX2V+OGUBaagpvzA1MYZKVnkaLZmmcdFj7UM+u8opKHvl4Jfe9t4xBXVoysEtL3pq3KXSO/z6tL09Mr77GcM6QzqzbWUS/js2ZumgLZRUH9xNZbx8xmdk44Dp3H7P/Z6ujR0wiEg9b9hTTvkUG6an1b7W9ykpn4aY9TJqzkbsuPIJKh9smfc3hnVrw41P7saeojI27i0hNsSrdclfk5HPW/dNC2wM7t+SoHq35y/eOqZc1iMnA9Wb2EoFG6j3h7Q/ApcCLSYlMRJq0Lq0zaz4oSVJSjKN6tOGoHoFaTKrB/3zn6ND+ts0zIvZo6tehOTeceTgXD+vB2p2FHNOzDa2z0vlLlGslLEGY2YvAKKCDmW0A7gLSAdz9UeAdAj2YVhDo5npl2GezgbOBnyQqPhGRpiQlxfj5mIEAVXoyRZOwBOHul9aw34HrqtlXCLRPRFwiIhKb+veATURE6gUlCBERiUgJQkREIlKCEBGRiJQgREQkIiUIERGJSAlCREQialQLBplZHrA02XHUEx2AyFNONj26F/voXuyjexHQ290jrt2azKk2EmFpdXOKNDVmNkv3IkD3Yh/di310L2qmR0wiIhKREoSIiETU2BLE48kOoB7RvdhH92If3Yt9dC9q0KgaqUVEJH4aWw1CRETiRAlCREQiahQJwszONbOlZrbCzG5NdjyJZmY9zewjM1tsZovM7MZgeTsze8/Mlgf/bRv2mduC92epmZ2TvOgTw8xSzWyumb0d3G6S98LM2pjZa2a2JPj/j5Oa8L24Ofjfx0Ize9HMMpvqvThUDT5BmFkq8DAwFjgCuNTMjkhuVAlXDvzC3QcDJwLXBb/zrcAH7t4f+CC4TXDfJcAQ4FzgH8H71pjcCCwO226q9+Ih4F13HwQcTeCeNLl7YWbdgRuA4e4+FEgl8F2b3L2ojQafIIARwAp3X+XupcBLwLgkx5RQ7r7Z3ecE3+cR+BHoTuB7Pxs87FlgfPD9OOAldy9x99UElnkdUbdRJ46Z9QDOB54MK25y98LMWgGnA08BuHupu++mCd6LoDQgy8zSgGxgE033XhySxpAgugPrw7Y3BMuaBDPrAxwLfAl0dvfNEEgiQKfgYY39Hj0I/AqoDCtriveiH7ANeDr4uO1JM2tOE7wX7r4R+AuwDtgM7HH3qTTBe1EbjSFBWISyJtF318xaAK8DN7l7brRDI5Q1intkZhcAOe4+O9aPRChrFPeCwF/Mw4BH3P1YoIDgI5RqNNp7EWxbGAf0BboBzc3ssmgfiVDWKO5FbTSGBLEB6Bm23YNAVbJRM7N0AsnhBXefFCzeamZdg/u7AjnB8sZ8j04BLjKzNQQeL55pZv+kad6LDcAGd/8yuP0agYTRFO/FWcBqd9/m7mXAJOBkmua9OGSNIUF8BfQ3s75mlkGgoWlykmNKKDMzAs+ZF7v7/WG7JgNXBN9fAbwVVn6JmTUzs75Af2BmXcWbSO5+m7v3cPc+BP63/9DdL6Np3ostwHozGxgsGg18QxO8FwQeLZ1oZtnB/15GE2ira4r34pA1+Nlc3b3czK4H/kOgp8JEd1+U5LAS7RTgcmCBmc0Llv0GmAC8Ymb/ReA/kO8CuPsiM3uFwI9FOXCdu1fUfdh1qqnei58BLwT/WFoFXEngD8EmdS/c/Uszew2YQ+C7zSUwtUYLmti9qA1NtSEiIhE1hkdMIiKSAEoQIiISkRKEiIhEpAQhIiIRKUGIiEhEShAiB8HMKsxsXtgrbrMHm1kfM1sYr/OJ1FaDHwchUseK3P2YZAchUhdUgxCJAzNbY2Z/NrOZwdfhwfLeZvaBmX0d/LdXsLyzmb1hZvODr5ODp0o1syeC6xhMNbOspH0pafKUIEQOTtZ+j5i+H7Yv191HAH8nMMMswffPuftRwAvAX4PlfwWmufvRBOZL2jv6vz/wsLsPAXYD307w9xGplkZSixwEM8t39xYRytcAZ7r7quBEilvcvb2ZbQe6untZsHyzu3cws21AD3cvCTtHH+C94GI2mNmvgXR3vyfx30zkQKpBiMSPV/O+umMiKQl7X4HaCSWJlCBE4uf7Yf9+EXz/OYFZZgF+AHwafP8BcC2E1tNuVVdBisRKf52IHJyssBl0IbD+896urs3M7EsCf3hdGiy7AZhoZrcQWO3tymD5jcDjwVlFKwgki80Jj17kIKgNQiQOgm0Qw919e7JjEYkXPWISEZGIVIMQEZGIVIMQEZGIlCBERCQiJQgREYlICUJERCJSghARkYj+P+AVmBM1kp/SAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(err).plot(logy=False)\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Reconstruction Error\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict ratings for validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputValid = ratings_valid\n",
    "inputValid = inputValid.astype(np.float32)\n",
    "\n",
    "_, reconstructedOutput_valid, _ = rbm.rbm_output(inputValid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate MSE on validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean squared error using RBM prediction: 9.33\n"
     ]
    }
   ],
   "source": [
    "predictionsArray = reconstructedOutput_valid\n",
    "pred_valid = predictionsArray[ratings_valid.nonzero()].flatten()\n",
    "actual_valid = ratings_valid[ratings_valid.nonzero()].flatten()\n",
    "\n",
    "rbm_prediction = mean_squared_error(pred_valid, actual_valid)\n",
    "print(f'Mean squared error using RBM prediction: {round(rbm_prediction,2)}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}