xgboost-regressor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_data = pd.read_csv('data/combined_data/cleaned_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>T</th>\n",
       "      <th>TM</th>\n",
       "      <th>Tm</th>\n",
       "      <th>H</th>\n",
       "      <th>PP</th>\n",
       "      <th>VV</th>\n",
       "      <th>V</th>\n",
       "      <th>VM</th>\n",
       "      <th>PM_2_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>23.4</td>\n",
       "      <td>30.3</td>\n",
       "      <td>19.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.3</td>\n",
       "      <td>4.3</td>\n",
       "      <td>5.4</td>\n",
       "      <td>284.795833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>22.4</td>\n",
       "      <td>30.3</td>\n",
       "      <td>16.9</td>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.9</td>\n",
       "      <td>3.3</td>\n",
       "      <td>7.6</td>\n",
       "      <td>239.695455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>24.0</td>\n",
       "      <td>31.8</td>\n",
       "      <td>16.9</td>\n",
       "      <td>51.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.9</td>\n",
       "      <td>2.8</td>\n",
       "      <td>5.4</td>\n",
       "      <td>182.187500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>22.8</td>\n",
       "      <td>30.2</td>\n",
       "      <td>17.6</td>\n",
       "      <td>55.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.7</td>\n",
       "      <td>3.7</td>\n",
       "      <td>7.6</td>\n",
       "      <td>264.064286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>23.1</td>\n",
       "      <td>30.6</td>\n",
       "      <td>17.4</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.9</td>\n",
       "      <td>3.3</td>\n",
       "      <td>5.4</td>\n",
       "      <td>223.208333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      T    TM    Tm     H   PP   VV    V   VM      PM_2_5\n",
       "0  23.4  30.3  19.0  59.0  0.0  6.3  4.3  5.4  284.795833\n",
       "1  22.4  30.3  16.9  57.0  0.0  6.9  3.3  7.6  239.695455\n",
       "2  24.0  31.8  16.9  51.0  0.0  6.9  2.8  5.4  182.187500\n",
       "3  22.8  30.2  17.6  55.0  0.0  7.7  3.7  7.6  264.064286\n",
       "4  23.1  30.6  17.4  46.0  0.0  6.9  3.3  5.4  223.208333"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = combined_data.iloc[:, :-1]\n",
    "y = combined_data.iloc[:, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=43)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGBOOST REGRESSOR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21:41:38] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "             importance_type='gain', learning_rate=0.1, max_delta_step=0,\n",
       "             max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "             n_jobs=1, nthread=None, objective='reg:linear', random_state=0,\n",
       "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "             silent=None, subsample=1, verbosity=1)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xgboost import XGBRegressor\n",
    "xgb = XGBRegressor()\n",
    "xgb.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Coefficient of determination R^2 on train set 0.8075419986616452'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f'Coefficient of determination R^2 on train set {xgb.score(X_train, y_train)}'\n",
    "# must be close to 1, 1 is perfect fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Coefficient of determination R^2 on test set 0.36292781401846796'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f'Coefficient of determination R^2 on test set {xgb.score(X_test, y_test)}'"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "### OVERFIT MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/opt/anaconda/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/opt/anaconda/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21:42:16] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[21:42:16] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[21:42:16] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "score = cross_val_score(xgb, X, y, cv = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.29975471126334907"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = xgb.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f599b363910>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEHCAYAAACncpHfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3Rc1X3o8e9vZvR+P0ayLcmWbNkGGbANwrwhYB4maXHaQmKSNLQhpWmhTZveB6yuy7rhlnVLb5M0915oLg1JaUpiHAKJmzg8zSNAkC3zsmVbtvySZMmWrPf7Nb/7xxyZQUjW2BrpaGZ+n7W0dGaffc78Nljzm7PPPnuLqmKMMSb+eNwOwBhjjDssARhjTJyyBGCMMXHKEoAxxsQpSwDGGBOnfG4HcDby8/O1tLTU7TCMMSZq7Nq165Sq+ifbF1UJoLS0lOrqarfDMMaYqCEix6baZ11AxhgTpywBGGNMnAorAYjIBhGpFZE6Ebl/kv1JIvK0s79KREpD9j3glNeKyC0h5X8tIjUiskdEfiIiyZFokDHGmPBMmwBExAs8CtwKVAB3ikjFhGp3Ax2qWg58B3jEObYC2ASsAjYAj4mIV0SKgL8EKlX1AsDr1DPGGDNHwrkCWAfUqephVR0GNgMbJ9TZCDzpbD8DrBcRcco3q+qQqh4B6pzzQfAGdIqI+IBUoGlmTTHGGHM2wkkARUBDyOtGp2zSOqo6CnQBeVMdq6rHgX8E6oFmoEtVX5zszUXkHhGpFpHq1tbWMMI1xhgTjnASgExSNnEK0anqTFouIjkErw7KgEVAmoh8abI3V9XHVbVSVSv9/kmHshpjjDkH4SSARqAk5HUxn+yuOV3H6dLJAtrPcOyNwBFVbVXVEeBZ4MpzaYAxxphzE04C2AksF5EyEUkkeLN264Q6W4G7nO3bge0aXGhgK7DJGSVUBiwHdhDs+rlcRFKdewXrgX0zb44xxphwTfsksKqOish9wAsER+v8QFVrROQhoFpVtwJPAD8SkTqC3/w3OcfWiMgWYC8wCtyrqmNAlYg8A7zrlL8HPB755plo9eOq+nM+9guXLY5gJMbELommFcEqKyvVpoKID5YAjIkMEdmlqpWT7bMngY0xJk5ZAjDGmDhlCcAYY+KUJQBjjIlTlgCMMSZOWQIwxpg4ZQnAGGPilCUAY4yJU5YAjDEmTlkCMMaYOGUJwBhj4pQlAGOMiVOWAIwxJk5ZAjDGmDhlCcAYY+LUtAvCGOOWhvZ+Drf2MjQWwCPCZWW5ZCQnuB2WMTHDEoCZd4ZHA7xQc4I3DrSigDjlbx86xS2rFnBpaS4ekTOdwhgThrC6gERkg4jUikidiNw/yf4kEXna2V8lIqUh+x5wymtF5BanbKWIvB/y0y0ifxWpRpno1TUwwu3fe5vXD7RSWZrDf/tMBX/32Qv4qxtXsCgrhV+838TPdjUSTSvZGTNfTXsFICJe4FHgJqAR2CkiW1V1b0i1u4EOVS0XkU3AI8DnRaSC4PrAq4BFwMsiskJVa4E1Iec/DjwXwXaZKKSqPPDsh9Q0dfOFdYu5oCjr9D5/RhJ3X13Gy/tO8mptKyW5qVy+NM/FaI2JfuFcAawD6lT1sKoOA5uBjRPqbASedLafAdaLiDjlm1V1SFWPAHXO+UKtBw6p6rFzbYSJDU9V1bNt9wn+8y0rP/bhP05EWH9+ISsLM/jVh83Ut/e7EKUxsSOcBFAENIS8bnTKJq2jqqNAF5AX5rGbgJ+EH7KJRfuau3nol3u5boWfe65ZOmU9jwifqywhM8XHT3bUMzwamMMojYkt4SSAye62TeyAnarOGY8VkUTgNuCnU765yD0iUi0i1a2trWGEa6LR3/1qLxlJPr71udV4PGe+wZuS6OX2S0roGhjhncNtcxShMbEnnATQCJSEvC4GmqaqIyI+IAtoD+PYW4F3VfXkVG+uqo+raqWqVvr9/jDCNdHmncNtvFXXxp99ahn56UlhHVOWn8bygnReP9DK4MjYLEdoTGwKJwHsBJaLSJnzjX0TsHVCna3AXc727cB2DQ7T2ApsckYJlQHLgR0hx92Jdf/ENVXl2y8doCAjiS9dvuSsjr2popCBkTHeqjs1S9EZE9umTQBOn/59wAvAPmCLqtaIyEMicptT7QkgT0TqgG8A9zvH1gBbgL3A88C9qjoGICKpBEcWPRvZJplo8vahNnYcaefe68tJTvCe1bHFOamsWpTJm3Wn6B8anaUIjYldYT0IpqrbgG0Tyh4M2R4E7pji2IeBhycp7yd4o9jEsX96+QALs5L5/KUl01eexI3nF1LT1M07R9q54byCCEdnTGyzuYCMa/Y1d7PzaAd3X1121t/+xxVmJrPMn0b1sXYC9nCYMWfFEoBxzdM7G0j0eviDi4tndJ51ZXl09o9w8GRvhCIzJj7YXEBmVv24qn7S8pGxAE/vbOC8hRn8es+JGb3H+QszSEvysfNoOysXZMzoXMbEE7sCMK7Y29TNwMgYlUtyZ3wun8fDJYtz2H+im+6BkQhEZ0x8sARgXLHzWDs5qQks9adF5HyXluYQUKg+1hGR8xkTDywBmDnX1jvE4dY+KiM4rXNeehJL/Wm8W99hM4UaEyZLAGbO7T7eBcDakuyInnd1UTbtfcPUNHVH9LzGxCpLAGbO1TR1U5KTQnZqYkTPW7EoE4/Att3NET2vMbHKEoCZU539wxzvHGDVok9O9zxTaUk+lvrT2ba72bqBjAmDJQAzp8a7ZyoWZc7K+S9clMXRtn7rBjImDJYAzJyqaepiQWZy2LN+nq2KRZl4PWLdQMaEwRKAmTM9gyMca+uftW//EOwGunJZnnUDGRMGSwBmzuxr7kGBC2ah/z/Upy9cyNG2fvaf6JnV9zEm2lkCMHNmX3M3uWmJFGbOTvfPuPXnB2cF3b6/ZVbfx5hoZwnAzInRsQCHT/WysjADidDDX1MpyEjmwqIsSwDGTMMSgJkTx9r7GRlTlhekz8n7XX9eAe/Vd9DRNzwn72dMNLIEYObEwZO9eEUoi9DcP9NZf14BAYXXD7TOyfsZE40sAZg5UdfSw+K8VJJ857bwy9m6sCiL/PQkXrFuIGOmFFYCEJENIlIrInUicv8k+5NE5Glnf5WIlIbse8AprxWRW0LKs0XkGRHZLyL7ROSKSDTIzD89gyM0dQ3OWfcPgMcjfGqln9drWxgdC8zZ+xoTTaZNACLiBR4FbgUqgDtFpGJCtbuBDlUtB74DPOIcWwFsAlYBG4DHnPMBfBd4XlXPA1YTXHDexKBDrcGVusrnMAEA3HBeAd2Do7xb3zmn72tMtAjnCmAdUKeqh1V1GNgMbJxQZyPwpLP9DLBegkM9NgKbVXVIVY8AdcA6EckErgWeAFDVYVW1v9IYdfBkL6mJXhZlp8zp+16zPB+fR2w0kDFTCGdJyCKgIeR1I3DZVHVUdVREuoA8p/ydCccWAQNAK/BDEVkN7AK+rqp9E99cRO4B7gFYvHhxGOGa+URVqWvppbwgPWJz/08ndBnK4pxU/uODJhbnpk573Bcus39fJr6EcwUw2V/txGfsp6ozVbkPuBj4Z1VdC/QBn7i3AKCqj6tqpapW+v3+MMI180lLzxA9Q6OU++e2+2dceUEaTZ0D9A+NuvL+xsxn4SSARqAk5HUx0DRVHRHxAVlA+xmObQQaVbXKKX+GYEIwMeZoW/Cirix/boZ/TlTuT0eBQ6c+cXFpTNwLJwHsBJaLSJmIJBK8qbt1Qp2twF3O9u3Adg3OxLUV2OSMEioDlgM7VPUE0CAiK51j1gN7Z9gWMw8dOdVHZrKP3LTILv4SrqKcVJJ8Hupael15f2Pms2nvATh9+vcBLwBe4AeqWiMiDwHVqrqV4M3cH4lIHcFv/pucY2tEZAvBD/dR4F5VHXNO/RfAU05SOQz8cYTbZlymqhw91UdpftqsT/8wFa9HWJqfdnokkjHmI+HcBEZVtwHbJpQ9GLI9CNwxxbEPAw9PUv4+UHk2wZro0tA+QPfgKKV57nT/jFtWkM6+Ez209w27diVizHxkTwKbWVN1pA1wr/9/3PjzB4esG8iYj7EEYGbNjiPtpCZ68WfM7vTP0/GnJ5GZ7KPOuoGM+RhLAGbW7DjaTmle2pyN/5+KiFBekM6h1l4CtkqYMadZAjCz4kTXIMfa+il1uftn3DJ/Ov3DY5zoGnQ7FGPmDUsAZlbsONoOQJnLN4DHLXPuA9hwUGM+YgnAzIrqo+2kJXpZkJXsdigAZCYnUJCRZPcBjAlhCcDMivfqO1ldko3X427/f6jygnSOnupjxKaHNgawBGBmwcDwGPuau1m7ONvtUD6mvCCd0YBS397vdijGzAuWAEzE7WnqYjSgrC3JcTuUjynLS8Mjdh/AmHGWAEzEvXusA4A18+wKICnBS0luqk0LYYzDEoCJuPfqO1mcm0p+ursPgE2m3J/O8Y4BBobHpq9sTIyzBGAiSlV5t76Di+fZt/9x5QXO9NB2FWCMJQATWU1dg7T0DLF28fzq/x9XnJNKos9jw0GNwRKAibD36oP9//NtBNC409ND241gYywBmMh6r76TJJ+H8xZkuh3KlMoL0mnrG6ajb9jtUIxxlSUAE1Hv1XdwUXEWib75+09rmbM+sd0HMPFu/v6VmqgzPBpgT1M3q4vnZ/fPuIKMJDJsemhjwksAIrJBRGpFpE5E7p9kf5KIPO3srxKR0pB9DzjltSJyS0j5URHZLSLvi0h1JBpj3HXgZA/DowEuKpnfCUBEKPenc6jFpoc28W3aJSFFxAs8CtwENAI7RWSrqoYu4n430KGq5SKyCXgE+LyIVBBcH3gVsAh4WURWhKwLfL2qnopge8ws+HFVfVj1dh4JzgB67FRf2Me4ZVlBOu81dHKye5CFWSluh2OMK8K5AlgH1KnqYVUdBjYDGyfU2Qg86Ww/A6yX4CrgG4HNqjqkqkeAOud8JgY1dvaTkuCNinV3x+8D2LQQJp6FkwCKgIaQ141O2aR1VHUU6ALypjlWgRdFZJeI3DPVm4vIPSJSLSLVra2tYYRr3HK8Y4CinBTE5RXAwpGVkoA/I8luBJu4Fk4CmOyveWLH6VR1znTsVap6MXArcK+IXDvZm6vq46paqaqVfr8/jHCNG0bGApzoHqQoO3q6U8r96Rw51ceoTQ9t4lQ4CaARKAl5XQw0TVVHRHxAFtB+pmNVdfx3C/Ac1jUU1U50DRJQoisBFKQzMqbUd9j00CY+hZMAdgLLRaRMRBIJ3tTdOqHOVuAuZ/t2YLuqqlO+yRklVAYsB3aISJqIZACISBpwM7Bn5s0xbml0PkSLc6InAZTlB6eHtqeCTbyadhSQqo6KyH3AC4AX+IGq1ojIQ0C1qm4FngB+JCJ1BL/5b3KOrRGRLcBeYBS4V1XHRKQQeM7pK/YBP1bV52ehfWaOHO8cIC3JR1ZKgtuhhC05wUtxTip1Lb3cVOF2NMbMvWkTAICqbgO2TSh7MGR7ELhjimMfBh6eUHYYWH22wZr5q7FjgOLs6LgBHGqZP53XalsYHLHpoU38sSeBzYwNjY7R2jNEURR1/4wbnx76cGuf26EYM+csAZgZa+ocRIHiKLoBPK4kN4UEr9i0ECYuWQIwM9bUOQDAoii8AvB5PJTlp9kDYSYuWQIwM9bUOUBGko/M5Oi5ARyq3J/Oqd6h04nMmHhhCcDMWHPXIIuisPtn3PLCDABeP2BPmpv4YgnAzMjIWICWnkEWZie7Hco5K8hIIjs1gVf3t7gdijFzyhKAmZHxJ4AXRfGMmiLCysIM3qo7xdCoDQc18cMSgJmRpi7nBnAUdwEBrCjMoG94jOqjHW6HYsycsQRgZqSpc5DkBA85qdF5A3jcMn86iV6PdQOZuGIJwMxIc9cAi7Ki7wngiRJ9Hi5bmsurtZYATPywBGDO2VhAORHlI4BCXb+ygEOtfdS32eygJj5YAjDnrLVniNGAsiiKRwCFuv68AgC7CjBxwxKAOWennwCO4hFAocry01ian8bL+066HYoxc8ISgDlnTV0DJHiF/Iwkt0OJmJtXLeC3h9ro6h9xOxRjZp0lAHPOmjoHWJiVgifKbwCHumVVIaMB5ZX9dhVgYp8lAHNOAqrOFBCx0f8/bnVxNgsyk3l+zwm3QzFm1lkCMOekvW+YodFAzPT/j/N4hJtXFfLGwVYGhu2pYBPbwkoAIrJBRGpFpE5E7p9kf5KIPO3srxKR0pB9DzjltSJyy4TjvCLynoj8cqYNMXPr9A3gGBkCGmrDqgUMjgRscjgT86ZNACLiBR4FbgUqgDtFZOIKqncDHapaDnwHeMQ5toLg+sCrgA3AY875xn0d2DfTRpi519Q5iFeEgszYuQE8bl1ZLtmpCbxQY91AJraFcwWwDqhT1cOqOgxsBjZOqLMReNLZfgZYL8FHQzcCm1V1SFWPAHXO+RCRYuAzwPdn3gwz15q6BijMTMLnib1eRJ/Xw43nF/LyvpM2OZyJaeH89RYBDSGvG52ySeuo6ijQBeRNc+w/Af8FCJx11MZVqhocARSD3T/jfnf1InoGR3l1v3UDmdgVTgKYbIyfhlln0nIR+R2gRVV3TfvmIveISLWIVLe22h/jfNA1MEL/8FhM9v+Pu2pZHvnpifzi/eNuh2LMrAknATQCJSGvi4GmqeqIiA/IAtrPcOxVwG0icpRgl9INIvLvk725qj6uqpWqWun3+8MI18y25q5BAIqyYmsIaCif18PvXLSIV/a10DVgD4WZ2BROAtgJLBeRMhFJJHhTd+uEOluBu5zt24HtqqpO+SZnlFAZsBzYoaoPqGqxqpY659uuql+KQHvMHDjeOYAAC2JsCOhEv7e2iOGxAM/vaXY7FGNmxbQJwOnTvw94geCInS2qWiMiD4nIbU61J4A8EakDvgHc7xxbA2wB9gLPA/eqqt1Vi3LNnQPkZySR6Iu9G8ChLirOoiw/jZ+/N/GC15jY4AunkqpuA7ZNKHswZHsQuGOKYx8GHj7DuV8DXgsnDjM/NHUNUpqX6nYYs05E2LhmEd995SDNXcFpL4yJJbH9Fc5EXO/QKF0DIzF9AzjUZ9cUoQo/29XodijGRJwlAHNWmmP4CeDJlOanceWyPH6yo4GxwMTBb8ZEN0sA5qwcj7E1AMLxxcuWcLxzgDcO2jBkE1ssAZiz0tQ5QG5aIimJ3ukrx4ibKgrJT0/kqXfq3Q7FmIiyBGDOSlPXIItiePz/ZBJ9Hu6oLGH7/pM0dw24HY4xEWMJwIRtYHiM9r5hiuKk/z/UnZcuRoHNOxqmrWtMtLAEYMLW1BVfN4BDLc5L5boVfp6qqmdwxB5lMbHBEoAJ2/gaALE8CdyZ/Mk1SznVO8TP37P5gUxssARgwtbUOUBWSgLpSWE9PxhzrlyWR8XCTB7/zWECNiTUxABLACZsxzsH47L7Z5yI8KfXLeVwax/b97e4HY4xM2YJwIRlaGSMtt6hmFsE/mx9+sKFFGWn8Pgbh90OxZgZswRgwtLcNYgCRXH0ANhkErwevnJ1GTuOtrPzaLvb4RgzI5YATFhOjwDKie8EAHDnuhLy0xP57ssH3Q7FmBmxBGDC0tQ5QEaSj8zkBLdDcV1qoo8/vXYZb9adsqsAE9UsAZiwNMX5DeCJvnT5EvLTE/nOSwfcDsWYcxaf4/nMWRkZC9DSM8j5CzPcDmXeSEn08rXrlvF3v9pH1eE2DrX2nfO5vnDZ4ghGZkz47ArATOtE1yABjc8ngM/ki5ctwZ+RxLdePEBwBVRjooslADOt43G2BkC4UhK9/MUN5ew42s7Bll63wzHmrIWVAERkg4jUikidiNw/yf4kEXna2V8lIqUh+x5wymtF5BanLFlEdojIByJSIyLfjFSDTOQ1dQ6QkuAlO8VuAE+06dLFFOek8GLNCQJ2FWCizLQJQES8wKPArUAFcKeIVEyodjfQoarlwHeAR5xjK4BNwCpgA/CYc74h4AZVXQ2sATaIyOWRaZKJtKauAYpyUhARt0OZdxJ9Hv76xhU0dQ1S09TtdjjGnJVwrgDWAXWqelhVh4HNwMYJdTYCTzrbzwDrJfhpsRHYrKpDqnoEqAPWadD4NXOC82Nfn+ah4dEAJ7uG4moFsLP12bVFFGQk8dLek7ZspIkq4SSAIiB0EvRGp2zSOqo6CnQBeWc6VkS8IvI+0AK8pKpVk725iNwjItUiUt3aakvyzbUDJ3sYU437KSDOxOsRbq4o5FTvEO/Vd7gdjjFhCycBTHbdP/FrzlR1pjxWVcdUdQ1QDKwTkQsme3NVfVxVK1W10u/3hxGuiaSapi6AuFwE5mycvzCT4pwUXtnfwshYwO1wjAlLOAmgESgJeV0MNE1VR0R8QBbQHs6xqtoJvEbwHoGZZ/Yc7ybJ5yEnLdHtUOY1EeHmigV0DYyw44g9HWyiQzgJYCewXETKRCSR4E3drRPqbAXucrZvB7ZrcGD0VmCTM0qoDFgO7BARv4hkA4hICnAjsH/mzTGRtvt4F4uyU/DYDeBplReks8yfxqu1LQzZqmEmCkybAJw+/fuAF4B9wBZVrRGRh0TkNqfaE0CeiNQB3wDud46tAbYAe4HngXtVdQxYCLwqIh8STDAvqeovI9s0M1MjYwH2Nndb989ZuLliAf3DY7x56JTboRgzrbCmglDVbcC2CWUPhmwPAndMcezDwMMTyj4E1p5tsGZu1Z7oYXg0QLHNABq2ktxUKhZm8ubBU1xRlkdqnK6eZqKDPQlspvR+QycAJTmpLkcSXW6qKGR4NMDrB2zUmpnfLAGYKb3f0EleWiLZqfYE8NkozExmTUk2vz3cRtfAiNvhGDMlSwBmSh80dLK6JNueAD4HN55fiCq8amsHm3nMEoCZVM/gCHWtvawuznY7lKiUk5bIpWW5VB9rp613yO1wjJmUJQAzqd3Hu1CF1SVZbocSta5f6cfrEV7ad9LtUIyZlCUAM6kPGoJPANsVwLnLSE7gymX5fNjYRbOzprIx84mNUTOT+qChk9K81Lh6AvjHVfURP+e1y/1UHWnjpb0n+fIVpRE/vzEzYVcAZlIfNAZvAJuZSUn0ct1yP/tP9NDQ3u92OMZ8jCUA8wknuwdp7hq07p8IuXxpHikJXl6z5wLMPGMJwHzCe/XBB8DsCiAykhK8XLksj33N3ZzoGnQ7HGNOswRgPmHXsXYSfR4uKMp0O5SYccWyPBJ9Hl47YM8FmPnDEoD5hOpjHVxUlEWSz+t2KDEjNdHHZWW57G7ssucCzLxhCcB8zODIGHuOd3FJaY7bocScq8vz8XrE5ggy84YlAPMxHzR0MjKmXLok1+1QYk5GcgKVpTm8V99JZ/+w2+EYYwnAfFz1seCatpcssSuA2XDtcj+K8ps6Wy/AuM8SgPmYXcc6WOZPi6sHwOZSdmoia0tyqD7aTu/QqNvhmDhnCcCcFggou451UGndP7PquhV+RseUt+wqwLgsrAQgIhtEpFZE6kTk/kn2J4nI087+KhEpDdn3gFNeKyK3OGUlIvKqiOwTkRoR+XqkGmTO3aHWXroGRuwG8CzLz0jigqIs3jncxsCwrR1s3DNtAhARL/AocCtQAdwpIhUTqt0NdKhqOfAd4BHn2AqCi8ivAjYAjznnGwX+RlXPBy4H7p3knGaOjff/V1r//6y7boWfodEAVUfa3A7FxLFwrgDWAXWqelhVh4HNwMYJdTYCTzrbzwDrJbiKyEZgs6oOqeoRoA5Yp6rNqvougKr2EFxsvmjmzTEzUX20g9y0RMry09wOJeYtyk5hRWE6bx1qY3DErgKMO8JJAEVAQ8jrRj75YX26jqqOAl1AXjjHOt1Fa4Gqyd5cRO4RkWoRqW5ttfHTs0VV+e2hU1xWlmsrgM2R61YU0Dc0ypbqhukrGzMLwkkAk30aaJh1znisiKQDPwP+SlW7J3tzVX1cVStVtdLv94cRrjkXx9r6aeoa5MryfLdDiRuleakszk3l/71+mJGxgNvhmDgUTgJoBEpCXhcDTVPVEREfkAW0n+lYEUkg+OH/lKo+ey7Bm8h561BwRMqVy/JcjiR+iAjXrfBzvHOAX3448U/KmNkXTgLYCSwXkTIRSSR4U3frhDpbgbuc7duB7aqqTvkmZ5RQGbAc2OHcH3gC2Keq345EQ8zMvH2ojQWZySy1/v85tXJBBisLM/jn1w4RCEy8sDZmdk2bAJw+/fuAFwjerN2iqjUi8pCI3OZUewLIE5E64BvA/c6xNcAWYC/wPHCvqo4BVwF/CNwgIu87P5+OcNtMmAIB5Z1DbVy5LM/6/+eYR4SvfWopB072sn2/zRRq5lZYS0Kq6jZg24SyB0O2B4E7pjj2YeDhCWVvMvn9AeOC2pM9tPUNc4V1/7jidy9axLdePMBjr9Wx/vwCS8JmztiTwIa3DwXHotsNYHf4vB7+9NqlvFvfSdWRdrfDMXHEEoDh7bpTlOalUpSd4nYoceuOyhLy0xN59NU6t0MxccQSQJwbHQtQdaTdvv27LDnBy1evWcpvDp7i3foOt8MxccISQJyrPtZB79Ao11gCcN0fXr6E3LRE/vcrB90OxcQJSwBxbvv+FhK8wjUr7CE7t6Ul+fjqNWW8VtvK+w2dbodj4oAlgDj38r6TXL40j/SksAaEmVn25StKyU5NsKsAMycsAcSxI6f6ONzax/rzCtwOxTjSk3z8yTVL2b6/xe4FmFlnCSCOvbLvJADrzy90ORIT6o+uLCU/PYlHfr2f4AP1xswOSwBxbPv+FlYUplOSm+p2KCZEWpKPv1xfTtWRdl4/YDPgmtljCSBOdQ+OsONIu337n6c2XbqYktwU/uH5WpsjyMwaSwBx6vXaVkYDav3/81Siz8Pf3LSSvc3dbP3AZgo1s8MSQJz61YfN5KcnsXaxLf84X922ehEXFGXy97/eT9/QqNvhmBhkCSAOdQ2MsH1/C7+7eiFej008Nl95PMI3b1vFie5BHnvNpogwkWcJIA49v6eZ4bEAn11jyzDPd5csyeX31xbxL28c4eipPrfDMTHGEkAc+sX7TZTmpXJRcZbboZgw3H/reSR4hW/+R40NCzURZQkgzpzoGuS3h9vYuKbI5p2PEgWZyfz1TSt4tbaVn79/3O1wTAyx5//jzDhhHWUAABG+SURBVC8/bEIVNq5Z5HYoxvHjqvpp6yQneFmcm8oDz+7mZNcQmSkJfOGyxXMQnYllYV0BiMgGEakVkToRuX+S/Uki8rSzv0pESkP2PeCU14rILSHlPxCRFhHZE4mGmOmpKs++e5yLirNY6k93OxxzFjwi3H5xMWMB5bn3jltXkImIaROAiHiBR4FbgQrgThGpmFDtbqBDVcuB7wCPOMdWEFxEfhWwAXjMOR/AvzplZo68W9/B3uZuPldZ4nYo5hzkZyRxc8UCak/28I6tHGYiIJwrgHVAnaoeVtVhYDOwcUKdjcCTzvYzwHoJdjBvBDar6pCqHgHqnPOhqm8A9q94Dv3wraNkJPv4/Ytt9E+0umJZHuctyOBXHzax65j9+ZiZCScBFAENIa8bnbJJ66jqKNAF5IV5rJkDJ7oGeX7PCT5fWUJqot36iVYeEe64pISc1ES+9u/v0tI96HZIJoqFkwAmGyoysQNyqjrhHHvmNxe5R0SqRaS6tdUmxjpXT1UdY0yVL19R6nYoZoZSEr188fIl9A6OcveT1XQPjrgdkolS4SSARiC007gYmDg5yek6IuIDsgh274Rz7Bmp6uOqWqmqlX6/rVp1LoZGx/hxVT3rzytgcZ7N/BkLFmQm8+gX17KvuZuv/HAn/cM2VYQ5e+EkgJ3AchEpE5FEgjd1t06osxW4y9m+HdiuwWEKW4FNziihMmA5sCMyoZtwbalupK1vmD+6ssztUEwE3XBeId/dtJZ36zv46pPV9Np8QeYsTZsAnD79+4AXgH3AFlWtEZGHROQ2p9oTQJ6I1AHfAO53jq0BtgB7geeBe1V1DEBEfgL8FlgpIo0icndkm2YABobH+D+vHOTS0hyuKs9zOxwTYZ+5aCHf+txqqo608/uPvUV9W7/bIZkoEtbdQFXdBmybUPZgyPYgcMcUxz4MPDxJ+Z1nFak5J//69lFaeoZ49IsX25O/Mer31hZTkJHMnz/1LhsffZNvfW41N5xn6zyY6dlUEDGsa2CE771+iOtX+rm0NNftcMwsuqo8n5/fexUFGcl85V+r+fOndnHSRgiZaVgCiGH//NohugZG+E+3rHQ7FDMHyvLT+I+/uJr/dPMKXt7XwrX/8CoPPPshdS09bodm5ikbEB6japq6+P5vDvMHFxezapHN+hkvEn0e7rthOb+7ehHfe/0QP3v3OD/Z0cB5CzL41MoC1pXlsLwgg6LsFDy2FkTcswQQg0bGAvznn35Idmoi/+13znc7HDNLpptE7sKibMry03mvvoPaEz08/sYhvvd6cJ/XI6QleklN9JHo8+DzCAleDz5v8HeCV/B5PKQl+chNSyAvLYmFWcl8+crS2W+YmTOWAGLQ/3v9EHubu/nely4hOzXR7XCMi9KTfFyz3M81y/0MjYxxonuQlu4h2vqG6B8eo394jOGxACNjAYaGRhkZCzAaUEacssGRwOlzeT3Cz98/zqdWFvDpCxdQXpDhYstMJFgCiDHVR9v57isH+cyFC9lwwQK3wzHzSFKClyV5aSzJSwv7mJGxAB19w7T0DNHQ3k/P0CjffukA337pAOcvzOTLVyxh45pFNr1IlLL/azGksaOfP/3RLoqyU3j49y5wOxwTAxK8HgoykynITOaCoiy+cNliZ16pZp6ubuSBZ3fzP7ft46vXLOUrV5eRnmQfKdHERgHFiL6hUb76ZDXDYwG+f9el1vVjZs2CrGT+6Koytv3l1fz0a1ewriyPb790gGv/4VX+5Y3DDI6MuR2iCZOl6xjQPTjC3f+6kwMne/jhH6+jvMAWezGzT0S4tDSXS0tzeb+hk2+9WMvD2/bxL785zF/cUM7nL11Mos++Y85nEk0rC1VWVmp1dbXbYbhq4siP/qFRfvj2UZq7BvhcZQkXFWe7FJkxcORUHy/uPcGxtn5y0xK58fwCLirOxiPiyhKW4Sy3OZVYWXJTRHapauVk++wKIIo1dQ6weWc9nf0jfOnyJZy3INPtkEycK8tP455rlnKwpZcXak6wpbqRNw6c4uaKQlTVpiOZZywBRKGAKlVH2vn17mZSE7388VVllOWHP7LDmNkkIqwozKC8IJ3dx7t4ee9J/u2dY+xu6uLPrlvGjecX2kNo84QlgChzrK2PX+1uprFjgJWFGfzBJcU28sLMSx4RVhdnc8GiLHYd66D6WDv3/GgXy/xpfOnyJfz+2mKyUhPcDjOu2SdHFAgElDcOtvLk20d5tbaVzGQff3BxMRcvzrZLajPveT3CurJc/vGOi/jV7mZ+8OYRvvkfe/n7X+/nxopCPn3BQq4/z2/PErjA/ovPUyNjAT5o6OTFvSd5fs8J6tv78WckceP5hVxdnm+jK0zU8Xk9bFxTxMY1RdQ0dbF5RwO/3tPMrz5sJtHr4eIl2Vy5LJ81JdlcWJRFTlpkhjIPjwbo7B+mc2CErv4ROgaG6eofoW94lOHRACNjikeCiSolwUt6cgJZKT6yUxNYXpDOUn863hjtsrJRQC6YODJhYHiM9r5hWnoGOdk9SEPHAI0d/YyMKV4RlvrTWLs4hwuKMvF57IPfxI6AKkdP9bH/RA+HW3tp7ho8vWh4ZrKPvPQk8tISuao8H39GEtmpCaQl+UjyehgJKKPOlBUjY0r34Agnu4do6Q7+HZ3sHuJoWx/9wx9/LsEjkJkcPE+iLzjvkSqMBpSB4TF6hkbpC1ldLSPZx6WluawrCw55vbAoK6q+gNkoIJeNjgWob+/nyKk+Gtr7eWnvSTr6R+joH6ajf/gT860syEymsjSX0rw0lhekk5zgdTF6Y2aPR4Sl/uC3bAh+GTreOUBT5wAnuwdp6xtm34kedtV3EM53VRHIT0+iMDOJBVnJpCf7yE5JIDs10fmdQEZywrTf6IdHA1SW5rD/RA+7jrVTdaSd7ftbAEhJ8HLZ0lxnjqV8lhekR21XrF0BRFD34AiHW/s41NLLodbxnz6OtfUxMvbRf+cEr5CTmhj8SUs4vV2QmUReWlLMXm4ac67uqCymrXeY7sEReoeCXTcJzsylPo+HRJ+QnpRAfnoiPu9H384j+RxAa88Q1UfbeedwG7+pO8Xh1j4ACjOTTieDq8rzyU9POuf3nA0zvgIQkQ3AdwEv8H1V/fsJ+5OAfwMuAdqAz6vqUWffA8DdwBjwl6r6QjjnnK+GRwM0dPRT3xb8Rn/4VC+HWvo41NpLS8/Q6Xo+j1Can8Yyfxo3VxSyzJ9OmT+NxbmpvLDnRNR+YzDGDQleDwuyklmQlexaDP6MJG69cCG3XrgQCM699ebBU/ym7hQv7zvJM7saASjNS+Wi4mwuKs5iTUk2qxZlkZI4P6/ip00AIuIFHgVuAhqBnSKyVVX3hlS7G+hQ1XIR2QQ8AnxeRCqATcAqYBHwsoiscI6Z7pyzSlUZCyijAWVwZIzeoVF6nb6/3qEx2vuGaO356Odk9xD17f00dw0QCLloykz2UV6QznUr/CwrSGeZP51l/jRKclNJ8E7eT2gf/sZEv+KcVDatW8ymdYsZCyg1TV28WXeKDxo62Xm0na0fNJ2uuyAzmZLcFEpyUynJSWVRdjJZKYlkpyaQ5XRNpSb4SPCNX9XInHxOhHMFsA6oU9XDACKyGdgIhH5YbwT+u7P9DPB/JRj9RmCzqg4BR0SkzjkfYZwzYq76++30Do0y6sx1Pv7BH47kBA/+jCQKMpK5tDSHxXnFlOalsiQvlSV5aeSlJdoHujFxzusR51v/R1OxtHQP8kFjF3ubuoO9Bu39vHOojee6j4d1P+OjLi6hMDOZl75xXcTjDicBFAENIa8bgcumqqOqoyLSBeQ55e9MOLbI2Z7unACIyD3APc7LXhGpDSPmmcoHTo2/mIs3nAMfa1OMsDZFhxm36YsRCiSC7zmn/592A/I353z4kql2hJMAJvt6OzF/TVVnqvLJ+kYmzYmq+jjw+JkCjDQRqZ7qpkm0sjZFB2tTdIiVNoUzmLURKAl5XQw0TVVHRHxAFtB+hmPDOacxxphZFE4C2AksF5EyEUkkeFN364Q6W4G7nO3bge0aHF+6FdgkIkkiUgYsB3aEeU5jjDGzaNouIKdP/z7gBYJDNn+gqjUi8hBQrapbgSeAHzk3edsJfqDj1NtC8ObuKHCvqo4BTHbOyDfvnM1pl9McsTZFB2tTdIiJNkXVg2DGGGMiJ3omtDDGGBNRlgCMMSZOxX0CEJH/JSL7ReRDEXlORLJD9j0gInUiUisit4SUb3DK6kTkfncin5qI3CEiNSISEJHKCfuisk0TRVu840TkByLSIiJ7QspyReQlETno/M5xykVE/rfTxg9F5GL3Ip+aiJSIyKsiss/5d/d1pzxq2yUiySKyQ0Q+cNr0Tae8TESqnDY97QxiwRno8rTTpioRKXUz/rCpalz/ADcDPmf7EeARZ7sC+ABIAsqAQwRvWHud7aVAolOnwu12TGjT+cBK4DWgMqQ8ats0oX1RFe+E2K8FLgb2hJT9A3C/s31/yL/BTwO/Jvg8zeVAldvxT9GmhcDFznYGcMD5txa17XJiS3e2E4AqJ9YtwCan/HvAnznbfw58z9neBDztdhvC+Yn7KwBVfVFVxyf/fofgMwkQMo2Fqh4BxqexOD01hqoOA+PTWMwbqrpPVSd7gDlq2zRBtMV7mqq+QXCkXKiNwJPO9pPAZ0PK/02D3gGyRWTh3EQaPlVtVtV3ne0eYB/BJ/6jtl1ObL3OywTnR4EbCE53A59s03hbnwHWSxTMERP3CWCCrxD8ZgKTT4FRdIbyaBArbYq2eKdTqKrNEPwwBQqc8qhrp9P1sZbgN+aobpeIeEXkfaAFeIngVWdnyBfG0Lg/Nh0OMD4dzrwWFwvCiMjLwIJJdv2tqv7CqfO3BJ9VeGr8sEnqn9U0FrMpnDZNdtgkZfOmTWchnOlJYkFUtVNE0oGfAX+lqt1n+AIcFe3S4DNLa5z7gs8R7Fr9RDXnd1S0aaK4SACqeuOZ9ovIXcDvAOvV6cTjzNNVuD6NxXRtmsK8btNZiLWpRE6KyEJVbXa6Qlqc8qhpp4gkEPzwf0pVn3WKo75dAKraKSKvEbwHkC0iPudbfmjc421qnDAdzrwW911AElyY5r8Ct6lqf8iuWJzGIlbaFG3xTid0KpW7gF+ElH/ZGTVzOdA13qUynzh93U8A+1T12yG7orZdIuIfHxEoIinAjQTvbbxKcLob+GSbJpsOZ35z+y602z8Eb4Q2AO87P98L2fe3BPv9aoFbQ8o/TXCkwyGCXS6ut2NCm36P4DeSIeAk8EK0t2mSNkZVvCFx/wRoBkac/0d3E+wrfgU46PzOdeoKwYWTDhGcEbjSzdjP0KarCXZ3fBjyd/TpaG4XcBHwntOmPcCDTvlSgl+a6oCfAklOebLzus7Zv9TtNoTzY1NBGGNMnIr7LiBjjIlXlgCMMSZOWQIwxpg4ZQnAGGPilCUAY4yJU5YAjDEmTlkCMHFNRMZE5H0R2SMiPxWRVKdcReRHIfV8ItIqIr88w7m+6Exv/KGIvC0iq6d576Mistt5/+rItcqY8FgCMPFuQFXXqOoFwDDwNae8D7jAeQoU4Cbg+DTnOgJcp6oXAf+D8NaNvd55/8rpqxoTWZYAjPnIb4DykNe/Bj7jbN9J8CneKanq26ra4bwMnVrcmHnJEoAxBLt4gFsJTk0wbjPBuZOSCU4NUHUWp7ybj6YWn4oCL4rILhG552ziNSYS4mI2UGPOIMWZ8x2CVwBPjO9Q1Q+d+e3vBLaFe0IRuZ5gArh6mqpXqWqTiBQAL4nIfg0uGGPMnLAEYOLdgKquOcP+rcA/Ap8ijAU+ROQi4PsEJ9prO1NdVW1yfreIyHMEVzqzBGDmjHUBGXNmPwAeUtXd01UUkcXAs8AfquqBaeqmiUjG+DbBtan3nOkYYyLNrgCMOQNVbQS+G2b1BwleJTzmrIY1eobRPYXAc049H/BjVX1+huEac1ZsOmhjjIlT1gVkjDFxyrqAjDlLIvLHwNcnFL+lqvdOUnd8VayJ1k93k9iY2WZdQMYYE6esC8gYY+KUJQBjjIlTlgCMMSZOWQIwxpg49f8BQeCEkmH3PaMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(y_test - pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### HYPERPARAMETER TUNING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_estimators = [int(x) for x in np.linspace(start=100, stop=1200, num=12)]\n",
    "learning_rate = [0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]\n",
    "max_depth = [int(x) for x in np.linspace(5, 30, num=6)]\n",
    "subsample = [0.7, 0.6, 0.8]\n",
    "min_child_weight = list(range(3, 8))\n",
    "objective = ['reg:squarederror']\n",
    "params = {\n",
    "    'n_estimators': n_estimators,\n",
    "    'learning_rate': learning_rate,\n",
    "    'max_depth': max_depth,\n",
    "    'subsample': subsample,\n",
    "    'min_child_weight': min_child_weight,\n",
    "    'objective': objective\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 100 candidates, totalling 500 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   30.1s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 500 out of 500 | elapsed:  2.7min finished\n",
      "/opt/anaconda/lib/python3.7/site-packages/sklearn/model_selection/_search.py:814: DeprecationWarning: The default of the `iid` parameter will change from True to False in version 0.22 and will be removed in 0.24. This will change numeric results when test-set sizes are unequal.\n",
      "  DeprecationWarning)\n",
      "/opt/anaconda/lib/python3.7/site-packages/xgboost/core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "/opt/anaconda/lib/python3.7/site-packages/xgboost/core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, error_score='raise-deprecating',\n",
       "                   estimator=XGBRegressor(base_score=0.5, booster='gbtree',\n",
       "                                          colsample_bylevel=1,\n",
       "                                          colsample_bynode=1,\n",
       "                                          colsample_bytree=1, gamma=0,\n",
       "                                          importance_type='gain',\n",
       "                                          learning_rate=0.1, max_delta_step=0,\n",
       "                                          max_depth=3, min_child_weight=1,\n",
       "                                          missing=None, n_estimators=100,\n",
       "                                          n_jobs=1, nthread=None,\n",
       "                                          objective='reg:linear',\n",
       "                                          random_st...\n",
       "                   param_distributions={'learning_rate': [0.05, 0.1, 0.2, 0.3,\n",
       "                                                          0.4, 0.5, 0.6],\n",
       "                                        'max_depth': [5, 10, 15, 20, 25, 30],\n",
       "                                        'min_child_weight': [3, 4, 5, 6, 7],\n",
       "                                        'n_estimators': [100, 200, 300, 400,\n",
       "                                                         500, 600, 700, 800,\n",
       "                                                         900, 1000, 1100,\n",
       "                                                         1200],\n",
       "                                        'objective': ['reg:squarederror'],\n",
       "                                        'subsample': [0.7, 0.6, 0.8]},\n",
       "                   pre_dispatch='2*n_jobs', random_state=43, refit=True,\n",
       "                   return_train_score=False, scoring='neg_mean_squared_error',\n",
       "                   verbose=True)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "search = RandomizedSearchCV(xgb, params, scoring='neg_mean_squared_error', \n",
    "                            cv=5, n_iter=100, random_state=43, n_jobs=-1, verbose=True)\n",
    "search.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 0.8,\n",
       " 'objective': 'reg:squarederror',\n",
       " 'n_estimators': 100,\n",
       " 'min_child_weight': 3,\n",
       " 'max_depth': 20,\n",
       " 'learning_rate': 0.05}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-5025.759472697369"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f5998eec550>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEHCAYAAACgHI2PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZxcZZ3v8c+vqrp637d0ekknJISEJAQIRAFXXMCFuIAGHcSBuegg6r2OV9EZURnvfQ0zV9G5og4DjAyCgAiXjEZRARURQgIJ2UM6na2X9Jqu3qq7a/ndP+pU7DTd6Uq6umv7vV+vvKg656mq36G7v/30c57zHFFVjDHGpC9XogswxhgzuyzojTEmzVnQG2NMmrOgN8aYNGdBb4wxac6T6AImqqio0MbGxkSXYYwxKeXll1/uVtXKyfYlXdA3NjayZcuWRJdhjDEpRUQOT7XPhm6MMSbNWdAbY0yas6A3xpg0Z0FvjDFpzoLeGGPSnAW9McakOQt6Y4xJcxb0xhiT5izojTEmzSXdlbHGnK6HNh2Zts3H1jbMQSXGJCfr0RtjTJqzoDfGmDRnQW+MMWnOgt4YY9KcBb0xxqQ5m3Vjkl4ss2qMMVOzHr0xxqQ5C3pjjElzFvTGGJPmLOiNMSbNWdAbY0yas6A3xpg0Z0FvjDFpzoLeGGPSXExBLyJXiMg+EWkSkVsn2Z8tIo84+zeJSOO4fatE5AUR2SUiO0QkJ37lG2OMmc60QS8ibuAu4EpgOXCtiCyf0OxG4LiqLgbuBO5wXusBfgJ8WlXPBd4KBOJWvTHGmGnF0qO/GGhS1WZVHQMeBtZNaLMOuN95/BhwuYgI8C5gu6q+CqCqPaoaik/pxhhjYhFL0NcCR8c9b3G2TdpGVYOADygHzgZURJ4SkVdE5EuTfYCI3CQiW0RkS1dX1+kegzHGmFOIJehlkm0aYxsPcBnwcee/HxSRy1/XUPVuVV2jqmsqKytjKMmYvxgJhDjYPURYJ35bGmMgttUrW4D6cc/rgLYp2rQ44/LFQK+z/Q+q2g0gIhuBC4CnZ1i3MbT1+dm4s51D3UOEFT6ypo7V9aWJLsuYpBNLj34zsEREFoqIF1gPbJjQZgNwvfP4auAZVVXgKWCViOQ5vwDeAuyOT+km0/1uTwdtfX4uW1xJYbaHXW39iS7JmKQ0bY9eVYMicguR0HYD96nqLhG5HdiiqhuAe4EHRKSJSE9+vfPa4yLyHSK/LBTYqKq/nKVjMRkkFFYOdg+xqq6EK1bMYyQQYtvRPgKhMFluuzzEmPFiuvGIqm4ENk7Ydtu4xyPANVO89idEplgaEzftPj+jwTCLKvMBWFZTxEuHemnuGmTpvKIEV2dMcrGuj0lJzV1DACysiAT9WZX5eD0u9rQPJLIsY5KSBb1JSc3dg1QWZFOUkwWAx+1iSVUBe4712+wbYyawoDcpJxRWDvUMs9AZtolaXlPEwEiQ1uP+BFVmTHKyoDcpp63Pz1gwzKKKk4N+6bxCXAJ72m32jTHjWdCblNPcNQjAosqCk7bneT3UleZxsGcoEWUZk7Qs6E3Kae4eoqowm4Ls108am1eUQ2f/KGrj9MacYEFvUkpYlcO9wydm20xUVZSNPxBiYDQ4x5UZk7ws6E1K6fcHGAuGmVc8+W0Nqosi2zv7R+eyLGOSmgW9SSk9Q2MAVBRkT7o/GvQd/SNzVpMxyc6C3qSU7sFIT7083zvp/oJsD3letwW9MeNY0JuU0jM4hsclFOVmTdmmuiiHzgEbujEmyoLepJSewVHK8r24ZLJbIERUFWbT0T9iM2+McVjQm5TSMzQ25fh8VHVRDqPBMP0jNvPGGLCgNykkrErv0BjlBZOPz0fZCVljTmZBb1KGzx8gGFYq8qfp0RdG9lvQGxNhQW9SRs9gZGpl2TQ9+rxsDwXZHptLb4zDgt6kjOjUyunG6CFyhWzHgPXojQELepNCegZHyXILhTnT3xitujAyxdJm3hhjQW9SSM/QGOX52aecWhlVVZTNWDCMzx+Yg8qMSW4W9CZldA9OP+Mmqtw5YRtdMsGYTGZBb1JCWJXjTo8+FtElEnot6I2xoDepoW84QEg15h59cV4WbhELemOIMehF5AoR2SciTSJy6yT7s0XkEWf/JhFpdLY3iohfRLY5/34U3/JNpuiJLmYWY9C7RCjNzzrxOmMy2bTTF0TEDdwFvBNoATaLyAZV3T2u2Y3AcVVdLCLrgTuAjzr7Dqjq6jjXbTJM73CkZx7r0A1AWb7XevTGEFuP/mKgSVWbVXUMeBhYN6HNOuB+5/FjwOUiMUyNMCZGvuEALiGmqZVRZfnZ9AyN2RRLk/FiCfpa4Oi45y3OtknbqGoQ8AHlzr6FIrJVRP4gIm+aYb0mQ/n8AYpysmKaWhlVnu9lNBhmeCw0i5UZk/xi6R5N9pM1sYs0VZt2oEFVe0TkQuD/ici5qtp/0otFbgJuAmhoaIihJJNp+vwBik+xBv1kojNvbIqlyXSx9OhbgPpxz+uAtqnaiIgHKAZ6VXVUVXsAVPVl4ABw9sQPUNW7VXWNqq6prKw8/aMwac/nD5zyZiOTKTsxxdJOyJrMFkvQbwaWiMhCEfEC64ENE9psAK53Hl8NPKOqKiKVzslcRGQRsARojk/pJlOoKv3+ACWnGfSl+V4E69EbM+3QjaoGReQW4CnADdynqrtE5HZgi6puAO4FHhCRJqCXyC8DgDcDt4tIEAgBn1bV3tk4EJO+hsZCBMNKcd7pBX2W20VRbha9gxb0JrPFNIVBVTcCGydsu23c4xHgmkle93Pg5zOs0WS46Ho1pztGD5HhG+vRm0xnV8aapOcbPvOgL7e59MZY0Jvk5xuZWY9+cDTI0KjdP9ZkLgt6k/R8wwHcIuRnx36xVFR05s2R3uF4l2VMyrCgN0nP5x+jKNdzWhdLRUWXTDjcY0FvMpcFvUl6vjO4WCoq2qM/3DMUz5KMSSkW9CbpzSToc71u8rxuDtvQjclgFvQmqYXDSr8/SHFubMsTT6Ys38sRG7oxGcyC3iS17qFRQnr6F0uNV5bv5XCvDd2YzGVBb5Jae98IAMU5Zx705fleWo/7GQuG41WWMSnFgt4ktXafH2BGPfry/GzCCq19/niVZUxKsaA3Sa3d5/Toz/BkLNjMG2Ms6E1Sa/eN4HEJ+V73Gb9HWYFdNGUymwW9SWptfX6KcrOYyZ0pC7M95Ga57aIpk7Es6E1Sa/eNzGjYBkBEWFCeZ0M3JmNZ0Jukdsw3cto3HJlMQ1me9ehNxrKgN0krFFaO9c+8Rw+woDyPI73DhMMTb3dsTPqzoDdJq2tglFBYT/tesZNpKM9nNBimc8DuH2syjwW9SVptzhz6eAzdLCjLA+CQjdObDGRBb5LWsegc+hlcLBXVWJ4PYGvemIxkQW+SVptzJWs8xujnl+TgcYmteWMykgW9SVrtvhFyslzkZp35xVJRHreL2tJcm3ljMpIFvUla7T4/84tzZ3Sx1Hg2xdJkKgt6k7Ta+kaoKcmJ2/vZRVMmU8UU9CJyhYjsE5EmEbl1kv3ZIvKIs3+TiDRO2N8gIoMi8sX4lG0ywTHfCDXFuXF7v8byfPpHgvQNj8XtPY1JBdMGvYi4gbuAK4HlwLUisnxCsxuB46q6GLgTuGPC/juBX828XJMpgqEwnQMj1BTHr0ff4EyxtOEbk2li6dFfDDSparOqjgEPA+smtFkH3O88fgy4XJyBVRH5ANAM7IpPySYTdAyMElbi2qNf4EyxtPvHmkwTS9DXAkfHPW9xtk3aRlWDgA8oF5F84MvAN0/1ASJyk4hsEZEtXV1dsdZu0li7M7UynmP0J3r03TZObzJLLEE/2ZSHiQuGTNXmm8Cdqjp4qg9Q1btVdY2qrqmsrIyhJJPuojccmR/HHn2u101VYbb16E3G8cTQpgWoH/e8Dmibok2LiHiAYqAXWAtcLSL/DJQAYREZUdXvz7hyk9aitxCsKcmBw/F738byfLs61mScWIJ+M7BERBYCrcB64GMT2mwArgdeAK4GnlFVBd4UbSAi3wAGLeRNLNr6Rsj3uinMjuVbNHYN5Xk8t9+GB01mmXboxhlzvwV4CtgDPKqqu0TkdhG5yml2L5Ex+SbgC8DrpmAaczrafX5qSuJ3sVTUgrI8OvpH8Y+F4vq+xiSzmLpLqroR2Dhh223jHo8A10zzHt84g/pMhorMoY/fidiohvLICdkjvcMsnVcY9/c3JhnZlbEmKbX5RuJ6IjbqxBRLu0LWZBALepN0xoJhugdHmTcLPfrGcT16YzKFBb1JOh39I6hGlhaOt5I8L0U5Hrs61mQUC3qTdKLr0MfzqtjxFpTn252mTEaxoDdJJ3oLwdrS2Qn6BudG4cZkCgt6k3Ta+uJ/Vex4jeV5tB73EwyFZ+X9jUk2FvQm6bT2+SnL95LrnfmdpSazoCyfYFhP/EIxJt1Z0Juk09bnn5UTsVHRufQHbZzeZAgLepN02vr8szZsA3BWZQEABzpPudaeMWnDgt4kFVWl9bif+SWzF/QVBV5K8rJo6rKgN5nBgt4klf6RIENjIWpnMehFhCVVBTR1WNCbzGBBb5JK6/HI1MrZ7NEDLK4q5LXOASKLrBqT3izoTVKJXiw1W3Poo5ZUFdA3HKBnyG4UbtKfBb1JKtGLpWZz1g3AkurICdn9NnxjMoAFvUkqrX1+vG4XFfnZs/o5S6oiSxQ3dQ7M6ucYkwws6E1SaesboaYkB5crvjccmai6KJuCbA/7bYqlyQAW9CapzPYc+igRYXFVAU0W9CYDWNCbpBK5Knb2gx4iJ2StR28ygQW9SRqBUJiO/hFqZ/lEbNSS6gK6BkbpG7aZNya9WdCbpNHRP0JYZ38OfdRfTshar96kNwt6kzROLE88R0G/uMqZYmlBb9KcBb1JGtGLpeYq6GtLcsnNcttcepP2Ygp6EblCRPaJSJOI3DrJ/mwRecTZv0lEGp3tF4vINuffqyLywfiWb9JJa/Sq2DkKepcrMvPmtQ6bS2/S27RBLyJu4C7gSmA5cK2ILJ/Q7EbguKouBu4E7nC27wTWqOpq4Arg30TEE6/iTXppOT67NxyZzLKaQva099uaNyatxdKjvxhoUtVmVR0DHgbWTWizDrjfefwYcLmIiKoOq2rQ2Z4D2E+TmdKR3iHqy/Lm9DOX1xTRMzRG58DonH6uMXMplqCvBY6Oe97ibJu0jRPsPqAcQETWisguYAfw6XHBb8xJjvQOs2Cug35+MQC72/vn9HONmUuxBP1k16JP7JlP2UZVN6nqucBFwFdE5HWTpEXkJhHZIiJburq6YijJpJtAKExb3wgLyuc26M+piUyx3N1mQW/SVyxB3wLUj3teB7RN1cYZgy8Gesc3UNU9wBCwYuIHqOrdqrpGVddUVlbGXr1JG63H/YTCSsMc9+iLcrKoL8u1Hr1Ja7GcGN0MLBGRhUArsB742IQ2G4DrgReAq4FnVFWd1xxV1aCILACWAofiVbxJH4d7hwFYUJ4/K+//0KYjU+4rzM5iU3PPrHyuMclg2qB3QvoW4CnADdynqrtE5HZgi6puAO4FHhCRJiI9+fXOyy8DbhWRABAGblbV7tk4EJPajvQMAcx5jx6gpjiHPe39DI8FyfPapDCTfmL6rlbVjcDGCdtuG/d4BLhmktc9ADwwwxpNBjjcM0y2x0VV4eyuQz+ZmuJcFNh7bIALGkrn/PONmW12ZaxJCkd6h2koy5v1degnU+MsomYnZE26sqA3SeFI7/Ccz7iJKsnNIifLxR47IWvSlAW9SThVdXr0s3MidjoiQk2xzbwx6cuC3iRc1+Aow2MhGsrmZo2bydQU57C3fYBQ2C7eNunHgt4k3JGe2Z1aGYua4lz8gRDNXbaSpUk/FvQm4Q47Qd+QoDF6gLrSyF8Tr7b4ElaDMbPFgt4k3JHeYUT+EraJUFmYTUG2h1eP9iWsBmNmiwW9SbgjvcPML84l2zN3yxNP5BJhZW0xr7ZY0Jv0Y0FvEu5wzxD1CTwRG3VefQl72vsZCYQSXYoxcWVBbxIusjxx4k7ERq2uLyYQUptPb9KOBb1JKN9wgO7BMRZWJkPQR5Y/sHF6k24s6E1CNXVF7td6dnVBgiuBecU5VBdl28wbk3Ys6E1C7e+IzFtfUlWY4EoizqsrsR69STsW9Cah9ncOkpPlorYk8SdjIXJCtrl7CN9wINGlGBM3tvi2Sag/vtZFWb6Xhzcfnb7xHFhdXwLA9tY+3rTE7nZm0oP16E1CdQ6MUlX4utsIJ8zKumJEYOsRG74x6cOC3iTMwEgAnz+QkJuNTKUoJ4ul1YW8dLB3+sbGpAgLepMwB7oitw9Mph49wBsWlbPlcC9jwXCiSzEmLizoTcLs74hMrawuSp4ePcAbFpUxEgizo9WGb0x6sKA3CbO/cxCPSyjN9ya6lJNcvLAcgBebbfjGpAcLepMw+zsGqCzMxiVzf5/YUynL97K0upBNNk5v0oQFvUmY/Z2DVCbRidjx1i4qY8uhXgIhG6c3qc+C3iTE8FiQluP+pDsRG7V2YTnDYyF2ttpyCCb1xRT0InKFiOwTkSYRuXWS/dki8oizf5OINDrb3ykiL4vIDue/b49v+SZVHeiMzrhJ3h49YMM3Ji1MG/Qi4gbuAq4ElgPXisjyCc1uBI6r6mLgTuAOZ3s38H5VXQlcDzwQr8JNatvnzLipSrIZN1EVBdksrirgxeaeRJdizIzF0qO/GGhS1WZVHQMeBtZNaLMOuN95/BhwuYiIqm5V1TZn+y4gR0SS8yfbzKmdrT7yvG4qCpL32+GSs8rZ1NxrNyIxKS+WtW5qgfELkbQAa6dqo6pBEfEB5UR69FEfBraq6ujEDxCRm4CbABoaGmIu3qSuHa0+VswvTqoZNw9tOnLSc4/LhT8Q4lu/2MPy+UUAfGytfX+a1BNLj36yn0Q9nTYici6R4ZxPTfYBqnq3qq5R1TWVlbaQVLoLhsLsavOxsq440aWc0sKKfHKz3OxqsxOyJrXFEvQtQP2453VA21RtRMQDFAO9zvM64AngE6p6YKYFm9TX1DXISCDMytrkDnq3S1hWU8ieY/0EwzbN0qSuWIJ+M7BERBaKiBdYD2yY0GYDkZOtAFcDz6iqikgJ8EvgK6r6fLyKNqlth3MHp2Tv0QOcO7+YkUCYg866PMakommDXlWDwC3AU8Ae4FFV3SUit4vIVU6ze4FyEWkCvgBEp2DeAiwGviYi25x/VXE/CpNSdrT6KMj2sLA88feJnc7iqgK8bhe72uyG4SZ1xXTjEVXdCGycsO22cY9HgGsmed23gG/NsEaTZra3+Dh3fhEuV/KciJ1KltvF0nmF7Grv56rV82N6zcSTuhPZCV0z1+zKWDOnAqEwe9r7WZUCwzZR584vYmg0yIGuwUSXYswZsaA3c2p/xyCjwTAr60oSXUrMltUUUZTj4dm9nahOnHBmTPKzoDdzKrrGe7LPuBkvy+3iLUurONQzzPNNdqWsST0W9GZO7Wj1UZjjYUFZXqJLOS0XLSilODeLb/92n/XqTcqxoDdzatvRPlbWFqfEidjxPG4Xb1taxdYjffx+X1eiyzHmtFjQmznj8wfY3dbPxQvLEl3KGblgQQkNZXl84dFtbD5kq1qa1GFBb+bMlkO9hDWy1nsq8rhc/OcNF1Oa5+Xj/76JhzYd4ZhvxIZyTNKLaR69MfGw6WAvXreL8xtSZ8bNRI0V+Tx+8yV8+icv89UndgCQm+WmpiSHqsJszm8opbowB6/H+lAmeVjQmzmzqbmH1fUl5GS5E13KjJTkeXngxrVsPtjLge4hDnYN0dE/wrH+EX74+wNUFGTzkTV11JWm1glnk74s6M2cGBwNsrOtn5vfelaiS4mLLLeLSxZXcMniipO2P9/Uzc0PvsKP/nCAGy9bxMKK5F/mwaQ/+/vSzIkth3oJhTVlx+djdeniCj739iUU52bx5LZWQmEbvzeJZz16MydebO7F4xIuWJC64/Mw/To2ALleN+9bNZ8HXjzMCwe6uWyJ3WPBJJb16M2c2HSwh1V1xeR5M6Nvcc68QpZWF/K7vZ34/IFEl2MynAW9mXXDY0F2tPhYuyi9h23GExHef958wmHld3s6El2OyXAW9GbWvXCgh2BYueSszAl6gLJ8Lxc0lLK9pc9uMG4SyoLezLrf7ekk3+tO2StiZ2JNYymBkPJqS1+iSzEZzILezCpV5Zm9Hbz57EqyPak9f/5M1JbkMq8ohy2Hjie6FJPBMuPMmEmIhzYdobXPT0f/KPleT0wzVtKNiLCmsZRfbG+n3eenpjg30SWZDGQ9ejOr9h7rR4Cz5xUmupSEWV1XgtslbDlsvXqTGBb0ZlbtbR+gviyPguzM/eMxL9vDufOL2Hakj2AonOhyTAayoDezpn8kQGufn3MyuDcfdX59Cf5AyO47axLCgt7Mmn3HBgA4Z15RgitJvLMqC8jJcrGztT/RpZgMFFPQi8gVIrJPRJpE5NZJ9meLyCPO/k0i0uhsLxeRZ0VkUES+H9/STbLb0eqjNC+L6qLsRJeScB63i2Xzitjd3k/Ahm/MHJs26EXEDdwFXAksB64VkeUTmt0IHFfVxcCdwB3O9hHga8AX41axSQnHfCMc6Bzk/IZSRFLrtoGzZUVtMf5AiBeb7QbjZm7F0qO/GGhS1WZVHQMeBtZNaLMOuN95/BhwuYiIqg6p6p+IBL7JIE9ua0WB1fWpvYhZPC2uKsDrcbFxx7FEl2IyTCxBXwscHfe8xdk2aRtVDQI+IObr3UXkJhHZIiJburrsxsupTlV5/JVWGsryqCiwYZuoLLeLc+YV8ptdx2z2jZlTsQT9ZH93T1xkO5Y2U1LVu1V1jaquqay0JV1T3e72fvZ1DFhvfhIr5hfTMzTGS3ZzcTOHYgn6FqB+3PM6oG2qNiLiAYoB+07OUI+/0kqWW1hVV5zoUpLO2dWF5GS5+PVOG74xcyeWoN8MLBGRhSLiBdYDGya02QBc7zy+GnhGVe3WOhloNBjiyW2tvP2cqoxZe/50eD0u3ra0il/tPEbY7j5l5si0Qe+Mud8CPAXsAR5V1V0icruIXOU0uxcoF5Em4AvAiSmYInII+A7wSRFpmWTGjkkjT7zSSvfgGNe9oTHRpSStK1fW0DUwystHbEkEMzdi6nKp6kZg44Rtt417PAJcM8VrG2dQn0kh4bBy9x+bWVFbxKWLyzny0nCiS0pKbz+nypl9085FjZm3dLOZe3ZlrImb3+zuoLl7iE+9+SybO38KBdke3rykkl/b8I2ZIxb0Ji5UlR/94QANZXlcuWJeostJeu9ZOY9234jdkMTMCQt6ExcvHOhh29E+/tubFuJx27fVdC5fVk2WW/iVzb4xc8CmRZgZGw2G+NqTO6krzeWaNfXTv8BQnJvFpYsr+MWrbXz5inNwuxIz1BXLzWA+trZhDioxs8m6XmbG7v5DMwe6hvjHD6wgJyvzbhd4pq65sJ423wh/fM2uBjezy4LezMjB7iH+77NNvHdVDW9bWpXoclLKO5dXU1Hg5cEMvMWimVsW9OaM+cdCfOHRbWS7XXz9fXZ5xOnyelxcs6aeZ/Z20O7zJ7ock8Ys6M0ZCYTC3Pzgy2w72scdV6+iqign0SWlpGsvaiCs8Mjmo9M3NuYMWdCb0xYIhfnSY9t5dl8X3/rACt6zsibRJaWshvI83rSkgkc2HyVkc+rNLLFZN+Z1TjUTo3NghJ9taaG1z8+7llcjSEwzN8zUPr52AZ/+ycs8ua2VD11QN6efHQyFOdw7jFuEwhwPJXnehM0ASjbpNCPJgt7EZGAkwAvNPfxpfzdej4uPXdzAilpbnTIe3rW8mlV1xdzx6728+9x55GfP/o/l/o4B7nnuIE++2spI4C9r4xfleHj3ufM4r74El13dnDYs6M0pdfSP8HxTN1uP9hEOK+fWFvP+VTUU5mQlurS04XIJX3//uXz4h3/mrmeb+NIV58zaZ40Fw/zg903c9WwTXuc+titqi3G7hH5/gE0He/nZyy282NzDtRc3UJLnnbVazNyxoDeTajk+zNN7OtnXMUCWW1izoJRLF1fYHaNmyYULSvnQBbXc89xBPnpRPQvK8+P+GU2dg9zy0CvsPTbAVefN57b3L+c3uzpOanPBglK2Henjv7a3cfdzzdx46cK412HmngW9OcnR3mF++tIRdrT6yPO6eceyKtYuLJ+T4YRMd+sV5/DUzmN89qdbeeCGtRTnxe+vpie2tvD3T+wkJ8vNPZ9YwzuWV0/aziXCBQtKqS7K4b7nD/LvzzVz5coaFlcVxK0WM/ck2e4PsmbNGt2yZUuiy8g4qsojm49y+y92EwiFuWxxJW9aUmFXus6CU53Ae3pPB3/7k1eoKPRywyULyTvFL9hYTgT++x+b2fBqGztafTSW5/HRixoozo3tF8ix/hHu+9NBinI8PH7zpcwrzqwptKl2MlZEXlbVNZPts+mVhv6RAJ964GVufXwH5zeU8D/ecTbvXF5tIZ8Aly+r5u5PXEhn/yj/9sdmdrb6CJ9BZywQCvPYyy1893evsbutn3csq+LGyxbFHPIA84py+OQljfj8AT75Hy8xMBI47TpMcrC/xzPcoe4hbrx/M4d7hvmH9y7jhksX8rBdvJNQb11axfWXNPLE1lYeeukI5fleltUUMb8kl3lFORTnZpGT9fo+WjAUZu+xAZ5v6ub+Px+izTdCbUkuH76g7ox74/NLcvnhX13IDT/ezN/+5BXu++RFeD2Z2z8cHA1yoHOQzoFRSvOyWFxVwOr6kqT/f2JBn8H+fKCbmx98BYAHblzLG88qT3BFJuqsygK+8M6z2dXWzwsHenixuYfguAuqstzC959tosiZ/TQcCNI9MIY/EALg4oVlfOuDK2jrG5nxNMk3n13JP314FV/82avc+vPtfPsj52XcjWWO+Ub45Y42DnQNnbT98a2t1JXm8rnLl/Ch82uTdoluC/oM9eCmw3z9yV00VuRz7/VrZmWWh5kZlwgra4tZWVtMKKx0DozQNTCKzx+g3x9gfkkuPn8Alwi5XjcleVmsri/hwgWl1JXmAbGNM8fi6gvraOvz853fvsb8kly++O6lcXnfZDcWDNmr6cYAAAzYSURBVPPb3cd4obmHbI+by8+p4uzqQmqKc/D5A5xVVcCP/nCALz22nXuea+Z7689nWU1Rost+HQv6DDMWDPO/N+7hx38+xFuXVvKv155/oldokpfbJdQU51JTnHti21yfCPzs2xfT1ufn+882ket1c/Nb0/uWke0+Pw+/dJTuwVEuaizjXcurTzo5Xl6QzXtW1nDlink8tauDrz25k3V3Pc8/vHcZ171hQVL9v7Ggn0aqnXk/lXafn888+AqvHOnj0rPKeceyan7xanuiyzJnaK6XnhARvvWBFQyPhfiXp/bRNTDKbe9bjivGJRNS5WdJVfnxnw/xg98fIM/r5obLFnJW5dTTS0WEK1bMY01jKX/36Kvc9uQu/rS/m3++elXSXHBmQZ8BVJVfbG/n6xt2MRoIsf6ielbVlSS6LJOCPG4X3/3oaioKsrnv+YO0HB/mm+tWUFuSO/2LU0DP4Cj/87HtPLO3k3PmFfLhC+pivoakoiCb//jkRdz3/EHu+PVervzec3z7mvO4ZHHFLFc9PQv6cXoGR9ne4mPb0T52tvoYHgvR0T+C1+OiLN9Leb6X2pJcakpyyUrSky4TNXUO8s3/2sVz+7tZWVvMd9evZlNzb6LLMinM5RK+9r5l1JXm8s9P7eXyb/+eT735LD5wfi2N5XmvG7IYGAlwqHuYV4/20T00Ss/gGD5/AFVFRMjP9lCe76WyMJu1i8pYWJ4f818J8RIMhXl0Swvf+e0++v1BvvH+5WS5Xac9/OJyCX/zpkWsXVjO5x7eysfu2cQHVs/nq+9dRlVh4q5DiCnoReQK4HuAG7hHVf9pwv5s4D+BC4Ee4KOqesjZ9xXgRiAEfE5Vn4pb9TPgHwuxqy0S6tuO9vFqSx9HeyM3f3AJLK4qoCTXS1iVvuEAzV1DjIUiiz+5RagpyaG+LI/60jwuW1xBfVlu0ozJBUJhXmzu4cfPH+LpvZ0UZnv45lXn8ldvWIDbJRb0ZsZEhBsuW8i7V8zjf/1yN997ej/fe3o/9WW51JXk4XEL/rEQh3qG6B4cO+m1xblZlOZlISKoQnufn91tPsIKj73cQlGOh9UNpZxfX8L5DSWsri+ZtSEQnz/Axh3t3Peng+zvHGTNglJuX7eC5fOLZjQ0trKumF99/k384NkmfvSHZn67u4MPX1jHJ964gMVVhXE8gthMe2WsiLiB14B3Ai3AZuBaVd09rs3NwCpV/bSIrAc+qKofFZHlwE+Bi4H5wO+As1U1NNXnxePKWFUlEFJGgiGGRoN09o/S0T9Cc/cQTZ2D7G7rZ1/HwIn1v2tLcjmvvpjz6ko4r76ElbXFJ/5ci36xVZX+kSAtx4c52jvMkV4/rX3DBEKR9yjP93J+QwnnN5RyzrxCqotyqCrMJj/bQ06We1aWfg2GwhwfDtA7NMbhniGaugbZftTH803dDIwGKc/38ldvWMB1b1xw0ho1tqxwYsUyDp1MX6NY6j3UPcRz+7v4U1M3vUNjBEKK1+OisTyPhRUFLKzIY3fbAGX53knnnIfCSvfgKLUluWw92sfWI8d5rWOA6IzSRRX5rHZ+vhZV5FNdlE1ZfjZ5XjfZnul73uGwMjAS5Fj/CG19fna0Rjp5f2rqZiwYZklVAX/3rqW8+9zqE+8Vr3MKzV2DfP+ZJn6xvZ2xUJizqwt446JyVjeU0FCWR11pHsW5WTEdx6mc6srYWHr0FwNNqtrsvNnDwDpg97g264BvOI8fA74vkYrXAQ+r6ihwUESanPd74UwO5FR2tvq47t5NjATCjAZDTHUPh6rCbJbOK+RvzzmL8+pLOK++OKY/qUSE4twsinOLOXd+ZHneUFjp6B9hXnEOW4/0sfXocX63p3PS13s9LnKzot+UIIjzviDjPuMvnxf5pwrBkBIMhwmGlWBICYQijye7UUVdaS7vO28+bzm7krcurbSrW82caKzIp7Ein+ve2Dhlm96hqa+sdbuE6qIcPnJRPR+5qB6IXJy0vaUv8rN1pI8/vtbF46+0Tvr63Cw3uV63c82AogoKhFUZDYRPXF8QJRK5VuHai+r50AV1rKornrW/yBdVFvCdj67m79+7jJ+/0sJz+7t5dEsL979w+KR2WW7h/efN5zsfWR33GmLp0V8NXKGqf+M8vw5Yq6q3jGuz02nT4jw/AKwlEv4vqupPnO33Ar9S1ccmfMZNwE3O06XAvpkf2qyrALoTXUScpMuxpMtxQPocS7ocByT/sSxQ1crJdsTSo5/s19zE3w5TtYnltajq3cDdMdSSNERky1R/JqWadDmWdDkOSJ9jSZfjgNQ+llimjrQA9eOe1wFtU7UREQ9QDPTG+FpjjDGzKJag3wwsEZGFIuIF1gMbJrTZAFzvPL4aeEYjY0IbgPUiki0iC4ElwEvxKd0YY0wsph26UdWgiNwCPEVkeuV9qrpLRG4HtqjqBuBe4AHnZGsvkV8GOO0eJXLiNgh85lQzblJMSg01TSNdjiVdjgPS51jS5TgghY8l6W48YowxJr5S4/JOY4wxZ8yC3hhj0pwF/RkSkS+KiIpIhfNcRORfRaRJRLaLyAWJrvFURORfRGSvU+sTIlIybt9XnOPYJyLvTmSdsRKRK5x6m0Tk1kTXEysRqReRZ0Vkj4jsEpHPO9vLROS3IrLf+W9pomuNhYi4RWSriPzCeb5QRDY5x/GIM6Ej6YlIiYg85vyM7BGRN6bq1wQs6M+IiNQTWRJi/DXSVxKZVbSEyMVfP0xAaafjt8AKVV1FZImLrwA4y1asB84FrgB+4CyDkbSc+u4i8jVYDlzrHEcqCAJ/p6rLgDcAn3FqvxV4WlWXAE87z1PB54E9457fAdzpHMdxIutepYLvAb9W1XOA84gcU6p+TSzoz9CdwJc4+eKvdcB/asSLQImI1CSkuhio6m9UNeg8fZHINQ4wbtkKVT0IRJetSGYnlulQ1TEgukxH0lPVdlV9xXk8QCRQaonUf7/T7H7gA4mpMHYiUge8F7jHeS7A24ksiwKpcxxFwJuJzCZEVcdUtY8U/JpEWdCfJhG5CmhV1Vcn7KoFxt9Vu8XZlgpuAH7lPE7F40jFml9HRBqB84FNQLWqtkPklwFQlbjKYvZdIh2gsPO8HOgb16FIla/LIqAL+A9nGOoeEcknNb8mgK1HPykR+R0wb5Jdfw98FXjXZC+bZFtC566e6jhU9Umnzd8TGT54MPqySdon+xzcVKz5JCJSAPwc+O+q2p8sS17HSkTeB3Sq6ssi8tbo5kmapsLXxQNcAHxWVTeJyPdIoWGayVjQT0JV3zHZdhFZCSwEXnV+EOuAV0TkYpJwuYepjiNKRK4H3gdcrn+5oCLpjiMGqVjzCSKSRSTkH1TVx53NHSJSo6rtzhDg5MuiJo9LgatE5D1ADlBEpIdfIiIep1efKl+XFqBFVTc5zx8jEvSp9jU5wYZuToOq7lDVKlVtVNVGIt8QF6jqMSLLPXzCmX3zBsAX/TMvGUnkZjJfBq5S1eFxu1Jx2YpYlulISs449r3AHlX9zrhd45cVuR54cq5rOx2q+hVVrXN+LtYTWQbl48CzRJZFgRQ4DgDn5/moiCx1Nl1O5Or+lPqajGc9+vjZCLyHyMnLYeCvE1vOtL4PZAO/df46eVFVP52Ky1ZMtUxHgsuK1aXAdcAOEdnmbPsq8E/AoyJyI5HZXdckqL6Z+jLwsIh8C9iKc4IzBXwWeNDpODQT+Xl2kaJfE1sCwRhj0pwN3RhjTJqzoDfGmDRnQW+MMWnOgt4YY9KcBb0xxqQ5C3pjjElzFvQmI4hISES2ichOEfmZiOQ521VEHhjXziMiXdFldqd4r487yztvF5E/i8h503z2IRHZ4Xz+lvgdlTGxsaA3mcKvqqtVdQUwBnza2T4ErBCRXOf5O4HWad7rIPAWZ4nnfyS2e4m+zfn8NWdQuzEzYkFvMtFzwOJxz39FZHldgGuBn57qxar6Z1U97jwdv8SzMUnJgt5kFBHxELlByY5xmx8msr5PDrCKyDLBsbqRvyzxPBUFfiMiL4vITadTrzHxYGvdmEyRO24tmecYt+aKqm531oK/lsiaRTERkbcRCfrLpml6qaq2iUgVkbWF9qrqH0+neGNmwoLeZAq/qq4+xf4NwP8B3krkhhmnJCKriNxJ6UpV7TlVW1Vtc/7bKSJPELkjlgW9mTM2dGNMxH3A7aq6Y7qGItIAPA5cp6qvTdM2X0QKo4+J3LRmZxzqNSZm1qM3BlDVFiI3hI7FbUR6/T9wlngOnmI2TTXwhNPOAzykqr+eYbnGnBZbptgYY9KcDd0YY0yas6EbY6YgIn8NfH7C5udV9TOTtC0Hnp7kbS6f7mStMbPNhm6MMSbN2dCNMcakOQt6Y4xJcxb0xhiT5izojTEmzf1/1Tc3QDLzPl8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred = search.predict(X_test)\n",
    "sns.distplot(y_test-pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Abs Error: 8.277064511406477\n",
      "Mean Sq Error: 192.08441507308692\n",
      "Root Mean Error: 13.859452192387941\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "print(f\"Mean Abs Error: {metrics.mean_absolute_error(y_test, pred)}\")\n",
    "print(f\"Mean Sq Error: {metrics.mean_squared_error(y_test, pred)}\")\n",
    "print(f\"Root Mean Error: {np.sqrt(metrics.mean_squared_error(y_test, pred))}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "             colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
       "             importance_type='gain', learning_rate=0.05, max_delta_step=0,\n",
       "             max_depth=20, min_child_weight=3, missing=None, n_estimators=100,\n",
       "             n_jobs=1, nthread=None, objective='reg:squarederror',\n",
       "             random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=1,\n",
       "             seed=None, silent=None, subsample=0.8, verbosity=1)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(search, open('xgb.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}