odsc-feature-enginering-pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Titanic Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/titanic/train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 12 columns):\n",
      "PassengerId    891 non-null int64\n",
      "Survived       891 non-null int64\n",
      "Pclass         891 non-null int64\n",
      "Name           891 non-null object\n",
      "Sex            891 non-null object\n",
      "Age            714 non-null float64\n",
      "SibSp          891 non-null int64\n",
      "Parch          891 non-null int64\n",
      "Ticket         891 non-null object\n",
      "Fare           891 non-null float64\n",
      "Cabin          204 non-null object\n",
      "Embarked       889 non-null object\n",
      "dtypes: float64(2), int64(5), object(5)\n",
      "memory usage: 83.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Select Features / Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features = [\"Sex\", \"Age\", \"Fare\", \"Pclass\", \"SibSp\",  \"Parch\", \"Embarked\"]\n",
    "X = df[features].copy()\n",
    "y = df[\"Survived\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 7 columns):\n",
      "Sex         891 non-null object\n",
      "Age         714 non-null float64\n",
      "Fare        891 non-null float64\n",
      "Pclass      891 non-null int64\n",
      "SibSp       891 non-null int64\n",
      "Parch       891 non-null int64\n",
      "Embarked    889 non-null object\n",
      "dtypes: float64(2), int64(3), object(2)\n",
      "memory usage: 48.8+ KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## To Emphasise One hot encoding, I am going to fill the missing values of Embarked with Frequently used values\n",
    "## Also since only to values of Embarked are missing, I am going to fill it with Most common Embarked Station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "S    644\n",
       "C    168\n",
       "Q     77\n",
       "Name: Embarked, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.Embarked.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X[\"Embarked\"] = X[\"Embarked\"].fillna(\"S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 891 entries, 0 to 890\n",
      "Data columns (total 7 columns):\n",
      "Sex         891 non-null object\n",
      "Age         714 non-null float64\n",
      "Fare        891 non-null float64\n",
      "Pclass      891 non-null int64\n",
      "SibSp       891 non-null int64\n",
      "Parch       891 non-null int64\n",
      "Embarked    891 non-null object\n",
      "dtypes: float64(2), int64(3), object(2)\n",
      "memory usage: 48.8+ KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Sex   Age     Fare  Pclass  SibSp  Parch Embarked\n",
       "0    male  22.0   7.2500       3      1      0        S\n",
       "1  female  38.0  71.2833       1      1      0        C"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pre-Processing Modules in Scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder, LabelBinarizer, Imputer, FunctionTransformer\n",
    "from sklearn.preprocessing import  OneHotEncoder, Imputer, StandardScaler, LabelBinarizer, LabelEncoder\n",
    "\n",
    "from sklearn.pipeline import Pipeline, make_pipeline, FeatureUnion, make_union"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Make a Pipeline of "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Encode Values - Pclass, Sex and Embarked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.],\n",
       "       [ 1.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# One Hot Encode Pclass\n",
    "\n",
    "class_encoder = OneHotEncoder(sparse=False)\n",
    "class_encoder.fit_transform(X[[\"Pclass\"]])[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "could not convert string to float: 'Q'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-0937c57b2332>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# One Hot Encoding doesn't work on Text Data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mgender_encoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mgender_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Embarked\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/preprocessing/data.py\u001b[0m in \u001b[0;36mfit_transform\u001b[0;34m(self, X, y)\u001b[0m\n\u001b[1;32m   1890\u001b[0m         \"\"\"\n\u001b[1;32m   1891\u001b[0m         return _transform_selected(X, self._fit_transform,\n\u001b[0;32m-> 1892\u001b[0;31m                                    self.categorical_features, copy=True)\n\u001b[0m\u001b[1;32m   1893\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1894\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/preprocessing/data.py\u001b[0m in \u001b[0;36m_transform_selected\u001b[0;34m(X, transform, selected, copy)\u001b[0m\n\u001b[1;32m   1695\u001b[0m     \u001b[0mX\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0marray\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0msparse\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_features_new\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1696\u001b[0m     \"\"\"\n\u001b[0;32m-> 1697\u001b[0;31m     \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'csc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFLOAT_DTYPES\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1698\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1699\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mselected\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring_types\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mselected\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"all\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[0;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[1;32m    380\u001b[0m                                       force_all_finite)\n\u001b[1;32m    381\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m         \u001b[0marray\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mensure_2d\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: could not convert string to float: 'Q'"
     ]
    }
   ],
   "source": [
    "# One Hot Encoding doesn't work on Text Data\n",
    "gender_encoder = OneHotEncoder(sparse=False)\n",
    "gender_encoder.fit_transform(X[[\"Embarked\"]])[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unfortunately One Hot Encoder Doesn't work with Categorical Values :-("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Let's try Label Encoder and Label Binarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, 0, 1])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gender_encoder = LabelEncoder()\n",
    "gender_encoder.fit_transform(X[\"Sex\"])[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 0, 2, 2, 2])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Let's do the same for Embarked Station\n",
    "\n",
    "embarked_encoder = LabelEncoder()\n",
    "embarked_values = embarked_encoder.fit_transform(X[\"Embarked\"])[:5]\n",
    "embarked_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## We will then need to pass those values to One Hot Encoder to encode into binary values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label Encoding and One Hot Encoding in one step via LabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 1],\n",
       "       [1, 0, 0],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1],\n",
       "       [0, 0, 1]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embarked_encoder = LabelBinarizer()\n",
    "embarked_encoder.fit_transform(X[\"Embarked\"])[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OK.  That looks good.  Let's put these together in a Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "fit_transform() takes 2 positional arguments but 3 were given",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-43651ac3df5d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmake_pipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLabelEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLogisticRegression\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Sex\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    266\u001b[0m             \u001b[0mThis\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    267\u001b[0m         \"\"\"\n\u001b[0;32m--> 268\u001b[0;31m         \u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    269\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    270\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36m_fit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    232\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"fit_transform\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                 \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m                 \u001b[0mXt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params_steps\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: fit_transform() takes 2 positional arguments but 3 were given"
     ]
    }
   ],
   "source": [
    "make_pipeline(LabelEncoder(), LogisticRegression()).fit(X[[\"Sex\"]], y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label Encoders Don't work with Pipelines :-(\n",
    "\n",
    "## Why?\n",
    "\n",
    "### Github Issues:\n",
    "    * https://github.com/scikit-learn/scikit-learn/pull/7327\n",
    "    * https://github.com/scikit-learn/scikit-learn/pull/7375\n",
    "    \n",
    "**Note**: Looks like they are looking to fix OneHotEncoding to take Categorical Values in Version 0.19 instead of fixing LabelEcoders to work with pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For now, We need to go back to Pandas and encode Gender and Embarked to Numbers before using OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>35.0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Sex   Age     Fare  Pclass  SibSp  Parch  Embarked\n",
       "0    1  22.0   7.2500       3      1      0         1\n",
       "1    0  38.0  71.2833       1      1      0         2\n",
       "2    0  26.0   7.9250       3      0      0         1\n",
       "3    0  35.0  53.1000       1      1      0         1\n",
       "4    1  35.0   8.0500       3      0      0         1"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# encode string features as numbers\n",
    "X['Sex'] = X['Sex'].map({\"male\": 1, \"female\": 0})\n",
    "X['Embarked'] = X['Embarked'].replace({'S': 1, 'C': 2, 'Q': 3})\n",
    "\n",
    "X.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now we can put them all together in a Pipeline :-)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('features', OneHotEncoder(categorical_features='all', dtype=<class 'numpy.float64'>,\n",
       "       handle_unknown='error', n_values='auto', sparse=True)), ('model', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Pipeline([\n",
    "        (\"features\", OneHotEncoder()),   # One Hot Encode Values\n",
    "        (\"model\", LogisticRegression())  # Model\n",
    "        ]).fit(X[[\"Pclass\", \"Sex\", \"Embarked\"]], y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's add other Features as well using FeatureUnion and make one Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from column_selector import ColumnSelector, ColumnImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "## Also Split data into train and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score on Test Data: 0.812\n"
     ]
    }
   ],
   "source": [
    "feature_mapper = make_union(\n",
    "    make_pipeline(ColumnSelector([\"Pclass\", \"Sex\", \"Embarked\"]), OneHotEncoder()),\n",
    "    make_pipeline(ColumnSelector([\"Age\", \"Fare\"]), Imputer(), StandardScaler()),  # Mean Zero\n",
    "    ColumnSelector([\"SibSp\", \"Parch\"])\n",
    ")\n",
    "\n",
    "# Put the combined Features into a Pipeline\n",
    "model_pipe = Pipeline([\n",
    "        (\"features\", feature_mapper),   # Combined Features\n",
    "        (\"model\", LogisticRegression())  # Model\n",
    "        ]).fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy Score on Test Data: {:.3f}\".format(model_pipe.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hurray - We Put it all together in a Pipeline\n",
    "\n",
    "#### But can we do better and not have to manually convert Labels into Numbers before passing into OneHotEncoder?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enter Sklearn-Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#! pip install sklearn-pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/Users/rameshsampath/miniconda2/envs/py35/lib/python3.5/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn_pandas import DataFrameMapper\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Sex   Age     Fare  Pclass  SibSp  Parch Embarked\n",
       "0    male  22.0   7.2500       3      1      0        S\n",
       "1  female  38.0  71.2833       1      1      0        C"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = [\"Sex\", \"Age\", \"Fare\", \"Pclass\", \"SibSp\",  \"Parch\", \"Embarked\"]\n",
    "X = df[features].copy()\n",
    "y = df[\"Survived\"]\n",
    "\n",
    "# Train / Test Split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "X.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score on Test Data: 0.825\n"
     ]
    }
   ],
   "source": [
    "mapper = DataFrameMapper([\n",
    "                ([\"Pclass\"], OneHotEncoder()),\n",
    "                ([\"Sex\"], LabelEncoder()),\n",
    "                ([\"Age\", \"Fare\"], [Imputer(strategy=\"mean\"), StandardScaler()]),\n",
    "                (\"Embarked\", [ColumnImputer(strategy=\"most_frequent\"), LabelBinarizer()]),\n",
    "                ([\"SibSp\", \"Parch\"], None)  # No Transformations necessary\n",
    "            ])\n",
    "\n",
    "pipeline = Pipeline(steps=[\n",
    "        (\"features\", mapper),\n",
    "        (\"model\", LogisticRegression())\n",
    "    ])\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy Score on Test Data: {:.3f}\".format(pipeline.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Awesome.  We were able to use LibaryBinarizer and Encoder without having to convert to Numbers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Engineering \n",
    "\n",
    "* Polynomial Features\n",
    "* Dimensionality Reduction\n",
    "* Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.feature_selection import SelectFromModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score on Test Data: 0.834\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline(steps=[\n",
    "        (\"features\", mapper),\n",
    "        (\"selection\", PCA(n_components=0.95)),\n",
    "        (\"model\", LogisticRegression())\n",
    "    ])\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy Score on Test Data: {:.3f}\".format(pipeline.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Put them all Together and Grid Search for Hyper Parameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.grid_search import GridSearchCV, RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Parameters:  {'rf_model__max_depth': 10, 'rf_model__min_samples_leaf': 5, 'rf_model__n_estimators': 200}\n",
      "Accuracy Score on Test Data: 0.830\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline(steps=[\n",
    "        (\"features\", mapper),\n",
    "        (\"rf_model\", RandomForestClassifier())\n",
    "    ])\n",
    "\n",
    "params = {\n",
    "    \"rf_model__n_estimators\": [10, 100, 200],\n",
    "    \"rf_model__min_samples_leaf\": [1, 3, 5],\n",
    "    \"rf_model__max_depth\": [None, 10, 7, 5, 3]\n",
    "}\n",
    "\n",
    "grid = RandomizedSearchCV(pipeline, param_distributions=params, n_iter=5, n_jobs=-1)\n",
    "\n",
    "grid.fit(X_train, y_train)\n",
    "print(\"Best Parameters: \", grid.best_params_)\n",
    "print(\"Accuracy Score on Test Data: {:.3f}\".format(grid.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load UCI Income Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "columns = [\"age\", \"workclass\", \"fnlwgt\", \"education\", \"education_num\", \"maritial_status\", \"occupation\", \"relationship\",\n",
    "          \"race\", \"sex\", \"capital_gain\", \"capital_loss\", \"\", \"native_country\", \"income\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data\", header=None, names=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 32561 entries, 0 to 32560\n",
      "Data columns (total 15 columns):\n",
      "age                32561 non-null int64\n",
      "workclass          32561 non-null object\n",
      "fnlwgt             32561 non-null int64\n",
      "education          32561 non-null object\n",
      "education_num      32561 non-null int64\n",
      "maritial_status    32561 non-null object\n",
      "occupation         32561 non-null object\n",
      "relationship       32561 non-null object\n",
      "race               32561 non-null object\n",
      "sex                32561 non-null object\n",
      "capital_gain       32561 non-null int64\n",
      "capital_loss       32561 non-null int64\n",
      "                   32561 non-null int64\n",
      "native_country     32561 non-null object\n",
      "income             32561 non-null object\n",
      "dtypes: int64(6), object(9)\n",
      "memory usage: 3.7+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>workclass</th>\n",
       "      <th>education_num</th>\n",
       "      <th>maritial_status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>native_country</th>\n",
       "      <th>income</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10382</th>\n",
       "      <td>22</td>\n",
       "      <td>State-gov</td>\n",
       "      <td>13</td>\n",
       "      <td>Married-civ-spouse</td>\n",
       "      <td>Exec-managerial</td>\n",
       "      <td>Husband</td>\n",
       "      <td>White</td>\n",
       "      <td>Male</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13776</th>\n",
       "      <td>64</td>\n",
       "      <td>Local-gov</td>\n",
       "      <td>9</td>\n",
       "      <td>Widowed</td>\n",
       "      <td>Other-service</td>\n",
       "      <td>Unmarried</td>\n",
       "      <td>White</td>\n",
       "      <td>Female</td>\n",
       "      <td>United-States</td>\n",
       "      <td>&lt;=50K</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       age   workclass  education_num      maritial_status        occupation  \\\n",
       "10382   22   State-gov             13   Married-civ-spouse   Exec-managerial   \n",
       "13776   64   Local-gov              9              Widowed     Other-service   \n",
       "\n",
       "      relationship    race      sex  native_country  income  \n",
       "10382      Husband   White     Male   United-States   <=50K  \n",
       "13776    Unmarried   White   Female   United-States   <=50K  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = [\"age\", \"workclass\", \"education_num\", \"maritial_status\", \"occupation\", \"relationship\",\n",
    "          \"race\", \"sex\", \"native_country\", \"income\"]\n",
    "\n",
    "X = df[features]\n",
    "y = df[\"income\"]\n",
    "\n",
    "# Train / Test Split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "X.sample(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Put togehter a Pipeline\n",
    "\n",
    "* We can Encode the Categories into Numbers (via Pandas - Map and Replace)\n",
    "* Then put them in a Pipeline\n",
    "\n",
    "OR\n",
    "\n",
    "#### Just use SKLearn-Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy Score on Test Data: 0.831\n"
     ]
    }
   ],
   "source": [
    "mapper = DataFrameMapper([\n",
    "                (\"workclass\", LabelBinarizer()),\n",
    "                (\"maritial_status\", LabelBinarizer()),\n",
    "                (\"occupation\", LabelBinarizer()),\n",
    "                (\"relationship\", LabelBinarizer()),\n",
    "                (\"race\", LabelBinarizer()),\n",
    "                (\"sex\", LabelBinarizer()),\n",
    "                (\"native_country\", LabelBinarizer()),\n",
    "                (\"workclass\", LabelBinarizer()),\n",
    "                (\"workclass\", LabelBinarizer()),\n",
    "                ([\"age\", \"education_num\"], None)  # No Transformations necessary\n",
    "            ])\n",
    "\n",
    "pipeline = Pipeline(steps=[\n",
    "        (\"features\", mapper),\n",
    "        (\"model\", LogisticRegression())\n",
    "    ])\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(\"Accuracy Score on Test Data: {:.3f}\".format(pipeline.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Example - Ham vs Spam\n",
    "\n",
    "* UCI Dataset - https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/smsspamcollection/SMSSpamCollection\", sep=\"\\t\", header=None, names=[\"target\", \"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df[\"text\"]\n",
    "y = df[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ham     4825\n",
       "spam     747\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert Target to Int 0/1\n",
    "y = (y == \"spam\").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Split into train/test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Build a Model\n",
    "log_reg_model = Pipeline(steps=[\n",
    "        (\"vect\", CountVectorizer()), \n",
    "        (\"model\", LogisticRegression())\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip...ty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Predictions\n",
    "y_pred = log_reg_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98504784688995217"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy\n",
    "log_reg_model.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1445,    3],\n",
       "       [  22,  202]])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confusion Matrix\n",
    "metrics.confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC / AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred_prob = log_reg_model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fpr, tpr, threshold = metrics.roc_curve(y_test, y_pred_prob[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "roc_auc = metrics.auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAF6CAYAAABcEv/JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VeXV9/HfYhLCJIMWJUwiFlBxqAKKQxQQ0Ocp2koL\nCIrFR9tKHaoCQluwb8ukItWOKBQcKlacsCoiKEQmbUtAZVCBGAEhgoAJERI4We8fOcQQknBIcrLP\nOfl+risX2cPZeyWHhMW+73Uvc3cBAAAgNtQIOgAAAAB8i+QMAAAghpCcAQAAxBCSMwAAgBhCcgYA\nABBDSM4AAABiSK2gA4iUmbHmBwAAiBvubuV5XdwkZ5LEmmzxa/z48Ro/fnzQYaAceO/iG+9ffOP9\ni19m5crLJDGsCQAAEFNIzgAAAGIIyRmqREpKStAhoJx47+Ib71984/2rnixe5nGZmcdLrAAAoHoz\ns3IXBPDkDAAAIIaQnAEAAMQQkjMAAIAYQnIGAAAQQ0jOAAAAYkhUkzMzm2FmmWb2QRnnPGpmn5rZ\najM7N5rxAAAAxLpoPzn7u6Q+pR00s36S2rt7B0m3SfprlOMBAACIaVFNztx9qaQ9ZZzSX9KT4XPf\nk9TYzL4TzZgAAABiWdBzzlpK2lJke1t4HwAAQLVUK+gAAAAx6MVrpPTXg44CiDupm9qoWf1vKnSN\noJOzbZJaFdlODu8r0fjx4ws/T0lJoecYkAhIAgAkgGdX1df/W9hG6788SW2b7JW0s9zXinpvTTNr\nK+lVdz+7hGNXS7rd3a8xs+6Sprl791KuQ29NIBE9XK7Wc6gK7a6WfvBa0FEAMS07O1cTJryrqVNX\nKi8vpKSk2ho1qofGjUspd2/NqCZnZvYPSSmSmknKlDROUh1J7u7Tw+f8UVJfSTmSbnb3VaVci+QM\n8Y+nRKW7h59vAPElFMpX585/1ieffCVJGjKkiyZO7Knk5EYVanwe1WFNdx8cwTkjohkDYgiJCUrT\n7uqgIwCA41azZg0NH36eXnppg6ZN66Nu3ZIr5bpRH9asLDw5SwAMXxVgqAgAEsahQ/mqWdNkduS/\ncTH75AzllOhPmBi+AgDEkaysXD355BrdfvuFRyVhtWpV/qpkJGflkejJUzQxfAUAiBOhUL5mzVqt\nsWPfVmZmjpo3T9LAgWdF/b6JnZzFcxLF0BcAAIFJTc3QXXfNV1raDklS9+7JOv30plVy78ROzqKZ\nmJE8AQCQkF5//VNdc80/JEnJyY00ZUovDRx41lFDmtGSuMnZi9d8+zlznAAAQIR69z5NF1xwqv73\nf8/QvfderKSk2lV6/8St1jxcGcgTLgAAcJzy8101apT/SVlFqjWDbnwefSRmAACgBEuWfKZXX/24\nxGMVScwqKnGGNeN58j8AAKgy6el7dN99b+mFF9arRYsG+uSTtmrY8ISgwyoU/8lZWUkZyzYAAICw\nkvpg/uxnF0RlrbKKiP/krGhixvwyAABQih/9aK7mz98o6cg+mLEm/pOzw6jIBAAAZRg9uoe+/vqA\nHnmk8vpgRkN8VmuWNJRJcgYAAI7B3atkvbLqV61ZPDFjbhkAAFDBvLLx4xdrz579JR6vqoVkKyK+\nhzV5WgYAAFSwLtmsWas1ZswiZWbmhIcv+wYdVrnEV3LGchkAAKCYkvpgVkWD8miJr+SseGUmAACo\n1j799CtdfvksScH0wYyG+ErOJJbLAAAAhTp0aKZbbz1fLVs2CqQPZjTEV7XmQ2KeGQAAiHnVr1oT\nAABUK6mpGXrwwWVBh1El4m9YEwAAVBvp6Xs0cuRCzZ27TjVqmK66qr3OOadF0GFFFckZAACIOdnZ\nuZo4cammTl2h3NyCPpijRvVQhw7Ngg4t6kjOAABAzBk3brEeeWSlpNjugxkNJGcAACDmjBrVQ2vX\n7tRvf5sS030wo4FqTQAAgEpGtSYAAIg72dm5uv/+hXr//W1BhxJTGNYEAABVKhTK1+zZawr7YKam\nfq6lS2+O61X9KxPJGQAAqDLF+2BedFGyHnmkD4lZESRnAACgSuTk5Om6657T7t371apVI02eHP99\nMKOB5AwAAFSJ+vXraNKkntq+fV/C9MGMBqo1AQAAKhnVmgAAIGakpmboppteViiUH3QocYlhTQAA\nUCmK9sGUpF692mno0HMCjir+kJwBAIAKyc7O1YQJ72rq1JXKy/u2D+YPf9g56NDiEskZAACokBdf\nXK9Jk5ZJqn59MKOB5AwAAFTIkCFd9O67n+uWW85X9+7Vqw9mNFCtCQAAUMmo1gQAAFF1uA/m9On/\nDTqUhMewJgAAKFXxPpjNmtXTkCFdWEA2ikjOAABAiYr3wezePVnTpvUhMYsykjMAAHAUd9eoUQuV\nlrZDyckFfTAHDaIPZlUgOQMAAEcxM02b1kdvvrmJPphVjGpNAACASlZ9qjXbXR10BAAAJJTU1Ayl\npMxSZua+oENBWHwlZz94LegIAABICOnpezRgwPO6/PJZWrIkQw89tDzokBDGnDMAAKqR4n0w69Wr\npVGjeui++3oEHRrC4mvOWZzECgBArFqzZofOO+9vcqcPZjRVZM4ZyRkAANXMww8v1yWXtFa3bvTB\njBaSMwAAcBR3Z12ygFSfak0AAHBMWVkFfTBvvfXVoENBOfDkDACABBEK5WvWrNUaO/ZtZWbmyEza\nuPEOnXZak6BDq3Yq8uSMak0AABJAaX0wScziD8kZAAAJ4MUX19MHM0EwrAkAQALYvXu//vrX/+iu\nu7rTBzMGUK0JAEA1QQVmfIjpak0z62tmG8zsEzMbVcLxRmY2z8xWm9mHZjYs2jEBABCPUlMz9L3v\nTdeSJZ8FHQqiKKrJmZnVkPRHSX0knSlpkJl1LHba7ZLWuvu5kq6Q9LCZMRcOAICwon0w09J26KGH\nVgQdEqIo2klQV0mfunuGJJnZHEn9JW0oco5Lahj+vKGkr9z9UJTjAgAg5uXk5On3v39XU6euUG5u\nSElJtTVqVA/de+/FQYeGKIp2ctZS0pYi21tVkLAV9UdJ88zsC0kNJP04yjEBABAX8vNdM2emKTc3\nRB/MaiQWhg/7SEpz9yvNrL2kt8ysi7vvCzowAACC1LDhCZox4/tq3jyJPpjVSLSTs22SWhfZTg7v\nK+pmSRMlyd03mVm6pI6S/lP8YuPHjy/8PCUlRSkpKZUbLQAAAQmF8lWz5tFTwa+55owAosHxWrx4\nsRYvXlwp14rqUhpmVlPSx5J6Stou6X1Jg9x9fZFz/iTpS3d/wMy+o4Kk7Bx3313sWiylAQBIONnZ\nuZo4canefHOT3nvvFtWqRdvrRBCz7ZvcPWRmIyQtUEFl6Ax3X29mtxUc9umSfidplpl9EH7ZyOKJ\nGQAAiSYUytfs2Ws0ZswiZWbmSJLefjtdV13VPuDIEDQWoQUAoIotX75FI0a8flQfTOaVJY6YfXIG\nAACO9tlne5WWtkOtWhX0wRw4kD6Y+BbJGQAAVWzQoLOUk5OnG27oQh9MHIVhTQAAoiQ/3xUK5at2\n7ZpBh4IqFtO9NQEAqI5SUzN0wQXTNW3ayqBDQZxhWBMAgEqUnr5HI0cu1Ny56yRJeXkh3XPPxapR\ngzlliAzJGQAAleDgwZB+85t3NHXqSuXlHdkHk8QMx4PkDACASlCrVg2999425eWFNHRoF02YQB9M\nlA8FAQAAVJL163cqKyuX9cpQoYIAkjMAAI5TTk6e6tevE3QYiGFUawIAUAWys3M1ZswitWr1iLZu\nzQo6HCQokjMAAI4hFMrXzJlp6tDhMU2cuFR79hzQa699EnRYSFAUBAAAUIbVq3foJz95hT6YqDIk\nZwAAlKFu3Vr68MMvlZxc0Adz0CD6YCK6KAgAAOAY5s/fqMsua0MfTESMak0AACooFMrXvn15aty4\nbtChIAFQrQkAQAWkpmbowgsf1y23vBp0KABzzgAA1VfxPpg7d36j3bv3q2nTegFHhuqM5AwAUC09\n8MBiTZiwVHl5IdWrV0ujRvXQfff1YF4ZAkdyBgColnJzQ8rLC2nIkC6aOJE+mIgdFAQAAKql7Oxc\nrVu3k/XKEBVUawIAUIrt27N1yikNgw4D1QzVmgAAFJOdnav771+otm3/oPnzNwYdDhAxkjMAQEIJ\nhfI1Y8YqdejwmCZNWqa8vJCWL98SdFhAxCgIAAAkjM2b9+j66/9JH0zENZIzAEDCaNGigXbt+kbJ\nyY00ZUovDRxIH0zEHwoCAAAJ5YMPMnX66U1ZrwyBoloTAFCthEL52ro1S23anBh0KECJqNYEAFQb\nh/tgXnnlk8rNPRR0OEClIzkDAMSFw5P9L798ltLSdigvL6SNG3cHHRZQ6SgIAADEvD/8YaVGjlyo\nvLyQkpJqa9SoHrr33ouZV4aERHIGAIh5bdqcSB9MVBsUBAAAYp67a+3anTrrrJODDgWICNWaAICE\nkJ6+RyefXF/169cJOhSgQqjWBADEtcN9MDt2/JMeemh50OEAgWLOGQAgMKFQvmbPXqMxYxYpMzNH\nkvTFF9kBRwUEi+QMABCIrKxcXXHFbK1atV0SfTCBw0jOAACBaNToBJ1ySgP6YALFUBAAAAjMjh37\n1KjRCaxXhoRDQQAAIGbl57v++98vSjzWokUDEjOgmIiSMzOrY2anRzsYAEBiSU3N0AUXTNfFF8/U\n5s17gg4HiAvHTM7M7BpJH0p6K7x9rpm9FO3AAADxKz19jwYMeL6wD+bJJ9fX1q1ZQYcFxIVICgJ+\nK6mbpHckyd1X8xQNAFCaOXM+0rBhLys3lz6YQHlEkpwddPe9xSpomJkPAChRt24tZWb0wQTKKZLk\nbL2Z/UhSDTNrJ+kOSSujGxYAIF61a9dEmzffoVNOaRh0KEBciqQgYISk70nKl/SipFxJd0YzKABA\n7EtP36P09JIn+ZOYAeUXSXLWx91Huft54Y/RkvpFOzAAQGwq2gfz9ttfDzocIOFEkpz9qoR9Yys7\nEABAbAuF8jVzZpo6dHhMkyYtU15eSM2bJyk391DQoQEJpdQ5Z2bWR1JfSS3NbGqRQ41UMMQJAKgm\n3F0pKbO1dOnnkqSLLkrWtGl91bVry4AjAxJPWQUBX0r6SNIBSWuL7M+WNDqaQQEAYouZ6aqrTlNG\nxl5NnkwfTCCajtlb08zquvuBKoqnrDjorQkAATpw4JDy8531yoAIRLu3Zkszm2NmH5jZJ4c/ynMz\nAEBsC4XyNW/exyrpP8N169YiMQOqQCTJ2SxJf5dkKqjS/Kek56IYEwAgAKmpGbrwwsfVv/8cvfzy\nhqDDAaqtSJKzJHd/U5LcfZO7/0ospQEACaN4H8xWrRqpVq1I/nkAEA2RdAjINbMakjaZ2U8lbZPE\n6oIAkADefTdDvXo9pby8gj6Yo0f30D330AcTCFIkBQHdJK2T1ETS7yU1ljTZ3ZdFP7wj4qAgAAAq\nWW7uIZ199l/UvXuyJkygDyZQWSpSEHDM5KyUG7Z0920RnttX0jQVDKHOcPfJJZyTIukRSbUl7XT3\nK0o4h+QMAKJg3748NWhQJ+gwgIQSteTMzC6U1FLSUnffZWZnShol6Up3T44gsBqSPpHUU9IXkv4t\naaC7byhyTmNJyyVd5e7bzKy5u+8q4VokZwBQTunpe7RlS5Yuu6xN0KEA1UJUltIws4mSnpF0g6T5\nZjZe0juS1kg6I8Lrd5X0qbtnuPtBSXMk9S92zmBJLxx+EldSYgYAKJ+ifTAHD35BOTl5QYcE4BjK\nKgjoL+kcd99vZk0lbZF0trtvPo7rtwy/7rCtKkjYijpDUm0ze0dSA0mPuvtTx3EPAEAxoVC+Zs9e\nozFjFikzM0eSdMUV7XTgwCHVr88QJhDLykrODrj7fkly991m9slxJmbHE8P5kq6UVF/SCjNb4e4b\ni584fvz4ws9TUlKUkpIShXAAIP7deOPL+sc/PpQkde+erGnT+qhbt2PORgFQTosXL9bixYsr5Vql\nzjkzs72S3j68KemKItty9x8c8+Jm3SWNd/e+4e3RBS/9tijAzEZJquvuD4S3n5D0hru/UOxazDkD\ngAgtWLBJw4fP0+TJvTRoEH0wgaoWlYIAM+tZ1gvdfVEEgdWU9LEKCgK2S3pf0iB3X1/knI6SHpPU\nV9IJkt6T9GN3X1fsWiRnAHAccnMP6YQTIlnOEkBlq0hyVupPbSTJ17G4e8jMRkhaoG+X0lhvZrcV\nHPbp7r7BzN6U9IGkkKTpxRMzAMDRQqF8Pf30B7ruuk5q1OiEo46TmAHxqVzrnAWBJ2cA8K3U1Azd\nddd8paXt0KhRPTRpUq+gQwJQRFSenAEAYk96+h6NHLlQc+cWDDAkJzfSeee1CDgqAJUp4uTMzE5w\n99xoBgMAKN2WLV+rU6c/KTc3pHr1amn06Et07730wQQSzTGTMzPrKmmGCnpqtjazcyTd4u6/iHZw\nAIBvtWrVWP37d1SdOjU1cSJ9MIFEFUnj85WSfizpZXc/L7zvI3c/qwriKxoHc84AVHuHDuWrVq1S\nm7sAiBFRad9U9Bx3zyi2L1SemwEAji09fY9mzVpd4jESMyDxRTLnbEt4aNPD65b9QgXNzAEAlSgr\nK1cTJ76rqVNXKhTKV7duLdWp00lBhwWgikWSnP1M0qOSWkvKlLQwvA8AUAlCoXzNmrVaY8e+XdgH\nc8iQLmrcuG7AkQEIQiRzzpq6++4qiqesOJhzBiAh/frXb+t3v3tXEn0wgUQRlfZNRS6+SQUtmJ6T\n9KK7Z5fnRhVFcgYgUW3blqVevZ7Sb35zmQYOpA8mkAiimpyFb3CxpIGSvi9ptaQ57j6nPDcsL5Iz\nAIksP99VowZJGZAool2tKXdf7u53SDpfUpakZ8pzMwCorkKhfM2cmaY1a3aUeJzEDMBhx0zOzKyB\nmd1gZq9Kel/STkkXRz0yAEgQqakZuvDCxzV8+Dzdeed8MQoAoCyRVGt+JOlVSVPc/d0oxwMACaOk\nPpi33vq9gKMCEOsiKQio4e75VRRPWXEw5wxA3MjNPaTWrafpyy9zlJRUW6NG9aAPJlCNRKUgwMwe\ndvd7zOwlSUed5O4/KM8Ny4vkDEC8efjh5Vq9OpM+mEA1FK3krKu7v29mPUs67u6LynPD8iI5AxBv\n3J1lMYBqKirVmu7+fvjTTu6+qOiHpE7luRkAJJr09D0aO3aR8vOP/s8jiRmA8ohkKY2flLBveGUH\nAgDxJDs7V2PGLFKnTn/ShAlL9Y9/fBh0SAASRKnVmmb2YxUsPNvOzF4scqihpL3RDgwAYlF+vmvW\nrNUaM2bREX0wU1LaBhsYgIRR1lIa70v6SlKypD8V2Z8tKS2aQQFArHr22Q81fPg8SfTBBBAdEbVv\nigUUBACIBYcO5et///dZ3XhjF/pgAihVtKo1l7j75Wa2R0cupWGS3N2blueG5UVyBgAA4kW0krMa\n7p5vZjVLOu7uofLcsLxIzgBUlVAoX7Nnr1FSUm0NHHhW0OEAiEMVSc5KnXNWpCtAK0lfuHuemV0i\nqYukp1XQAB0AEkpqaobuumu+0tJ26OST6+uaazqoYcMTgg4LQDUSyVIaL0tyM2sv6e+SOkj6R1Sj\nAoAqlp6+RwMGPK/LL5+ltLQdatWqkaZN66MGDeoEHRqAaiaSxuf57n7QzH4g6TF3f9TMqNYEkFCG\nDn1Jy5ZtoQ8mgMBFkpwdMrMBkoZKuja8j99YABLKxIk9NX36KvpgAgjcMZfSMLOzJP1c0nJ3f9rM\n2kka7O6/r4oAi8RBQQAAAIgLUemteZi7fyTpDkn/MbOOkrZUdWIGAJUhPX2Phg17Wbt37w86FAAo\n1TGHNc3sUklPSdqmgjXOWpjZUHdfFu3gAKAyZGfnasKEdzV16krl5YXUpEldPfJI36DDAoASRTLn\n7BFJV7v7Okkys04qSNYuiGZgAFBRJfXBHDq0i+655+KAIwOA0kWSnNU5nJhJkruvNzNqywHEvA8/\nzCzsg3nRRcmaNq2vunZtGXBUAFC2SAoCZkk6oIKFZyXpBklJ7n5TdEM7Kg4KAgAct/vvX6guXb5D\nH0wAVSoq7ZuKXLyuCgoCLgnvelcF650dKM8Ny4vkDAAAxIuoJWdmdrak9pLWuvun5YyvUpCcASjJ\n4T6Yn376lSZO7BV0OAAgKXqNz8dIGi5plaQLJf3W3WeWO8oKIjkDUFzRPphm0kcf/VydO58UdFgA\nEJ3G5yqYW9bF3XPM7CRJr0sKLDkDgMPS0/do5MiFmju3oFYpObmRJk/upU6dmgccGQBUXFnJWa67\n50iSu+80s0iapANA1D344HLNnbtO9erV0qhRPXTffT3ogwkgYZSVnJ1mZi+GPzdJ7Ytsy91/ENXI\nAKAU48en6ODBkMaNS6EPJoCEU9acs55lvdDdF0UlolIw5wwAAMSLqC6lEStIzoDq5fC8sl/+srsu\nuqhV0OEAwHGJVkEAAFS54n0wv/rqG739dpWueQ0AgSI5AxATDq9XVrQP5pAhXTRxYpkzLAAg4UQ8\nrGlmJ7h7bpTjKev+DGsCCWzv3gM6/fRH9dVX+9W9e7KmTeujbt2Sgw4LAMol2u2bukqaIamxu7c2\ns3Mk3eLuvyjPDcuL5AxIfM8884HMTIMG0QcTQHyLdnK2UtKPJb3s7ueF933k7meV54blRXIGAADi\nRUWSs0gWlq3h7hnF9oXKczMA1VsolK8ZM1apb9+nFQrlBx0OAMSkSAoCtoSHNt3Makr6haRPohsW\ngERTtA+mJL388gb98IedA44KAGJPJMOaJ0t6VFKv8K6Fkka4+64ox1Y8DoY1gThUUh/MKVN6aeBA\n5pUBSFxRXefM3b+UNLA8FweAJUsyNHfuOiUl1daoUT10770X0wcTAMpwzOTMzB6XdNQjK3e/NSoR\nAUgoN954jjZt2q3bbruAPpgAEIFIhjV/XGSzrqTrJG1hKQ0Axbk7Q5UAoCrurWlmNSQtdfeLy3PD\n8iI5A2LX5s17NHLkW7rssja6445uQYcDAIGr6uSsvaQF7t6+PDcsL5IzIPYU74OZnNxImzffodq1\nawYdGgAEKqrrnJnZHjPbHf7YK+ktSfcfR3B9zWyDmX1iZqPKOO9CMztoZj+I9NoAgpGf75o5M00d\nOjymSZOWKS8vpCFDumjFiuEkZgBQQWUWBFjB5JFzJG0L78o/nsdX4SHQP0rqKekLSf82s1fcfUMJ\n502S9OZxxA4gIGbSzJlpyszMoQ8mAFSyMpMzd3cze70CrZq6Svr0cIcBM5sjqb+kDcXO+4WkuZIu\nLOd9AFQhM9Ojj/bThg276IMJAJUskg4Bq83sPHdPK8f1W0raUmR7qwoStkJmdqqka939inAnAgAx\nJD/fVaPG0cnX+eefovPPPyWAiAAgsZU658zMDidu56lgOPJjM1tlZmlmtqoSY5gmqehcNP4LDsSA\nw/PKOnb8o3bs2Bd0OABQbZT15Ox9SedL+n4Frr9NUusi28n6dv7aYRdImhOe39ZcUj8zO+ju84pf\nbPz48YWfp6SkKCUlpQKhAShN8T6YM2as0tixlwUcFQDErsWLF2vx4sWVcq1Sl9IwszR3P69CFy9o\nlP6xCgoCtqsg4Rvk7utLOf/vkl519xdLOMZSGkCUff7517rnngX0wQSACopWb82TzOyXpR1096nH\nuri7h8xshKQFKhhCneHu683stoLDPr34SyIJGkB0fP31Ab344nr6YAJAgMp6crZd0l9Uyhwwd38g\ninGVFA9PzoAqMHv2avXseRp9MAGgAqLSIcDMVrn7+RWKrBKRnAGV6+DBEAvGAkCURKtDABNMgASU\nnr5HAwY8r2HDXgk6FABACcqac9azyqIAEHXZ2bmaOHGppk5dodzckOrXr60dO/apRYsGQYcGACii\n1OTM3XdXZSAAoufJJ9do5Mi3lJmZI0kaMqSLJk7sSWIGADEokg4BAOLc6tU76IMJAHGi1IKAWENB\nAFB+e/ce0Pz5G/XjH5/JemUAUAWiUq0Za0jOgGPbv/+g6tVjXTIACFq0qjUBxIlQKF8zZ6bptNMe\n1TvvpAcdDgCgAkjOgDiXmpqhCy98XMOHz9OOHfv0zDMfBh0SAKACKAgA4tSXX+bo9ttfL+yD2apV\nI02eXNAHEwAQv0jOgDiVlFRbS5d+rqSk2ho9uofuuYc+mACQCCgIAOJYamqGTjutCX0wASDGUK0J\nJLisrFw1anRC0GEAACJEtSaQoA73wezW7QkdPBgKOhwAQBVgzhkQg7KzczVhwruaOnWl8vJCqlev\nllat2s7K/gBQDZCcATFm7tx1GjHi9aP6YDKvDACqB5IzIMbk5zt9MAGgGqMgAIgx7q433tiofv1O\npw8mAMQpqjWBOJSdnavatWuqbl0eYANAoqFaE4gjh/tgdujwmP7wh5VBhwMAiDEkZ0AVKtoHMzMz\nR2+//Zl4IgwAKIrxFKAKfPPNQd1008uFfTCTkwv6YA4adBbzygAARyA5A6pAvXq1tHfvASUl1dao\nUT107730wQQAlIyCAKCKbNq0WyecUIv1ygCgGqBaE4ghW7dmkYABQDVHtSYQAw73wTzjjMf0+edf\nBx0OACBOMecMqKCS+mD+979fqHXrxkGHBgCIQyRnQAUsXLhZQ4a8SB9MAEClITkDKqB168bavXs/\nfTABAJWGggCggtLStuvcc1uwXhkAoBDVmkCUZWXl6ptvDqpFiwZBhwIAiANUawJREgrla8aMVTrj\njMf085+/FnQ4AIBqgDlnQCmWLPlMd9/9ptLSdkiStm/fp5ycPNWvXyfgyAAAiYzkDCjG3XXjjS/r\n6ac/kFTQB3PKlF4aOJA+mACA6CM5A4oxM7Vp05g+mACAQFAQAJRg37487d17gPXKAADlQkEAUE5r\n1uwocX+DBnVIzAAAgSA5Q7V0uA/muef+Ta+//mnQ4QAAUIg5Z6hWsrNzNXHiUk2dukK5uSElJdXW\n1q1ZQYcFAEAhkjNUG2vW7FCfPk/TBxMAENNIzlBtnHFGM9WtW4s+mACAmEa1JqqVLVu+VnJyI9Yr\nAwBEFdWaQBHZ2bn64IPMEo+1atWYxAwAENNIzpAw8vNdM2emqUOHx9S//xwdOHAo6JAAADhuJGdI\nCKmpGbq6FqwsAAAa5UlEQVTggukaPnyeMjNz1KJFA335ZU7QYQEAcNwoCEDcu+++BXrooRWSpFat\nGmnyZPpgAgDiF0/OEPcuu6yNkpJq67e/TdGGDSM0aNDZJGYAgLhFtSbinrtr585vdPLJ9YMOBQAA\nSVRropp4990M5eTkHbXfzEjMAAAJg+QMMe9wH8zLLpulKVOWBR0OAABRRUEAYlZ2dq4mTHhXjzyy\nsrAPZv36dYIOCwCAqCI5Q0z68sscdenyl8I+mEOHdtGECfTBBAAkPpIzxKSTT66vrl1bateubzRt\nWl917doy6JAAAKgSVGsiZmVl5aphwzosiwEAiDtUayJuZWfn6o03Pi3xWKNGJ5CYAQCqnagnZ2bW\n18w2mNknZjaqhOODzWxN+GOpmZ0d7ZgQvFAo/4g+mBs37g46JAAAYkJU55yZWQ1Jf5TUU9IXkv5t\nZq+4+4Yip22WdJm7f21mfSU9Lql7NONCsFJTM3TXXfOVlrZDktS9e7Jyc2lSDgCAFP2CgK6SPnX3\nDEkyszmS+ksqTM7cfWWR81dKYuZ3AvvDH1bqrrvelCQlJxf0wRw0iD6YAAAcFu1hzZaSthTZ3qqy\nk69bJL0R1YgQqGuv7aimTetp/PjL9fHHIzR4MH0wAQAoKmaW0jCzKyTdLOmSoGNB9LRpc6K2bLlb\nSUm1gw4FAICYFO3kbJuk1kW2k8P7jmBmXSRNl9TX3feUdrHx48cXfp6SkqKUlJTKihOVLDU1Qyef\nXF8dOzY/6hiJGQAg0SxevFiLFy+ulGtFdZ0zM6sp6WMVFARsl/S+pEHuvr7IOa0lLZI0tNj8s+LX\nYp2zOJCevkcjRy7U3LnrdNVV7TV//g0MWwIAqp2KrHMW1Sdn7h4ysxGSFqhgftsMd19vZrcVHPbp\nkn4tqamkP1vBv+IH3b1rNONC5TvcB3Pq1JXKywupXr1auvjiZOXnu2rWJDkDACBSdAhAhR06lK9O\nnf5UuFbZkCFdNHEifTABANVXzD45Q/VQq1YN3XzzuXr11U80bVofdeuWHHRIAADELZ6coVIcPBhS\nrVo1mF8GAIDorYkqkp2dq7/85d8qKUmuXbsmiRkAAJWAYU0cUyiUr1mzVmvs2LeVmZmjk06qr+uv\n7xx0WAAAJCSSM5RpyZLPdPfdbx7RB7Nt2xMDjgoAgMRFcoZSzZv3sfr3nyOpoA/mlCm9NHAgfTAB\nAIgmCgJQqry8kLp1e0LXXddR9957MSv7AwAQoYoUBJCcoUyhUL5q1qRuBACA40G1JiokNTVDr732\nSYnHSMwAAKha/MtbjW3evEfXX/9PXX75LN1667+Uk5MXdEgAAFR7FARUQ1lZuZo48ds+mElJtXXb\nbd9TjRpM9AcAIGjMOauGevd+SgsXbpZEH0wAAKKB3po4LqNH99C+fXn0wQQAIAbx5KyacnfWKwMA\nIEqo1sRRsrNzNX78Yu3de6DE4yRmAADEJoY1E0wolK/Zs9dozJhFyszMUU5Onh588KqgwwIAABEi\nOUsgqakZuuuu+Uf0waRBOQAA8YU5Zwliw4Zd6tTpT5LogwkAQNBo3wRJ0vDhr6hNmxPpgwkAQMBI\nzgAAAGII1ZrVSGpqhqZOXRF0GAAAIEooCIgT6el7NHLkQs2du041a5r69GmvM888OeiwAABAJSM5\ni3HZ2bmaOHGppk5dodzcgj6Yo0b1ULt2TYIODQAARAFzzmLcnXe+oUcffV8SfTABAIgXFAQksO3b\ns3XjjS/rd7+7gj6YAADECZIzAACAGEK1ZpzLzs7VmDGLtGrV9qBDAQAAAaMgIEDF+2AuX75FixcP\nCzosAAAQIJKzgBTvg3nRRcmaMqV3wFEBAICgMecsANnZuWrdepr27j2gVq0aafJk+mACAJBIKAiI\nQ3/5y7+1c+c39MEEACABkZwBAALVtm1bZWRkBB0GUOXatGmjzz777Kj9JGcxKjU1Q7Nnr9bjj39f\nNWowZAkgcYX/IQo6DKDKlfZ3vyLJGQUBUVC0D6Yk9ex5mgYPPjvgqAAAQDwgOatE2dm5mjDhXU2d\nulJ5ed/2wbz22o5BhwYAAOIEyVkleu65tZo0aZkk+mACAIDyoUNAJRo27FwNG3auVq4crqeeuo7E\nDAAQc9atW6cLL7ww6DDiwpdffqnOnTvr4MGDVXpfkrNKVKtWDf397/1pUA4AMaRt27ZKSkpSo0aN\ndOqpp+rmm2/WN998c8Q5y5cvV8+ePdWoUSM1adJE/fv31/r16484Jzs7W3fddZfatGmjRo0aqUOH\nDvrlL3+p3bt3V+WXU2G/+c1vNHLkyKDDqJC8vDz95Cc/UePGjXXqqafqkUceKfP83//+92rTpo1O\nPPFEDR48WPv27Ss89sUXX+jaa69Vs2bN1Lp1a/3tb38rPHbyySfryiuvPGJfVSA5O06H+2DOnJkW\ndCgAgAiYmV577TVlZWVp9erVSktL08SJEwuPr1ixQn369NF1112n7du3Kz09XV26dFGPHj0Kl0g4\nePCgrrzySq1fv14LFixQVlaWVqxYoebNm+v999+PWuyhUKhSr7djxw4tXrxY/fv3j4l4ymvcuHHa\ntGmTtmzZorfffltTpkzRggULSjx39uzZeuaZZ7RixQp98cUX+uabbzRixIjC40OGDFH79u21c+dO\n/etf/9KYMWO0ZMmSwuODBw+u8uRM7h4XHwWhBufQoZDPmLHKv/OdB10a7yef/KDv338w0JgAIFYE\n/Tu6LG3btvVFixYVbo8cOdL/53/+p3D70ksv9REjRhz1un79+vlNN93k7u6PP/64t2jRwr/55puI\n7/vRRx957969vWnTpt6iRQufOHGiu7sPGzbMf/3rXxeet3jxYk9OTj4i3smTJ3uXLl28bt26Pnny\nZL/++uuPuPYdd9zhd955p7u7f/311z58+HA/5ZRTPDk52X/1q195fn5+iTE9+eST3rt37yP2TZo0\nydu3b+8NGzb0M88801966aXCY7NmzfIePXr43Xff7c2aNSuMe8aMGd6pUydv2rSp9+3b1zMyMgpf\nc+edd3qrVq28UaNGfsEFF/i7774b8fcsUqeeeqovXLiwcPs3v/mNDxo0qMRzr7/+en/wwQcLt5cv\nX+5169b1/fv3+759+9zMfNeuXYXHb731Vr/xxhsLtw8dOuRJSUn++eefl3j90v7uh/eXK+fhyVkE\nUlMzdOGFj2v48HnKzMxR9+7JmjdvoOrWpZ4CAOLJ1q1b9cYbb6hDhw6SpP3792v58uW6/vrrjzr3\nRz/6kd566y1J0qJFi9S3b1/Vq1cvovvs27dPvXv31tVXX63t27dr48aN6tmzZ6nnF2/fN2fOHL3x\nxhvau3evBg4cqDfeeEM5OTmSpPz8fD3//PO64YYbJEk33XST6tSpo82bNystLU1vvfWWnnjiiRLv\n8+GHH+q73/3uEftOP/10LVu2TFlZWRo3bpyGDBmizMzMwuPvvfeeTj/9dH355ZcaO3asXnnlFU2a\nNEkvv/yydu7cqUsvvVSDBg0qPL9r16764IMPtGfPHg0ePFgDBgxQXl5eifFMnjxZTZo0UdOmTdWk\nSZMjPm/atGmJr9m7d6+2b9+uLl26FO4755xztHbt2tK+vUfIz89XXl6ePv30U7n7UeuUubs++uij\nwu2aNWvq9NNP15o1ayK6fqUob1ZX1R8K6H9l+fn5fuGF010a78nJU/2ZZz4o9X8kAFBdlfk7+iFV\n3kc5tG3b1hs2bOgNGzZ0M/NevXr5119/7e7uW7dudTPzjz/++KjXzZ8/3+vUqePu7r179/b7778/\n4ns+++yzfv7555d4rKQnZ61atToi3lmzZh3xmksvvdSfeuopd3dfsGCBn3766e7uvmPHDj/hhBP8\nwIEDR9z7iiuuKPHe//d//3fMr+Pcc8/1efPmuXvBk7M2bdoccbxfv34+c+bMwu1QKFTmk6UmTZr4\nBx98UOY9j8eWLVu8Ro0anpubW7jvrbfe8nbt2pV4/hNPPOHf/e53/bPPPvO9e/f697//fa9Ro4av\nXLnS3Qu+t3fccYcfOHDA//vf/3rTpk29Y8eOR1yjR48ehd//4kr7uy+enEWPmWnatL4aP/5yffzx\nCA0efDYNygEgzrzyyivKysrSkiVLtGHDBu3atUuS1KRJE9WoUUPbt28/6jXbt29X8+bNJUnNmjUr\n8ZzSbNmyRe3bty93vMnJRxaWDRo0SM8++6wk6dlnn9XgwYMlSZ9//rkOHjyoU045pfCJ009/+tPC\nr6+4Jk2aKDs7+4h9Tz75pM4777zCJ1dr16494vWtWrU64vyMjAzdeeedatq0qZo2bapmzZrJzLRt\n2zZJ0kMPPaTOnTsXXi8rK6vUeMqjQYMGkqSsrKzCfV9//bUaNmxY4vk/+clPNGjQIKWkpOjss8/W\nlVdeKenb7/EzzzyjzZs3q3Xr1rr99ts1dOjQo77/2dnZOvHEEyvtazgWxuUicPHFrXTxxa2OfSIA\n4Gj3BN/WycPDVpdeeqluuukm3XPPPXrppZeUlJSkiy66SM8//7wuv/zyI17zz3/+U7169ZIk9erV\nS7/+9a+1f//+iIY2W7VqpTlz5pR4rH79+kdUi5aU9BV/CDBgwADde++92rZtm1566SWtXLmy8D51\n69bVV199FdGDgy5duujJJ58s3P78889166236p133tFFF10kSTrvvPOOGOYrft3WrVvrV7/61RFD\nmYctXbpUDz74oN555x117txZktS0adMjrlfUxIkTNWHChKPu4eHhxqIJ2GEnnniiTjnlFK1Zs6Zw\nqHjNmjU688wzS7yHmWncuHEaN26cJGnBggVq2bKlWrZsKange/jqq68Wnn/DDTeoa9euhduhUEgb\nN27UOeecU+L1o6K8j9yq+kNRHtZcsuQzv+KKWb5zZ05U7wMAiSjav6MronhBwM6dO71+/fqFQ21L\nly71Bg0a+GOPPebZ2dm+e/duHzt2rDdp0sQ3btzo7u65ubnetWtX79evn2/YsMHz8/N9165dPmHC\nBH/jjTeOumd2drafeuqp/oc//MFzc3M9Ozvb33vvPXcvKC7o1KmT796927dv3+7du3c/alizaLyH\n9evXz3v37n3UcOm1117rd955p2dlZXl+fr5v2rTJlyxZUuL3IjMz05s3b144JLhu3TqvV6+ef/LJ\nJx4KhXzmzJleq1YtnzFjhrsXDGteeumlR1zjpZde8rPOOsvXrl3r7u579+71559/3t3dX3/9dW/Z\nsqXv2LHDc3Nz/YEHHvBatWqV+PVUxOjRoz0lJcX37Nnj69at8xYtWviCBQtKPHf37t2+adMmd3df\nu3atn3XWWf7EE08UHl+/fr1nZ2d7Xl6eP/XUU37SSScdUSCwfPlyP/PMM0uNpbS/+2JYs/zS0/do\nwIDndfnls/TOO5/p4YeXBx0SAKASFX8q07x5c91000367W9/K0nq0aOH3nzzTb3wwgs65ZRT1K5d\nO61Zs0bLli0rHJqsU6eOFi5cqI4dO6p3795q3Lixunfvrq+++krdunU76p4NGjTQW2+9pXnz5qlF\nixY644wztHjxYknS0KFD1aVLF7Vt21Z9+/bVwIEDy4z3sMGDB2vRokWFhQCHPfnkk8rLy1Pnzp3V\ntGlTDRgwQDt27CjxGofX7Xr55ZclSZ06ddI999yj7t27q0WLFlq7dq0uueSSMr+f1157rUaPHq2B\nAwfqxBNPVJcuXTR//nxJUp8+fdSnTx+dccYZateunZKSko4aFq0MDzzwgE477TS1adNGV155pUaP\nHq3evXsXHm/YsKGWLSvo2LNr1y5dffXVatCgga655hrdcsstGj58eOG5b775pk477TQ1bdpU06dP\n15tvvqlmzZoVHn/mmWf005/+tNK/hrKYl/KoMdaYmVdmrKX1wbz33ouVlFS70u4DANVB8Yo3xK71\n69dr2LBheu+994IOJebt3LlTKSkpSktLU506dUo8p7S/++H95ZqkXm2Ts1WrtuuCC6bLnT6YAFBR\nJGeorkjOKjnWKVOW6fLL29BuCQAqiOQM1RXJWZzECgDVDckZqqtoJGcJXRCQlZWr++9fqJ/97F9B\nhwIAABCRhHxyFgrla9as1Ro79m1lZuaoRg3Tpk13qG3bqltADgCqE56cobqKxpOzhFuENjU1Q3fd\nNV9paQVlxN27J2vatD4kZgAAIC4kXHL23HMfKS1th5KTG2nKlF4aOPAs2i0BAIC4kXDJ2QMPXKFT\nT22ou+++iPXKAKCKtGnThv8Io1pq06ZNpV8z6nPOzKyvpGkqKD6Y4e6TSzjnUUn9JOVIGubuq0s4\n54g5Zx7uuwUAABBrYrZa08xqSPqjpD6SzpQ0yMw6Fjunn6T27t5B0m2S/nqs66amZuiCCx7X8uVb\nohA1ouFw2xLEH967+Mb7F994/6qnaC+l0VXSp+6e4e4HJc2R1L/YOf0lPSlJ7v6epMZm9p2SLla0\nD+aqVds1ZcqyaMaOSsQvmPjFexffeP/iG+9f9RTtOWctJRV9vLVVBQlbWedsC+/LLH6xjh3/dFQf\nTAAAgEQSVwUBeXkh+mACAICEFtWCADPrLmm8u/cNb4+W5EWLAszsr5LecffnwtsbJF3u7pnFrsXq\nhgAAIG7E6iK0/5Z0upm1kbRd0kBJg4qdM0/S7ZKeCydze4snZlL5v0AAAIB4EtXkzN1DZjZC0gJ9\nu5TGejO7reCwT3f3183sajPbqIKlNG6OZkwAAACxLG56awIAAFQH0V5K47iZWV8z22Bmn5jZqFLO\nedTMPjWz1WZ2blXHiJId670zs8Fmtib8sdTMzg4iTpQskp+98HkXmtlBM/tBVcaHskX4uzPFzNLM\n7CMze6eqY0TJIvjd2cjM5oX/zfvQzIYFECZKYGYzzCzTzD4o45zjzlliKjmL1qK1iL5I3jtJmyVd\n5u7nSPqdpMerNkqUJsL37/B5kyS9WbURoiwR/u5sLOlPkv7H3c+SNKDKA8VRIvzZu13SWnc/V9IV\nkh42s7habSGB/V0F712JypuzxFRypkpetBZV6pjvnbuvdPevw5srVbCeHWJDJD97kvQLSXMlfVmV\nweGYInn/Bkt6wd23SZK776riGFGySN47l9Qw/HlDSV+5+6EqjBGlcPelkvaUcUq5cpZYS85KWrS2\n+D/gpS1ai2BF8t4VdYukN6IaEY7HMd8/MztV0rXu/hdJVE/Hlkh+/s6Q1NTM3jGzf5vZ0CqLDmWJ\n5L37o6TOZvaFpDWS7qyi2FBx5cpZeCyKKmdmV6igKveSoGPBcZkmqeh8GBK0+FJL0vmSrpRUX9IK\nM1vh7huDDQsR6CMpzd2vNLP2kt4ysy7uvi/owBAdsZacbZPUush2cnhf8XNaHeMcVL1I3juZWRdJ\n0yX1dfeyHgWjakXy/l0gaY6ZmaTmkvqZ2UF3n1dFMaJ0kbx/WyXtcvcDkg6YWaqkcySRnAUrkvfu\nZkkTJcndN5lZuqSOkv5TJRGiIsqVs8TasGbhorVmVkcFi9YW/8U/T9KNUmEHghIXrUWVO+Z7Z2at\nJb0gaai7bwogRpTumO+fu58W/mingnlnPycxixmR/O58RdIlZlbTzJIkdZO0vorjxNEiee8yJPWS\npPB8pTNUUGCF2GAqfSShXDlLTD05Y9Ha+BXJeyfp15KaSvpz+OnLQXfvGlzUOCzC9++Il1R5kChV\nhL87N5jZm5I+kBSSNN3d1wUYNhTxz97vJM0qslzDSHffHVDIKMLM/iEpRVIzM/tc0jhJdVTBnIVF\naAEAAGJIrA1rAgAAVGskZwAAADGE5AwAACCGkJwBAADEEJIzAACAGEJyBgAAEENIzgBUCjMLmdkq\nM0sL/9m6jHPbmNmHlXDPd8xsg5mtNrN3zaxDOa5xm5kNCX9+k5m1KHJsupl1rOQ43wt3yjjWa+40\ns7oVvTeA+ENyBqCy5Lj7+e5+XvjPz49xfmUtsjjI3c+V9KSkh473xe7+N3d/Orw5TEWaErv7re6+\noVKi/DbOvyiyOO+SlFRJ9wYQR0jOAFSWo9qXhJ+QpZrZf8If3Us4p3P4adKq8JOl9uH9NxTZ/5dw\nV4my7psq6fBre4Zft8bMnjCz2uH9k8zso/B9poT3jTOze8zshyroH/p0+LV1w0+8zg8/XZtSJOab\nzOzRcsa5QtKpRa71ZzN738w+NLNx4X2/CJ/zjpktCu+7ysyWh7+Pz4VbMAFIQCRnACpLvSLDmi+E\n92VK6uXuF6igZ+BjJbzup5Kmufv5KkiOtoaHEn8s6eLw/nxJNxzj/t+X9KGZnSDp75IGuPs5kmpL\n+pmZNZV0rbufFX6C9bsir3V3f0EFjaQHh5/8HShy/AVJ1xXZ/rEKmsCXJ86+kl4usj0m3MbsHEkp\nZnaWuz+mgubIKe7e08yaSRorqWf4e/lfSfcc4z4A4lRM9dYEENe+CScoRdWR9EczO1cF/RxLmhO2\nQtJYM2sl6UV332hmPSWdL+nf4SdRdVWQ6JXkGTPbL+kzSb+Q9F1Jm919U/j4bEk/l/QnSfvN7AlJ\nr0n6VynXO+rJl7vvMrNNZtZV0kZJ33X35WZ2+3HGeYKk+pLOLbJ/oJn9nwp+H7eQ1FnSRzqymXL3\n8P5l4fvUVsH3DUACIjkDEE13S9rh7l3MrKak/cVPcPdnzWylpP+R9Fq44bNJmu3uYyO4x2B3Tzu8\nEX7KVFKCFQonVz0lDZA0Ivx5pJ5TwVOyDZJeOny7440zPDz6R0k/NLO2KngC9j13zzKzv6sgwSvO\nJC1w92M9lQOQABjWBFBZSppr1VjS9vDnN0qqedSLzNq5e3p4KG+epC6SFkm63sxOCp/TpIzqz+L3\n/VhSGzM7Lbw9VNKS8BytE919vqRfhu9TXLakRqXc5yVJ/VUwPDsnvK88cf5GUjczOyN8r32Sss3s\nO5L6FTk/q0gsKyX1KDIfL6k8lakA4gPJGYDKUlL15Z8lDTOzNElnSMop4ZwfhSfpp0k6U9KT7r5e\n0q8kLTCzNZIWqGDI75j3dPdcSTdLmht+bUjSX1WQ6PwrvC9VBU/1ipsl6a+HCwKKXt/d90paL6m1\nu/8nvO+44wzPZXtY0n3u/oGk1eHrPi1paZHXPC5pvpktcvdd4a/p2fB9lqtg+BZAAjL3yqpmBwAA\nQEXx5AwAACCGkJwBAADEEJIzAACAGEJyBgAAEENIzgAAAGIIyRkAAEAMITkDAACIISRnAAAAMeT/\nA2qLihSeIyDuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x116922898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(fpr, tpr, color='darkorange',\n",
    "         lw=2, label='ROC curve (area = %0.2f)' % roc_auc)\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend(loc=\"lower right\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Precision - Recall Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.99      1.00      0.99      1448\n",
      "          1       0.99      0.90      0.94       224\n",
      "\n",
      "avg / total       0.99      0.99      0.98      1672\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Precision / Recall\n",
    "print(metrics.classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred_prob = log_reg_model.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "precision, recall, threshold = metrics.precision_recall_curve(y_test, y_pred_prob[:, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAF/CAYAAAAM3256AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VPXd/vH7k5UASUwISwh7WFRUVhFcA1QFeRQFUaBa\nRW2tK9q60qcKLVYfW/ur4FasVVsFrCsooFgRRUUBkVVFZIeEJSQQlhCSzPf3R0IMJIHJMjkzyft1\nXXNlzpnvnHOPE+D2rOacEwAAAIJDmNcBAAAA8BPKGQAAQBChnAEAAAQRyhkAAEAQoZwBAAAEEcoZ\nAABAEAloOTOzF8xsh5mtOM6YSWa21syWmVn3QOYBAAAIdoHecvaipIsretHMBktKdc51knSzpOcC\nnAcAACCoBbScOec+k5R9nCFDJf2reOxXkuLNrHkgMwEAAAQzr485S5G0pdT0tuJ5AAAA9ZLX5QwA\nAAClRHi8/m2SWpeablU8rwwz4yagAAAgZDjnrCrvq41yZsWP8syUdJuk18ysr6Q9zrkdFS3I5/PJ\n53xyckU/nZOTU5iFKSo8qsz4QwWHtP/wfjnnSt7nnFODiAZKiEkoMz4nL0c7D+wsWfaR98RFx6lV\nXKsy4zMPZmpD9oYyeZo2bKpOTTqVGb8tZ5tW71pdJk9KXIq6tyh7ouqG7A1anL5YPucreeQX5is1\nMVXntz2/zPhl25dp5pqZKvQVqtAVlvzsldxLV592dZnxn2z8RC9884IKXaF8zqdCX9HPtHZpur3P\n7WXGz1k7R3/+4s8qdIXKyctR5sFMZedm6xfdfqFnhjxTZvzzXz+vX733q6KJjyX1L3p6U4+b9Pxl\nzx9/fCk39rhR/7jsH36Pv6H7DXph6AsBWX6YhemmHjfp75f+vcz4TzZ+oueXPq+mDZvqpAYnKSIs\nQpHhkerRoocuTL2wzPh1Wev0zfZvFBkWWTI2IixCreNal/v7k5OXo+zc7JJxR94XHRFd7u9/TRk/\nfrzGjx8fsOUjsPj+Qlttfn/OSRERks9X+fdOnSqNGlXzmUKZWZV6maQAlzMzmyopTVITM9ss6WFJ\nUZKcc26Kc262mV1iZj9KOiBpzAmWp3AL93v9DSIaqEFEA7/Hx0XHKS46zu/xSQ2TlNQwye/xKXEp\nSonz/5C69gnt1T6hvd/ju7foXm7Jq8gF7S7QBe0u8Hv84E6DNbjTYL/HX9vtWl3W5TIV+Ar059w/\n6/Y7bleBr6DC/8aDOg7Se6PeU4GvQAW+AuX78lXgK1BqQmq543u17KXxF4zXoYJDOlRwSLkFucot\nyNU5bc4pd3xiTKK6t+iuQl+hCnwFJQW2eaPyz0EJszA1iGhQUnKPFOSKrNixQq+ufLXM/Ft631Ju\nOXv/x/d1+5yyJfiW3reUW3ZfWfGKbpt9m9/jX13xqsZ/Ml6RYZEKDwtXuIUrPCxcV3e9Wvedc1+Z\n8e/98J6eW/KcIsMjjyqMOdtzyv28X239Su/98J4iwiLUKKqREhokKC46Tp2bdFa3Ft3KjN+Ws03r\nstcp3MLVIKKB2sS3UVLDpGr9BQag5phJjzwizZ1b9Lyix5GxZtJ330kbNkhPPCHNmVNU7I48nKt4\n+uqrpWuv9fbzBrOAljPn3Gg/xpT91wl1QoOIBmrQuKgcn9TgJHVM7Hjc8a3jW6t1fOvjjimtZ3JP\n9Uzu6ff44acO1/BTh/s9/saeN+rGnjeWTJfe4lmeizterJcvf1k7D+xUTl5OScns16pfueM7JHTQ\nsFOG/VRGC4vKaJcmXcod3zCyoVrHtT6quOYX5lf4PyCZBzP1Y9aPZeaf3erscsevz16vWWtnlZl/\n5r4zyx2/aNsiTVwwscz828+8XZMvmVxm/pvfvamx748t85nu6HOHHvvZY2XGb9yzUZv2bFJCTIIS\nYxLVMralwozDZIFAeuCBooe/Hn1UGjdO+vrrooe/li6VLrywqKgVFh7980Tzjn3dOalPH6lx48p/\n3mDl9TFnqCfS0tK8jlBtJ9py27lJZ3Vu0tnv5VV2S+T13a/X9d2vr9T4wZ0GK78w/6jd3E0bNi13\n/KWdL1X7k9or35dfUhTzffk6uPZguePPanWWJqRNUIGvQPsP79eeQ3uUk5dT4dbb5MbJOrfNuSr0\nFepA/gFt3rtZew7tUXR4dLnjp6+argc/erBkOjEmUf3b9de9Z9+rs1qdVWb8jv07dKjgkKIjomUy\nhVmYwixMjaIaVWoLel1TF/7s1WfB/v2NHSt16CDl5kphYUUPs5+eHzu9d690/fVSRoaUnFxzOS66\nSPrgg5pbntfMudA4zt7MXKhkBeCfvYf2yud85R4D+vKyl/XCNy8oKzdLOw7sUObBTEnSk4Oe1J1n\n3Vlm/N3v362/ffW3MvMnD55c7jGU//fZ/+nt799WdES0EmMS1SSmiRpENNDI00bq3Dbnlhk/d91c\nrdixQlHhUSWPFo1baGD7gYqOKL9gAjiazycNGVK0le1IYQsPP/pnRc/Lm5ebKy1ZIp16qrR6tdef\n7mhmFtQnBABAueIbxFf42nXdr9N13a+TVLRLeX32es3bME8nJ51c7vhGUY3UKq6V8gvzjzpxqKKt\nZuuz1+urbV+VmX9G8zPKLWdvfPuGnl9a9kSWhy94WOPTxpeZv3rnapmZmjVqpsSYRHbJAioqVHPm\n1NzyVq+WTjvtp5KWlCS1a1dzy/cKW84A1Evrs9eX7ArdnbtbWblZyivIU//2/XVas9PKjH999ev6\ncuuXOlx4WPm+fOUV5mnZ9mX69xX/Lnf8gJcH6OONH0uSIsIi1LRhUzVv3FzPDXmu3N2yACrv22+l\nrl2Pnvf111JP/w9HDhi2nAFAJXVI6KAOCR38Hj+i6wiN6DqiUsvftm+bdh7YqT2H9ihjf4Yy9mdU\nuAVt5Y6VahnbUhFhESXHy5mZYiJiOKMVqECXLtKIEdIPP0gbNxYd07ZhQ3CUs+pgyxkABFheQZ52\nHtipnQd26pSmp6hhZMOjXi/0FSr5iWTtOrirzHvdw2X/3vM5ny6ffrliImN0QdsLdMXJVyg5tgaP\nrgZC0PDh0ltvSW+8UfTca2w5A4AgFh0RfdxLxWzfv12t41vrcOHhkmPljndNvfzCfL37w7uSpP+s\n/o9un3272ie0V2pCquZeO7fM+Nz8XD216Cl1SOiglLgUJcYkKjEmseRiyQCCC38qAcBjKXEp+vpX\n/l8kKjwsXO9c/Y52Hdyld394Vx/8+IHWZ69XZFhkueM37Nmg+/5b9sLDnZt01prb15SZn5ufqz2H\n9rA1DvAI5QwAQkxEWISGnjxUknRTz5t04PABpe9Lr/B4toaRDTX2rLFan71e2/dvV1ZulrJysyq8\nw8nKnSvV74V+GtRxkK45/Ro1b9xcEWERSk1IrdRdTgBUDeUMAEJco6hG5d6P9Yh2J7XT3waVvQZc\nRcfx5ubnKtzCNXvtbM1eO7tk/pBOQ/Te6PfKjN+8d7N2HdiluOg4xUbHKqlhErtLgWrghAAAQBmZ\nBzP16opX9cG6D0ruX3vrmbfqmjOuKTP2wf8+qMc+/+kWXDERMereoruu7nq1xvYdW2Y8EAhHTgho\n00Zq1Ei64w7pllu8y8MJAQCAGpXUMElj+471q1w1b9xcPVr0UE5ejnLycrTr4C4t3LpQv+r1q3LH\nZ+dmy8wUGxWr8LCKb4kGVEaH4ivjbN5c9HPiRCknp+jCtNdfX3RXgVDBljMAQI3afXC3lmYs1flt\nzy/31lYdJ3XUuux1CrdwJccmKyU2Rc0bN9dTg58q94xW5xzXesMJFRZKK1ZIW7ZIQ4ce/dq8eVL/\n/rWbpzpbzihnAIBa1ePvPbQua532Hd531Pxtv9mmlrEty4w/95/nKiYyRme3OluJMYmKjY5V46jG\nGnbKMI5tQxnOSU8+Ka1fL82aVfTznXfKFrZAY7cmACBkfHPzN5KKLs6bvi9d6fvStePADjVt2LTM\n2L2H9mrh1oXyOZ/+u/6/R712+H8P10pehBYz6a67ip5v2lRUzkIN5QwA4InoiGi1T2iv9gntKxwT\n3yBe6b9J1/yN87VixwrtO7xP+w7vU25+riLDy17X7WD+Qf1n9X/UM7mnmjZsqqSGSeWOA4IZ5QwA\nENSaN26uq0+7WlefdvUJx9479149s+SZo+YlNEjQxAETdeuZtwYqIlCjKGcAgDqjd8veGtB+gHbs\n36GdB3Yq82Cmsg9lK9zKP1Xv38v/rd25u3X5yZcrJTaFrWwICpwQAACoswp9hcrKzVKDiAaKjY49\n6rW8gjylTkrVtn3bJElhFqaWsS3VNr6tXh32qtqe1NaLyKhBQ4dKM2eG3gkB5d/rAwCAOiA8LFxN\nGzUtU8wkKSo8Ss8OeVY/6/AzpcSmyDmnrTlb9fmWz9U4qnG5y/tq61cq9BUGOjZq2Nq10mefSbm5\nXifxD1vOAACQdLjwsLblbNOmvZt0QdsLylxbbcGmBRrwrwG6OPVivTLsFZ3U4CSPksJfl18uzZjx\n0/QVV0hvvll0uY2wAG+e4jpnAAAE2PyN8zX8P8OVlZulxJhENW3YVDGRMbqzz50a02OM1/FQjjff\nlCZMkFau/GmemdS0qbR0qZSSErh1U84AAKgFG7I3aNh/hmnZ9mWSio5T+/T6T3VOm3M8Tobjyc6W\nOnWSdu/+ad6sWdIllwRunZQzAABqic/5tGnPJuUV5mn/4f3q3bK315Hgh8OHi445u+oqae7c4C5n\nXEoDAIBKCLOw4144V5KmfD1Fk76apNObn66/XvRXJccm11I6VCQqqugRCjdA52xNAABq2MY9G7V6\n12pNXzVdPf7eQ59s/MTrSAghlDMAAGrYPWffoyW/XKK0dmnacWCHBvxrgAa/OtjrWAgRlDMAAGpY\nYkyierXspQ+v/VAPnPOAfM6nrTlbyx17qOCQ3l3zrjiuGkdQzgAACJCIsAg9+rNHlf6bdE0fPr3c\nMW98+4Yum36ZLnjpAi3NWFrLCeuvL76Qpk6Vtm/3OklZlDMAAAIsOTZZXZt1Lfe16PBoJTVM0oLN\nC9R7Sm/dO/deFfgKajlh/XHkhIBHHpF+/nPpl7/0Nk95KGcAAHhoRNcRWnvHWt3T7x6Fh4XrLwv/\noov+fZGycrO8jlYn3X67dOGF0jnFl6bbtcvbPOXhOmcAAASJBZsWaMTrI9Q6vrUWjFmgBhENvI5U\nZ335pdSvn3TWWUXPaxrXOQMAoA44r+15WnrzUhX6Cilm9RjlDACAINIytmW589fuXqv9h/erR3KP\nWk6E2sYxZwAAhIA5P85Rrym9NHXlVK+jIMAoZwAAhIBwC5eT0y/e/oVeX/2613EQQJQzAABCwG19\nbtPvz/+9Cl2hRr81WvM2zPM6EgKEcgYAQIiYkDZBd/a5UwW+Aj29+Gmv4yBAKGcAAIQIM9MdZ90h\nSVq1c5XHaRAonK0JAEAISU1I1Ypfr6jwjgOonM2biy5Mm5AgPfywFBEEzYiL0AIAgHpn5UrpjDOO\nntezp9S9u3T99dJ551Vv+dW5CC3lDAAA1DvOSc89J+3YIU2YUPb1a6+VmjSR/vznqm1No5wBAABU\n0fffS6+9Jq1fL/3rX0e/tmCBdO65lV8m5QwAAKCanJP+/W8pM7Noq9ratdK8eVL//pVfFvfWBACg\nnnHOacOeDfI5nzomdvQ6Tp1gJv3iF0XP3323qJx5gUtpAAAQgp5d8qxSJ6Xq0QWPeh0FNYxyBgBA\nCOqT0keSNH/TfPmcz+M0qEmUMwAAQlD3Ft2V0CBB67PXa/Crg5WxL8PrSKghlDMAAEJQRFiEpg2f\npiYxTTR33Vyd8dwZWrFjhdexUAM4IQAAgBB1cceLteKWFbr+nevVKKqRTkk6xetIqAGUMwAAQljL\n2JZ6/5r3VeArUGR4pNdxUAPYrQkAQIgLszBFhUd5HQM1hHIGAEAdVegr9DoCqoByBgBAHbQvb58u\neOkCvffDe15HQSVRzgAAqINeXPaiPt/yuaZ8PcXrKKgkyhkAAHXQsFOGSZI+3vixDhce9jgNKoNy\nBgBAHdQqrpW6Nu2q/Yf364stX3gdB5VAOQMAoI66OPViSdL7P77vcRJURsDLmZkNMrPvzewHM7u/\nnNfjzGymmS0zs5Vmdn2gMwEAUB8M6jhIkjR33VyPk6AyAnoRWjMLk/SUpIGS0iUtNrMZzrnvSw27\nTdJq59xlZpYkaY2ZveKcKwhkNgAA6rrz2p6nV4e9qgs7XOh1FFRCoO8Q0EfSWufcJkkys+mShkoq\nXc6cpNji57GSdlPMAACovgYRDTT69NFex0AlBXq3ZoqkLaWmtxbPK+0pSaeaWbqk5ZLGBjgTAABA\n0AqGEwIulvSNc66lpB6Snjazxh5nAgAA8ESgd2tuk9Sm1HSr4nmljZH0qCQ559aZ2QZJJ0tacuzC\nxo8fX/I8LS1NaWlpNZsWAIA6rNBXqFU7V6lbi25eR6lz5s+fr/nz59fIssw5VyMLKnfhZuGS1qjo\nhIAMSYskjXLOfVdqzNOSdjrnJphZcxWVsm7OuaxjluUCmRUAgLrugf8+oCcWPqFpw6fpylOv9DpO\nUOvfX5o/X5o3r+h5ZZmZnHNWlXUHdLemc65Q0u2S5kpaLWm6c+47M7vZzH5VPGyipLPNbIWkDyXd\nd2wxAwAA1eOcU35hvgp8BbrmrWu4MG0QC+iWs5rEljMAAKrHOadbZ92q575+TkkNk7TwxoXqmNjR\n61hBqc5uOQMAAMHDzDT5kska3HGwMg9mauj0ofI5n9excAzKGQAA9UhEWIReu/I1NWvUTDl5OUrf\nl+51JBwj0GdrAgCAIBMbHatVt6xSUsMkmVVpzxsCiHIGAEA91LRRU68joALs1gQAAAgilDMAAIAg\nQjkDAAAIIpQzAADqKeecVu5YqVdWvOJ1FJTCCQEAANRTB/MPqteUXip0hRrSaYgSYhK8jgSx5QwA\ngHqrUVQjnd36bPmcTx9v/NjrOChGOQMAoB7r367o3kSfbf7M4yQ4gnIGAEA91jO5pyRpxY4VHifB\nEZQzAADqsW4tukmSlu9YLuecx2kgcUIAAAD1Wuu41rry1CvVKbGT8n35igqP8jpSvUc5AwCgHjMz\nvT7ida9joBR2awIAAAQRyhkAAEAQoZwBAAAEEcoZAABAEKGcAQAAzfh+hu56/y6tyVzjdZR6j3IG\nAAA0bdU0PfnVk/piyxdeR6n3KGcAAEDdmv90MVp4i3IGAACOulMAvEU5AwAAJVvOuMem9yhnAABA\nybHJkqSs3CyPk4DbNwEAAJlMkwZN8joGRDkDAAAqusfmHWfd4XUMiN2aAAAAFbrhBql5c2lSLW5U\npJwBAAAco3nzop8bN0o7d0pvv11766acAQCAcu3L26cCX4HXMTzx9NPSjBnS5Mm1v27KGQAAKOOW\n925R0p+T9Nnmz7yO4okmTaTLLpO6dq39dVPOAABAGbHRsTpceFgzvp/hdZR6h3IGAADKGNplqCRp\nxpoZcs55nKZ+oZwBAIAy+rbqq6YNm2rDng1atXOV13HqFcoZAAAoIzwsXJd2vlSSNHPNTI/T1C+U\nMwAAUK6hJw/VSQ1OUqEr9DpKvcIdAgAAQLku6XSJdt6zU5HhkV5HqVcoZwAAoFwRYdQEL7BbEwAA\nIIhQzgAAgN++3fWtLnn1Eq3PXu91lDqLcgYAAPz2+OePa86Pc3TX+3d5HaXOopwBAAC/PTrwUTWO\naqx3f3hXH2/42Os4dRLlDAAA+C05Nln3n3O/JOneD++Vz/k8TlT3UM4AAECl/Kbfb9QytqW+zvha\nr616zes4dQ7lDAAAVErDyIb6Q9ofFBsVq5y8HK/j1DlcwAQAAFTa9d2v16VdLlWzRs28jlLnsOUM\nAABUWnhYOMUsQChnAAAAQYRyBgAAEEQoZwAAAEGEcgYAAKrsu13f6Tcf/EZvffeW11HqDMoZAACo\nss82f6b/9+X/07NLnvU6Sp1BOQMAAFV25alXKio8SvM2zFP6vnSv49QJlDMAAFBlCTEJGtJpiHzO\np+mrpnsdp06gnAEAgGq55oxrJElTvp6ivYf2epwm9FHOAABAtVzS6RK1O6mdMvZnaNPeTV7HCXnc\nvgkAAFRLg4gG+vi6j5W+L11nND/D6zghj3IGAACqrd1J7dTupHZex6gTAr5b08wGmdn3ZvaDmd1f\nwZg0M/vGzFaZ2ceBzgQAABCsArrlzMzCJD0laaCkdEmLzWyGc+77UmPiJT0t6SLn3DYzSwpkJgAA\ngGAW6C1nfSStdc5tcs7lS5ouaegxY0ZLetM5t02SnHOZAc4EAAACzOd8XkcIWYEuZymStpSa3lo8\nr7TOkhLN7GMzW2xm1wY4EwAACKB+L/RToz81UnZuttdRQlIwXEojQlJPSYMlDZL0ezPr6G0kAABQ\nVbn5uTpUcEg/7P7B6yghKdBna26T1KbUdKvieaVtlZTpnDsk6ZCZfSqpm6Qfj13Y+PHjS56npaUp\nLS2thuMCAIDq6pLURct3LNcPu3/QWa3O8jpOrZg/f77mz59fI8sKdDlbLKmjmbWVlCFppKRRx4yZ\nIWmymYVLipZ0lqS/lrew0uUMAAAEp86JnSVJa3av8ThJ7Tl2o9GECROqvKyAljPnXKGZ3S5prop2\nob7gnPvOzG4uetlNcc59b2YfSFohqVDSFOfct4HMBQAAAqdLUhdJqnO7Nffvl8ykRo0Cu56AX4TW\nOfe+pC7HzPv7MdN/kfSXQGcBAACB16VJ0T/7W3K2nGBk6Pj0Uyk2VoqJkdatk5KTA7cuc84Fbuk1\nyMxcqGQFAKA+O1x4WLsP7laLxi1kZl7HqZYNG6TUVKl0BVmwQDr33OO/z8zknKvShw+GszUBAEAd\nEhUepeTY5JAvZpLUvr20ZUtRSevbt3bWyb01AQAAjiOl+Aqt4eG1sz62nAEAAAQRyhkAAEAQoZwB\nAICAKPAV6Ntd3+pg/kHutVkJHHMGAABq3IbsDeryVBfl+/IlSQ0jG2rW6FlKa5fmbbAQwJYzAABQ\n49rEt9HADgN1UoOTFB0erYP5B/Xbub8Vl8U6McoZAACoceFh4Zrz8znKvj9bWfdnKblxspZmLNWH\n6z/0OlrQ83u3ppmlSGpb+j3OuU8DEQoAANQdDSMbatLgSYoOj9aFHS70Ok7Q86ucmdn/Sbpa0rcq\nuv+lJDlJlDMAAHBCV556pdcRQoa/W84ul9TFOZcXyDAAAAD1nb/HnK2XFBnIIAAAAPB/y9lBScvM\n7CNJJVvPnHN3BiQVAABAPeVvOZtZ/AAAAKgSn/MpJy9HJlN8g3iv4wQtv8qZc+5lM4uS1Ll41hrn\nXH7gYgEAgLpm54GdSn4iWc0bNdf2e7Z7HSdo+Xu2ZpqklyVtlGSSWpvZdVxKAwAAoGb5u1vzCUkX\nOefWSJKZdZY0TVKvQAUDAACoj/w9WzPySDGTJOfcD+LsTQAAgBrn75azJWb2D0mvFE//XNKSwEQC\nAACov/wtZ7dIuk3SkUtnLJD0TEASAQCAOslkio+OV1x0nNdRgpq/Z2vmSfpr8QMAAKDSmjdurj0P\n7PE6RtA7bjkzs/84564ys5UqupfmUZxzZwQsGQAAQD10oi1nY4t//k+ggwAAAOAEZ2s65zKKn2ZK\n2uKc2yQpWlI3SekBzgYAAFDv+HspjU8lNTCzFElzJV0r6aVAhQIAAKiv/C1n5pw7KGmYpGeccyMk\ndQ1cLAAAUBc557Rj/w4tzVjqdZSg5Xc5M7N+Krq+2azieeGBiQQAAOqqrTlb1eKJFhr0yiCvowQt\nf8vZXZIelPS2c261mXWQ9HHgYgEAgLqoVVwrxUbFatfBXdp5YKfXcYKSX+XMOfeJc+4y59z/FU+v\nd87deaL3AQAAlGZm6tqs6Mio1TtXe5wmOB23nJnZ34p/vmtmM4991E5EAABQl3RtWlzOdlHOynOi\n65z9u/jnXwIdBAAA1A8l5YwtZ+U6bjlzzn1d/HSJpFznnE+SzCxcRdc7AwAAqJRuLbqpa9OuatG4\nhddRgpK/Nz7/SNLPJO0vno5R0fXOzg5EKAAAUHcNaD9Aq25d5XWMoOXv2ZoNnHNHipmKnzcMTCQA\nAID6y99ydsDMeh6ZMLNeknIDEwkAAKD+8ne35l2SXjezdEkmqYWkqwOWCgAA1BtLM5bqllm3aFDq\nIF3a5VL1btnb60ie8qucOecWm9nJkroUz1rjnMsPXCwAAFBfzFk7R4u2LdKibYv0h0//oNmjZ2tw\np8Fex/KMX7s1zayhpPsljXXOrZLUzsz+J6DJAABAvXBX37v03qj3NLD9QEnSJ5s+8TiRt/w95uxF\nSYcl9Sue3iZpYkASAQCAeqVRVCMN6TxEt555qyRp+Y7lHifylr/lLNU597ikfElyzh1U0bFnAAAA\nNaJ7i+6KDo9WmPlbT+omf08IOGxmMZKcJJlZqqS8gKUCAAD1TvuT2mv/uP2KCPO3ntRN/n76hyW9\nL6m1mb0q6RxJ1wcqFAAAqH/MTBFWv4uZ5Ec5MzOT9L2kYZL6qmh35ljnXGaAswEAANQ7Jyxnzjln\nZrOdc6dLmlULmQAAAOotf4+4W2pmZwY0CQAAAPwuZ2dJ+tLM1pnZCjNbaWYrAhkMAADUT+n70jV7\n7WwdKjjkdRRP+HvU3cUBTQEAAFDson9fpNW7VmvRTYt0Zkr923F33C1nZtbAzO6SdK+kQZK2Oec2\nHXnUSkIAAFCvdG/RXZK0bPsyj5N440S7NV+W1FvSSkmDJT0R8EQAAKBeq+/l7ES7NU8tPktTZvaC\npEWBjwQAAOqzbs27SZIWbF6gvII8RUdEe5yodp1oy1n+kSfOuYIAZwEAAFCflD5KaJCglTtX6i9f\n/MXrOLXuRFvOuplZTvFzkxRTPG0qugRaXEDTAQCAeie+QbzmXTdPj332mO45+x6v45SxYYN0+LB0\n5plSbGzNL9+cczW/1AAwMxcqWQEAQN1z7rnS55//NH399dKLL5Y/1szknLOqrKd+3/YdAADAT2lp\nUni4lJhYNL1tW2DWQzkDAADww8SJUl6eNG1aYNdDOQMAAPBTeHjg10E5AwAAQavQV6jnljyncR+N\nU3059jyX7x8LAAAajElEQVTg5czMBpnZ92b2g5ndf5xxZ5pZvpkNC3QmAAAQGsIsTA/89wE9+tmj\nyjyY6XWcWhHQcmZmYZKeUtG9ObtKGmVmJ1cw7jFJHwQyDwAACC1mptTEVEnSuux1HqepHYHectZH\n0trie3HmS5ouaWg54+6Q9IaknQHOAwAAQkxqQnE5y6Kc1YQUSVtKTW8tnlfCzFpKutw596yKLm4L\nAABQoqScseWs1vxNUulj0ShoAACgRH3brXmi2zdV1zZJbUpNtyqeV1pvSdPNzCQlSRpsZvnOuZnH\nLmz8+PElz9PS0pSWllbTeQEAQJDp16qffnfe73R267O9jlKh+fPna/78+TWyrIDevsnMwiWtkTRQ\nUoakRZJGOee+q2D8i5Ledc69Vc5r3L4JAAB4bu5c6eKLpQsvLHpenurcvimgW86cc4VmdrukuSra\nhfqCc+47M7u56GU35di3BDIPAABAsAv0bk05596X1OWYeX+vYOwNgc4DAAAQzILhhAAAAAAUo5wB\nAAAEEcoZAAAIesu2L9PYOWP17OJnvY4ScJQzAAAQ9LbmbNWkRZP0zpp3vI4ScJQzAAAQ9E5OKro1\n97wN8/Taqtc8ThNYlDMAABD0OiZ21H1n36cCX4FGvTlKU74+9mpcdQflDAAAhITHfvaYHhnwiJyc\nbn7vZn259UuvIwVEwK9zBgAAUBPMTOPOG6eEBglq0biF+rbq63WkgKCcAQCAkHLLmbd4HSGg2K0J\nAAAQRChnAAAAQYRyBgAAEEQ45gwAAISk3Qd3a+rKqUqMSdTPz/i513FqDFvOAABASNq+f7vufP9O\n/emzP3kdpUZRzgAAAIII5QwAACCIUM4AAACCCOUMAAAgiHC2JgAACEmJMYm6/czb1aJxC6+j1CjK\nGQAACEnJscmafMlkr2PUOHZrAgAAVMGePdLMmdKnn9bsctlyBgAAUAWLF0tDhxY9X7tW6tixZpbL\nljMAAIBK6N1bOv10qXNnqXHjonmZmTW3fMoZAABAJSQmSitWSGvWSF271vzyKWcAACAkOef0xrdv\naOycsSrwFXgdp8ZQzgAAQEgyMz340YOatGiSlmYs9TpOjaGcAQCAkDWg3QBJ0kfrP/I4Sc2hnAEA\ngJA1oH1ROZu3cZ7HSWoO5QwAAISs/u37S5I+2/yZ8gryPE5TMyhnAAAgZDVr1EynNztdhwoO6cut\nX3odp0ZwEVoAABDSxp03ToW+Qp3e/HSvo9QIyhkAAAhpI08b6XWEGsVuTQAAgCBCOQMAAAgilDMA\nAIAgQjkDAAAIIpQzAAAQ8h7++GENeHmAPt30qddRqo1yBgAAQt6qXav08caPtevALq+jVBvlDAAA\nIIhQzgAAAIII5QwAACCIUM4AAACCCLdvAgAAIW/8BeN125m36bRmp3my/uXLpf37pb59pcaNq7cs\nc87VTKoAMzMXKlkBAED90Lev9NVXP03/8pfSlCmSmck5Z1VZJrs1AQAAqmjoUCkxUWrZsmh627bq\nL5NyBgAAUEUPPijt3l20taymUM4AAACCCOUMAAAgiFDOAABAyOPemgAAAEGEe2sCAAAgIChnAAAA\nQYRyBgAAEEQoZwAAAEGEe2sCAICQ5/W9NWsS5QwAAIS805uf7nWEGsNuTQAAgCBCOQMAAAgiAS9n\nZjbIzL43sx/M7P5yXh9tZsuLH5+ZWd3ZLgkAAFBJAS1nZhYm6SlJF0vqKmmUmZ18zLD1ks53znWT\nNFHS84HMBAAA6i7nnPIL872OUS2B3nLWR9Ja59wm51y+pOmShpYe4Jz70jm3t3jyS0kpAc4EAADq\noH8s/Yda/b9WemLhE15HqZZAl7MUSVtKTW/V8cvXTZLmBDQRAACok2IiYpS+L12fb/nc6yjVEjSX\n0jCz/pLGSDrX6ywAACD0nNPmHEnSF1u+kM/5FGa1f97jrl3S669XbxmBLmfbJLUpNd2qeN5RzOwM\nSVMkDXLOZVe0sPHjx5c8T0tLU1paWk3lBAAAIa5tfFu1jG2p9H3pWpO5Rqc0PaXW1r1y5XxJ87V4\nsXTVVdVbljnnaiJT+Qs3C5e0RtJASRmSFkka5Zz7rtSYNpI+knStc+7L4yzLBTIrAAAIfSNeH6E3\nvn1D/7j0H7qx5421tt69e6Urr5QOHJBatJDeftvknLOqLCugW86cc4VmdrukuSo6vu0F59x3ZnZz\n0ctuiqTfS0qU9IyZmaR851yfQOYCAAB10zmtz9Fb372lzXs31+p64+OlDz/8adqqVMuK3xsqW6PY\ncgYAAE4kJy9HJlNsdKynOcyqvuWMcgYAAFDDqlPOuH0TAABAEKGcAQAABBHKGQAAqJP++c0/9dsP\nfqtVO1d5HaVSKGcAAKBOmrFmhv765V+1Lmud11EqhXIGAAAQRChnAAAAQYRyBgAAEEQoZwAAAEEk\n0Dc+BwAA8MSY7mN0fpvz1bVZV6+jVAp3CAAAAKhh3CEAAACgjqCcAQAABBHKGQAAQBChnAEAgDpv\nwaYFSt+X7nUMv4R8OWvXrp3MjAePox7t2rXz+lcTABAk/rrwrzr/pfN1/3/v9zqKX0L+bE0zU6h8\nBtQefi8AAEesy1qnrs90VV5hnt4b9Z6GdB4S8HUW/zvE2ZoAAADHSk1M1cQBEyVJv3rvV9pzaI/H\niY6PcgYAAOq8u/verb6t+ip9X7ru/uBur+McF+UMAADUeeFh4Xpx6IuKCo/SS8te0vLty72OVCGO\nOUOdxO8FAKA8+/L2KSIsQlHhUQoPCw/YejjmLIi1a9dODRs2VFxcnJKTkzVmzBgdPHhQkpSWlqaY\nmBjFxcWpWbNmGj58uHbs2HHc5S1atEhDhgxRQkKCkpKS1LdvX7300ku18EkAAAh9sdGxiomMCWgx\nqy7KWYCZmWbNmqWcnBwtXbpUS5Ys0cSJE0tee+aZZ5STk6Mff/xR+/fv1z333FPhshYuXKiBAweq\nf//+WrdunTIzM/Xss8/qgw8+qFI2n89XpfcBAIDAoZzVgiO715KTkzV48GCtWrWqzGtxcXG6/PLL\ntWzZsgqXc99992nMmDG65557lJiYKEnq0aOHpk2bJkl6+eWXdd555x31nrCwMK1fv16SNGbMGN16\n660aMmSIYmNj9Ze//EXJyclH7f57++231a1bt5Jsjz32mDp27KimTZtq5MiR2rMnuM9wAQAg1FHO\natGWLVs0e/Zs9ezZs8xru3fv1ltvvaVOnTqV+97c3FwtXLhQw4cPP+46zOy409OmTdPvf/977du3\nT2PHjlXjxo01b968o16/5pprJEmTJk3SzJkztWDBAqWnpyshIUG33nqrX58VAABUTZ0vZ2Y186iO\nyy+/XImJiTr//PPVv39/PfjggyWv3XnnnUpISFDTpk21e/duTZo0qdxlZGdny+fzKTk5uVLrPvag\n+KFDh6pv376SpOjoaI0cOVJTp06VJO3bt0+zZ8/WqFGjJEl///vf9cgjjyg5OVmRkZF66KGH9MYb\nb7A7FAAQ0tbuXqv+L/fXmsw1XkcpV50vZ8FgxowZysrK0oYNGzR58mRFR0eXvDZp0iRlZ2dr5cqV\nys7O1tatW8tdRkJCgsLCwpSRkVGtLK1btz5qevTo0Xr77beVn5+vt956S7169VKrVq0kSZs2bdIV\nV1yhxMREJSYm6tRTT1VkZOQJT1oAACCYjZs3TvM3zlePv/fQa6te8zpOGXW+nDlXM4/qZTjxArp2\n7arf/e53Fe42jImJUb9+/fTmm29WuIxGjRqVnAkqSdu3by8z5tjdnKeccoratm2r2bNna9q0aRo9\nenTJa23atNGcOXOUlZWlrKwsZWdn68CBA5XeegcAQDD5x6X/0MjTRiq3IFdPfvWk13HKqPPlLJRc\nd9112rlzp959991yX3/88cf10ksv6YknnlBWVpYkafny5SW7Ibt166bVq1drxYoVysvL04QJE8qU\nsfKMHj1aTz75pBYsWKARI0aUzL/55ps1btw4bd68WZK0a9cuzZw5s7ofEwAAT8U3iNfYs8ZKknwu\n+A7VoZwF2PHK0bGvRUZG6s4779Qf//jHcsf369dP8+bN00cffaTU1FQlJSXp17/+tYYMKbqBa6dO\nnfTQQw9p4MCB6ty5c5kzNysycuRIffrppxo4cGDJWaCSNHbsWA0dOlQXXXSR4uPjdfbZZ2vRokV+\nLRMAAFQNdwhAncTvBQDgeL7b9Z3ufP9OndzkZE2+ZHKNL786dwignKFO4vcCAOAlbt8EAABQR1DO\nAABAvZe+L11LM5YGxQkClDMAAFCvPbv4WY37aJz6PN9HKX9N0b+W/8vTPJQzAABQrw3sMFDR4dEq\ndIXavn+7rnvnOr2++nXP8nBCAOokfi8AAJXlcz4Ne22YZqyZIUl6bshzurn3zVVaFicEAAAAVFOY\nhentq9/WU4OfUoeEDhp2yjBPcrDlDHUSvxcAgOrIK8hTdET0iQdWgC1nIWrChAm69tprA76eTZs2\nKSwsTD5f5c9A+eSTT8rcLL20MWPG6KGHHqpOPL+FhYVp/fr1tbIuAED9Vp1iVl2UswCKjY1VXFyc\n4uLiFB4eroYNG5bMmzZtmqTj396pJlVnPVV978svv6yIiAjFxcXppJNOUs+ePTVr1qxazwEAQCih\nnAXQvn37lJOTo5ycHLVt21azZs0qmXfkZuX+KiwsDFDKwDr77LOVk5OjPXv26IYbbtBVV12lvXv3\nlhnnz+djNyUAoD6gnNUS51y55SIvL0/XXXed4uLidPrpp2vp0qUlr7Vv316PP/64unXrpsaNG8vn\n8ykjI0NXXnmlmjVrptTUVE2e/NP9wBYvXqwzzzxT8fHxSk5O1j333HPU+l955RW1bdtWzZo105/+\n9KeS1w4fPqy77rpLKSkpatWqle6++27l5+eX+zm++eYb9erVS/Hx8Ro5cqQOHTrk93+DG264Qbm5\nuVq3bl3J7tLHH39cycnJuuGGGyRJzz//vDp16qSkpCRdfvnlysjIOGoZs2bNUmpqqpo1a6b77rvP\n73UDABAqKGcee/fddzV69Gjt3btXl156qW677bajXp8+fbrmzJmjPXv2yMx06aWXqkePHsrIyNBH\nH32kJ598Uh9++KEkaezYsbrrrru0d+9erVu3TlddddVRy/r888+1du1a/fe//9Uf/vAHrVmzRpI0\nceJELVq0SCtWrNDy5cu1aNEiTZw4sUzW/Px8XXHFFbruuuuUlZWlESNG6M033/TrcxYUFOj5559X\nbGysOnXqJEnavn279uzZo82bN2vKlCmaN2+exo0bpzfeeEMZGRlq06aNRo4cedRy3nnnHS1dulRL\nly7VjBkz9M9//tO//9AAAFTCvXPv1UX/vkjLti+r9XVH1PoaPWATyj9WyT1c/m6yY8dXNK4mnHvu\nubr44oslSddee62efPLJo14fO3asWrZsKUlatGiRMjMz9bvf/U6S1K5dO910002aPn26LrzwQkVG\nRurHH3/U7t271aRJE/Xp0+enz2Sm8ePHKyoqSmeccYa6deum5cuXq0uXLpo6daqefvppNWnSRJL0\n8MMP69e//rUmTJhwVJaFCxeqoKBAd955pyRp+PDhOvPMM4/7+RYuXKjExERFRESoY8eOeueddxQb\nGytJCg8P14QJExQZGSlJmjp1qm688UZ169ZNkvToo48qISFBmzdvVps2bSRJDzzwgOLj4xUfH6+7\n7rpL06ZNK9nqBgBATVmSsUTzN85Xdm52ra+bLWcea9GiRcnzhg0b6tChQ0edVdmqVauS55s2bdK2\nbduUmJioxMREJSQk6NFHH9XOnTslSf/85z+1Zs0anXzyyTrrrLPKHHzfvHnzo9a1f/9+SVJ6enpJ\n+ZGktm3bKj09vUzWjIwMpaSkHDWvbdu2x/18/fr1U1ZWlnbu3KkvvvhC/fv3L3mtadOmJcXsSI7S\ny2vUqJGaNGmibdu2lfvfo6KcAACEsnqx5ayyW74CuaWsskqfodi6dWt16NChZHfksVJTUzV16lRJ\n0ptvvqkrr7xSWVlZJ1xHy5YttWnTJp1yyimSikrgka11pSUnJx9VlCRp8+bN6tixo9+fp7Rjz748\nkuOIAwcOaPfu3UcVsi1btpTk3Lx5c7k5AQAIZWw5CzLHOyOxT58+io2N1eOPP65Dhw6psLBQq1ev\n1pIlSyRJr776qjIzMyVJ8fHxMjOFhYWdcLmjRo3SxIkTlZmZqczMTP3xj38s9/pr/fr1U0REhCZP\nnqyCggK99dZbWrRoUXU+bpkcL774olasWKG8vDyNGzdOffv2Peo6a3/+85+1Z88ebdmyRU8++WSZ\nY9IAAAh1lLNa4u81ukqPO/Y9YWFheu+997Rs2TK1b99ezZo10y9/+Uvl5ORIkt5//3117dpVcXFx\nuvvuu/Xaa68pOjq63GWVnv7f//1f9e7du+RYtN69e5cc11ZaZGSk3nrrLb344otq0qSJXn/9dQ0f\nPty//wB+GDhwoP74xz9q2LBhSklJ0YYNGzR9+vSjMg8dOlS9evVSz549demll3K8GQCgzuH2TaiT\n+L0AAFTH4m2LlX0oW71b9lZiTGKl31+d2zdRzlAn8XsBAPAS99YEAACoI+rF2ZoAAABVsS9vn37Y\n/YMaRzVWl6QutbJOyhkAAEAFlqQv0YB/DZAk9WjRQ12bddULl72gqPCogK2T3ZoAAAAV6Nyks+Ki\n4yRJ32z/Rq+seEVfp38d0HVSzgAAACqQEpeijN9m6Mc7flT3Ft0lSYWuMKDrZLcmAADAcTSMbKjU\nxFT9tt9vtTVnq9rGH//WhdUV8pfSaNeu3VG3/AGkovtubty40esYAIB6Kqivc2ZmgyT9TUW7UF9w\nzv1fOWMmSRos6YCk651zy8oZU245AwAA8MLcdXP10rKXdErSKfrd+b9TmP10tFjQXufMzMIkPSXp\nYkldJY0ys5OPGTNYUqpzrpOkmyU9F8hM8Mb8+fO9joAq4rsLbXx/oY3vL7it3rla01ZN00PzH1K3\n57rp9tm3a+WOldVebqBPCOgjaa1zbpNzLl/SdElDjxkzVNK/JMk595WkeDNrHuBcqGX8BRO6+O5C\nG99faOP7C26XdLqk5Pmqnav09OKnNeGTCdVebqBPCEiRtKXU9FYVFbbjjdlWPG9HYKMBAABUXZek\nLtpz/x7N2zBP32d+r+xD2RrccXC1l8vZmgAAAFUU3yBeV5xyRY0uM6AnBJhZX0njnXODiqcfkORK\nnxRgZs9J+tg591rx9PeSLnDO7ThmWZwNAAAAQkZVTwgI9JazxZI6mllbSRmSRkoadcyYmZJuk/Ra\ncZnbc2wxk6r+AQEAAEJJQMuZc67QzG6XNFc/XUrjOzO7uehlN8U5N9vMLjGzH1V0KY0xgcwEAAAQ\nzELmIrQAAAD1QdDdW9PMBpnZ92b2g5ndX8GYSWa21syWmVn32s6I8p3ouzOz0Wa2vPjxmZmd7kVO\nlM+fP3vF4840s3wzG1ab+XB8fv7dmWZm35jZKjP7uLYzonx+/N0ZZ2Yzi//NW2lm13sQE+UwsxfM\nbIeZrTjOmEp3lqAqZ1y0NnT5891JWi/pfOdcN0kTJT1fuylRET+/vyPjHpP0Qe0mxPH4+XdnvKSn\nJf2Pc+40SSNqPSjK8PPP3m2SVjvnukvqL+kJM+NqC8HhRRV9d+WqamcJqnImLlobyk743TnnvnTO\n7S2e/FJF17NDcPDnz54k3SHpDUk7azMcTsif72+0pDedc9skyTmXWcsZUT5/vjsnKbb4eayk3c65\nglrMiAo45z6TlH2cIVXqLMFWzsq7aO2x/4BXdNFaeMuf7660myTNCWgiVMYJvz8zaynpcufcs5I4\nezq4+PPnr7OkRDP72MwWm9m1tZYOx+PPd/eUpFPNLF3Sckljaykbqq9KnYXNoqh1ZtZfRWflnut1\nFlTK3ySVPh6GghZaIiT1lDRAUiNJC81soXPuR29jwQ8XS/rGOTfAzFIlfWhmZzjn9nsdDIERbOVs\nm6Q2paZbFc87dkzrE4xB7fPnu5OZnSFpiqRBzrnjbQpG7fLn++stabqZmaQkSYPNLN85N7OWMqJi\n/nx/WyVlOucOSTpkZp9K6iaJcuYtf767MZIelSTn3Doz2yDpZElLaiUhqqNKnSXYdmuWXLTWzKJU\ndNHaY//inynpF1LJHQjKvWgtat0JvzszayPpTUnXOufWeZARFTvh9+ec61D8aK+i485upZgFDX/+\n7pwh6VwzCzezhpLOkvRdLedEWf58d5sk/UySio9X6qyiE6wQHEwV70moUmcJqi1nXLQ2dPnz3Un6\nvaRESc8Ub33Jd8718S41jvDz+zvqLbUeEhXy8+/O783sA0krJBVKmuKc+9bD2JDff/YmSnqp1OUa\n7nPOZXkUGaWY2VRJaZKamNlmSQ9LilI1OwsXoQUAAAgiwbZbEwAAoF6jnAEAAAQRyhkAAEAQoZwB\nAAAEEcoZAABAEKGcAQAABBHKGYA6wcwKzWypma00sxlmFlfDy7/OzCYVP3/YzH5Tk8sHgCMoZwDq\nigPOuZ7OudMlZUu6zetAAFAVlDMAddFCSSlHJszsHjNbZGbLzOzhUvN/YWbLzewbM3u5eN7/mNmX\nZva1mc01s6Ye5AdQjwXV7ZsAoBpMkswsXNJASf8onr5QUifnXJ/i24bNNLNzJWVJGiepn3Mu28xO\nKl7OAudc3+L33ijpfkn31O5HAVCfUc4A1BUxZrZUUitJ30r6sHj+RZIuLH7NJDWS1Kn45+vOuWxJ\ncs7tKR7f2sz+IylZUqSkDbX3EQCA3ZoA6o6DzrmektqoqIQdOebMJD1afDxaD+dcZ+fci8dZzmRJ\nk5xzZ0j6taQGAU0NAMegnAGoK0ySnHOHJI2VdI+ZhUn6QNINZtZIksysZfFxZPMkjTCzxOL5CcXL\niZOUXvz8ulrMDwCS2K0JoO5wJU+cW2ZmyyWNcs69amanSFpYdMiZ9km6xjn3rZk9IukTMyuQ9I2k\nGyRNkPSGmWWpqMC1q+XPAaCeM+fciUcBAACgVrBbEwAAIIhQzgAAAIII5QwAACCIUM4AAACCCOUM\nAAAgiFDOAAAAggjlDAAAIIhQzgAAAILI/wf6rFjaGaOP3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1169a4240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot precision-recall curve\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(recall, precision, label='PR Curve', lw=2)\n",
    "plt.plot(recall[1:],threshold, 'g--', lw=2, label=\"Threshold Prob\") # threshold \n",
    "plt.legend(loc=\"lower left\")\n",
    "plt.xlabel(\"Recall\")\n",
    "plt.ylabel(\"Precision\")\n",
    "plt.xlim([0,1])\n",
    "plt.ylim([0,1]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scikit-Learn Pipelines make it easy to keep Feature Engineering and Model building together.  Use them."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}