Naive Bayes Classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing the libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import statsmodels.formula.api as sm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Email No.</th>\n",
       "      <th>the</th>\n",
       "      <th>to</th>\n",
       "      <th>ect</th>\n",
       "      <th>and</th>\n",
       "      <th>for</th>\n",
       "      <th>of</th>\n",
       "      <th>a</th>\n",
       "      <th>you</th>\n",
       "      <th>hou</th>\n",
       "      <th>...</th>\n",
       "      <th>connevey</th>\n",
       "      <th>jay</th>\n",
       "      <th>valued</th>\n",
       "      <th>lay</th>\n",
       "      <th>infrastructure</th>\n",
       "      <th>military</th>\n",
       "      <th>allowing</th>\n",
       "      <th>ff</th>\n",
       "      <th>dry</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Email 1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Email 2</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>24</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Email 3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Email 4</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Email 5</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3002 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Email No.  the  to  ect  and  for  of    a  you  hou  ...  connevey  jay  \\\n",
       "0   Email 1    0   0    1    0    0   0    2    0    0  ...         0    0   \n",
       "1   Email 2    8  13   24    6    6   2  102    1   27  ...         0    0   \n",
       "2   Email 3    0   0    1    0    0   0    8    0    0  ...         0    0   \n",
       "3   Email 4    0   5   22    0    5   1   51    2   10  ...         0    0   \n",
       "4   Email 5    7   6   17    1    5   2   57    0    9  ...         0    0   \n",
       "\n",
       "   valued  lay  infrastructure  military  allowing  ff  dry  Prediction  \n",
       "0       0    0               0         0         0   0    0           0  \n",
       "1       0    0               0         0         0   1    0           0  \n",
       "2       0    0               0         0         0   0    0           0  \n",
       "3       0    0               0         0         0   0    0           0  \n",
       "4       0    0               0         0         0   1    0           0  \n",
       "\n",
       "[5 rows x 3002 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset=pd.read_csv('emails.csv') \n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>the</th>\n",
       "      <th>to</th>\n",
       "      <th>ect</th>\n",
       "      <th>and</th>\n",
       "      <th>for</th>\n",
       "      <th>of</th>\n",
       "      <th>a</th>\n",
       "      <th>you</th>\n",
       "      <th>hou</th>\n",
       "      <th>in</th>\n",
       "      <th>...</th>\n",
       "      <th>connevey</th>\n",
       "      <th>jay</th>\n",
       "      <th>valued</th>\n",
       "      <th>lay</th>\n",
       "      <th>infrastructure</th>\n",
       "      <th>military</th>\n",
       "      <th>allowing</th>\n",
       "      <th>ff</th>\n",
       "      <th>dry</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "      <td>5172.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>6.640565</td>\n",
       "      <td>6.188128</td>\n",
       "      <td>5.143852</td>\n",
       "      <td>3.075599</td>\n",
       "      <td>3.124710</td>\n",
       "      <td>2.627030</td>\n",
       "      <td>55.517401</td>\n",
       "      <td>2.466551</td>\n",
       "      <td>2.024362</td>\n",
       "      <td>10.600155</td>\n",
       "      <td>...</td>\n",
       "      <td>0.005027</td>\n",
       "      <td>0.012568</td>\n",
       "      <td>0.010634</td>\n",
       "      <td>0.098028</td>\n",
       "      <td>0.004254</td>\n",
       "      <td>0.006574</td>\n",
       "      <td>0.004060</td>\n",
       "      <td>0.914733</td>\n",
       "      <td>0.006961</td>\n",
       "      <td>0.290023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>11.745009</td>\n",
       "      <td>9.534576</td>\n",
       "      <td>14.101142</td>\n",
       "      <td>6.045970</td>\n",
       "      <td>4.680522</td>\n",
       "      <td>6.229845</td>\n",
       "      <td>87.574172</td>\n",
       "      <td>4.314444</td>\n",
       "      <td>6.967878</td>\n",
       "      <td>19.281892</td>\n",
       "      <td>...</td>\n",
       "      <td>0.105788</td>\n",
       "      <td>0.199682</td>\n",
       "      <td>0.116693</td>\n",
       "      <td>0.569532</td>\n",
       "      <td>0.096252</td>\n",
       "      <td>0.138908</td>\n",
       "      <td>0.072145</td>\n",
       "      <td>2.780203</td>\n",
       "      <td>0.098086</td>\n",
       "      <td>0.453817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>8.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>62.250000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>210.000000</td>\n",
       "      <td>132.000000</td>\n",
       "      <td>344.000000</td>\n",
       "      <td>89.000000</td>\n",
       "      <td>47.000000</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>1898.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>167.000000</td>\n",
       "      <td>223.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 3001 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               the           to          ect          and          for  \\\n",
       "count  5172.000000  5172.000000  5172.000000  5172.000000  5172.000000   \n",
       "mean      6.640565     6.188128     5.143852     3.075599     3.124710   \n",
       "std      11.745009     9.534576    14.101142     6.045970     4.680522   \n",
       "min       0.000000     0.000000     1.000000     0.000000     0.000000   \n",
       "25%       0.000000     1.000000     1.000000     0.000000     1.000000   \n",
       "50%       3.000000     3.000000     1.000000     1.000000     2.000000   \n",
       "75%       8.000000     7.000000     4.000000     3.000000     4.000000   \n",
       "max     210.000000   132.000000   344.000000    89.000000    47.000000   \n",
       "\n",
       "                of            a          you          hou           in  ...  \\\n",
       "count  5172.000000  5172.000000  5172.000000  5172.000000  5172.000000  ...   \n",
       "mean      2.627030    55.517401     2.466551     2.024362    10.600155  ...   \n",
       "std       6.229845    87.574172     4.314444     6.967878    19.281892  ...   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
       "25%       0.000000    12.000000     0.000000     0.000000     1.000000  ...   \n",
       "50%       1.000000    28.000000     1.000000     0.000000     5.000000  ...   \n",
       "75%       2.000000    62.250000     3.000000     1.000000    12.000000  ...   \n",
       "max      77.000000  1898.000000    70.000000   167.000000   223.000000  ...   \n",
       "\n",
       "          connevey          jay       valued          lay  infrastructure  \\\n",
       "count  5172.000000  5172.000000  5172.000000  5172.000000     5172.000000   \n",
       "mean      0.005027     0.012568     0.010634     0.098028        0.004254   \n",
       "std       0.105788     0.199682     0.116693     0.569532        0.096252   \n",
       "min       0.000000     0.000000     0.000000     0.000000        0.000000   \n",
       "25%       0.000000     0.000000     0.000000     0.000000        0.000000   \n",
       "50%       0.000000     0.000000     0.000000     0.000000        0.000000   \n",
       "75%       0.000000     0.000000     0.000000     0.000000        0.000000   \n",
       "max       4.000000     7.000000     2.000000    12.000000        3.000000   \n",
       "\n",
       "          military     allowing           ff          dry   Prediction  \n",
       "count  5172.000000  5172.000000  5172.000000  5172.000000  5172.000000  \n",
       "mean      0.006574     0.004060     0.914733     0.006961     0.290023  \n",
       "std       0.138908     0.072145     2.780203     0.098086     0.453817  \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "75%       0.000000     0.000000     1.000000     0.000000     1.000000  \n",
       "max       4.000000     3.000000   114.000000     4.000000     1.000000  \n",
       "\n",
       "[8 rows x 3001 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Email No.</th>\n",
       "      <th>the</th>\n",
       "      <th>to</th>\n",
       "      <th>ect</th>\n",
       "      <th>and</th>\n",
       "      <th>for</th>\n",
       "      <th>of</th>\n",
       "      <th>a</th>\n",
       "      <th>you</th>\n",
       "      <th>hou</th>\n",
       "      <th>...</th>\n",
       "      <th>connevey</th>\n",
       "      <th>jay</th>\n",
       "      <th>valued</th>\n",
       "      <th>lay</th>\n",
       "      <th>infrastructure</th>\n",
       "      <th>military</th>\n",
       "      <th>allowing</th>\n",
       "      <th>ff</th>\n",
       "      <th>dry</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 3002 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Email No., the, to, ect, and, for, of, a, you, hou, in, on, is, this, enron, i, be, that, will, have, with, your, at, we, s, are, it, by, com, as, from, gas, or, not, me, deal, if, meter, hpl, please, re, e, any, our, corp, can, d, all, has, was, know, need, an, forwarded, new, t, may, up, j, mmbtu, should, do, am, get, out, see, no, there, price, daren, but, been, company, l, these, let, so, would, m, into, xls, farmer, attached, us, information, they, message, day, time, my, one, what, only, http, th, volume, mail, contract, which, month, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 3002 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# verify that there are no null columns in the dataset\n",
    "dataset[dataset.isnull().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "categorical = [var for var in dataset.columns if dataset[var].dtype=='O']\n",
    "numerical = [var for var in dataset.columns if dataset[var].dtype!='O']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 1 categorical variables : \n",
      " ['Email No.']\n"
     ]
    }
   ],
   "source": [
    "print('There are {} categorical variables : \\n'.format(len(categorical)), categorical)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Email No.  contains  5172  labels\n"
     ]
    }
   ],
   "source": [
    "# check for cardinality in categorical variables\n",
    "for var in categorical:\n",
    "    print(var, ' contains ', len(dataset[var].unique()), ' labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          the      to     ect     and     for      of       a     you     hou  \\\n",
      "count  5172.0  5172.0  5172.0  5172.0  5172.0  5172.0  5172.0  5172.0  5172.0   \n",
      "mean      7.0     6.0     5.0     3.0     3.0     3.0    56.0     2.0     2.0   \n",
      "std      12.0    10.0    14.0     6.0     5.0     6.0    88.0     4.0     7.0   \n",
      "min       0.0     0.0     1.0     0.0     0.0     0.0     0.0     0.0     0.0   \n",
      "25%       0.0     1.0     1.0     0.0     1.0     0.0    12.0     0.0     0.0   \n",
      "50%       3.0     3.0     1.0     1.0     2.0     1.0    28.0     1.0     0.0   \n",
      "75%       8.0     7.0     4.0     3.0     4.0     2.0    62.0     3.0     1.0   \n",
      "max     210.0   132.0   344.0    89.0    47.0    77.0  1898.0    70.0   167.0   \n",
      "\n",
      "           in  ...  connevey     jay  valued     lay  infrastructure  \\\n",
      "count  5172.0  ...    5172.0  5172.0  5172.0  5172.0          5172.0   \n",
      "mean     11.0  ...       0.0     0.0     0.0     0.0             0.0   \n",
      "std      19.0  ...       0.0     0.0     0.0     1.0             0.0   \n",
      "min       0.0  ...       0.0     0.0     0.0     0.0             0.0   \n",
      "25%       1.0  ...       0.0     0.0     0.0     0.0             0.0   \n",
      "50%       5.0  ...       0.0     0.0     0.0     0.0             0.0   \n",
      "75%      12.0  ...       0.0     0.0     0.0     0.0             0.0   \n",
      "max     223.0  ...       4.0     7.0     2.0    12.0             3.0   \n",
      "\n",
      "       military  allowing      ff     dry  Prediction  \n",
      "count    5172.0    5172.0  5172.0  5172.0      5172.0  \n",
      "mean        0.0       0.0     1.0     0.0         0.0  \n",
      "std         0.0       0.0     3.0     0.0         0.0  \n",
      "min         0.0       0.0     0.0     0.0         0.0  \n",
      "25%         0.0       0.0     0.0     0.0         0.0  \n",
      "50%         0.0       0.0     0.0     0.0         0.0  \n",
      "75%         0.0       0.0     1.0     0.0         1.0  \n",
      "max         4.0       3.0   114.0     4.0         1.0  \n",
      "\n",
      "[8 rows x 3001 columns] 2\n"
     ]
    }
   ],
   "source": [
    "# view summary statistics in numerical variables to check for outliers\n",
    "print(round(dataset[numerical].describe()),2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Email No.</th>\n",
       "      <th>the</th>\n",
       "      <th>to</th>\n",
       "      <th>ect</th>\n",
       "      <th>and</th>\n",
       "      <th>for</th>\n",
       "      <th>of</th>\n",
       "      <th>a</th>\n",
       "      <th>you</th>\n",
       "      <th>hou</th>\n",
       "      <th>...</th>\n",
       "      <th>connevey</th>\n",
       "      <th>jay</th>\n",
       "      <th>valued</th>\n",
       "      <th>lay</th>\n",
       "      <th>infrastructure</th>\n",
       "      <th>military</th>\n",
       "      <th>allowing</th>\n",
       "      <th>ff</th>\n",
       "      <th>dry</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1111</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>24</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>102</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2222</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3333</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4444</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>57</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 3002 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Email No.  the  to  ect  and  for  of    a  you  hou  ...  connevey  jay  \\\n",
       "0          0    0   0    1    0    0   0    2    0    0  ...         0    0   \n",
       "1       1111    8  13   24    6    6   2  102    1   27  ...         0    0   \n",
       "2       2222    0   0    1    0    0   0    8    0    0  ...         0    0   \n",
       "3       3333    0   5   22    0    5   1   51    2   10  ...         0    0   \n",
       "4       4444    7   6   17    1    5   2   57    0    9  ...         0    0   \n",
       "\n",
       "   valued  lay  infrastructure  military  allowing  ff  dry  Prediction  \n",
       "0       0    0               0         0         0   0    0           0  \n",
       "1       0    0               0         0         0   1    0           0  \n",
       "2       0    0               0         0         0   0    0           0  \n",
       "3       0    0               0         0         0   0    0           0  \n",
       "4       0    0               0         0         0   1    0           0  \n",
       "\n",
       "[5 rows x 3002 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use LabelEncoder to replace purchased (dependent variable) with 0 and 1 \n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "dataset['Email No.']= LabelEncoder().fit_transform(dataset['Email No.'])\n",
    "dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = dataset['Prediction']\n",
    "x = dataset.drop(['Prediction'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Email No.  the  to  ect  and  for  of    a  you  hou  ...  enhancements  \\\n",
      "0          0    0   0    1    0    0   0    2    0    0  ...             0   \n",
      "1       1111    8  13   24    6    6   2  102    1   27  ...             0   \n",
      "2       2222    0   0    1    0    0   0    8    0    0  ...             0   \n",
      "3       3333    0   5   22    0    5   1   51    2   10  ...             0   \n",
      "4       4444    7   6   17    1    5   2   57    0    9  ...             0   \n",
      "\n",
      "   connevey  jay  valued  lay  infrastructure  military  allowing  ff  dry  \n",
      "0         0    0       0    0               0         0         0   0    0  \n",
      "1         0    0       0    0               0         0         0   1    0  \n",
      "2         0    0       0    0               0         0         0   0    0  \n",
      "3         0    0       0    0               0         0         0   0    0  \n",
      "4         0    0       0    0               0         0         0   1    0  \n",
      "\n",
      "[5 rows x 3001 columns]\n"
     ]
    }
   ],
   "source": [
    "print(x.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    0\n",
      "1    0\n",
      "2    0\n",
      "3    0\n",
      "4    0\n",
      "Name: Prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting the dataset into training and test set  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=0.3,random_state = 0) # func returns train and test data. It takes dataset and then split size test_size =0.3 means 30% data is for test and rest for training and random_state \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Email No.  the  to  ect  and  for  of    a  you  hou  ...  enhancements  \\\n",
      "3459       2735    5   4    1    2    2   3   27    7    0  ...             0   \n",
      "1385        430    3   6    4    6    0   2   57    2    0  ...             0   \n",
      "1380        425    1   0    1    0    0   1    8    0    0  ...             0   \n",
      "4462       3849   37  16    5   22    5  19  228    6    2  ...             0   \n",
      "3840       3158    2   1    1    0    2   0   10    0    0  ...             0   \n",
      "\n",
      "      connevey  jay  valued  lay  infrastructure  military  allowing  ff  dry  \n",
      "3459         0    0       0    0               0         0         0   0    0  \n",
      "1385         0    0       0    0               0         0         0   1    0  \n",
      "1380         0    0       0    0               0         0         0   0    0  \n",
      "4462         0    0       0    2               0         0         0   1    0  \n",
      "3840         0    0       0    0               0         0         0   0    0  \n",
      "\n",
      "[5 rows x 3001 columns]\n"
     ]
    }
   ],
   "source": [
    "print(x_train.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Email No.  the  to  ect  and  for  of    a  you  hou  ...  enhancements  \\\n",
      "3324       2585   14   7    2    2    2   4   37    0    1  ...             0   \n",
      "15          667    6   2    1    0    2   0   36    3    1  ...             0   \n",
      "4950       4391   40   6    4    6    5  11  130    3    0  ...             0   \n",
      "3964       3295   42  19    1    8    7   7  214    1    3  ...             0   \n",
      "2315       1464    8  14    8    3    3   0   87    0    4  ...             0   \n",
      "\n",
      "      connevey  jay  valued  lay  infrastructure  military  allowing  ff  dry  \n",
      "3324         0    0       0    0               0         0         0   0    0  \n",
      "15           0    0       0    0               0         0         0   0    0  \n",
      "4950         0    0       0    0               0         0         0   1    0  \n",
      "3964         0    0       0    1               0         4         0   3    0  \n",
      "2315         0    0       0    0               0         0         0   5    0  \n",
      "\n",
      "[5 rows x 3001 columns]\n"
     ]
    }
   ],
   "source": [
    "print(x_test.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3459    1\n",
      "1385    0\n",
      "1380    0\n",
      "4462    1\n",
      "3840    0\n",
      "1452    1\n",
      "2849    0\n",
      "3145    0\n",
      "1271    0\n",
      "1021    0\n",
      "Name: Prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y_train[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3324    0\n",
      "15      0\n",
      "4950    0\n",
      "3964    1\n",
      "2315    0\n",
      "861     1\n",
      "2350    0\n",
      "1767    0\n",
      "965     0\n",
      "2630    1\n",
      "Name: Prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y_test[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "x_train=scaler.fit_transform(x_train) # apply on whole x data \n",
    "x_test=scaler.transform(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB()"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "classifier = GaussianNB()  \n",
    "classifier.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = classifier.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate Model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1034   77]\n",
      " [  23  418]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report, average_precision_score\n",
    "cm = confusion_matrix(y_test,y_pred)\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.93      0.95      1111\n",
      "           1       0.84      0.95      0.89       441\n",
      "\n",
      "    accuracy                           0.94      1552\n",
      "   macro avg       0.91      0.94      0.92      1552\n",
      "weighted avg       0.94      0.94      0.94      1552\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cr = classification_report(y_test,y_pred)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9355670103092784"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8152227118415141\n"
     ]
    }
   ],
   "source": [
    "average_precision= average_precision_score(y_test,y_pred)\n",
    "print(average_precision)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, '2-class Precision-Recall curve: AP=0.82')"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5gU1Z3/8fdH7lFRI6NR7igqoEJ0IppE4/0WkVWMgkRBzbokYrLGmPj4rJt4ifGSVZMfKpoVSAyKwXhB18ga2YgYE0EdjaAIKMqIF0QFQVEGvr8/qmbS9PTM9MD0jDP1eT1PP9N16tTp7+nprm/VqeoqRQRmZpZdW7V0AGZm1rKcCMzMMs6JwMws45wIzMwyzonAzCzjnAjMzDLOiaCVkzRW0pyWjqMpSRot6X+LqDdR0qXNEVNzkLRU0pHp859J+n1Lx2TZ4ETQAiR1knS7pNclfSTpOUnHtXRcxUhXVp9IWiPpHUmTJW3TlK8REVMj4ugi6o2LiCua8rWrSQpJa9N+vinpekntSvFabZ2kKZKqJO2aV/4zSevT9/hDSX+VdNBmtH96+l1aK+l+SV+sp+4QSU9IWiWpUtJ/5sz7pqQ5aSxvS/qNpG0bG09r5ETQMtoDy4BvANsBlwJ/kNSnBWNqjGERsQ2wH/AV4D/yK0hq3+xRNb3BaT+/AZwGnN3C8TSp5vgfSdoaGAGsAkYXqHJ3+h6XAXOAeyWpEe0PAm4FzgB2Bj4Gbq5nkTuB2cAXSf6v35V0YjpvO+BKYFdgANADuK7YWFozJ4IWEBFrI+JnEbE0IjZGxEPAa8D+dS0jqaekeyWtkLRS0oQ66v1K0jJJqyU9I+ngnHkHSJqXzntH0vVpeWdJv0/b/VDSXEk7F9GPN4E/AXun7YSk8yQtAhalZSdIqsjZ4tu3oT7lDncpcYOkd9OtuBckVb/eFElX5rT3r5IWS3pf0ozcLdA0tnGSFkn6QNJNxa5wImIx8CQwJKe9zenXbpJmpWXvSZoqaftiYsgnaXj6+qslLZF0bFpeM7yUTtcMMUnqk74P50h6A5gl6RFJ4/Pafl7SyenzvSQ9mr6nCyWd2shQRwAfApcDY+qqFBHrgd8CXwJ2bET7o4EHI2J2RKwh2ag6uZ4t+T7A1IjYEBFLSJLPoDSGOyPikYj4OCI+AH4DfK0RsbRaTgSfA+lKdw9gfh3z2wEPAa+TfJC7A9PqaG4uyQrriyRbP9MldU7n/Qr4VUR0BXYD/pCWjyHZGupJ8iUcB3xSRNw9geOB53KK/wUYCgyUtB8wCfi3tN1bgRlKhsaK7dPRwCEk78/2JFvmKwvEcjjwC+BUYJe03fz2TiDZgxmc1jumoT6mbe8FHAwsTqc3t19KY6ze4uwJ/KyYGPLiOQD4HXARyXtyCLC0EU18I339Y0g+I6Ny2h4I9Ab+J92afzSts1Na7+Z0K7x6SOaFBl5rDHAXyXuwV/reFepTJ2AsUBkR70n6eppk63p8PV10EPB8dTvpyv0zks9LITcCZ0rqIGlP4CDgz3XUPYQ6vpNtTkT40YIPoAPJB/HWeuocBKwA2heYNxaYU8+yH5AMcUCyS3wZ0C2vztnAX4F9i4h3KbCGZCvvdZLd8C7pvAAOz6l7C3BF3vILSVZERfUJOBx4BTgQ2Cqv3hTgyvT57cC1OfO2AdYDfXJi+3rO/D8AF9fTzwBWA2vT53cBnbakXwVe41+A5/Le2yPT5z8Dfl/HcrcCN9Tz/zkyZ7qmHZLEFEC/nPnbpn3snU7/HJiUPj8NeKLAa/+0yM92L2AjMCSdnkmyIZIb22fpZ+ldYBawfyO/P48B4/LK3gQOraP+V0kSelX6XlxWR72jSL47ezQmntb68B5BC5K0FXAHyZdhfE75n5QcQFsjaTTJluPrEVFVRJsXSnopHUb5kGRLv1s6+xySLaWX0+GfE9LyO0i+pNMkLZd0raQO9bzMv0TE9hHROyK+FxG5ew/Lcp73Bi7M3ZJL+7JrsX2KiFnABOAm4B1Jt0nqWqDqriSJqXq5NSR7Dt1z6ryd8/xjkmSBpPk57/fBOXX2S+ucRrKXs/WW9EvSTpKmKTn4vBr4Pf/83zRGT2DJZixXreZ/FBEfAf8DjEyLRgJT0+e9gaF5/RxNMnxTjDOAlyKiIp2eCpye99n6Q/pZ2ikiDo+IZxrZlzVA/uehK/BRfkUlB5EfIRmm6kzyPh4j6Xt59Q4k2Qs6JSJeaWQ8rZITQQtJx6dvJznANSKSMVIAIuK4iNgmfUwl+eL2UgMH99KV2E9Ihj12iIjtSQ7SKW13UUSMItnNvwa4R9LWEbE+Ii6LiIEkW0wnAGduZtdyL2e7DPh5+kWvfnwhIu4qtk9p3L+OiP1JhgH2IBkSybecZMUF1Byk3JFk67Ch9gflvN9P5M2LiPgD8BRQfYbJ5vbrFyTvz76RDM99m/R/00jLSIb2ClkLfCFnutBKO/+Sw3cBo5ScsdMF+L+c13k8r5/bRMR3i4zzTKCfkjNw3gauJ0l8DZ4hJ+ngnORc6FGdsOeTDPVVL9cP6ESyF5mvH7AhIn4XEVURUUkyZHV8zvJfBmYAZ0fEY0X2s9VzImg5t5CM0w7L26Iu5GngLeBqSVsrObhb6CDWtiS7vCuA9kpOjavZWpL0bUllEbGRZHccYIOkwyTtk45vryYZUtmwRb1L/AYYJ2moElsrOUVv22L7JOkr6fIdSFZy6+qI7U7gLCWnB3YCrgL+HhFLm6AfAFcD50r60hb0a1vSYTVJ3Smc0IpxO0lfj5C0laTu6XEMgApgZDoGXg6cUkR7D5Mk0ctJzuLZmJY/BOwh6Yy0vQ7p/2NAQw2mSWU34ACSY1ZDSE4quJN6DhpXi4gncpJzoUd1wp4KDEsTx9ZpH+5N93TyvZKEptPT9+1LJHt7z6cx702yx3B+RDzYUIxtiRNBC5DUm+RA4xDg7bxhoFoiYgMwDNgdeAOoJPkA55tJchbPKyTDJOvYdKjmWGC+pDUkB45HRsQ6kq3Ge0iSwEvA4yTDFlskIuYB/0oytPMBydjs2Eb2qSvJiveDtE8rgV8WeK3HSM4Y+SPJing3/jncscUi4h8k78tFW9Cvy0iGm1aRDMfcu5mxPA2cBdyQtvU4/9wbupSk7x+kr3dnEe19msZyZG79dGV6NMn7uJxkaO0aki3u6h/+1XUwdQzwQET8IyLern6QfO5OUD3n+jdGRMwnOblhKslxhm2BmqEeJT86nJjWXQ2cDFxA8v5UAC+SHBcBuJDkNNbbc76TmThYrAjfmMbMLMu8R2BmlnFOBGZmGedEYGaWcU4EZmYZ1+ouDNatW7fo06dPS4dhZtaqPPPMM+9FRFmhea0uEfTp04d58+a1dBhmZq2KpNfrmuehITOzjHMiMDPLOCcCM7OMcyIwM8s4JwIzs4wrWSKQNEnJ7QVfrGO+JP1aya0FX1Addy4yM7PSKuUewRSSq13W5Tigf/o4l+SyzGZm1sxKlggiYjbwfj1VhgO/S2/88Tdge0m7lCqeyx6cz2UPZuKKsmZmjdKSPyjrzqbXyq9My97KryjpXJK9Bnr16rVZL7Zg+erNWs7MrK1ryYPFhW7RV/DmCBFxW0SUR0R5WVnBX0ibmdlmaslEUEly8+hqPUjugmRmZs2oJRPBDODM9OyhA4FVEVFrWMjMzEqrZMcIJN0FHAp0k1QJ/BToABARE0lumH08yf1ePya5B6uZmTWzkiWCiBjVwPwAzivV65uZWXH8y2Izs4xzIjAzyzgnAjOzjHMiMDPLOCcCM7OMcyIwM8s4JwIzs4xzIjAzyzgnAjOzjHMiMDPLOCcCM7OMcyIwM8s4JwIzs4xzIjAzyzgnAjOzjHMiMDPLOCcCM7OMcyIwM8s4JwIzs4wraSKQdKykhZIWS7q4wPwdJN0n6QVJT0vau5TxmJlZbSVLBJLaATcBxwEDgVGSBuZVuwSoiIh9gTOBX5UqHjMzK6yUewQHAIsj4tWI+AyYBgzPqzMQeAwgIl4G+kjauYQxmZlZnlImgu7AspzpyrQs1/PAyQCSDgB6Az3yG5J0rqR5kuatWLGiROGamWVTKROBCpRF3vTVwA6SKoDzgeeAqloLRdwWEeURUV5WVtb0kZqZZVj7ErZdCfTMme4BLM+tEBGrgbMAJAl4LX2YmVkzKeUewVygv6S+kjoCI4EZuRUkbZ/OA/gOMDtNDmZm1kxKtkcQEVWSxgMzgXbApIiYL2lcOn8iMAD4naQNwALgnFLFY2ZmhZVyaIiIeBh4OK9sYs7zp4D+pYzBzMzq518Wm5llnBOBmVnGORGYmWWcE4GZWcY5EZiZZZwTgZlZxjkRmJllnBOBmVnGlfQHZZ83C95azWm3PtXSYZhZRg0f0p3Th/Zq6TBqyUwiGD4k/wrYZmbN5++vvQ/gRNCSTh/a63P5DzCzbPg8j0b4GIGZWcY5EZiZZZwTgZlZxjkRmJllnBOBmVnGORGYmWWcE4GZWcaVNBFIOlbSQkmLJV1cYP52kh6U9Lyk+ZLOKmU8ZmZWW8kSgaR2wE3AccBAYJSkgXnVzgMWRMRg4FDgvyR1LFVMZmZWWyn3CA4AFkfEqxHxGTANGJ5XJ4BtJQnYBngfqCphTGZmlqeUiaA7sCxnujItyzUBGAAsB/4B/CAiNuY3JOlcSfMkzVuxYkWp4jUzy6RSJgIVKIu86WOACmBXYAgwQVLXWgtF3BYR5RFRXlZW1vSRmpllWCkTQSXQM2e6B8mWf66zgHsjsRh4DdirhDGZmVmeUiaCuUB/SX3TA8AjgRl5dd4AjgCQtDOwJ/BqCWMyM7M8JbsMdURUSRoPzATaAZMiYr6kcen8icAVwBRJ/yAZSvpJRLxXqpjMzKy2kt6PICIeBh7OK5uY83w5cHQpYzAzs/r5l8VmZhnnRGBmlnFOBGZmGedEYGaWcU4EZmYZ50RgZpZxTgRmZhnnRGBmlnFOBGZmGedEYGaWcU4EZmYZ50RgZpZxTgRmZhnnRGBmlnFOBGZmGedEYGaWcU4EZmYZ50RgZpZxTgRmZhlXVCKQ9DVJj0p6RdKrkl6T9GoRyx0raaGkxZIuLjD/IkkV6eNFSRskfXFzOmJmZpun2JvX3w5cADwDbChmAUntgJuAo4BKYK6kGRGxoLpORFwHXJfWHwZcEBHvFx++mZltqWITwaqI+FMj2z4AWBwRrwJImgYMBxbUUX8UcFcjX8PMzLZQsYng/yRdB9wLfFpdGBHP1rNMd2BZznQlMLRQRUlfAI4Fxtcx/1zgXIBevXoVGbKZmRWj2ERQvQIvzykL4PB6llGBsqij7jDgybqGhSLiNuA2gPLy8rraMDOzzVBUIoiIwzaj7UqgZ850D2B5HXVH4mEhM7MWUexZQ9tJul7SvPTxX5K2a2CxuUB/SX0ldSRZ2c8o1DbwDeCBxgZvZmZbrtjfEUwCPgJOTR+rgcn1LRARVSRj/jOBl4A/RMR8SeMkjcupehLwvxGxtrHBm5nZliv2GMFuETEiZ/oySRUNLRQRDwMP55VNzJueAkwpMg4zM2tixe4RfCLp69UTkr4GfFKakMzMrDkVu0fwXeC36Xi+gPeBsaUKyszMmk+xZw1VAIMldU2nV5c0KjMzazb1JgJJ346I30v6YV45ABFxfQljMzOzZtDQHsHW6d9tSx2ImZm1jHoTQUTcmv69rHnCMTOz5lbsD8quldRVUgdJj0l6T9K3Sx2cmZmVXrGnjx6dHiA+geTSEXsAF5UsKjMzazbFJoIO6d/jgbt8zwAzs7aj2N8RPCjpZZIfkX1PUhmwrnRhmZlZcylqjyAiLgYOAsojYj2wluQmM2Zm1so19DuCwyNilqSTc8pyq9xbqsDMzKx5NDQ09A1gFsmNY/IFTgRmZq1eQ78j+Gn696zmCcfMzJpbsb8juErS9jnTO0i6snRhmZlZcyn29NHjIuLD6omI+IDkVFIzM2vlik0E7SR1qp6Q1AXoVE99MzNrJYr9HcHvgcckTSY5SHw28NuSRWVmZs2m2PsRXCvpBeBIkhvTXBERM0samZmZNYtih4YguQH9IxFxIfCEpAYvTS3pWEkLJS2WdHEddQ6VVCFpvqTHGxGPmZk1gWLPGvpX4B7g1rSoO3B/A8u0A24CjgMGAqMkDcyrsz1wM3BiRAwCvtWo6M3MbIsVu0dwHvA1YDVARCwCdmpgmQOAxRHxakR8Bkyj9mUpTgfujYg30nbfLTZwMzNrGsUmgk/TlTkAktqTHDSuT3dgWc50ZVqWaw9gB0l/kfSMpDMLNSTpXEnzJM1bsWJFkSGbmVkxik0Ej0u6BOgi6ShgOvBgA8uoQFl+8mgP7A98EzgGuFTSHrUWirgtIsojorysrKzIkM3MrBjFJoKfACuAfwD/BjwM/EcDy1QCPXOmewDLC9R5JCLWRsR7wGxgcJExmZlZE2jw9FFJWwEvRMTewG8a0fZcoL+kvsCbwEiSYwK5HgAmpENNHYGhwA2NeA0zM9tCDSaCiNgo6XlJvaoP6hYjIqokjQdmAu2ASRExX9K4dP7EiHhJ0iPAC8BG4L8j4sXN64qZmW2OYn9ZvAswX9LTJDelASAiTqxvoYh4mGQYKbdsYt70dcB1RcZhZmZNrNhEcFlJozAzsxbT0B3KOgPjgN1JDhTfHhFVzRGYmZk1j4bOGvotUE6SBI4D/qvkEZmZWbNqaGhoYETsAyDpduDp0odkZmbNqaE9gvXVTzwkZGbWNjW0RzBY0ur0uUh+Wbw6fR4R0bWk0ZmZWck1dPP6ds0ViJmZtYzG3I/AzMzaICcCM7OMcyIwM8s4JwIzs4xzIjAzyzgnAjOzjHMiMDPLOCcCM7OMcyIwM8s4JwIzs4xzIjAzy7iSJgJJx0paKGmxpIsLzD9U0ipJFenjP0sZj5mZ1VbsrSobTVI74CbgKKASmCtpRkQsyKv6REScUKo4zMysfqXcIzgAWBwRr0bEZ8A0YHgJX8/MzDZDKRNBd2BZznRlWpbvIEnPS/qTpEGFGpJ0rqR5kuatWLGiFLGamWVWKROBCpRF3vSzQO+IGAz8P+D+Qg1FxG0RUR4R5WVlZU0cpplZtpUyEVQCPXOmewDLcytExOqIWJM+fxjoIKlbCWMyM7M8pUwEc4H+kvpK6giMBGbkVpD0JUlKnx+QxrOyhDGZmVmekp01FBFVksYDM4F2wKSImC9pXDp/InAK8F1JVcAnwMiIyB8+MjOzEipZIoCa4Z6H88om5jyfAEwoZQxmZlY//7LYzCzjnAjMzDLOicDMLOOcCMzMMs6JwMws45wIzMwyzonAzCzjnAjMzDLOicDMLOOcCMzMMs6JwMws45wIzMwyzonAzCzjnAjMzDLOicDMLOOcCMzMMs6JwMws45wIzMwyzonAzCzjSpoIJB0raaGkxZIurqfeVyRtkHRKKeMxM7PaSpYIJLUDbgKOAwYCoyQNrKPeNcDMUsViZmZ1K+UewQHA4oh4NSI+A6YBwwvUOx/4I/BuCWMxM7M6lDIRdAeW5UxXpmU1JHUHTgIm1teQpHMlzZM0b8WKFU0eqJlZlpUyEahAWeRN3wj8JCI21NdQRNwWEeURUV5WVtZkAZqZGbQvYduVQM+c6R7A8rw65cA0SQDdgOMlVUXE/SWMy8zMcpQyEcwF+kvqC7wJjAROz60QEX2rn0uaAjzkJGBm1rxKlggiokrSeJKzgdoBkyJivqRx6fx6jwuYmVnzKOUeARHxMPBwXlnBBBARY0sZi5mZFeZfFpuZZZwTgZlZxjkRmJllnBOBmVnGORGYmWWcE4GZWcY5EZiZZZwTgZlZxjkRmJllnBOBmVnGORGYmWWcE4GZWcY5EZiZZVxJrz7aXNavX09lZSXr1q1r6VAsAzp37kyPHj3o0KFDS4di1iTaRCKorKxk2223pU+fPqR3OzMriYhg5cqVVFZW0rdv34YXMGsF2sTQ0Lp169hxxx2dBKzkJLHjjjt679PalDaRCAAnAWs2/qxZW9NmEoGZmW2ekiYCScdKWihpsaSLC8wfLukFSRWS5kn6einjKaV33nmH008/nX79+rH//vtz0EEHcd9995X8defNm8f3v//9zV6+T58+jBgxomb6nnvuYezYsQBMmTKFsrIyhgwZwqBBgzjllFP4+OOPC7Zz//33c/nll29SNnjwYEaNGrVJ2dixY+nbty9Dhgxhv/3246mnntrs2Ks98sgj7Lnnnuy+++5cffXVBeusWrWKYcOGMXjwYAYNGsTkyZMBWLZsGYcddhgDBgxg0KBB/OpXv6pZ5kc/+hGzZs3a4vjMPvcioiQPkhvWLwH6AR2B54GBeXW2AZQ+3xd4uaF2999//8i3YMGCWmXNaePGjXHggQfGLbfcUlO2dOnS+PWvf92CURWnd+/e0atXr3jxxRcjImL69OkxZsyYiIiYPHlynHfeeTV1R40aFZMmTSrYzkEHHRQrVqyomV6wYEHsvffeseuuu8aaNWtqyseMGRPTp0+PiIiZM2fGPvvss0XxV1VVRb9+/WLJkiXx6aefxr777hvz58+vVe/nP/95/PjHP46IiHfffTd22GGH+PTTT2P58uXxzDPPRETE6tWro3///jXLL126NI466qiCr9vSnzlrfU6d+Nc4deJfW+z1gXlRx3q1lGcNHQAsjohXASRNA4YDC3KS0Jqc+lsDsaUvetmD81mwfPWWNrOJgbt25afDBtU5f9asWXTs2JFx48bVlPXu3Zvzzz8fgKVLl3LGGWewdu1aACZMmMBXv/pV/vKXv/DLX/6Shx56CIDx48dTXl7O2LFjufjii5kxYwbt27fn6KOP5pe//CXTp0/nsssuo127dmy33XbMnj17kzaefvpp/v3f/51PPvmELl26MHnyZPbcc0+mTJnCjBkz+Pjjj1myZAknnXQS1157bU2sP/rRj7jqqquYOnVqnX2sqqpi7dq17LDDDrXmvfLKK3Tq1Ilu3brVlN15552cccYZvPTSS8yYMaPWngHAIYccwuLFi+t8zWI8/fTT7L777vTr1w+AkSNH8sADDzBw4MBN6knio48+IiJYs2YNX/ziF2nfvj277LILu+yyCwDbbrstAwYM4M0332TgwIH07t2blStX8vbbb/OlL31pi+I0+zwrZSLoDizLma4EhuZXknQS8AtgJ+CbJYynZObPn89+++1X5/yddtqJRx99lM6dO7No0SJGjRrFvHnz6qz//vvvc9999/Hyyy8jiQ8//BCAyy+/nJkzZ9K9e/easlx77bUXs2fPpn379vz5z3/mkksu4Y9//CMAFRUVPPfcc3Tq1Ik999yT888/n549ewJw6qmncvPNNxdcKd99993MmTOHt956iz322INhw4bVqvPkk0/W6v/dd9/No48+ysKFC5kwYULBRPDggw+yzz771CqfOnUq1113Xa3y3XffnXvuuWeTsjfffLOmHwA9evTg73//e61lx48fz4knnsiuu+7KRx99xN13381WW206Mrp06VKee+45hg7958d0v/3248knn9xk+MysrSllIih0akWtLf6IuA+4T9IhwBXAkbUaks4FzgXo1atXvS9a35Z7cznvvPOYM2cOHTt2ZO7cuaxfv57x48dTUVFBu3bteOWVV+pdvmvXrnTu3JnvfOc7fPOb3+SEE04A4Gtf+xpjx47l1FNP5eSTT6613KpVqxgzZgyLFi1CEuvXr6+Zd8QRR7DddtsBMHDgQF5//fWaFWi7du246KKL+MUvfsFxxx23SZunnXYaEyZMICI477zzuO6667j44k0P97z11luUlZXVTM+dO5eysjJ69+5Njx49OPvss/nggw9q9iYuuugirrzySsrKyrj99ttr9WP06NGMHj263veoWrLHu6lCZ/XMnDmTIUOGMGvWLJYsWcJRRx3FwQcfTNeuXQFYs2YNI0aM4MYbb6wpgySJL1++vKhYzFqrUh4srgR65kz3AOr8RkXEbGA3Sd0KzLstIsojojx3hfN5MWjQIJ599tma6ZtuuonHHnuMFStWAHDDDTew88478/zzzzNv3jw+++wzANq3b8/GjRtrlqs+N719+/Y8/fTTjBgxgvvvv59jjz0WgIkTJ3LllVeybNkyhgwZwsqVKzeJ49JLL+Wwww7jxRdf5MEHH9zkXPdOnTrVPG/Xrh1VVVWbLHvGGWcwe/Zs3njjjYJ9lMSwYcOYPXt2rXldunTZ5LXuuusuXn75Zfr06cNuu+3G6tWra/ZMAK677joqKip49NFH2XvvvWu1N3XqVIYMGVLrccopp9Sq26NHD5Yt++eOZ2VlJbvuumutepMnT+bkk09GErvvvjt9+/bl5ZdfBpJfpo8YMYLRo0fXSrDr1q2jS5cuBd8Ts7ailIlgLtBfUl9JHYGRwIzcCpJ2V7r5Jmk/koPKK2u19Dl3+OGHs27dOm655Zaastyza1atWsUuu+zCVlttxR133MGGDRuA5DjCggUL+PTTT1m1ahWPPfYYkGydrlq1iuOPP54bb7yRiooKAJYsWcLQoUO5/PLL6dat2yYrwOrX6d69O5Cc8dMYHTp04IILLuDGG2+ss86cOXPYbbfdapUPGDCgZlhp48aNTJ8+nRdeeIGlS5eydOlSHnjgAe66666iYxk9ejQVFRW1HvnDQgBf+cpXWLRoEa+99hqfffYZ06ZN48QTT6xVr1evXjXv7zvvvMPChQvp168fEcE555zDgAED+OEPf1hruVdeeaVgsjJrS0o2NBQRVZLGAzNJziCaFBHzJY1L508ERgBnSloPfAKcFoX29T/nJHH//fdzwQUXcO2111JWVsbWW2/NNddcA8D3vvc9RowYwfTp0znssMPYeuutAejZsyennnoq++67L/379+fLX/4yAB999BHDhw9n3bp1RAQ33HADkAypLFq0iIjgiCOOYPDgwTz++OM1cfz4xz9mzJgxXH/99Rx++OGN7sc555zDlVdeuUlZ9TGCjRs30qNHj4IJ5pBDDuHCCy8kIpg9ezbdu3evSUjV8xcsWMBbb73V6Jga0r59eyZMmMAxxxzDhg0bOPvssxk0KBkenDhxIgDjxo3j0ksvZezYseyzzz5EBNdccw3dunVjzpw53HHHHeyzzyvPkUYAAAY1SURBVD4MGTIEgKuuuorjjz+e9evXs3jxYsrLy5s8bsumBW+t5rRbN/+U6YZOXNlcam3r3fLy8sg/0PrSSy8xYMCAForIAH7wgx8wbNgwjjyy1iGeVuu+++7j2Wef5Yorrqg1z585a6w7//4GD1S8uUVtbEkikPRMRBTcqmkTF52zlnfJJZcUPFunNauqquLCCy9s6TCsjTh9aC9OH1r/yS4txYnAmsTOO+9ccGy+NfvWt77V0iGYNYs2c62h1jbEZa2XP2vW1rSJRNC5c2dWrlzpL6iVXKT3I+jcuXNLh2LWZNrE0FCPHj2orKysOW/frJSq71Bm1la0iUTQoUMH3y3KzGwztYmhITMz23xOBGZmGedEYGaWca3ul8WSVgCvb+bi3YD3mjCc1sB9zgb3ORu2pM+9I6LgVTtbXSLYEpLm1fUT67bKfc4G9zkbStVnDw2ZmWWcE4GZWcZlLRHc1tIBtAD3ORvc52woSZ8zdYzAzMxqy9oegZmZ5XEiMDPLuDaZCCQdK2mhpMWSLi4wX5J+nc5/Ib1fcqtWRJ9Hp319QdJfJQ1uiTibUkN9zqn3FUkbJJ3SnPGVQjF9lnSopApJ8yU9XqhOa1LEZ3s7SQ9Kej7t81ktEWdTkTRJ0ruSXqxjftOvvyKiTT1I7o+8BOgHdASeBwbm1Tke+BMg4EDg7y0ddzP0+avADunz47LQ55x6s4CHgVNaOu5m+D9vDywAeqXTO7V03M3Q50uAa9LnZcD7QMeWjn0L+nwIsB/wYh3zm3z91Rb3CA4AFkfEqxHxGTANGJ5XZzjwu0j8Ddhe0i7NHWgTarDPEfHXiPggnfwb0Nqvo1zM/xngfOCPwLvNGVyJFNPn04F7I+INgIho7f0ups8BbCtJwDYkiaCqecNsOhExm6QPdWny9VdbTATdgWU505VpWWPrtCaN7c85JFsUrVmDfZbUHTgJmNiMcZVSMf/nPYAdJP1F0jOSzmy26EqjmD5PAAYAy4F/AD+IiI3NE16LaPL1V5u4H0EeFSjLP0e2mDqtSdH9kXQYSSL4ekkjKr1i+nwj8JOI2JBsLLZ6xfS5PbA/cATQBXhK0t8i4pVSB1cixfT5GKACOBzYDXhU0hMRsbrUwbWQJl9/tcVEUAn0zJnuQbKl0Ng6rUlR/ZG0L/DfwHERsbKZYiuVYvpcDkxLk0A34HhJVRFxf/OE2OSK/Wy/FxFrgbWSZgODgdaaCIrp81nA1ZEMoC+W9BqwF/B084TY7Jp8/dUWh4bmAv0l9ZXUERgJzMirMwM4Mz36fiCwKiLeau5Am1CDfZbUC7gXOKMVbx3marDPEdE3IvpERB/gHuB7rTgJQHGf7QeAgyW1l/QFYCjwUjPH2ZSK6fMbJHtASNoZ2BN4tVmjbF5Nvv5qc3sEEVElaTwwk+SMg0kRMV/SuHT+RJIzSI4HFgMfk2xRtFpF9vk/gR2Bm9Mt5KpoxVduLLLPbUoxfY6IlyQ9ArwAbAT+OyIKnobYGhT5f74CmCLpHyTDJj+JiFZ7eWpJdwGHAt0kVQI/BTpA6dZfvsSEmVnGtcWhITMzawQnAjOzjHMiMDPLOCcCM7OMcyIwM8s4JwKzAtKrlVZIejG9suX2Tdz+Uknd0udrmrJts8ZyIjAr7JOIGBIRe5NcAOy8lg7IrFScCMwa9hTpRb0k7SbpkfSCbk9I2ist31nSfek18Z+X9NW0/P607nxJ57ZgH8zq1OZ+WWzWlCS1I7l8we1p0W3AuIhYJGkocDPJxc5+DTweESely2yT1j87It6X1AWYK+mPbeA6T9bGOBGYFdZFUgXQB3iG5IqW25Dc4Gd6ztVMO6V/DwfOBIiIDcCqtPz7kk5Kn/cE+gNOBPa54kRgVtgnETFE0nbAQyTHCKYAH0bEkGIakHQocCRwUER8LOkvQOfShGu2+XyMwKweEbEK+D7wI+AT4DVJ34Kae8dW3/v5MeC7aXk7SV2B7YAP0iSwF8ltBc0+d5wIzBoQEc+R3Ct3JDAaOEfS88B8/nnbxB8Ah6VXwHwGGAQ8ArSX9ALJFTL/1tyxmxXDVx81M8s47xGYmWWcE4GZWcY5EZiZZZwTgZlZxjkRmJllnBOBmVnGORGYmWXc/wfF1bG5MbskQAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import plot_precision_recall_curve\n",
    "disp = plot_precision_recall_curve(classifier, x_test, y_test)\n",
    "disp.ax_.set_title('2-class Precision-Recall curve: '\n",
    "                   'AP={0:0.2f}'.format(average_precision))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}