XGBoost on application_train.csv + manual feature Engineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix,classification_report\n",
    "#np.set_printoptions(threshold=np.nan)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Used only application_train.csv and some manual feature Engineering- Gave good results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"application_train.csv\")\n",
    "# df_test = pd.read_csv(\"application_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "2      100004       0    Revolving loans           M            Y   \n",
       "3      100006       0         Cash loans           F            N   \n",
       "4      100007       0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "              ...              FLAG_DOCUMENT_18 FLAG_DOCUMENT_19  \\\n",
       "0             ...                             0                0   \n",
       "1             ...                             0                0   \n",
       "2             ...                             0                0   \n",
       "3             ...                             0                0   \n",
       "4             ...                             0                0   \n",
       "\n",
       "  FLAG_DOCUMENT_20 FLAG_DOCUMENT_21 AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                0                0                        0.0   \n",
       "1                0                0                        0.0   \n",
       "2                0                0                        0.0   \n",
       "3                0                0                        NaN   \n",
       "4                0                0                        0.0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  \\\n",
       "0                       0.0                         0.0   \n",
       "1                       0.0                         0.0   \n",
       "2                       0.0                         0.0   \n",
       "3                       NaN                         NaN   \n",
       "4                       0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                         1.0  \n",
       "1                         0.0  \n",
       "2                         0.0  \n",
       "3                         NaN  \n",
       "4                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(307511, 122)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 columns were label encoded.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "le = LabelEncoder()\n",
    "le_count = 0\n",
    "\n",
    "# Iterate through the columns\n",
    "for col in df_train:\n",
    "    if df_train[col].dtype == 'object':\n",
    "        # If 2 or fewer unique categories\n",
    "        if len(list(df_train[col].unique())) <= 2:\n",
    "            # Train on the training data\n",
    "            le.fit(df_train[col])\n",
    "            # Transform both training and testing data\n",
    "            df_train[col] = le.transform(df_train[col])\n",
    "            df_test[col] = le.transform(df_test[col])\n",
    "            \n",
    "            # Keep track of how many columns were label encoded\n",
    "            le_count += 1\n",
    "            \n",
    "print('%d columns were label encoded.' % le_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 243)\n",
      "Testing Features shape:  (48744, 239)\n",
      "Index(['SK_ID_CURR', 'TARGET', 'NAME_CONTRACT_TYPE', 'FLAG_OWN_CAR',\n",
      "       'FLAG_OWN_REALTY', 'CNT_CHILDREN', 'AMT_INCOME_TOTAL', 'AMT_CREDIT',\n",
      "       'AMT_ANNUITY', 'AMT_GOODS_PRICE',\n",
      "       ...\n",
      "       'HOUSETYPE_MODE_terraced house', 'WALLSMATERIAL_MODE_Block',\n",
      "       'WALLSMATERIAL_MODE_Mixed', 'WALLSMATERIAL_MODE_Monolithic',\n",
      "       'WALLSMATERIAL_MODE_Others', 'WALLSMATERIAL_MODE_Panel',\n",
      "       'WALLSMATERIAL_MODE_Stone, brick', 'WALLSMATERIAL_MODE_Wooden',\n",
      "       'EMERGENCYSTATE_MODE_No', 'EMERGENCYSTATE_MODE_Yes'],\n",
      "      dtype='object', length=243)\n"
     ]
    }
   ],
   "source": [
    "# one-hot encoding of categorical variables\n",
    "df_train = pd.get_dummies(df_train)\n",
    "df_test = pd.get_dummies(df_test)\n",
    "list1=df_train.columns\n",
    "\n",
    "print('Training Features shape: ', df_train.shape)\n",
    "print('Testing Features shape: ', df_test.shape)\n",
    "print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"TARGET\"].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    282686\n",
       "1     24825\n",
       "Name: TARGET, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[\"TARGET\"].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, eval_metric='auc', gamma=16, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=6, min_child_weight=1, missing=None,\n",
       "       n_estimators=100, n_jobs=16, nthread=None,\n",
       "       objective='binary:logistic', random_state=0, reg_alpha=40,\n",
       "       reg_lambda=40, scale_pos_weight=4, seed=None, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split( df_train.drop('TARGET',axis=1) , df_train['TARGET'] , test_size=0.2, random_state=42)\n",
    "model = XGBClassifier(max_depth=6, learning_rate=0.1,n_estimators=100,n_jobs=16,scale_pos_weight=4,missing=np.nan,gamma=16, eval_metric=\"auc\",reg_lambda=40,reg_alpha=40)\n",
    "model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train AUC 0.7756\n",
      "Test AUC 0.7588\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_train_predicted = model.predict_proba(X_train)[:,1]\n",
    "y_test_predicted = model.predict_proba(X_test)[:,1]\n",
    "print(\"Train AUC %.4f\"%roc_auc_score(y_train,y_train_predicted))\n",
    "print(\"Test AUC %.4f\"%roc_auc_score(y_test,y_test_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy 0.8903\n",
      "Test Accuracy 0.8863\n"
     ]
    }
   ],
   "source": [
    "y_train_pred = model.predict(X_train)\n",
    "y_test_pred = model.predict(X_test)\n",
    "print(\"Train Accuracy %.4f\" % accuracy_score(y_train,y_train_pred))\n",
    "print(\"Test Accuracy %.4f\" % accuracy_score(y_test,y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(100):\n",
    "#     print(y_test_pred[i],\" \",y_test_predicted[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[53121  3433]\n",
      " [ 3560  1389]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confmat=confusion_matrix(y_test,y_test_pred)\n",
    "print(confmat)\n",
    "# plt.imshow(confmat, cmap='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Part 2 : Dealing with some Missing Values and removing 15 cols and checking result---(It improved)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Function to calculate missing values by column# Funct \n",
    "def missing_values_table(df):\n",
    "        # Total missing values\n",
    "        mis_val = df.isnull().sum()\n",
    "        \n",
    "        # Percentage of missing values\n",
    "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "        \n",
    "        # Make a table with the results\n",
    "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "        # Rename the columns\n",
    "        mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
    "        \n",
    "        # Sort the table by percentage of missing descending\n",
    "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "        '% of Total Values', ascending=False).round(1)\n",
    "        \n",
    "        # Print some summary information\n",
    "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
    "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
    "              \" columns that have missing values.\")\n",
    "        \n",
    "        # Return the dataframe with missing information\n",
    "        return mis_val_table_ren_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 243 columns.\n",
      "There are 61 columns that have missing values.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['COMMONAREA_MODE', 'COMMONAREA_MEDI', 'COMMONAREA_AVG',\n",
       "       'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAPARTMENTS_AVG',\n",
       "       'NONLIVINGAPARTMENTS_MEDI', 'LIVINGAPARTMENTS_MEDI',\n",
       "       'LIVINGAPARTMENTS_AVG', 'LIVINGAPARTMENTS_MODE', 'FLOORSMIN_MEDI',\n",
       "       'FLOORSMIN_MODE', 'FLOORSMIN_AVG', 'YEARS_BUILD_AVG',\n",
       "       'YEARS_BUILD_MODE', 'YEARS_BUILD_MEDI', 'OWN_CAR_AGE', 'LANDAREA_MEDI',\n",
       "       'LANDAREA_MODE', 'LANDAREA_AVG', 'BASEMENTAREA_MODE',\n",
       "       'BASEMENTAREA_AVG', 'BASEMENTAREA_MEDI', 'EXT_SOURCE_1',\n",
       "       'NONLIVINGAREA_MODE', 'NONLIVINGAREA_AVG', 'NONLIVINGAREA_MEDI',\n",
       "       'ELEVATORS_MEDI', 'ELEVATORS_MODE', 'ELEVATORS_AVG', 'APARTMENTS_AVG',\n",
       "       'APARTMENTS_MEDI', 'APARTMENTS_MODE', 'ENTRANCES_AVG', 'ENTRANCES_MEDI',\n",
       "       'ENTRANCES_MODE', 'LIVINGAREA_MEDI', 'LIVINGAREA_AVG',\n",
       "       'LIVINGAREA_MODE', 'FLOORSMAX_MODE', 'FLOORSMAX_MEDI', 'FLOORSMAX_AVG',\n",
       "       'YEARS_BEGINEXPLUATATION_MEDI', 'YEARS_BEGINEXPLUATATION_AVG',\n",
       "       'YEARS_BEGINEXPLUATATION_MODE', 'TOTALAREA_MODE'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Missing values statistics\n",
    "missing_values = missing_values_table(df_train)\n",
    "missing_values.index[:45]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, eval_metric='auc', gamma=16, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=6, min_child_weight=1, missing=None,\n",
       "       n_estimators=100, n_jobs=16, nthread=None,\n",
       "       objective='binary:logistic', random_state=0, reg_alpha=40,\n",
       "       reg_lambda=40, scale_pos_weight=4, seed=None, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "newX=df_train.drop(missing_values.index[:15],axis=1)\n",
    "X_train, X_test, y_train, y_test = train_test_split( newX.drop('TARGET',axis=1) , df_train['TARGET'] , test_size=0.2, random_state=42)\n",
    "model = XGBClassifier(max_depth=6, learning_rate=0.1,n_estimators=100,n_jobs=16,scale_pos_weight=4,missing=np.nan,gamma=16, eval_metric=\"auc\",reg_lambda=40,reg_alpha=40)\n",
    "model.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_train.drop(missing_values.index[:30],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train AUC 0.7755\n",
      "Test AUC 0.7586\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_train_predicted = model.predict_proba(X_train)[:,1]\n",
    "y_test_predicted = model.predict_proba(X_test)[:,1]\n",
    "print(\"Train AUC %.4f\"%roc_auc_score(y_train,y_train_predicted))\n",
    "print(\"Test AUC %.4f\"%roc_auc_score(y_test,y_test_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy 0.8900\n",
      "Test Accuracy 0.8864\n"
     ]
    }
   ],
   "source": [
    "y_train_pred = model.predict(X_train)\n",
    "y_test_pred = model.predict(X_test)\n",
    "print(\"Train Accuracy %.4f\" % accuracy_score(y_train,y_train_pred))\n",
    "print(\"Test Accuracy %.4f\" % accuracy_score(y_test,y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[53110  3444]\n",
      " [ 3541  1408]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confmat=confusion_matrix(y_test,y_test_pred)\n",
    "print(confmat)\n",
    "# plt.imshow(confmat, cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(100):\n",
    "#     print(y_test_pred[i],\" \",y_test_predicted[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 3: In addition to Part 2, I did parameter tuning and the results improved again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, eval_metric='auc', gamma=16, learning_rate=0.1,\n",
       "       max_delta_step=0, max_depth=20, min_child_weight=1, missing=None,\n",
       "       n_estimators=100, n_jobs=30, nthread=None,\n",
       "       objective='binary:logistic', random_state=0, reg_alpha=40,\n",
       "       reg_lambda=40, scale_pos_weight=4, seed=None, silent=True,\n",
       "       subsample=1)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "newX=df_train.drop(missing_values.index[:15],axis=1)\n",
    "X_train, X_test, y_train, y_test = train_test_split( newX.drop('TARGET',axis=1) , df_train['TARGET'] , test_size=0.2, random_state=42)\n",
    "model = XGBClassifier(max_depth=20, learning_rate=0.1,n_estimators=100,n_jobs=30,scale_pos_weight=4,missing=np.nan,gamma=16, eval_metric=\"auc\",reg_lambda=40,reg_alpha=40)\n",
    "model.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train AUC 0.7904\n",
      "Test AUC 0.7607\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_train_predicted = model.predict_proba(X_train)[:,1]\n",
    "y_test_predicted = model.predict_proba(X_test)[:,1]\n",
    "print(\"Train AUC %.4f\"%roc_auc_score(y_train,y_train_predicted))\n",
    "print(\"Test AUC %.4f\"%roc_auc_score(y_test,y_test_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy 0.8903\n",
      "Test Accuracy 0.8839\n"
     ]
    }
   ],
   "source": [
    "y_train_pred = model.predict(X_train)\n",
    "y_test_pred = model.predict(X_test)\n",
    "print(\"Train Accuracy %.4f\" % accuracy_score(y_train,y_train_pred))\n",
    "print(\"Test Accuracy %.4f\" % accuracy_score(y_test,y_test_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[52896  3658]\n",
      " [ 3483  1466]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confmat=confusion_matrix(y_test,y_test_pred)\n",
    "print(confmat)\n",
    "# plt.imshow(confmat, cmap='binary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}