FR_HOME_CREDIT_missing value 999 without agg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Preprocessing\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import accuracy_score\n",
    "from xgboost import plot_importance\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import os\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.externals import joblib\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import dask.dataframe as dd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score,recall_score,precision_score\n",
    "from dask.distributed import Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "2      100004       0    Revolving loans           M            Y   \n",
       "3      100006       0         Cash loans           F            N   \n",
       "4      100007       0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "              ...              FLAG_DOCUMENT_18 FLAG_DOCUMENT_19  \\\n",
       "0             ...                             0                0   \n",
       "1             ...                             0                0   \n",
       "2             ...                             0                0   \n",
       "3             ...                             0                0   \n",
       "4             ...                             0                0   \n",
       "\n",
       "  FLAG_DOCUMENT_20 FLAG_DOCUMENT_21 AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                0                0                        0.0   \n",
       "1                0                0                        0.0   \n",
       "2                0                0                        0.0   \n",
       "3                0                0                        NaN   \n",
       "4                0                0                        0.0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  \\\n",
       "0                       0.0                         0.0   \n",
       "1                       0.0                         0.0   \n",
       "2                       0.0                         0.0   \n",
       "3                       NaN                         NaN   \n",
       "4                       0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                         1.0  \n",
       "1                         0.0  \n",
       "2                         0.0  \n",
       "3                         NaN  \n",
       "4                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = pd.read_csv(\"application_train.csv\")\n",
    "df_test = pd.read_csv(\"application_test.csv\")\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "F    202448\n",
       "M    105059\n",
       "Name: CODE_GENDER, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X['CODE_GENDER'].replace('XNA',np.nan, inplace=True)\n",
    "X['CODE_GENDER'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 columns were label encoded.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "le = LabelEncoder()\n",
    "le_count = 0\n",
    "\n",
    "# Iterate through the columns\n",
    "for col in X:\n",
    "    if X[col].dtype == 'object':\n",
    "        # If 2 or fewer unique categories\n",
    "        if len(list(X[col].unique())) <= 2:\n",
    "            # Train on the training data\n",
    "            le.fit(X[col])\n",
    "            # Transform both training and testing data\n",
    "            X[col] = le.transform(X[col])\n",
    "           # df_test[col] = le.transform(df_test[col])\n",
    "            \n",
    "            # Keep track of how many columns were label encoded\n",
    "            le_count += 1\n",
    "            \n",
    "print('%d columns were label encoded.' % le_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 242)\n",
      "Index(['SK_ID_CURR', 'TARGET', 'NAME_CONTRACT_TYPE', 'FLAG_OWN_CAR',\n",
      "       'FLAG_OWN_REALTY', 'CNT_CHILDREN', 'AMT_INCOME_TOTAL', 'AMT_CREDIT',\n",
      "       'AMT_ANNUITY', 'AMT_GOODS_PRICE',\n",
      "       ...\n",
      "       'HOUSETYPE_MODE_terraced house', 'WALLSMATERIAL_MODE_Block',\n",
      "       'WALLSMATERIAL_MODE_Mixed', 'WALLSMATERIAL_MODE_Monolithic',\n",
      "       'WALLSMATERIAL_MODE_Others', 'WALLSMATERIAL_MODE_Panel',\n",
      "       'WALLSMATERIAL_MODE_Stone, brick', 'WALLSMATERIAL_MODE_Wooden',\n",
      "       'EMERGENCYSTATE_MODE_No', 'EMERGENCYSTATE_MODE_Yes'],\n",
      "      dtype='object', length=242)\n"
     ]
    }
   ],
   "source": [
    "# one-hot encoding of categorical variables\n",
    "X = pd.get_dummies(X)\n",
    "#df_test = pd.get_dummies(df_test)\n",
    "list1=X.columns\n",
    "\n",
    "print('Training Features shape: ', X.shape)\n",
    "#print('Testing Features shape: ', df_test.shape)\n",
    "print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate missing values by column# Funct \n",
    "def missing_values_table(df):\n",
    "        # Total missing values\n",
    "        mis_val = df.isnull().sum()\n",
    "        \n",
    "        # Percentage of missing values\n",
    "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "        \n",
    "        # Make a table with the results\n",
    "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "        # Rename the columns\n",
    "        mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
    "        \n",
    "        # Sort the table by percentage of missing descending\n",
    "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "        '% of Total Values', ascending=False).round(1)\n",
    "        \n",
    "        # Print some summary information\n",
    "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
    "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
    "              \" columns that have missing values.\")\n",
    "        \n",
    "        # Return the dataframe with missing information\n",
    "        return mis_val_table_ren_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 242 columns.\n",
      "There are 62 columns that have missing values.\n"
     ]
    }
   ],
   "source": [
    "# Missing values statistics\n",
    "missing_values = missing_values_table(X)\n",
    "mis_col=list(missing_values.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X.replace(np.nan,-999, inplace=True)\n",
    "#X.astype('float32',inplace=True)\n",
    "#print(newX[].describe())\n",
    "y=X['TARGET']\n",
    "X=X.drop('TARGET',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate below functions only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_train_test(X , y):\n",
    "    return(train_test_split( X , y , test_size=0.2, random_state=42))\n",
    "\n",
    "X_train, X_test, y_train, y_test =split_train_test(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improved number of features-> 82\n"
     ]
    }
   ],
   "source": [
    "def feature_eng(X_train,y_train,X_test,y_test):\n",
    "    train_col=list(X_train.columns)\n",
    "    test_col=list(X_test.columns)    \n",
    "    clf = ExtraTreesClassifier(n_estimators=50)\n",
    "    clf = clf.fit(X_train, y_train)\n",
    "    model = SelectFromModel(clf, prefit=True)\n",
    "    new_trainX = model.transform(X_train)\n",
    "    n=np.shape(new_trainX)[1]\n",
    "    importances = clf.feature_importances_\n",
    "    indices = np.argsort(importances)[::-1]\n",
    "    clist=[]\n",
    "    for f in range(n):     \n",
    "        clist.append(indices[f])\n",
    "    new_train_col=[train_col[i] for i in clist]\n",
    "    new_test_col=[test_col[i] for i in clist]\n",
    "    X_train=pd.DataFrame(X_train)\n",
    "    X_test=pd.DataFrame(X_test)\n",
    "    y_train=pd.DataFrame(y_train)\n",
    "    y_test=pd.DataFrame(y_test)\n",
    "    df_new_train=pd.DataFrame(X_train.loc[:,new_train_col],columns=new_train_col)            \n",
    "    df_new_test=pd.DataFrame(X_test.loc[:,new_test_col],columns=new_test_col)\n",
    "    print(\"Improved number of features->\",len(clist))    \n",
    "    return(df_new_train,df_new_test)\n",
    "\n",
    "df_new_train,df_new_test=feature_eng(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model_eval_metric(modelname,modelobj,X_train,Y_train,X_test,Y_test):\n",
    "    # In order to let the models run faster we use dask dataframe\n",
    "    \n",
    "    modelobj.fit(X_train,Y_train)\n",
    "    ypred_train=modelobj.predict(X_train)\n",
    "    ypred_test=modelobj.predict(X_test)\n",
    "    y_train_proba = modelobj.predict_proba(X_train)[:,1]\n",
    "    y_test_proba = modelobj.predict_proba(X_test)[:,1]\n",
    "    print('\\n')\n",
    "    print(modelname,\"model:\")\n",
    "    confmat=confusion_matrix(Y_train,ypred_train)\n",
    "    print(\"Train Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    confmat=confusion_matrix(Y_test,ypred_test)\n",
    "    print(\"Test Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    metric=pd.DataFrame([[0]*5]*2,columns=['PRECISION','RECALL','F1_SCORE','ROC_AUC_SCORE','ACCURACY'],index=['Train','Test'],dtype='float32')\n",
    "    metric['PRECISION']['Train']=round(precision_score(Y_train, ypred_train),3)\n",
    "    metric['RECALL']['Train']=recall_score(Y_train, ypred_train)\n",
    "    metric['F1_SCORE']['Train']=round(f1_score(Y_train, ypred_train),3)\n",
    "    metric['ROC_AUC_SCORE']['Train']=roc_auc_score(Y_train,y_train_proba)\n",
    "    metric['ACCURACY']['Train']=accuracy_score(Y_train,ypred_train)\n",
    "    metric['PRECISION']['Test']=precision_score(Y_test, ypred_test)\n",
    "    metric['RECALL']['Test']=recall_score(Y_test, ypred_test)\n",
    "    metric['F1_SCORE']['Test']=f1_score(Y_test, ypred_test)\n",
    "    metric['ROC_AUC_SCORE']['Test']=roc_auc_score(Y_test,y_test_proba)\n",
    "    metric['ACCURACY']['Test']=accuracy_score(Y_test,ypred_test)\n",
    "    print(metric)\n",
    "   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "K-Nearest Neighbor 1 model:\n",
      "Train Confusion Matrix:\n",
      "[[226132      0]\n",
      " [     0  19876]]\n",
      "Test Confusion Matrix:\n",
      "[[51975  4579]\n",
      " [ 4412   537]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   1.000000  1.000000  1.000000        1.00000  1.000000\n",
      "Test    0.104965  0.108507  0.106706        0.51377  0.853812\n",
      "\n",
      "\n",
      "K-Nearest Neighbor 3 model:\n",
      "Train Confusion Matrix:\n",
      "[[224329   1803]\n",
      " [ 16083   3793]]\n",
      "Test Confusion Matrix:\n",
      "[[55357  1197]\n",
      " [ 4776   173]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.678000  0.190833  0.298000       0.933968  0.927295\n",
      "Test    0.126277  0.034957  0.054755       0.528867  0.902883\n",
      "\n",
      "\n",
      "Random Forest model:\n",
      "Train Confusion Matrix:\n",
      "[[225936    196]\n",
      " [  3630  16246]]\n",
      "Test Confusion Matrix:\n",
      "[[55734   820]\n",
      " [ 4726   223]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.988000  0.817368  0.895000       0.994287  0.984448\n",
      "Test    0.213806  0.045060  0.074433       0.594956  0.909826\n",
      "\n",
      "\n",
      "Decision Tree model:\n",
      "Train Confusion Matrix:\n",
      "[[226116     16]\n",
      " [    98  19778]]\n",
      "Test Confusion Matrix:\n",
      "[[51663  4891]\n",
      " [ 4222   727]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.999000  0.995069  0.997000       0.999995  0.999537\n",
      "Test    0.129405  0.146898  0.137598       0.530103  0.851828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Logistic Regression model:\n",
      "Train Confusion Matrix:\n",
      "[[169804  56328]\n",
      " [ 10211   9665]]\n",
      "Test Confusion Matrix:\n",
      "[[42538 14016]\n",
      " [ 2490  2459]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.146000  0.486265  0.225000       0.670771  0.729525\n",
      "Test    0.149256  0.496868  0.229556       0.675434  0.731623\n"
     ]
    }
   ],
   "source": [
    "def run_all_models(df_new_train,y_train,df_new_test,y_test):\n",
    "    # K Nearest Neighbor\n",
    "    knearest = KNeighborsClassifier(algorithm= 'auto', n_jobs= -1, n_neighbors= 1)\n",
    "    run_model_eval_metric('K-Nearest Neighbor 1',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "    \n",
    "    knearest = KNeighborsClassifier(algorithm= 'auto', n_jobs= -1, n_neighbors= 3)\n",
    "    run_model_eval_metric('K-Nearest Neighbor 3',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Random Forest\n",
    "    randfor = RandomForestClassifier(n_estimators= 5, max_features= 'auto', max_depth= 60, bootstrap= True, n_jobs= -1)\n",
    "    run_model_eval_metric('Random Forest',randfor,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Decision Trees\n",
    "    dectree = DecisionTreeClassifier(random_state= 42, max_features= 'auto', max_depth= 40, criterion= 'gini')\n",
    "    run_model_eval_metric('Decision Tree',dectree,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Logistic Regression\n",
    "    logreg = LogisticRegression(penalty = 'l1', class_weight= {0: 0.1, 1: 0.9}, C= 0.5)\n",
    "    run_model_eval_metric('Logistic Regression',logreg,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "run_all_models(df_new_train,y_train,df_new_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}