FR_HOME_CREDIT_missing value meanmode Imputer with agg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Preprocessing\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import accuracy_score\n",
    "from xgboost import plot_importance\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import os\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.externals import joblib\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import dask.dataframe as dd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score,recall_score,precision_score\n",
    "from dask.distributed import Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "2      100004       0    Revolving loans           M            Y   \n",
       "3      100006       0         Cash loans           F            N   \n",
       "4      100007       0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "              ...              FLAG_DOCUMENT_18 FLAG_DOCUMENT_19  \\\n",
       "0             ...                             0                0   \n",
       "1             ...                             0                0   \n",
       "2             ...                             0                0   \n",
       "3             ...                             0                0   \n",
       "4             ...                             0                0   \n",
       "\n",
       "  FLAG_DOCUMENT_20 FLAG_DOCUMENT_21 AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                0                0                        0.0   \n",
       "1                0                0                        0.0   \n",
       "2                0                0                        0.0   \n",
       "3                0                0                        NaN   \n",
       "4                0                0                        0.0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  \\\n",
       "0                       0.0                         0.0   \n",
       "1                       0.0                         0.0   \n",
       "2                       0.0                         0.0   \n",
       "3                       NaN                         NaN   \n",
       "4                       0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                         1.0  \n",
       "1                         0.0  \n",
       "2                         0.0  \n",
       "3                         NaN  \n",
       "4                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = pd.read_csv(\"application_train.csv\")\n",
    "df_test = pd.read_csv(\"application_test.csv\")\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "F    202448\n",
       "M    105059\n",
       "Name: CODE_GENDER, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X['CODE_GENDER'].replace('XNA',np.nan, inplace=True)\n",
    "X['CODE_GENDER'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "544478d917c541d2809d9f129c7da98a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "584b7f284f8b4f889b1c80adb785bfb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b701de971ffb4717af8ad0799649288f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d56bf816651b479eb2b72f899cd33c63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b4a4bde6b0d44048edd70da58d535ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=7), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0b7e24a3af345c8bb011f6fce8f946b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "AGGREGATION_RECIPIES = [\n",
    "    (['CODE_GENDER', 'NAME_EDUCATION_TYPE'], [('AMT_ANNUITY', 'max')                                             \n",
    "                                              \n",
    "                                              ]),\n",
    "    (['CODE_GENDER', 'ORGANIZATION_TYPE'], [\n",
    "                                            ('AMT_INCOME_TOTAL', 'mean'),\n",
    "                                            ('DAYS_REGISTRATION', 'mean')\n",
    "                                            ]),\n",
    "    (['CODE_GENDER', 'REG_CITY_NOT_WORK_CITY'], [\n",
    "                                                 ('CNT_CHILDREN', 'mean')]),\n",
    " \n",
    "    (['NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE'], [('AMT_CREDIT', 'mean'),\n",
    "                                                  ('AMT_REQ_CREDIT_BUREAU_YEAR', 'mean'),\n",
    "                                                  ('APARTMENTS_AVG', 'mean'),\n",
    "                                                  ('BASEMENTAREA_AVG', 'mean'),\n",
    "                                                  ('NONLIVINGAREA_AVG', 'mean'),\n",
    "                                                  ('OWN_CAR_AGE', 'mean'),\n",
    "                                                  ('YEARS_BUILD_AVG', 'mean')]),\n",
    "    (['NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE', 'REG_CITY_NOT_WORK_CITY'], [('ELEVATORS_AVG', 'mean')\n",
    "                                                                            ])\n",
    "]\n",
    "\n",
    "groupby_aggregate_names = []\n",
    "for groupby_cols, specs in tqdm(AGGREGATION_RECIPIES):\n",
    "    group_object = X.groupby(groupby_cols)\n",
    "    for select, agg in tqdm(specs):\n",
    "        groupby_aggregate_name = '{}_{}_{}'.format('_'.join(groupby_cols), agg, select)\n",
    "        X = X.merge(group_object[select]\n",
    "                              .agg(agg)\n",
    "                              .reset_index()\n",
    "                              .rename(index=str,\n",
    "                                      columns={select: groupby_aggregate_name})\n",
    "                              [groupby_cols + [groupby_aggregate_name]],\n",
    "                              on=groupby_cols,\n",
    "                              how='left')\n",
    "        groupby_aggregate_names.append(groupby_aggregate_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TARGET                                                                           1.000000\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_REG_CITY_NOT_WORK_CITY_mean_ELEVATORS_AVG    0.078057\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_AMT_REQ_CREDIT_BUREAU_YEAR              0.074528\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_YEARS_BUILD_AVG                         0.073816\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_NONLIVINGAREA_AVG                       0.073730\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_OWN_CAR_AGE                             0.073535\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_APARTMENTS_AVG                          0.072854\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_BASEMENTAREA_AVG                        0.072231\n",
       "NAME_EDUCATION_TYPE_OCCUPATION_TYPE_mean_AMT_CREDIT                              0.071023\n",
       "CODE_GENDER_REG_CITY_NOT_WORK_CITY_mean_CNT_CHILDREN                             0.066094\n",
       "CODE_GENDER_NAME_EDUCATION_TYPE_max_AMT_ANNUITY                                  0.064214\n",
       "CODE_GENDER_ORGANIZATION_TYPE_mean_DAYS_REGISTRATION                             0.052410\n",
       "CODE_GENDER_ORGANIZATION_TYPE_mean_AMT_INCOME_TOTAL                              0.050272\n",
       "Name: TARGET, dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_agg = X[groupby_aggregate_names + ['TARGET']]\n",
    "X_agg_corr = abs(X_agg.corr())\n",
    "X_agg_corr.sort_values('TARGET', ascending=False)['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=X.drop(['AMT_ANNUITY','OWN_CAR_AGE','AMT_INCOME_TOTAL','DAYS_REGISTRATION','CNT_CHILDREN','AMT_CREDIT',\n",
    "          'AMT_REQ_CREDIT_BUREAU_YEAR','APARTMENTS_AVG','BASEMENTAREA_AVG','NONLIVINGAREA_AVG',\n",
    "          'YEARS_BUILD_AVG','ELEVATORS_AVG'\n",
    "         ],axis=1)\n",
    "#Dropped 12 columns and replaced them with their corresponding aggregated columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 columns were label encoded.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "le = LabelEncoder()\n",
    "le_count = 0\n",
    "\n",
    "# Iterate through the columns\n",
    "for col in X:\n",
    "    if X[col].dtype == 'object':\n",
    "        # If 2 or fewer unique categories\n",
    "        if len(list(X[col].unique())) <= 2:\n",
    "            # Train on the training data\n",
    "            le.fit(X[col])\n",
    "            # Transform both training and testing data\n",
    "            X[col] = le.transform(X[col])\n",
    "           # df_test[col] = le.transform(df_test[col])\n",
    "            \n",
    "            # Keep track of how many columns were label encoded\n",
    "            le_count += 1\n",
    "            \n",
    "print('%d columns were label encoded.' % le_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 242)\n",
      "Index(['SK_ID_CURR', 'TARGET', 'NAME_CONTRACT_TYPE', 'FLAG_OWN_CAR',\n",
      "       'FLAG_OWN_REALTY', 'AMT_GOODS_PRICE', 'REGION_POPULATION_RELATIVE',\n",
      "       'DAYS_BIRTH', 'DAYS_EMPLOYED', 'DAYS_ID_PUBLISH',\n",
      "       ...\n",
      "       'HOUSETYPE_MODE_terraced house', 'WALLSMATERIAL_MODE_Block',\n",
      "       'WALLSMATERIAL_MODE_Mixed', 'WALLSMATERIAL_MODE_Monolithic',\n",
      "       'WALLSMATERIAL_MODE_Others', 'WALLSMATERIAL_MODE_Panel',\n",
      "       'WALLSMATERIAL_MODE_Stone, brick', 'WALLSMATERIAL_MODE_Wooden',\n",
      "       'EMERGENCYSTATE_MODE_No', 'EMERGENCYSTATE_MODE_Yes'],\n",
      "      dtype='object', length=242)\n"
     ]
    }
   ],
   "source": [
    "# one-hot encoding of categorical variables\n",
    "X = pd.get_dummies(X)\n",
    "#df_test = pd.get_dummies(df_test)\n",
    "list1=X.columns\n",
    "\n",
    "print('Training Features shape: ', X.shape)\n",
    "#print('Testing Features shape: ', df_test.shape)\n",
    "print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate missing values by column# Funct \n",
    "def missing_values_table(df):\n",
    "        # Total missing values\n",
    "        mis_val = df.isnull().sum()\n",
    "        \n",
    "        # Percentage of missing values\n",
    "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "        \n",
    "        # Make a table with the results\n",
    "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "        # Rename the columns\n",
    "        mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
    "        \n",
    "        # Sort the table by percentage of missing descending\n",
    "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "        '% of Total Values', ascending=False).round(1)\n",
    "        \n",
    "        # Print some summary information\n",
    "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
    "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
    "              \" columns that have missing values.\")\n",
    "        \n",
    "        # Return the dataframe with missing information\n",
    "        return mis_val_table_ren_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 242 columns.\n",
      "There are 66 columns that have missing values.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          SK_ID_CURR         TARGET  NAME_CONTRACT_TYPE   FLAG_OWN_CAR  \\\n",
      "count  307511.000000  307511.000000       307511.000000  307511.000000   \n",
      "mean   278180.518577       0.080729            0.095213       0.340108   \n",
      "std    102790.175348       0.272419            0.293509       0.473746   \n",
      "min    100002.000000       0.000000            0.000000       0.000000   \n",
      "25%    189145.500000       0.000000            0.000000       0.000000   \n",
      "50%    278202.000000       0.000000            0.000000       0.000000   \n",
      "75%    367142.500000       0.000000            0.000000       1.000000   \n",
      "max    456255.000000       1.000000            1.000000       1.000000   \n",
      "\n",
      "       FLAG_OWN_REALTY  AMT_GOODS_PRICE  REGION_POPULATION_RELATIVE  \\\n",
      "count    307511.000000     3.072330e+05               307511.000000   \n",
      "mean          0.693673     5.383962e+05                    0.020868   \n",
      "std           0.460968     3.694465e+05                    0.013831   \n",
      "min           0.000000     4.050000e+04                    0.000290   \n",
      "25%           0.000000     2.385000e+05                    0.010006   \n",
      "50%           1.000000     4.500000e+05                    0.018850   \n",
      "75%           1.000000     6.795000e+05                    0.028663   \n",
      "max           1.000000     4.050000e+06                    0.072508   \n",
      "\n",
      "          DAYS_BIRTH  DAYS_EMPLOYED  DAYS_ID_PUBLISH          ...           \\\n",
      "count  307511.000000  252137.000000    307511.000000          ...            \n",
      "mean   -16036.995067   -2384.169325     -2994.202373          ...            \n",
      "std      4363.988632    2338.360162      1509.450419          ...            \n",
      "min    -25229.000000  -17912.000000     -7197.000000          ...            \n",
      "25%    -19682.000000   -3175.000000     -4299.000000          ...            \n",
      "50%    -15750.000000   -1648.000000     -3254.000000          ...            \n",
      "75%    -12413.000000    -767.000000     -1720.000000          ...            \n",
      "max     -7489.000000       0.000000         0.000000          ...            \n",
      "\n",
      "       OBS_60_CNT_SOCIAL_CIRCLE_miss  DEF_30_CNT_SOCIAL_CIRCLE_miss  \\\n",
      "count                  307511.000000                  307511.000000   \n",
      "mean                        0.003320                       0.003320   \n",
      "std                         0.057526                       0.057526   \n",
      "min                         0.000000                       0.000000   \n",
      "25%                         0.000000                       0.000000   \n",
      "50%                         0.000000                       0.000000   \n",
      "75%                         0.000000                       0.000000   \n",
      "max                         1.000000                       1.000000   \n",
      "\n",
      "       OBS_30_CNT_SOCIAL_CIRCLE_miss  EXT_SOURCE_2_miss  AMT_GOODS_PRICE_miss  \\\n",
      "count                  307511.000000      307511.000000         307511.000000   \n",
      "mean                        0.003320           0.002146              0.000904   \n",
      "std                         0.057526           0.046278              0.030054   \n",
      "min                         0.000000           0.000000              0.000000   \n",
      "25%                         0.000000           0.000000              0.000000   \n",
      "50%                         0.000000           0.000000              0.000000   \n",
      "75%                         0.000000           0.000000              0.000000   \n",
      "max                         1.000000           1.000000              1.000000   \n",
      "\n",
      "       CODE_GENDER_NAME_EDUCATION_TYPE_max_AMT_ANNUITY_miss  \\\n",
      "count                                      307511.000000      \n",
      "mean                                            0.000013      \n",
      "std                                             0.003607      \n",
      "min                                             0.000000      \n",
      "25%                                             0.000000      \n",
      "50%                                             0.000000      \n",
      "75%                                             0.000000      \n",
      "max                                             1.000000      \n",
      "\n",
      "       CODE_GENDER_ORGANIZATION_TYPE_mean_AMT_INCOME_TOTAL_miss  \\\n",
      "count                                      307511.000000          \n",
      "mean                                            0.000013          \n",
      "std                                             0.003607          \n",
      "min                                             0.000000          \n",
      "25%                                             0.000000          \n",
      "50%                                             0.000000          \n",
      "75%                                             0.000000          \n",
      "max                                             1.000000          \n",
      "\n",
      "       CODE_GENDER_ORGANIZATION_TYPE_mean_DAYS_REGISTRATION_miss  \\\n",
      "count                                      307511.000000           \n",
      "mean                                            0.000013           \n",
      "std                                             0.003607           \n",
      "min                                             0.000000           \n",
      "25%                                             0.000000           \n",
      "50%                                             0.000000           \n",
      "75%                                             0.000000           \n",
      "max                                             1.000000           \n",
      "\n",
      "       CODE_GENDER_REG_CITY_NOT_WORK_CITY_mean_CNT_CHILDREN_miss  \\\n",
      "count                                      307511.000000           \n",
      "mean                                            0.000013           \n",
      "std                                             0.003607           \n",
      "min                                             0.000000           \n",
      "25%                                             0.000000           \n",
      "50%                                             0.000000           \n",
      "75%                                             0.000000           \n",
      "max                                             1.000000           \n",
      "\n",
      "       CNT_FAM_MEMBERS_miss  \n",
      "count         307511.000000  \n",
      "mean               0.000007  \n",
      "std                0.002550  \n",
      "min                0.000000  \n",
      "25%                0.000000  \n",
      "50%                0.000000  \n",
      "75%                0.000000  \n",
      "max                1.000000  \n",
      "\n",
      "[8 rows x 308 columns]\n"
     ]
    }
   ],
   "source": [
    "# Missing values statistics\n",
    "missing_values = missing_values_table(X)\n",
    "mis_col=list(missing_values.index)\n",
    "Mis=X[mis_col][:]\n",
    "\n",
    "for col in mis_col:\n",
    "    X[col+'_miss']=[0]*len(X[col])\n",
    "    X[col+'_miss'][list(X[np.isnan(X[col])].index)]=1\n",
    "\n",
    "print(X.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before...\n",
      "Mis -          COMMONAREA_MEDI  COMMONAREA_AVG  COMMONAREA_MODE  \\\n",
      "0                0.0144          0.0143           0.0144   \n",
      "1                0.0608          0.0605           0.0497   \n",
      "2                   NaN             NaN              NaN   \n",
      "3                   NaN             NaN              NaN   \n",
      "4                   NaN             NaN              NaN   \n",
      "5                   NaN             NaN              NaN   \n",
      "6                   NaN             NaN              NaN   \n",
      "7                   NaN             NaN              NaN   \n",
      "8                   NaN             NaN              NaN   \n",
      "9                   NaN             NaN              NaN   \n",
      "10                  NaN             NaN              NaN   \n",
      "11                  NaN             NaN              NaN   \n",
      "12                  NaN             NaN              NaN   \n",
      "13               0.0585          0.0582           0.0587   \n",
      "14               0.1150          0.1143           0.1153   \n",
      "15                  NaN             NaN              NaN   \n",
      "16                  NaN             NaN              NaN   \n",
      "17                  NaN             NaN              NaN   \n",
      "18               0.0018          0.0018           0.0018   \n",
      "19                  NaN             NaN              NaN   \n",
      "20                  NaN             NaN              NaN   \n",
      "21                  NaN             NaN              NaN   \n",
      "22               0.0120          0.0120           0.0121   \n",
      "23               0.1071          0.1064           0.0000   \n",
      "24               0.0214          0.0212           0.0214   \n",
      "25               0.0010          0.0010           0.0010   \n",
      "26                  NaN             NaN              NaN   \n",
      "27                  NaN             NaN              NaN   \n",
      "28                  NaN             NaN              NaN   \n",
      "29               0.0189          0.0188           0.0190   \n",
      "...                 ...             ...              ...   \n",
      "307481              NaN             NaN              NaN   \n",
      "307482           0.0038          0.0037           0.0038   \n",
      "307483           0.0000          0.0000           0.0000   \n",
      "307484              NaN             NaN              NaN   \n",
      "307485              NaN             NaN              NaN   \n",
      "307486              NaN             NaN              NaN   \n",
      "307487              NaN             NaN              NaN   \n",
      "307488              NaN             NaN              NaN   \n",
      "307489           0.0009          0.0009           0.0009   \n",
      "307490              NaN             NaN              NaN   \n",
      "307491              NaN             NaN              NaN   \n",
      "307492              NaN             NaN              NaN   \n",
      "307493              NaN             NaN              NaN   \n",
      "307494              NaN             NaN              NaN   \n",
      "307495           0.0137          0.0136           0.0138   \n",
      "307496              NaN             NaN              NaN   \n",
      "307497              NaN             NaN              NaN   \n",
      "307498              NaN             NaN              NaN   \n",
      "307499              NaN             NaN              NaN   \n",
      "307500              NaN             NaN              NaN   \n",
      "307501              NaN             NaN              NaN   \n",
      "307502              NaN             NaN              NaN   \n",
      "307503              NaN             NaN              NaN   \n",
      "307504              NaN             NaN              NaN   \n",
      "307505           0.1441          0.1432           0.1445   \n",
      "307506           0.0203          0.0202           0.0172   \n",
      "307507           0.0022          0.0022           0.0022   \n",
      "307508           0.0124          0.0123           0.0124   \n",
      "307509              NaN             NaN              NaN   \n",
      "307510           0.0177          0.0176           0.0178   \n",
      "\n",
      "        NONLIVINGAPARTMENTS_MEDI  NONLIVINGAPARTMENTS_MODE  \\\n",
      "0                         0.0000                    0.0000   \n",
      "1                         0.0039                    0.0000   \n",
      "2                            NaN                       NaN   \n",
      "3                            NaN                       NaN   \n",
      "4                            NaN                       NaN   \n",
      "5                            NaN                       NaN   \n",
      "6                            NaN                       NaN   \n",
      "7                            NaN                       NaN   \n",
      "8                            NaN                       NaN   \n",
      "9                            NaN                       NaN   \n",
      "10                           NaN                       NaN   \n",
      "11                           NaN                       NaN   \n",
      "12                           NaN                       NaN   \n",
      "13                        0.0000                    0.0000   \n",
      "14                        0.0194                    0.0195   \n",
      "15                           NaN                       NaN   \n",
      "16                           NaN                       NaN   \n",
      "17                           NaN                       NaN   \n",
      "18                        0.0000                    0.0000   \n",
      "19                           NaN                       NaN   \n",
      "20                        0.0000                    0.0000   \n",
      "21                           NaN                       NaN   \n",
      "22                        0.0078                    0.0078   \n",
      "23                        0.0019                    0.0000   \n",
      "24                        0.0000                    0.0000   \n",
      "25                        0.0000                    0.0000   \n",
      "26                           NaN                       NaN   \n",
      "27                           NaN                       NaN   \n",
      "28                           NaN                       NaN   \n",
      "29                        0.0078                    0.0078   \n",
      "...                          ...                       ...   \n",
      "307481                       NaN                       NaN   \n",
      "307482                    0.0039                    0.0039   \n",
      "307483                    0.0000                    0.0000   \n",
      "307484                       NaN                       NaN   \n",
      "307485                       NaN                       NaN   \n",
      "307486                       NaN                       NaN   \n",
      "307487                       NaN                       NaN   \n",
      "307488                       NaN                       NaN   \n",
      "307489                    0.0000                    0.0000   \n",
      "307490                       NaN                       NaN   \n",
      "307491                       NaN                       NaN   \n",
      "307492                       NaN                       NaN   \n",
      "307493                       NaN                       NaN   \n",
      "307494                       NaN                       NaN   \n",
      "307495                    0.0039                    0.0039   \n",
      "307496                       NaN                       NaN   \n",
      "307497                       NaN                       NaN   \n",
      "307498                       NaN                       NaN   \n",
      "307499                       NaN                       NaN   \n",
      "307500                       NaN                       NaN   \n",
      "307501                       NaN                       NaN   \n",
      "307502                       NaN                       NaN   \n",
      "307503                       NaN                       NaN   \n",
      "307504                       NaN                       NaN   \n",
      "307505                    0.0155                    0.0156   \n",
      "307506                    0.0757                    0.0000   \n",
      "307507                    0.0000                    0.0000   \n",
      "307508                    0.0000                    0.0000   \n",
      "307509                       NaN                       NaN   \n",
      "307510                       NaN                       NaN   \n",
      "\n",
      "        NONLIVINGAPARTMENTS_AVG  LIVINGAPARTMENTS_MEDI  LIVINGAPARTMENTS_AVG  \\\n",
      "0                        0.0000                 0.0205                0.0202   \n",
      "1                        0.0039                 0.0787                0.0773   \n",
      "2                           NaN                    NaN                   NaN   \n",
      "3                           NaN                    NaN                   NaN   \n",
      "4                           NaN                    NaN                   NaN   \n",
      "5                           NaN                    NaN                   NaN   \n",
      "6                           NaN                    NaN                   NaN   \n",
      "7                           NaN                    NaN                   NaN   \n",
      "8                           NaN                    NaN                   NaN   \n",
      "9                           NaN                    NaN                   NaN   \n",
      "10                          NaN                    NaN                   NaN   \n",
      "11                          NaN                    NaN                   NaN   \n",
      "12                          NaN                    NaN                   NaN   \n",
      "13                       0.0000                 0.1223                0.1202   \n",
      "14                       0.0193                 0.2899                0.2849   \n",
      "15                          NaN                    NaN                   NaN   \n",
      "16                          NaN                    NaN                   NaN   \n",
      "17                          NaN                    NaN                   NaN   \n",
      "18                       0.0000                 0.0231                0.0227   \n",
      "19                          NaN                    NaN                   NaN   \n",
      "20                       0.0000                 0.0599                0.0588   \n",
      "21                          NaN                    NaN                   NaN   \n",
      "22                       0.0077                 0.0735                0.0723   \n",
      "23                       0.0019                 0.1193                0.1173   \n",
      "24                       0.0000                 0.1189                0.1168   \n",
      "25                       0.0000                 0.0599                0.0588   \n",
      "26                          NaN                    NaN                   NaN   \n",
      "27                          NaN                    NaN                   NaN   \n",
      "28                          NaN                    NaN                   NaN   \n",
      "29                       0.0077                 0.1231                0.1210   \n",
      "...                         ...                    ...                   ...   \n",
      "307481                      NaN                    NaN                   NaN   \n",
      "307482                   0.0039                 0.0316                0.0311   \n",
      "307483                   0.0000                 0.0068                0.0067   \n",
      "307484                      NaN                    NaN                   NaN   \n",
      "307485                      NaN                    NaN                   NaN   \n",
      "307486                      NaN                    NaN                   NaN   \n",
      "307487                      NaN                    NaN                   NaN   \n",
      "307488                      NaN                    NaN                   NaN   \n",
      "307489                   0.0000                 0.0094                0.0092   \n",
      "307490                      NaN                    NaN                   NaN   \n",
      "307491                      NaN                    NaN                   NaN   \n",
      "307492                      NaN                    NaN                   NaN   \n",
      "307493                      NaN                    NaN                   NaN   \n",
      "307494                      NaN                    NaN                   NaN   \n",
      "307495                   0.0039                 0.0607                0.0597   \n",
      "307496                      NaN                    NaN                   NaN   \n",
      "307497                      NaN                    NaN                   NaN   \n",
      "307498                      NaN                    NaN                   NaN   \n",
      "307499                      NaN                    NaN                   NaN   \n",
      "307500                      NaN                    NaN                   NaN   \n",
      "307501                      NaN                    NaN                   NaN   \n",
      "307502                      NaN                    NaN                   NaN   \n",
      "307503                      NaN                    NaN                   NaN   \n",
      "307504                      NaN                    NaN                   NaN   \n",
      "307505                   0.0154                 0.1796                0.1765   \n",
      "307506                   0.0753                 0.1509                0.1484   \n",
      "307507                   0.0000                 0.0205                0.0202   \n",
      "307508                   0.0000                 0.0855                0.0841   \n",
      "307509                      NaN                    NaN                   NaN   \n",
      "307510                      NaN                    NaN                   NaN   \n",
      "\n",
      "        LIVINGAPARTMENTS_MODE  FLOORSMIN_MEDI       ...         \\\n",
      "0                      0.0220          0.1250       ...          \n",
      "1                      0.0790          0.3333       ...          \n",
      "2                         NaN             NaN       ...          \n",
      "3                         NaN             NaN       ...          \n",
      "4                         NaN             NaN       ...          \n",
      "5                         NaN             NaN       ...          \n",
      "6                         NaN             NaN       ...          \n",
      "7                         NaN             NaN       ...          \n",
      "8                         NaN             NaN       ...          \n",
      "9                         NaN             NaN       ...          \n",
      "10                        NaN             NaN       ...          \n",
      "11                        NaN             NaN       ...          \n",
      "12                        NaN             NaN       ...          \n",
      "13                     0.1313          0.3750       ...          \n",
      "14                     0.3113          0.7083       ...          \n",
      "15                        NaN             NaN       ...          \n",
      "16                        NaN             NaN       ...          \n",
      "17                        NaN             NaN       ...          \n",
      "18                     0.0248          0.1250       ...          \n",
      "19                        NaN             NaN       ...          \n",
      "20                     0.0643          0.0417       ...          \n",
      "21                        NaN             NaN       ...          \n",
      "22                     0.0790          0.2083       ...          \n",
      "23                     0.1102          0.4167       ...          \n",
      "24                     0.1276          0.2083       ...          \n",
      "25                     0.0643          0.0417       ...          \n",
      "26                        NaN             NaN       ...          \n",
      "27                        NaN             NaN       ...          \n",
      "28                        NaN             NaN       ...          \n",
      "29                     0.1322          0.3750       ...          \n",
      "...                       ...             ...       ...          \n",
      "307481                    NaN             NaN       ...          \n",
      "307482                 0.0340          0.2083       ...          \n",
      "307483                 0.0073          0.0417       ...          \n",
      "307484                    NaN             NaN       ...          \n",
      "307485                    NaN             NaN       ...          \n",
      "307486                    NaN             NaN       ...          \n",
      "307487                    NaN             NaN       ...          \n",
      "307488                    NaN             NaN       ...          \n",
      "307489                 0.0101          0.0417       ...          \n",
      "307490                    NaN             NaN       ...          \n",
      "307491                    NaN             NaN       ...          \n",
      "307492                    NaN             NaN       ...          \n",
      "307493                    NaN             NaN       ...          \n",
      "307494                    NaN             NaN       ...          \n",
      "307495                 0.0652             NaN       ...          \n",
      "307496                    NaN             NaN       ...          \n",
      "307497                    NaN             NaN       ...          \n",
      "307498                    NaN          0.7083       ...          \n",
      "307499                    NaN             NaN       ...          \n",
      "307500                    NaN             NaN       ...          \n",
      "307501                    NaN             NaN       ...          \n",
      "307502                    NaN             NaN       ...          \n",
      "307503                    NaN             NaN       ...          \n",
      "307504                    NaN             NaN       ...          \n",
      "307505                 0.1928          0.3750       ...          \n",
      "307506                 0.0882          0.2708       ...          \n",
      "307507                 0.0220          0.1250       ...          \n",
      "307508                 0.0918          0.2083       ...          \n",
      "307509                    NaN             NaN       ...          \n",
      "307510                    NaN             NaN       ...          \n",
      "\n",
      "        OBS_60_CNT_SOCIAL_CIRCLE  DEF_30_CNT_SOCIAL_CIRCLE  \\\n",
      "0                            2.0                       2.0   \n",
      "1                            1.0                       0.0   \n",
      "2                            0.0                       0.0   \n",
      "3                            2.0                       0.0   \n",
      "4                            0.0                       0.0   \n",
      "5                            0.0                       0.0   \n",
      "6                            1.0                       0.0   \n",
      "7                            2.0                       0.0   \n",
      "8                            1.0                       0.0   \n",
      "9                            2.0                       0.0   \n",
      "10                           0.0                       0.0   \n",
      "11                           0.0                       0.0   \n",
      "12                           0.0                       0.0   \n",
      "13                           0.0                       0.0   \n",
      "14                           0.0                       0.0   \n",
      "15                           0.0                       0.0   \n",
      "16                           0.0                       0.0   \n",
      "17                           4.0                       0.0   \n",
      "18                           8.0                       0.0   \n",
      "19                           0.0                       0.0   \n",
      "20                           0.0                       0.0   \n",
      "21                           0.0                       0.0   \n",
      "22                           0.0                       0.0   \n",
      "23                           0.0                       0.0   \n",
      "24                           1.0                       1.0   \n",
      "25                           0.0                       0.0   \n",
      "26                          10.0                       1.0   \n",
      "27                           2.0                       0.0   \n",
      "28                           0.0                       0.0   \n",
      "29                           0.0                       0.0   \n",
      "...                          ...                       ...   \n",
      "307481                       0.0                       0.0   \n",
      "307482                       0.0                       0.0   \n",
      "307483                       1.0                       1.0   \n",
      "307484                       0.0                       0.0   \n",
      "307485                       0.0                       0.0   \n",
      "307486                       0.0                       0.0   \n",
      "307487                       0.0                       0.0   \n",
      "307488                       1.0                       0.0   \n",
      "307489                       1.0                       1.0   \n",
      "307490                       0.0                       0.0   \n",
      "307491                       4.0                       0.0   \n",
      "307492                       0.0                       0.0   \n",
      "307493                       0.0                       0.0   \n",
      "307494                      11.0                       0.0   \n",
      "307495                       0.0                       0.0   \n",
      "307496                       0.0                       0.0   \n",
      "307497                       0.0                       0.0   \n",
      "307498                       0.0                       0.0   \n",
      "307499                       4.0                       0.0   \n",
      "307500                       0.0                       0.0   \n",
      "307501                       2.0                       1.0   \n",
      "307502                       0.0                       0.0   \n",
      "307503                       9.0                       3.0   \n",
      "307504                       0.0                       0.0   \n",
      "307505                       0.0                       0.0   \n",
      "307506                       0.0                       0.0   \n",
      "307507                       0.0                       0.0   \n",
      "307508                       6.0                       0.0   \n",
      "307509                       0.0                       0.0   \n",
      "307510                       0.0                       0.0   \n",
      "\n",
      "        OBS_30_CNT_SOCIAL_CIRCLE  EXT_SOURCE_2  AMT_GOODS_PRICE  \\\n",
      "0                            2.0      0.262949         351000.0   \n",
      "1                            1.0      0.622246        1129500.0   \n",
      "2                            0.0      0.555912         135000.0   \n",
      "3                            2.0      0.650442         297000.0   \n",
      "4                            0.0      0.322738         513000.0   \n",
      "5                            0.0      0.354225         454500.0   \n",
      "6                            1.0      0.724000        1395000.0   \n",
      "7                            2.0      0.714279        1530000.0   \n",
      "8                            1.0      0.205747         913500.0   \n",
      "9                            2.0      0.746644         405000.0   \n",
      "10                           0.0      0.651862         652500.0   \n",
      "11                           0.0      0.555183         135000.0   \n",
      "12                           0.0      0.715042          67500.0   \n",
      "13                           0.0      0.566907         697500.0   \n",
      "14                           0.0      0.642656         679500.0   \n",
      "15                           0.0      0.346634         247500.0   \n",
      "16                           0.0      0.236378         387000.0   \n",
      "17                           4.0      0.683513         270000.0   \n",
      "18                           8.0      0.706428         157500.0   \n",
      "19                           0.0      0.586617         454500.0   \n",
      "20                           0.0      0.113375         427500.0   \n",
      "21                           0.0      0.233767         927000.0   \n",
      "22                           0.0      0.457143         450000.0   \n",
      "23                           0.0      0.624305         225000.0   \n",
      "24                           1.0      0.786179         247500.0   \n",
      "25                           0.0      0.651406         225000.0   \n",
      "26                          10.0      0.548477         702000.0   \n",
      "27                           2.0      0.541124         270000.0   \n",
      "28                           0.0      0.685011         675000.0   \n",
      "29                           0.0      0.502779         180000.0   \n",
      "...                          ...           ...              ...   \n",
      "307481                       0.0      0.713524         297000.0   \n",
      "307482                       0.0      0.737905         472500.0   \n",
      "307483                       1.0      0.589906         225000.0   \n",
      "307484                       0.0      0.739703         450000.0   \n",
      "307485                       0.0      0.702403         180000.0   \n",
      "307486                       0.0      0.661025         270000.0   \n",
      "307487                       0.0      0.307082         936000.0   \n",
      "307488                       1.0      0.619662         135000.0   \n",
      "307489                       1.0      0.615261         450000.0   \n",
      "307490                       0.0      0.378421         135000.0   \n",
      "307491                       4.0      0.561158         900000.0   \n",
      "307492                       0.0      0.599173        1575000.0   \n",
      "307493                       0.0      0.424683         765000.0   \n",
      "307494                      11.0      0.699727         450000.0   \n",
      "307495                       0.0      0.267834         675000.0   \n",
      "307496                       0.0      0.512081         337500.0   \n",
      "307497                       0.0      0.651306         193500.0   \n",
      "307498                       0.0      0.746259        1125000.0   \n",
      "307499                       4.0      0.649123         225000.0   \n",
      "307500                       0.0      0.789389        1138500.0   \n",
      "307501                       2.0      0.575481         225000.0   \n",
      "307502                       0.0      0.313334         225000.0   \n",
      "307503                       9.0      0.501221         247500.0   \n",
      "307504                       0.0      0.530725         225000.0   \n",
      "307505                       0.0      0.346391         225000.0   \n",
      "307506                       0.0      0.681632         225000.0   \n",
      "307507                       0.0      0.115992         225000.0   \n",
      "307508                       6.0      0.535722         585000.0   \n",
      "307509                       0.0      0.514163         319500.0   \n",
      "307510                       0.0      0.708569         675000.0   \n",
      "\n",
      "        CODE_GENDER_NAME_EDUCATION_TYPE_max_AMT_ANNUITY  \\\n",
      "0                                              173704.5   \n",
      "1                                              230161.5   \n",
      "2                                              173704.5   \n",
      "3                                              216589.5   \n",
      "4                                              173704.5   \n",
      "5                                              173704.5   \n",
      "6                                              230161.5   \n",
      "7                                              258025.5   \n",
      "8                                              216589.5   \n",
      "9                                              173704.5   \n",
      "10                                             230161.5   \n",
      "11                                             216589.5   \n",
      "12                                             216589.5   \n",
      "13                                             173704.5   \n",
      "14                                             216589.5   \n",
      "15                                             173704.5   \n",
      "16                                             173704.5   \n",
      "17                                             216589.5   \n",
      "18                                             216589.5   \n",
      "19                                             230161.5   \n",
      "20                                             173704.5   \n",
      "21                                             216589.5   \n",
      "22                                             216589.5   \n",
      "23                                             216589.5   \n",
      "24                                             173704.5   \n",
      "25                                             216589.5   \n",
      "26                                             216589.5   \n",
      "27                                             173704.5   \n",
      "28                                             258025.5   \n",
      "29                                             258025.5   \n",
      "...                                                 ...   \n",
      "307481                                         173704.5   \n",
      "307482                                         216589.5   \n",
      "307483                                         216589.5   \n",
      "307484                                         216589.5   \n",
      "307485                                         216589.5   \n",
      "307486                                         230161.5   \n",
      "307487                                         173704.5   \n",
      "307488                                         230161.5   \n",
      "307489                                         216589.5   \n",
      "307490                                         258025.5   \n",
      "307491                                         173704.5   \n",
      "307492                                         173704.5   \n",
      "307493                                         216589.5   \n",
      "307494                                         173704.5   \n",
      "307495                                         258025.5   \n",
      "307496                                         230161.5   \n",
      "307497                                         216589.5   \n",
      "307498                                         173704.5   \n",
      "307499                                         216589.5   \n",
      "307500                                         230161.5   \n",
      "307501                                         216589.5   \n",
      "307502                                         216589.5   \n",
      "307503                                         230161.5   \n",
      "307504                                         216589.5   \n",
      "307505                                         216589.5   \n",
      "307506                                         173704.5   \n",
      "307507                                         216589.5   \n",
      "307508                                         230161.5   \n",
      "307509                                         216589.5   \n",
      "307510                                         230161.5   \n",
      "\n",
      "        CODE_GENDER_ORGANIZATION_TYPE_mean_AMT_INCOME_TOTAL  \\\n",
      "0                                           208566.954548     \n",
      "1                                           153236.991435     \n",
      "2                                           180212.879701     \n",
      "3                                           175499.426888     \n",
      "4                                           179284.090909     \n",
      "5                                           193671.537279     \n",
      "6                                           175499.426888     \n",
      "7                                           193671.537279     \n",
      "8                                           133968.829812     \n",
      "9                                           187450.135714     \n",
      "10                                          153800.183702     \n",
      "11                                          133968.829812     \n",
      "12                                          158044.886446     \n",
      "13                                          191266.266794     \n",
      "14                                          162098.580357     \n",
      "15                                          186480.962105     \n",
      "16                                          180212.879701     \n",
      "17                                          186236.624073     \n",
      "18                                          154769.765015     \n",
      "19                                          136766.214801     \n",
      "20                                          191266.266794     \n",
      "21                                          157412.599851     \n",
      "22                                          155619.546251     \n",
      "23                                          133968.829812     \n",
      "24                                          208566.954548     \n",
      "25                                          175499.426888     \n",
      "26                                          175499.426888     \n",
      "27                                          178311.022388     \n",
      "28                                          249654.328249     \n",
      "29                                          208566.954548     \n",
      "...                                                   ...     \n",
      "307481                                      208566.954548     \n",
      "307482                                      163790.094340     \n",
      "307483                                      133968.829812     \n",
      "307484                                      175499.426888     \n",
      "307485                                      175499.426888     \n",
      "307486                                      158044.886446     \n",
      "307487                                      147269.919668     \n",
      "307488                                      153252.513841     \n",
      "307489                                      175499.426888     \n",
      "307490                                      199545.773196     \n",
      "307491                                      191266.266794     \n",
      "307492                                      191266.266794     \n",
      "307493                                      153236.991435     \n",
      "307494                                      191266.266794     \n",
      "307495                                      208566.954548     \n",
      "307496                                      155619.546251     \n",
      "307497                                      157412.599851     \n",
      "307498                                      206252.611111     \n",
      "307499                                      153800.183702     \n",
      "307500                                      175499.426888     \n",
      "307501                                      153963.327068     \n",
      "307502                                      153236.991435     \n",
      "307503                                      184169.782577     \n",
      "307504                                      155619.546251     \n",
      "307505                                      133968.829812     \n",
      "307506                                      213583.146067     \n",
      "307507                                      133968.829812     \n",
      "307508                                      153236.991435     \n",
      "307509                                      175089.798898     \n",
      "307510                                      175499.426888     \n",
      "\n",
      "        CODE_GENDER_ORGANIZATION_TYPE_mean_DAYS_REGISTRATION  \\\n",
      "0                                            -4442.857420      \n",
      "1                                            -4965.608351      \n",
      "2                                            -4677.001719      \n",
      "3                                            -4709.777998      \n",
      "4                                            -4276.931818      \n",
      "5                                            -4587.493886      \n",
      "6                                            -4709.777998      \n",
      "7                                            -4587.493886      \n",
      "8                                            -6692.617415      \n",
      "9                                            -4651.734694      \n",
      "10                                           -4977.160798      \n",
      "11                                           -6692.617415      \n",
      "12                                           -5036.512502      \n",
      "13                                           -4383.874254      \n",
      "14                                           -4791.561508      \n",
      "15                                           -4658.636306      \n",
      "16                                           -4677.001719      \n",
      "17                                           -4667.722195      \n",
      "18                                           -5214.004334      \n",
      "19                                           -4710.026227      \n",
      "20                                           -4383.874254      \n",
      "21                                           -4651.860327      \n",
      "22                                           -4557.933663      \n",
      "23                                           -6692.617415      \n",
      "24                                           -4442.857420      \n",
      "25                                           -4709.777998      \n",
      "26                                           -4709.777998      \n",
      "27                                           -4860.949956      \n",
      "28                                           -3170.101297      \n",
      "29                                           -4442.857420      \n",
      "...                                                   ...      \n",
      "307481                                       -4442.857420      \n",
      "307482                                       -4781.776730      \n",
      "307483                                       -6692.617415      \n",
      "307484                                       -4709.777998      \n",
      "307485                                       -4709.777998      \n",
      "307486                                       -5036.512502      \n",
      "307487                                       -6091.431951      \n",
      "307488                                       -4939.548232      \n",
      "307489                                       -4709.777998      \n",
      "307490                                       -4706.889543      \n",
      "307491                                       -4383.874254      \n",
      "307492                                       -4383.874254      \n",
      "307493                                       -4965.608351      \n",
      "307494                                       -4383.874254      \n",
      "307495                                       -4442.857420      \n",
      "307496                                       -4557.933663      \n",
      "307497                                       -4651.860327      \n",
      "307498                                       -4330.576415      \n",
      "307499                                       -4977.160798      \n",
      "307500                                       -4709.777998      \n",
      "307501                                       -5054.867168      \n",
      "307502                                       -4965.608351      \n",
      "307503                                       -4041.638848      \n",
      "307504                                       -4557.933663      \n",
      "307505                                       -6692.617415      \n",
      "307506                                       -4288.348315      \n",
      "307507                                       -6692.617415      \n",
      "307508                                       -4965.608351      \n",
      "307509                                       -4905.762167      \n",
      "307510                                       -4709.777998      \n",
      "\n",
      "        CODE_GENDER_REG_CITY_NOT_WORK_CITY_mean_CNT_CHILDREN  CNT_FAM_MEMBERS  \n",
      "0                                                0.442195                 1.0  \n",
      "1                                                0.365735                 2.0  \n",
      "2                                                0.442195                 1.0  \n",
      "3                                                0.365735                 2.0  \n",
      "4                                                0.514099                 1.0  \n",
      "5                                                0.442195                 2.0  \n",
      "6                                                0.365735                 3.0  \n",
      "7                                                0.514099                 2.0  \n",
      "8                                                0.365735                 2.0  \n",
      "9                                                0.442195                 1.0  \n",
      "10                                               0.365735                 3.0  \n",
      "11                                               0.365735                 2.0  \n",
      "12                                               0.365735                 2.0  \n",
      "13                                               0.442195                 3.0  \n",
      "14                                               0.365735                 2.0  \n",
      "15                                               0.514099                 1.0  \n",
      "16                                               0.514099                 2.0  \n",
      "17                                               0.507015                 3.0  \n",
      "18                                               0.365735                 1.0  \n",
      "19                                               0.365735                 2.0  \n",
      "20                                               0.442195                 2.0  \n",
      "21                                               0.365735                 3.0  \n",
      "22                                               0.365735                 3.0  \n",
      "23                                               0.365735                 2.0  \n",
      "24                                               0.514099                 4.0  \n",
      "25                                               0.365735                 2.0  \n",
      "26                                               0.365735                 1.0  \n",
      "27                                               0.514099                 3.0  \n",
      "28                                               0.442195                 1.0  \n",
      "29                                               0.442195                 1.0  \n",
      "...                                                   ...                 ...  \n",
      "307481                                           0.442195                 2.0  \n",
      "307482                                           0.365735                 2.0  \n",
      "307483                                           0.365735                 1.0  \n",
      "307484                                           0.365735                 2.0  \n",
      "307485                                           0.507015                 2.0  \n",
      "307486                                           0.365735                 3.0  \n",
      "307487                                           0.442195                 2.0  \n",
      "307488                                           0.507015                 1.0  \n",
      "307489                                           0.365735                 2.0  \n",
      "307490                                           0.442195                 1.0  \n",
      "307491                                           0.442195                 4.0  \n",
      "307492                                           0.442195                 2.0  \n",
      "307493                                           0.507015                 2.0  \n",
      "307494                                           0.442195                 3.0  \n",
      "307495                                           0.442195                 2.0  \n",
      "307496                                           0.365735                 2.0  \n",
      "307497                                           0.507015                 2.0  \n",
      "307498                                           0.442195                 2.0  \n",
      "307499                                           0.365735                 2.0  \n",
      "307500                                           0.365735                 2.0  \n",
      "307501                                           0.365735                 5.0  \n",
      "307502                                           0.507015                 3.0  \n",
      "307503                                           0.365735                 1.0  \n",
      "307504                                           0.365735                 1.0  \n",
      "307505                                           0.365735                 1.0  \n",
      "307506                                           0.442195                 1.0  \n",
      "307507                                           0.365735                 1.0  \n",
      "307508                                           0.507015                 1.0  \n",
      "307509                                           0.507015                 2.0  \n",
      "307510                                           0.507015                 2.0  \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[307511 rows x 66 columns]\n",
      "after...\n",
      "        COMMONAREA_MEDI  COMMONAREA_AVG  COMMONAREA_MODE  \\\n",
      "0              0.014400        0.014300         0.014400   \n",
      "1              0.060800        0.060500         0.049700   \n",
      "2              0.044595        0.044621         0.042553   \n",
      "3              0.044595        0.044621         0.042553   \n",
      "4              0.044595        0.044621         0.042553   \n",
      "5              0.044595        0.044621         0.042553   \n",
      "6              0.044595        0.044621         0.042553   \n",
      "7              0.044595        0.044621         0.042553   \n",
      "8              0.044595        0.044621         0.042553   \n",
      "9              0.044595        0.044621         0.042553   \n",
      "10             0.044595        0.044621         0.042553   \n",
      "11             0.044595        0.044621         0.042553   \n",
      "12             0.044595        0.044621         0.042553   \n",
      "13             0.058500        0.058200         0.058700   \n",
      "14             0.115000        0.114300         0.115300   \n",
      "15             0.044595        0.044621         0.042553   \n",
      "16             0.044595        0.044621         0.042553   \n",
      "17             0.044595        0.044621         0.042553   \n",
      "18             0.001800        0.001800         0.001800   \n",
      "19             0.044595        0.044621         0.042553   \n",
      "20             0.044595        0.044621         0.042553   \n",
      "21             0.044595        0.044621         0.042553   \n",
      "22             0.012000        0.012000         0.012100   \n",
      "23             0.107100        0.106400         0.000000   \n",
      "24             0.021400        0.021200         0.021400   \n",
      "25             0.001000        0.001000         0.001000   \n",
      "26             0.044595        0.044621         0.042553   \n",
      "27             0.044595        0.044621         0.042553   \n",
      "28             0.044595        0.044621         0.042553   \n",
      "29             0.018900        0.018800         0.019000   \n",
      "...                 ...             ...              ...   \n",
      "307481         0.044595        0.044621         0.042553   \n",
      "307482         0.003800        0.003700         0.003800   \n",
      "307483         0.000000        0.000000         0.000000   \n",
      "307484         0.044595        0.044621         0.042553   \n",
      "307485         0.044595        0.044621         0.042553   \n",
      "307486         0.044595        0.044621         0.042553   \n",
      "307487         0.044595        0.044621         0.042553   \n",
      "307488         0.044595        0.044621         0.042553   \n",
      "307489         0.000900        0.000900         0.000900   \n",
      "307490         0.044595        0.044621         0.042553   \n",
      "307491         0.044595        0.044621         0.042553   \n",
      "307492         0.044595        0.044621         0.042553   \n",
      "307493         0.044595        0.044621         0.042553   \n",
      "307494         0.044595        0.044621         0.042553   \n",
      "307495         0.013700        0.013600         0.013800   \n",
      "307496         0.044595        0.044621         0.042553   \n",
      "307497         0.044595        0.044621         0.042553   \n",
      "307498         0.044595        0.044621         0.042553   \n",
      "307499         0.044595        0.044621         0.042553   \n",
      "307500         0.044595        0.044621         0.042553   \n",
      "307501         0.044595        0.044621         0.042553   \n",
      "307502         0.044595        0.044621         0.042553   \n",
      "307503         0.044595        0.044621         0.042553   \n",
      "307504         0.044595        0.044621         0.042553   \n",
      "307505         0.144100        0.143200         0.144500   \n",
      "307506         0.020300        0.020200         0.017200   \n",
      "307507         0.002200        0.002200         0.002200   \n",
      "307508         0.012400        0.012300         0.012400   \n",
      "307509         0.044595        0.044621         0.042553   \n",
      "307510         0.017700        0.017600         0.017800   \n",
      "\n",
      "        NONLIVINGAPARTMENTS_MEDI  NONLIVINGAPARTMENTS_MODE  \\\n",
      "0                       0.000000                  0.000000   \n",
      "1                       0.003900                  0.000000   \n",
      "2                       0.008651                  0.008076   \n",
      "3                       0.008651                  0.008076   \n",
      "4                       0.008651                  0.008076   \n",
      "5                       0.008651                  0.008076   \n",
      "6                       0.008651                  0.008076   \n",
      "7                       0.008651                  0.008076   \n",
      "8                       0.008651                  0.008076   \n",
      "9                       0.008651                  0.008076   \n",
      "10                      0.008651                  0.008076   \n",
      "11                      0.008651                  0.008076   \n",
      "12                      0.008651                  0.008076   \n",
      "13                      0.000000                  0.000000   \n",
      "14                      0.019400                  0.019500   \n",
      "15                      0.008651                  0.008076   \n",
      "16                      0.008651                  0.008076   \n",
      "17                      0.008651                  0.008076   \n",
      "18                      0.000000                  0.000000   \n",
      "19                      0.008651                  0.008076   \n",
      "20                      0.000000                  0.000000   \n",
      "21                      0.008651                  0.008076   \n",
      "22                      0.007800                  0.007800   \n",
      "23                      0.001900                  0.000000   \n",
      "24                      0.000000                  0.000000   \n",
      "25                      0.000000                  0.000000   \n",
      "26                      0.008651                  0.008076   \n",
      "27                      0.008651                  0.008076   \n",
      "28                      0.008651                  0.008076   \n",
      "29                      0.007800                  0.007800   \n",
      "...                          ...                       ...   \n",
      "307481                  0.008651                  0.008076   \n",
      "307482                  0.003900                  0.003900   \n",
      "307483                  0.000000                  0.000000   \n",
      "307484                  0.008651                  0.008076   \n",
      "307485                  0.008651                  0.008076   \n",
      "307486                  0.008651                  0.008076   \n",
      "307487                  0.008651                  0.008076   \n",
      "307488                  0.008651                  0.008076   \n",
      "307489                  0.000000                  0.000000   \n",
      "307490                  0.008651                  0.008076   \n",
      "307491                  0.008651                  0.008076   \n",
      "307492                  0.008651                  0.008076   \n",
      "307493                  0.008651                  0.008076   \n",
      "307494                  0.008651                  0.008076   \n",
      "307495                  0.003900                  0.003900   \n",
      "307496                  0.008651                  0.008076   \n",
      "307497                  0.008651                  0.008076   \n",
      "307498                  0.008651                  0.008076   \n",
      "307499                  0.008651                  0.008076   \n",
      "307500                  0.008651                  0.008076   \n",
      "307501                  0.008651                  0.008076   \n",
      "307502                  0.008651                  0.008076   \n",
      "307503                  0.008651                  0.008076   \n",
      "307504                  0.008651                  0.008076   \n",
      "307505                  0.015500                  0.015600   \n",
      "307506                  0.075700                  0.000000   \n",
      "307507                  0.000000                  0.000000   \n",
      "307508                  0.000000                  0.000000   \n",
      "307509                  0.008651                  0.008076   \n",
      "307510                  0.008651                  0.008076   \n",
      "\n",
      "        NONLIVINGAPARTMENTS_AVG  LIVINGAPARTMENTS_MEDI  LIVINGAPARTMENTS_AVG  \\\n",
      "0                      0.000000               0.020500              0.020200   \n",
      "1                      0.003900               0.078700              0.077300   \n",
      "2                      0.008809               0.101954              0.100775   \n",
      "3                      0.008809               0.101954              0.100775   \n",
      "4                      0.008809               0.101954              0.100775   \n",
      "5                      0.008809               0.101954              0.100775   \n",
      "6                      0.008809               0.101954              0.100775   \n",
      "7                      0.008809               0.101954              0.100775   \n",
      "8                      0.008809               0.101954              0.100775   \n",
      "9                      0.008809               0.101954              0.100775   \n",
      "10                     0.008809               0.101954              0.100775   \n",
      "11                     0.008809               0.101954              0.100775   \n",
      "12                     0.008809               0.101954              0.100775   \n",
      "13                     0.000000               0.122300              0.120200   \n",
      "14                     0.019300               0.289900              0.284900   \n",
      "15                     0.008809               0.101954              0.100775   \n",
      "16                     0.008809               0.101954              0.100775   \n",
      "17                     0.008809               0.101954              0.100775   \n",
      "18                     0.000000               0.023100              0.022700   \n",
      "19                     0.008809               0.101954              0.100775   \n",
      "20                     0.000000               0.059900              0.058800   \n",
      "21                     0.008809               0.101954              0.100775   \n",
      "22                     0.007700               0.073500              0.072300   \n",
      "23                     0.001900               0.119300              0.117300   \n",
      "24                     0.000000               0.118900              0.116800   \n",
      "25                     0.000000               0.059900              0.058800   \n",
      "26                     0.008809               0.101954              0.100775   \n",
      "27                     0.008809               0.101954              0.100775   \n",
      "28                     0.008809               0.101954              0.100775   \n",
      "29                     0.007700               0.123100              0.121000   \n",
      "...                         ...                    ...                   ...   \n",
      "307481                 0.008809               0.101954              0.100775   \n",
      "307482                 0.003900               0.031600              0.031100   \n",
      "307483                 0.000000               0.006800              0.006700   \n",
      "307484                 0.008809               0.101954              0.100775   \n",
      "307485                 0.008809               0.101954              0.100775   \n",
      "307486                 0.008809               0.101954              0.100775   \n",
      "307487                 0.008809               0.101954              0.100775   \n",
      "307488                 0.008809               0.101954              0.100775   \n",
      "307489                 0.000000               0.009400              0.009200   \n",
      "307490                 0.008809               0.101954              0.100775   \n",
      "307491                 0.008809               0.101954              0.100775   \n",
      "307492                 0.008809               0.101954              0.100775   \n",
      "307493                 0.008809               0.101954              0.100775   \n",
      "307494                 0.008809               0.101954              0.100775   \n",
      "307495                 0.003900               0.060700              0.059700   \n",
      "307496                 0.008809               0.101954              0.100775   \n",
      "307497                 0.008809               0.101954              0.100775   \n",
      "307498                 0.008809               0.101954              0.100775   \n",
      "307499                 0.008809               0.101954              0.100775   \n",
      "307500                 0.008809               0.101954              0.100775   \n",
      "307501                 0.008809               0.101954              0.100775   \n",
      "307502                 0.008809               0.101954              0.100775   \n",
      "307503                 0.008809               0.101954              0.100775   \n",
      "307504                 0.008809               0.101954              0.100775   \n",
      "307505                 0.015400               0.179600              0.176500   \n",
      "307506                 0.075300               0.150900              0.148400   \n",
      "307507                 0.000000               0.020500              0.020200   \n",
      "307508                 0.000000               0.085500              0.084100   \n",
      "307509                 0.008809               0.101954              0.100775   \n",
      "307510                 0.008809               0.101954              0.100775   \n",
      "\n",
      "        LIVINGAPARTMENTS_MODE  FLOORSMIN_MEDI       ...         \\\n",
      "0                    0.022000        0.125000       ...          \n",
      "1                    0.079000        0.333300       ...          \n",
      "2                    0.105645        0.231625       ...          \n",
      "3                    0.105645        0.231625       ...          \n",
      "4                    0.105645        0.231625       ...          \n",
      "5                    0.105645        0.231625       ...          \n",
      "6                    0.105645        0.231625       ...          \n",
      "7                    0.105645        0.231625       ...          \n",
      "8                    0.105645        0.231625       ...          \n",
      "9                    0.105645        0.231625       ...          \n",
      "10                   0.105645        0.231625       ...          \n",
      "11                   0.105645        0.231625       ...          \n",
      "12                   0.105645        0.231625       ...          \n",
      "13                   0.131300        0.375000       ...          \n",
      "14                   0.311300        0.708300       ...          \n",
      "15                   0.105645        0.231625       ...          \n",
      "16                   0.105645        0.231625       ...          \n",
      "17                   0.105645        0.231625       ...          \n",
      "18                   0.024800        0.125000       ...          \n",
      "19                   0.105645        0.231625       ...          \n",
      "20                   0.064300        0.041700       ...          \n",
      "21                   0.105645        0.231625       ...          \n",
      "22                   0.079000        0.208300       ...          \n",
      "23                   0.110200        0.416700       ...          \n",
      "24                   0.127600        0.208300       ...          \n",
      "25                   0.064300        0.041700       ...          \n",
      "26                   0.105645        0.231625       ...          \n",
      "27                   0.105645        0.231625       ...          \n",
      "28                   0.105645        0.231625       ...          \n",
      "29                   0.132200        0.375000       ...          \n",
      "...                       ...             ...       ...          \n",
      "307481               0.105645        0.231625       ...          \n",
      "307482               0.034000        0.208300       ...          \n",
      "307483               0.007300        0.041700       ...          \n",
      "307484               0.105645        0.231625       ...          \n",
      "307485               0.105645        0.231625       ...          \n",
      "307486               0.105645        0.231625       ...          \n",
      "307487               0.105645        0.231625       ...          \n",
      "307488               0.105645        0.231625       ...          \n",
      "307489               0.010100        0.041700       ...          \n",
      "307490               0.105645        0.231625       ...          \n",
      "307491               0.105645        0.231625       ...          \n",
      "307492               0.105645        0.231625       ...          \n",
      "307493               0.105645        0.231625       ...          \n",
      "307494               0.105645        0.231625       ...          \n",
      "307495               0.065200        0.231625       ...          \n",
      "307496               0.105645        0.231625       ...          \n",
      "307497               0.105645        0.231625       ...          \n",
      "307498               0.105645        0.708300       ...          \n",
      "307499               0.105645        0.231625       ...          \n",
      "307500               0.105645        0.231625       ...          \n",
      "307501               0.105645        0.231625       ...          \n",
      "307502               0.105645        0.231625       ...          \n",
      "307503               0.105645        0.231625       ...          \n",
      "307504               0.105645        0.231625       ...          \n",
      "307505               0.192800        0.375000       ...          \n",
      "307506               0.088200        0.270800       ...          \n",
      "307507               0.022000        0.125000       ...          \n",
      "307508               0.091800        0.208300       ...          \n",
      "307509               0.105645        0.231625       ...          \n",
      "307510               0.105645        0.231625       ...          \n",
      "\n",
      "        OBS_60_CNT_SOCIAL_CIRCLE  DEF_30_CNT_SOCIAL_CIRCLE  \\\n",
      "0                            2.0                       2.0   \n",
      "1                            1.0                       0.0   \n",
      "2                            0.0                       0.0   \n",
      "3                            2.0                       0.0   \n",
      "4                            0.0                       0.0   \n",
      "5                            0.0                       0.0   \n",
      "6                            1.0                       0.0   \n",
      "7                            2.0                       0.0   \n",
      "8                            1.0                       0.0   \n",
      "9                            2.0                       0.0   \n",
      "10                           0.0                       0.0   \n",
      "11                           0.0                       0.0   \n",
      "12                           0.0                       0.0   \n",
      "13                           0.0                       0.0   \n",
      "14                           0.0                       0.0   \n",
      "15                           0.0                       0.0   \n",
      "16                           0.0                       0.0   \n",
      "17                           4.0                       0.0   \n",
      "18                           8.0                       0.0   \n",
      "19                           0.0                       0.0   \n",
      "20                           0.0                       0.0   \n",
      "21                           0.0                       0.0   \n",
      "22                           0.0                       0.0   \n",
      "23                           0.0                       0.0   \n",
      "24                           1.0                       1.0   \n",
      "25                           0.0                       0.0   \n",
      "26                          10.0                       1.0   \n",
      "27                           2.0                       0.0   \n",
      "28                           0.0                       0.0   \n",
      "29                           0.0                       0.0   \n",
      "...                          ...                       ...   \n",
      "307481                       0.0                       0.0   \n",
      "307482                       0.0                       0.0   \n",
      "307483                       1.0                       1.0   \n",
      "307484                       0.0                       0.0   \n",
      "307485                       0.0                       0.0   \n",
      "307486                       0.0                       0.0   \n",
      "307487                       0.0                       0.0   \n",
      "307488                       1.0                       0.0   \n",
      "307489                       1.0                       1.0   \n",
      "307490                       0.0                       0.0   \n",
      "307491                       4.0                       0.0   \n",
      "307492                       0.0                       0.0   \n",
      "307493                       0.0                       0.0   \n",
      "307494                      11.0                       0.0   \n",
      "307495                       0.0                       0.0   \n",
      "307496                       0.0                       0.0   \n",
      "307497                       0.0                       0.0   \n",
      "307498                       0.0                       0.0   \n",
      "307499                       4.0                       0.0   \n",
      "307500                       0.0                       0.0   \n",
      "307501                       2.0                       1.0   \n",
      "307502                       0.0                       0.0   \n",
      "307503                       9.0                       3.0   \n",
      "307504                       0.0                       0.0   \n",
      "307505                       0.0                       0.0   \n",
      "307506                       0.0                       0.0   \n",
      "307507                       0.0                       0.0   \n",
      "307508                       6.0                       0.0   \n",
      "307509                       0.0                       0.0   \n",
      "307510                       0.0                       0.0   \n",
      "\n",
      "        OBS_30_CNT_SOCIAL_CIRCLE  EXT_SOURCE_2  AMT_GOODS_PRICE  \\\n",
      "0                            2.0      0.262949         351000.0   \n",
      "1                            1.0      0.622246        1129500.0   \n",
      "2                            0.0      0.555912         135000.0   \n",
      "3                            2.0      0.650442         297000.0   \n",
      "4                            0.0      0.322738         513000.0   \n",
      "5                            0.0      0.354225         454500.0   \n",
      "6                            1.0      0.724000        1395000.0   \n",
      "7                            2.0      0.714279        1530000.0   \n",
      "8                            1.0      0.205747         913500.0   \n",
      "9                            2.0      0.746644         405000.0   \n",
      "10                           0.0      0.651862         652500.0   \n",
      "11                           0.0      0.555183         135000.0   \n",
      "12                           0.0      0.715042          67500.0   \n",
      "13                           0.0      0.566907         697500.0   \n",
      "14                           0.0      0.642656         679500.0   \n",
      "15                           0.0      0.346634         247500.0   \n",
      "16                           0.0      0.236378         387000.0   \n",
      "17                           4.0      0.683513         270000.0   \n",
      "18                           8.0      0.706428         157500.0   \n",
      "19                           0.0      0.586617         454500.0   \n",
      "20                           0.0      0.113375         427500.0   \n",
      "21                           0.0      0.233767         927000.0   \n",
      "22                           0.0      0.457143         450000.0   \n",
      "23                           0.0      0.624305         225000.0   \n",
      "24                           1.0      0.786179         247500.0   \n",
      "25                           0.0      0.651406         225000.0   \n",
      "26                          10.0      0.548477         702000.0   \n",
      "27                           2.0      0.541124         270000.0   \n",
      "28                           0.0      0.685011         675000.0   \n",
      "29                           0.0      0.502779         180000.0   \n",
      "...                          ...           ...              ...   \n",
      "307481                       0.0      0.713524         297000.0   \n",
      "307482                       0.0      0.737905         472500.0   \n",
      "307483                       1.0      0.589906         225000.0   \n",
      "307484                       0.0      0.739703         450000.0   \n",
      "307485                       0.0      0.702403         180000.0   \n",
      "307486                       0.0      0.661025         270000.0   \n",
      "307487                       0.0      0.307082         936000.0   \n",
      "307488                       1.0      0.619662         135000.0   \n",
      "307489                       1.0      0.615261         450000.0   \n",
      "307490                       0.0      0.378421         135000.0   \n",
      "307491                       4.0      0.561158         900000.0   \n",
      "307492                       0.0      0.599173        1575000.0   \n",
      "307493                       0.0      0.424683         765000.0   \n",
      "307494                      11.0      0.699727         450000.0   \n",
      "307495                       0.0      0.267834         675000.0   \n",
      "307496                       0.0      0.512081         337500.0   \n",
      "307497                       0.0      0.651306         193500.0   \n",
      "307498                       0.0      0.746259        1125000.0   \n",
      "307499                       4.0      0.649123         225000.0   \n",
      "307500                       0.0      0.789389        1138500.0   \n",
      "307501                       2.0      0.575481         225000.0   \n",
      "307502                       0.0      0.313334         225000.0   \n",
      "307503                       9.0      0.501221         247500.0   \n",
      "307504                       0.0      0.530725         225000.0   \n",
      "307505                       0.0      0.346391         225000.0   \n",
      "307506                       0.0      0.681632         225000.0   \n",
      "307507                       0.0      0.115992         225000.0   \n",
      "307508                       6.0      0.535722         585000.0   \n",
      "307509                       0.0      0.514163         319500.0   \n",
      "307510                       0.0      0.708569         675000.0   \n",
      "\n",
      "        CODE_GENDER_NAME_EDUCATION_TYPE_max_AMT_ANNUITY  \\\n",
      "0                                              173704.5   \n",
      "1                                              230161.5   \n",
      "2                                              173704.5   \n",
      "3                                              216589.5   \n",
      "4                                              173704.5   \n",
      "5                                              173704.5   \n",
      "6                                              230161.5   \n",
      "7                                              258025.5   \n",
      "8                                              216589.5   \n",
      "9                                              173704.5   \n",
      "10                                             230161.5   \n",
      "11                                             216589.5   \n",
      "12                                             216589.5   \n",
      "13                                             173704.5   \n",
      "14                                             216589.5   \n",
      "15                                             173704.5   \n",
      "16                                             173704.5   \n",
      "17                                             216589.5   \n",
      "18                                             216589.5   \n",
      "19                                             230161.5   \n",
      "20                                             173704.5   \n",
      "21                                             216589.5   \n",
      "22                                             216589.5   \n",
      "23                                             216589.5   \n",
      "24                                             173704.5   \n",
      "25                                             216589.5   \n",
      "26                                             216589.5   \n",
      "27                                             173704.5   \n",
      "28                                             258025.5   \n",
      "29                                             258025.5   \n",
      "...                                                 ...   \n",
      "307481                                         173704.5   \n",
      "307482                                         216589.5   \n",
      "307483                                         216589.5   \n",
      "307484                                         216589.5   \n",
      "307485                                         216589.5   \n",
      "307486                                         230161.5   \n",
      "307487                                         173704.5   \n",
      "307488                                         230161.5   \n",
      "307489                                         216589.5   \n",
      "307490                                         258025.5   \n",
      "307491                                         173704.5   \n",
      "307492                                         173704.5   \n",
      "307493                                         216589.5   \n",
      "307494                                         173704.5   \n",
      "307495                                         258025.5   \n",
      "307496                                         230161.5   \n",
      "307497                                         216589.5   \n",
      "307498                                         173704.5   \n",
      "307499                                         216589.5   \n",
      "307500                                         230161.5   \n",
      "307501                                         216589.5   \n",
      "307502                                         216589.5   \n",
      "307503                                         230161.5   \n",
      "307504                                         216589.5   \n",
      "307505                                         216589.5   \n",
      "307506                                         173704.5   \n",
      "307507                                         216589.5   \n",
      "307508                                         230161.5   \n",
      "307509                                         216589.5   \n",
      "307510                                         230161.5   \n",
      "\n",
      "        CODE_GENDER_ORGANIZATION_TYPE_mean_AMT_INCOME_TOTAL  \\\n",
      "0                                           208566.954548     \n",
      "1                                           153236.991435     \n",
      "2                                           180212.879701     \n",
      "3                                           175499.426888     \n",
      "4                                           179284.090909     \n",
      "5                                           193671.537279     \n",
      "6                                           175499.426888     \n",
      "7                                           193671.537279     \n",
      "8                                           133968.829812     \n",
      "9                                           187450.135714     \n",
      "10                                          153800.183702     \n",
      "11                                          133968.829812     \n",
      "12                                          158044.886446     \n",
      "13                                          191266.266794     \n",
      "14                                          162098.580357     \n",
      "15                                          186480.962105     \n",
      "16                                          180212.879701     \n",
      "17                                          186236.624073     \n",
      "18                                          154769.765015     \n",
      "19                                          136766.214801     \n",
      "20                                          191266.266794     \n",
      "21                                          157412.599851     \n",
      "22                                          155619.546251     \n",
      "23                                          133968.829812     \n",
      "24                                          208566.954548     \n",
      "25                                          175499.426888     \n",
      "26                                          175499.426888     \n",
      "27                                          178311.022388     \n",
      "28                                          249654.328249     \n",
      "29                                          208566.954548     \n",
      "...                                                   ...     \n",
      "307481                                      208566.954548     \n",
      "307482                                      163790.094340     \n",
      "307483                                      133968.829812     \n",
      "307484                                      175499.426888     \n",
      "307485                                      175499.426888     \n",
      "307486                                      158044.886446     \n",
      "307487                                      147269.919668     \n",
      "307488                                      153252.513841     \n",
      "307489                                      175499.426888     \n",
      "307490                                      199545.773196     \n",
      "307491                                      191266.266794     \n",
      "307492                                      191266.266794     \n",
      "307493                                      153236.991435     \n",
      "307494                                      191266.266794     \n",
      "307495                                      208566.954548     \n",
      "307496                                      155619.546251     \n",
      "307497                                      157412.599851     \n",
      "307498                                      206252.611111     \n",
      "307499                                      153800.183702     \n",
      "307500                                      175499.426888     \n",
      "307501                                      153963.327068     \n",
      "307502                                      153236.991435     \n",
      "307503                                      184169.782577     \n",
      "307504                                      155619.546251     \n",
      "307505                                      133968.829812     \n",
      "307506                                      213583.146067     \n",
      "307507                                      133968.829812     \n",
      "307508                                      153236.991435     \n",
      "307509                                      175089.798898     \n",
      "307510                                      175499.426888     \n",
      "\n",
      "        CODE_GENDER_ORGANIZATION_TYPE_mean_DAYS_REGISTRATION  \\\n",
      "0                                            -4442.857420      \n",
      "1                                            -4965.608351      \n",
      "2                                            -4677.001719      \n",
      "3                                            -4709.777998      \n",
      "4                                            -4276.931818      \n",
      "5                                            -4587.493886      \n",
      "6                                            -4709.777998      \n",
      "7                                            -4587.493886      \n",
      "8                                            -6692.617415      \n",
      "9                                            -4651.734694      \n",
      "10                                           -4977.160798      \n",
      "11                                           -6692.617415      \n",
      "12                                           -5036.512502      \n",
      "13                                           -4383.874254      \n",
      "14                                           -4791.561508      \n",
      "15                                           -4658.636306      \n",
      "16                                           -4677.001719      \n",
      "17                                           -4667.722195      \n",
      "18                                           -5214.004334      \n",
      "19                                           -4710.026227      \n",
      "20                                           -4383.874254      \n",
      "21                                           -4651.860327      \n",
      "22                                           -4557.933663      \n",
      "23                                           -6692.617415      \n",
      "24                                           -4442.857420      \n",
      "25                                           -4709.777998      \n",
      "26                                           -4709.777998      \n",
      "27                                           -4860.949956      \n",
      "28                                           -3170.101297      \n",
      "29                                           -4442.857420      \n",
      "...                                                   ...      \n",
      "307481                                       -4442.857420      \n",
      "307482                                       -4781.776730      \n",
      "307483                                       -6692.617415      \n",
      "307484                                       -4709.777998      \n",
      "307485                                       -4709.777998      \n",
      "307486                                       -5036.512502      \n",
      "307487                                       -6091.431951      \n",
      "307488                                       -4939.548232      \n",
      "307489                                       -4709.777998      \n",
      "307490                                       -4706.889543      \n",
      "307491                                       -4383.874254      \n",
      "307492                                       -4383.874254      \n",
      "307493                                       -4965.608351      \n",
      "307494                                       -4383.874254      \n",
      "307495                                       -4442.857420      \n",
      "307496                                       -4557.933663      \n",
      "307497                                       -4651.860327      \n",
      "307498                                       -4330.576415      \n",
      "307499                                       -4977.160798      \n",
      "307500                                       -4709.777998      \n",
      "307501                                       -5054.867168      \n",
      "307502                                       -4965.608351      \n",
      "307503                                       -4041.638848      \n",
      "307504                                       -4557.933663      \n",
      "307505                                       -6692.617415      \n",
      "307506                                       -4288.348315      \n",
      "307507                                       -6692.617415      \n",
      "307508                                       -4965.608351      \n",
      "307509                                       -4905.762167      \n",
      "307510                                       -4709.777998      \n",
      "\n",
      "        CODE_GENDER_REG_CITY_NOT_WORK_CITY_mean_CNT_CHILDREN  CNT_FAM_MEMBERS  \n",
      "0                                                0.442195                 1.0  \n",
      "1                                                0.365735                 2.0  \n",
      "2                                                0.442195                 1.0  \n",
      "3                                                0.365735                 2.0  \n",
      "4                                                0.514099                 1.0  \n",
      "5                                                0.442195                 2.0  \n",
      "6                                                0.365735                 3.0  \n",
      "7                                                0.514099                 2.0  \n",
      "8                                                0.365735                 2.0  \n",
      "9                                                0.442195                 1.0  \n",
      "10                                               0.365735                 3.0  \n",
      "11                                               0.365735                 2.0  \n",
      "12                                               0.365735                 2.0  \n",
      "13                                               0.442195                 3.0  \n",
      "14                                               0.365735                 2.0  \n",
      "15                                               0.514099                 1.0  \n",
      "16                                               0.514099                 2.0  \n",
      "17                                               0.507015                 3.0  \n",
      "18                                               0.365735                 1.0  \n",
      "19                                               0.365735                 2.0  \n",
      "20                                               0.442195                 2.0  \n",
      "21                                               0.365735                 3.0  \n",
      "22                                               0.365735                 3.0  \n",
      "23                                               0.365735                 2.0  \n",
      "24                                               0.514099                 4.0  \n",
      "25                                               0.365735                 2.0  \n",
      "26                                               0.365735                 1.0  \n",
      "27                                               0.514099                 3.0  \n",
      "28                                               0.442195                 1.0  \n",
      "29                                               0.442195                 1.0  \n",
      "...                                                   ...                 ...  \n",
      "307481                                           0.442195                 2.0  \n",
      "307482                                           0.365735                 2.0  \n",
      "307483                                           0.365735                 1.0  \n",
      "307484                                           0.365735                 2.0  \n",
      "307485                                           0.507015                 2.0  \n",
      "307486                                           0.365735                 3.0  \n",
      "307487                                           0.442195                 2.0  \n",
      "307488                                           0.507015                 1.0  \n",
      "307489                                           0.365735                 2.0  \n",
      "307490                                           0.442195                 1.0  \n",
      "307491                                           0.442195                 4.0  \n",
      "307492                                           0.442195                 2.0  \n",
      "307493                                           0.507015                 2.0  \n",
      "307494                                           0.442195                 3.0  \n",
      "307495                                           0.442195                 2.0  \n",
      "307496                                           0.365735                 2.0  \n",
      "307497                                           0.507015                 2.0  \n",
      "307498                                           0.442195                 2.0  \n",
      "307499                                           0.365735                 2.0  \n",
      "307500                                           0.365735                 2.0  \n",
      "307501                                           0.365735                 5.0  \n",
      "307502                                           0.507015                 3.0  \n",
      "307503                                           0.365735                 1.0  \n",
      "307504                                           0.365735                 1.0  \n",
      "307505                                           0.365735                 1.0  \n",
      "307506                                           0.442195                 1.0  \n",
      "307507                                           0.365735                 1.0  \n",
      "307508                                           0.507015                 1.0  \n",
      "307509                                           0.507015                 2.0  \n",
      "307510                                           0.507015                 2.0  \n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[307511 rows x 66 columns]\n"
     ]
    }
   ],
   "source": [
    "# Shashank imputer\n",
    "from sklearn.base import TransformerMixin\n",
    "\n",
    "class DataFrameImputer(TransformerMixin):\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Impute missing values.\n",
    "\n",
    "        Columns of dtype object are imputed with the most frequent value \n",
    "        in column.\n",
    "\n",
    "        Columns of other types are imputed with mean of column.\n",
    "\n",
    "        \"\"\"\n",
    "    def fit(self, X, y=None):\n",
    "\n",
    "        self.fill = pd.Series([X[c].value_counts().index[0]\n",
    "            if X[c].dtype == np.dtype('O') else X[c].mean() for c in X],\n",
    "            index=X.columns)\n",
    "\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        return X.fillna(self.fill)\n",
    "    \n",
    "print('before...')    \n",
    "Mis=X[mis_col][:]\n",
    "print('Mis - ',Mis)\n",
    "\n",
    "xt = DataFrameImputer().fit_transform(Mis)\n",
    "\n",
    "print('after...')\n",
    "print(xt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(307511, 308)\n",
      "          SK_ID_CURR         TARGET  NAME_CONTRACT_TYPE   FLAG_OWN_CAR  \\\n",
      "count  307511.000000  307511.000000       307511.000000  307511.000000   \n",
      "mean   278180.518577       0.080729            0.095213       0.340108   \n",
      "std    102790.175348       0.272419            0.293509       0.473746   \n",
      "min    100002.000000       0.000000            0.000000       0.000000   \n",
      "25%    189145.500000       0.000000            0.000000       0.000000   \n",
      "50%    278202.000000       0.000000            0.000000       0.000000   \n",
      "75%    367142.500000       0.000000            0.000000       1.000000   \n",
      "max    456255.000000       1.000000            1.000000       1.000000   \n",
      "\n",
      "       FLAG_OWN_REALTY  AMT_GOODS_PRICE  REGION_POPULATION_RELATIVE  \\\n",
      "count    307511.000000     3.075110e+05               307511.000000   \n",
      "mean          0.693673     5.383962e+05                    0.020868   \n",
      "std           0.460968     3.692794e+05                    0.013831   \n",
      "min           0.000000     4.050000e+04                    0.000290   \n",
      "25%           0.000000     2.385000e+05                    0.010006   \n",
      "50%           1.000000     4.500000e+05                    0.018850   \n",
      "75%           1.000000     6.795000e+05                    0.028663   \n",
      "max           1.000000     4.050000e+06                    0.072508   \n",
      "\n",
      "          DAYS_BIRTH  DAYS_EMPLOYED  DAYS_ID_PUBLISH          ...           \\\n",
      "count  307511.000000  307511.000000    307511.000000          ...            \n",
      "mean   -16036.995067   -2384.169325     -2994.202373          ...            \n",
      "std      4363.988632    2117.381973      1509.450419          ...            \n",
      "min    -25229.000000  -17912.000000     -7197.000000          ...            \n",
      "25%    -19682.000000   -2760.000000     -4299.000000          ...            \n",
      "50%    -15750.000000   -2219.000000     -3254.000000          ...            \n",
      "75%    -12413.000000    -933.000000     -1720.000000          ...            \n",
      "max     -7489.000000       0.000000         0.000000          ...            \n",
      "\n",
      "       OBS_60_CNT_SOCIAL_CIRCLE_miss  DEF_30_CNT_SOCIAL_CIRCLE_miss  \\\n",
      "count                  307511.000000                  307511.000000   \n",
      "mean                        0.003320                       0.003320   \n",
      "std                         0.057526                       0.057526   \n",
      "min                         0.000000                       0.000000   \n",
      "25%                         0.000000                       0.000000   \n",
      "50%                         0.000000                       0.000000   \n",
      "75%                         0.000000                       0.000000   \n",
      "max                         1.000000                       1.000000   \n",
      "\n",
      "       OBS_30_CNT_SOCIAL_CIRCLE_miss  EXT_SOURCE_2_miss  AMT_GOODS_PRICE_miss  \\\n",
      "count                  307511.000000      307511.000000         307511.000000   \n",
      "mean                        0.003320           0.002146              0.000904   \n",
      "std                         0.057526           0.046278              0.030054   \n",
      "min                         0.000000           0.000000              0.000000   \n",
      "25%                         0.000000           0.000000              0.000000   \n",
      "50%                         0.000000           0.000000              0.000000   \n",
      "75%                         0.000000           0.000000              0.000000   \n",
      "max                         1.000000           1.000000              1.000000   \n",
      "\n",
      "       CODE_GENDER_NAME_EDUCATION_TYPE_max_AMT_ANNUITY_miss  \\\n",
      "count                                      307511.000000      \n",
      "mean                                            0.000013      \n",
      "std                                             0.003607      \n",
      "min                                             0.000000      \n",
      "25%                                             0.000000      \n",
      "50%                                             0.000000      \n",
      "75%                                             0.000000      \n",
      "max                                             1.000000      \n",
      "\n",
      "       CODE_GENDER_ORGANIZATION_TYPE_mean_AMT_INCOME_TOTAL_miss  \\\n",
      "count                                      307511.000000          \n",
      "mean                                            0.000013          \n",
      "std                                             0.003607          \n",
      "min                                             0.000000          \n",
      "25%                                             0.000000          \n",
      "50%                                             0.000000          \n",
      "75%                                             0.000000          \n",
      "max                                             1.000000          \n",
      "\n",
      "       CODE_GENDER_ORGANIZATION_TYPE_mean_DAYS_REGISTRATION_miss  \\\n",
      "count                                      307511.000000           \n",
      "mean                                            0.000013           \n",
      "std                                             0.003607           \n",
      "min                                             0.000000           \n",
      "25%                                             0.000000           \n",
      "50%                                             0.000000           \n",
      "75%                                             0.000000           \n",
      "max                                             1.000000           \n",
      "\n",
      "       CODE_GENDER_REG_CITY_NOT_WORK_CITY_mean_CNT_CHILDREN_miss  \\\n",
      "count                                      307511.000000           \n",
      "mean                                            0.000013           \n",
      "std                                             0.003607           \n",
      "min                                             0.000000           \n",
      "25%                                             0.000000           \n",
      "50%                                             0.000000           \n",
      "75%                                             0.000000           \n",
      "max                                             1.000000           \n",
      "\n",
      "       CNT_FAM_MEMBERS_miss  \n",
      "count         307511.000000  \n",
      "mean               0.000007  \n",
      "std                0.002550  \n",
      "min                0.000000  \n",
      "25%                0.000000  \n",
      "50%                0.000000  \n",
      "75%                0.000000  \n",
      "max                1.000000  \n",
      "\n",
      "[8 rows x 308 columns]\n"
     ]
    }
   ],
   "source": [
    "# # #Have commented the dropping (not dropping any columns for now) Replace missing with -999\n",
    "# # Mis=X[mis_col][:]\n",
    "# # X_mis=(Mis.isnull().sum() / Mis.shape[0] * 100.00)\n",
    "# # df_mis=pd.DataFrame(np.c_[X_mis.index,X_mis],columns=['Colname','miss_p'])\n",
    "# # drop_col=df_mis[df_mis['miss_p']>65]['Colname'].values\n",
    "# # print('number of columns before dropping->',len(X.columns))\n",
    "# # #X=X.drop(drop_col,axis=1)\n",
    "# # print('number of columns after dropping->',len(X.columns))\n",
    "# X.replace(np.nan,-999, inplace=True)\n",
    "# #X.astype('float32',inplace=True)\n",
    "# #print(newX[].describe())\n",
    "# y=X['TARGET']\n",
    "# X=X.drop('TARGET',axis=1)\n",
    "for mis in mis_col:\n",
    "    X[mis]=xt[mis]\n",
    "print(X.shape)\n",
    "print(X.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate below functions only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=X['TARGET']\n",
    "X=X.drop('TARGET',axis=1)\n",
    "\n",
    "def split_train_test(X , y):\n",
    "    return(train_test_split( X , y , test_size=0.2, random_state=42))\n",
    "\n",
    "X_train, X_test, y_train, y_test =split_train_test(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improved number of features-> 94\n"
     ]
    }
   ],
   "source": [
    "def feature_eng(X_train,y_train,X_test,y_test):\n",
    "    train_col=list(X_train.columns)\n",
    "    test_col=list(X_test.columns)    \n",
    "    clf = ExtraTreesClassifier(n_estimators=50)\n",
    "    clf = clf.fit(X_train, y_train)\n",
    "    model = SelectFromModel(clf, prefit=True)\n",
    "    new_trainX = model.transform(X_train)\n",
    "    n=np.shape(new_trainX)[1]\n",
    "    importances = clf.feature_importances_\n",
    "    indices = np.argsort(importances)[::-1]\n",
    "    clist=[]\n",
    "    for f in range(n):     \n",
    "        clist.append(indices[f])\n",
    "    new_train_col=[train_col[i] for i in clist]\n",
    "    new_test_col=[test_col[i] for i in clist]\n",
    "    X_train=pd.DataFrame(X_train)\n",
    "    X_test=pd.DataFrame(X_test)\n",
    "    y_train=pd.DataFrame(y_train)\n",
    "    y_test=pd.DataFrame(y_test)\n",
    "    df_new_train=pd.DataFrame(X_train.loc[:,new_train_col],columns=new_train_col)            \n",
    "    df_new_test=pd.DataFrame(X_test.loc[:,new_test_col],columns=new_test_col)\n",
    "    print(\"Improved number of features->\",len(clist))    \n",
    "    return(df_new_train,df_new_test)\n",
    "\n",
    "df_new_train,df_new_test=feature_eng(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model_eval_metric(modelname,modelobj,X_train,Y_train,X_test,Y_test):\n",
    "    # In order to let the models run faster we use dask dataframe\n",
    "    \n",
    "    modelobj.fit(X_train,Y_train)\n",
    "    ypred_train=modelobj.predict(X_train)\n",
    "    ypred_test=modelobj.predict(X_test)\n",
    "    y_train_proba = modelobj.predict_proba(X_train)[:,1]\n",
    "    y_test_proba = modelobj.predict_proba(X_test)[:,1]\n",
    "    print('\\n')\n",
    "    print(modelname,\"model:\")\n",
    "    confmat=confusion_matrix(Y_train,ypred_train)\n",
    "    print(\"Train Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    confmat=confusion_matrix(Y_test,ypred_test)\n",
    "    print(\"Test Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    metric=pd.DataFrame([[0]*5]*2,columns=['PRECISION','RECALL','F1_SCORE','ROC_AUC_SCORE','ACCURACY'],index=['Train','Test'],dtype='float32')\n",
    "    metric['PRECISION']['Train']=round(precision_score(Y_train, ypred_train),3)\n",
    "    metric['RECALL']['Train']=recall_score(Y_train, ypred_train)\n",
    "    metric['F1_SCORE']['Train']=round(f1_score(Y_train, ypred_train),3)\n",
    "    metric['ROC_AUC_SCORE']['Train']=roc_auc_score(Y_train,y_train_proba)\n",
    "    metric['ACCURACY']['Train']=accuracy_score(Y_train,ypred_train)\n",
    "    metric['PRECISION']['Test']=precision_score(Y_test, ypred_test)\n",
    "    metric['RECALL']['Test']=recall_score(Y_test, ypred_test)\n",
    "    metric['F1_SCORE']['Test']=f1_score(Y_test, ypred_test)\n",
    "    metric['ROC_AUC_SCORE']['Test']=roc_auc_score(Y_test,y_test_proba)\n",
    "    metric['ACCURACY']['Test']=accuracy_score(Y_test,ypred_test)\n",
    "    print(metric)\n",
    "   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "K-Nearest Neighbor 1 model:\n",
      "Train Confusion Matrix:\n",
      "[[226132      0]\n",
      " [     0  19876]]\n",
      "Test Confusion Matrix:\n",
      "[[52009  4545]\n",
      " [ 4481   468]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   1.000000  1.000000  1.000000       1.000000  1.000000\n",
      "Test    0.093357  0.094565  0.093957       0.507099  0.853243\n",
      "\n",
      "\n",
      "K-Nearest Neighbor 3 model:\n",
      "Train Confusion Matrix:\n",
      "[[224327   1805]\n",
      " [ 16038   3838]]\n",
      "Test Confusion Matrix:\n",
      "[[55294  1260]\n",
      " [ 4797   152]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.680000  0.193097  0.301000        0.93468  0.927470\n",
      "Test    0.107649  0.030713  0.047791        0.51888  0.901517\n",
      "\n",
      "\n",
      "Random Forest model:\n",
      "Train Confusion Matrix:\n",
      "[[225949    183]\n",
      " [  3699  16177]]\n",
      "Test Confusion Matrix:\n",
      "[[55692   862]\n",
      " [ 4707   242]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.989000  0.813896   0.89300       0.993799  0.984220\n",
      "Test    0.219203  0.048899   0.07996       0.601270  0.909452\n",
      "\n",
      "\n",
      "Decision Tree model:\n",
      "Train Confusion Matrix:\n",
      "[[226113     19]\n",
      " [   472  19404]]\n",
      "Test Confusion Matrix:\n",
      "[[51859  4695]\n",
      " [ 4210   739]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.999000  0.976253  0.988000       0.999879  0.998004\n",
      "Test    0.135996  0.149323  0.142348       0.533789  0.855210\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Logistic Regression model:\n",
      "Train Confusion Matrix:\n",
      "[[172839  53293]\n",
      " [  8273  11603]]\n",
      "Test Confusion Matrix:\n",
      "[[43161 13393]\n",
      " [ 2044  2905]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.179000  0.583769   0.27400       0.743543  0.749740\n",
      "Test    0.178243  0.586987   0.27345       0.744460  0.749004\n"
     ]
    }
   ],
   "source": [
    "def run_all_models(df_new_train,y_train,df_new_test,y_test):\n",
    "    # K Nearest Neighbor\n",
    "    knearest = KNeighborsClassifier(algorithm= 'auto', n_jobs= -1, n_neighbors= 1)\n",
    "    run_model_eval_metric('K-Nearest Neighbor 1',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "    \n",
    "    knearest = KNeighborsClassifier(algorithm= 'auto', n_jobs= -1, n_neighbors= 3)\n",
    "    run_model_eval_metric('K-Nearest Neighbor 3',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Random Forest\n",
    "    randfor = RandomForestClassifier(n_estimators= 5, max_features= 'auto', max_depth= 60, bootstrap= True, n_jobs= -1)\n",
    "    run_model_eval_metric('Random Forest',randfor,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Decision Trees\n",
    "    dectree = DecisionTreeClassifier(random_state= 42, max_features= 'auto', max_depth= 40, criterion= 'gini')\n",
    "    run_model_eval_metric('Decision Tree',dectree,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Logistic Regression\n",
    "    logreg = LogisticRegression(penalty = 'l1', class_weight= {0: 0.1, 1: 0.9}, C= 0.5)\n",
    "    run_model_eval_metric('Logistic Regression',logreg,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "run_all_models(df_new_train,y_train,df_new_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}