preprocessed_data_exploration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fb21b772",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "47fd82f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>requested_amnt</th>\n",
       "      <th>annual_income</th>\n",
       "      <th>debt_to_income_ratio</th>\n",
       "      <th>public_bankruptcies</th>\n",
       "      <th>state</th>\n",
       "      <th>delinquency_last_2yrs</th>\n",
       "      <th>fico_inquired_last_6mths</th>\n",
       "      <th>months_since_last_delinq</th>\n",
       "      <th>revolving_balance</th>\n",
       "      <th>total_revolving_limit</th>\n",
       "      <th>...</th>\n",
       "      <th>home_ownership_status_MORTGAGE</th>\n",
       "      <th>home_ownership_status_NONE</th>\n",
       "      <th>home_ownership_status_OTHER</th>\n",
       "      <th>home_ownership_status_OWN</th>\n",
       "      <th>home_ownership_status_RENT</th>\n",
       "      <th>zipcode_bad</th>\n",
       "      <th>zipcode_good</th>\n",
       "      <th>zipcode_mid</th>\n",
       "      <th>zipcode_mid_bad</th>\n",
       "      <th>zipcode_mid_good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.015939</td>\n",
       "      <td>2.459194</td>\n",
       "      <td>-0.172762</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>0.276568</td>\n",
       "      <td>1.993483</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.966066</td>\n",
       "      <td>-0.088885</td>\n",
       "      <td>-0.274116</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.170826</td>\n",
       "      <td>-0.441404</td>\n",
       "      <td>-0.482499</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.172512</td>\n",
       "      <td>-0.556033</td>\n",
       "      <td>0.243417</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.265686</td>\n",
       "      <td>-0.522719</td>\n",
       "      <td>-0.599367</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.538836</td>\n",
       "      <td>-0.442785</td>\n",
       "      <td>1.747582</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>0.242150</td>\n",
       "      <td>-0.194404</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.098399</td>\n",
       "      <td>0.137611</td>\n",
       "      <td>1.239915</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.479890</td>\n",
       "      <td>0.628560</td>\n",
       "      <td>1.952715</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>-0.736681</td>\n",
       "      <td>-0.074729</td>\n",
       "      <td>0.686504</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.919941</td>\n",
       "      <td>-0.059481</td>\n",
       "      <td>-0.042203</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999996</th>\n",
       "      <td>-1.195451</td>\n",
       "      <td>-0.541877</td>\n",
       "      <td>-0.443637</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.537514</td>\n",
       "      <td>-0.508019</td>\n",
       "      <td>-0.637418</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999997</th>\n",
       "      <td>-0.392603</td>\n",
       "      <td>-0.265835</td>\n",
       "      <td>-0.795237</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.240628</td>\n",
       "      <td>-0.525985</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999998</th>\n",
       "      <td>-0.002649</td>\n",
       "      <td>-0.400317</td>\n",
       "      <td>-0.159308</td>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.022532</td>\n",
       "      <td>-0.319426</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999999</th>\n",
       "      <td>-0.048526</td>\n",
       "      <td>-0.400317</td>\n",
       "      <td>1.665960</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.808787</td>\n",
       "      <td>0.240596</td>\n",
       "      <td>-0.058511</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        requested_amnt  annual_income  debt_to_income_ratio  \\\n",
       "0             2.015939       2.459194             -0.172762   \n",
       "1            -0.966066      -0.088885             -0.274116   \n",
       "2            -1.172512      -0.556033              0.243417   \n",
       "3            -0.538836      -0.442785              1.747582   \n",
       "4             1.098399       0.137611              1.239915   \n",
       "...                ...            ...                   ...   \n",
       "999995       -0.736681      -0.074729              0.686504   \n",
       "999996       -1.195451      -0.541877             -0.443637   \n",
       "999997       -0.392603      -0.265835             -0.795237   \n",
       "999998       -0.002649      -0.400317             -0.159308   \n",
       "999999       -0.048526      -0.400317              1.665960   \n",
       "\n",
       "        public_bankruptcies  state  delinquency_last_2yrs  \\\n",
       "0                         0     15                      0   \n",
       "1                         0      3                      0   \n",
       "2                         0      7                      1   \n",
       "3                         0     22                      0   \n",
       "4                         0     27                      0   \n",
       "...                     ...    ...                    ...   \n",
       "999995                    1     29                      1   \n",
       "999996                    0     15                      0   \n",
       "999997                    0     11                      0   \n",
       "999998                    0     43                      0   \n",
       "999999                    0     25                      0   \n",
       "\n",
       "        fico_inquired_last_6mths  months_since_last_delinq  revolving_balance  \\\n",
       "0                              0                 -0.113201           0.276568   \n",
       "1                              1                 -0.170826          -0.441404   \n",
       "2                              0                 -1.265686          -0.522719   \n",
       "3                              0                 -0.113201           0.242150   \n",
       "4                              0                  2.479890           0.628560   \n",
       "...                          ...                       ...                ...   \n",
       "999995                         2                 -0.919941          -0.059481   \n",
       "999996                         0                  2.537514          -0.508019   \n",
       "999997                         1                 -0.113201          -0.240628   \n",
       "999998                         1                 -0.113201          -0.022532   \n",
       "999999                         0                  0.808787           0.240596   \n",
       "\n",
       "        total_revolving_limit  ...  home_ownership_status_MORTGAGE  \\\n",
       "0                    1.993483  ...                               0   \n",
       "1                   -0.482499  ...                               1   \n",
       "2                   -0.599367  ...                               0   \n",
       "3                   -0.194404  ...                               0   \n",
       "4                    1.952715  ...                               1   \n",
       "...                       ...  ...                             ...   \n",
       "999995              -0.042203  ...                               1   \n",
       "999996              -0.637418  ...                               0   \n",
       "999997              -0.525985  ...                               1   \n",
       "999998              -0.319426  ...                               0   \n",
       "999999              -0.058511  ...                               1   \n",
       "\n",
       "        home_ownership_status_NONE  home_ownership_status_OTHER  \\\n",
       "0                                0                            0   \n",
       "1                                0                            0   \n",
       "2                                0                            0   \n",
       "3                                0                            0   \n",
       "4                                0                            0   \n",
       "...                            ...                          ...   \n",
       "999995                           0                            0   \n",
       "999996                           0                            0   \n",
       "999997                           0                            0   \n",
       "999998                           0                            0   \n",
       "999999                           0                            0   \n",
       "\n",
       "        home_ownership_status_OWN  home_ownership_status_RENT  zipcode_bad  \\\n",
       "0                               0                           1            0   \n",
       "1                               0                           0            0   \n",
       "2                               1                           0            0   \n",
       "3                               1                           0            0   \n",
       "4                               0                           0            0   \n",
       "...                           ...                         ...          ...   \n",
       "999995                          0                           0            0   \n",
       "999996                          0                           1            0   \n",
       "999997                          0                           0            0   \n",
       "999998                          0                           1            0   \n",
       "999999                          0                           0            0   \n",
       "\n",
       "        zipcode_good  zipcode_mid  zipcode_mid_bad  zipcode_mid_good  \n",
       "0                  0            0                1                 0  \n",
       "1                  0            1                0                 0  \n",
       "2                  0            0                0                 1  \n",
       "3                  0            0                0                 1  \n",
       "4                  0            1                0                 0  \n",
       "...              ...          ...              ...               ...  \n",
       "999995             0            1                0                 0  \n",
       "999996             0            0                1                 0  \n",
       "999997             0            0                1                 0  \n",
       "999998             1            0                0                 0  \n",
       "999999             0            1                0                 0  \n",
       "\n",
       "[1000000 rows x 42 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reading in the training dataset\n",
    "train_dataset = pd.read_csv(\"data/train_preprocessed_data.csv\")\n",
    "train_dataset_loan_paid = train_dataset['loan_paid']\n",
    "train_dataset = train_dataset.drop(columns=['loan_paid'])\n",
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "46b1214e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['ID', 'requested_amnt', 'annual_income', 'debt_to_income_ratio',\n",
      "       'public_bankruptcies', 'state', 'delinquency_last_2yrs',\n",
      "       'fico_inquired_last_6mths', 'months_since_last_delinq',\n",
      "       'revolving_balance', 'total_revolving_limit', 'any_tax_liens',\n",
      "       'missing_vals', 'income_to_loan_amount_ratio', 'fico_score',\n",
      "       'fico_score_range', 'employment_length', 'employment_verified',\n",
      "       'type_of_application', 'loan_duration', 'reason_for_loan_credit_card',\n",
      "       'reason_for_loan_debt_consolidation',\n",
      "       'reason_for_loan_home_improvement', 'reason_for_loan_major_purchase',\n",
      "       'reason_for_loan_other', 'reason_for_loan_small_business',\n",
      "       'employment_Manager', 'employment_Owner', 'employment_RN',\n",
      "       'employment_Registered Nurse', 'employment_Teacher', 'employment_other',\n",
      "       'home_ownership_status_ANY', 'home_ownership_status_MORTGAGE',\n",
      "       'home_ownership_status_NONE', 'home_ownership_status_OTHER',\n",
      "       'home_ownership_status_OWN', 'home_ownership_status_RENT',\n",
      "       'zipcode_bad', 'zipcode_good', 'zipcode_mid', 'zipcode_mid_bad',\n",
      "       'zipcode_mid_good'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>requested_amnt</th>\n",
       "      <th>annual_income</th>\n",
       "      <th>debt_to_income_ratio</th>\n",
       "      <th>public_bankruptcies</th>\n",
       "      <th>state</th>\n",
       "      <th>delinquency_last_2yrs</th>\n",
       "      <th>fico_inquired_last_6mths</th>\n",
       "      <th>months_since_last_delinq</th>\n",
       "      <th>revolving_balance</th>\n",
       "      <th>total_revolving_limit</th>\n",
       "      <th>...</th>\n",
       "      <th>home_ownership_status_MORTGAGE</th>\n",
       "      <th>home_ownership_status_NONE</th>\n",
       "      <th>home_ownership_status_OTHER</th>\n",
       "      <th>home_ownership_status_OWN</th>\n",
       "      <th>home_ownership_status_RENT</th>\n",
       "      <th>zipcode_bad</th>\n",
       "      <th>zipcode_good</th>\n",
       "      <th>zipcode_mid</th>\n",
       "      <th>zipcode_mid_bad</th>\n",
       "      <th>zipcode_mid_good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.553173</td>\n",
       "      <td>-0.655125</td>\n",
       "      <td>0.065823</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>0.099462</td>\n",
       "      <td>-0.256915</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.736681</td>\n",
       "      <td>-0.591423</td>\n",
       "      <td>1.168160</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.494430</td>\n",
       "      <td>-0.172661</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.754321</td>\n",
       "      <td>-0.442785</td>\n",
       "      <td>-0.459782</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.474001</td>\n",
       "      <td>-0.642853</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.080758</td>\n",
       "      <td>-0.655125</td>\n",
       "      <td>-1.034719</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.549809</td>\n",
       "      <td>-0.278658</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.310143</td>\n",
       "      <td>-0.541877</td>\n",
       "      <td>-1.568397</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.710661</td>\n",
       "      <td>-0.640135</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345305</th>\n",
       "      <td>-1.310143</td>\n",
       "      <td>-0.740061</td>\n",
       "      <td>-1.123516</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.286074</td>\n",
       "      <td>-0.560600</td>\n",
       "      <td>-0.324862</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345306</th>\n",
       "      <td>-0.277911</td>\n",
       "      <td>-0.159665</td>\n",
       "      <td>0.092731</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.804692</td>\n",
       "      <td>-0.277222</td>\n",
       "      <td>-0.384655</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345307</th>\n",
       "      <td>1.798023</td>\n",
       "      <td>0.152163</td>\n",
       "      <td>0.935853</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.380935</td>\n",
       "      <td>-0.371548</td>\n",
       "      <td>-0.623828</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345308</th>\n",
       "      <td>1.442476</td>\n",
       "      <td>0.341457</td>\n",
       "      <td>0.264047</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>0.126863</td>\n",
       "      <td>-0.004153</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345309</th>\n",
       "      <td>-0.369665</td>\n",
       "      <td>-0.591423</td>\n",
       "      <td>-0.079481</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.113201</td>\n",
       "      <td>-0.026351</td>\n",
       "      <td>-0.205276</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>345310 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        requested_amnt  annual_income  debt_to_income_ratio  \\\n",
       "0            -0.553173      -0.655125              0.065823   \n",
       "1            -0.736681      -0.591423              1.168160   \n",
       "2             0.754321      -0.442785             -0.459782   \n",
       "3            -1.080758      -0.655125             -1.034719   \n",
       "4            -1.310143      -0.541877             -1.568397   \n",
       "...                ...            ...                   ...   \n",
       "345305       -1.310143      -0.740061             -1.123516   \n",
       "345306       -0.277911      -0.159665              0.092731   \n",
       "345307        1.798023       0.152163              0.935853   \n",
       "345308        1.442476       0.341457              0.264047   \n",
       "345309       -0.369665      -0.591423             -0.079481   \n",
       "\n",
       "        public_bankruptcies  state  delinquency_last_2yrs  \\\n",
       "0                         0     10                      0   \n",
       "1                         0     29                      0   \n",
       "2                         1     15                      0   \n",
       "3                         0     26                      0   \n",
       "4                         0     29                      0   \n",
       "...                     ...    ...                    ...   \n",
       "345305                    0     29                      0   \n",
       "345306                    0      7                      1   \n",
       "345307                    0     34                      2   \n",
       "345308                    0     29                      0   \n",
       "345309                    0     15                      0   \n",
       "\n",
       "        fico_inquired_last_6mths  months_since_last_delinq  revolving_balance  \\\n",
       "0                              0                 -0.113201           0.099462   \n",
       "1                              0                 -0.113201          -0.494430   \n",
       "2                              0                 -0.113201          -0.474001   \n",
       "3                              1                 -0.113201          -0.549809   \n",
       "4                              0                 -0.113201          -0.710661   \n",
       "...                          ...                       ...                ...   \n",
       "345305                         0                 -0.286074          -0.560600   \n",
       "345306                         2                 -0.804692          -0.277222   \n",
       "345307                         1                 -1.380935          -0.371548   \n",
       "345308                         0                 -0.113201           0.126863   \n",
       "345309                         0                 -0.113201          -0.026351   \n",
       "\n",
       "        total_revolving_limit  ...  home_ownership_status_MORTGAGE  \\\n",
       "0                   -0.256915  ...                               0   \n",
       "1                   -0.172661  ...                               0   \n",
       "2                   -0.642853  ...                               1   \n",
       "3                   -0.278658  ...                               0   \n",
       "4                   -0.640135  ...                               0   \n",
       "...                       ...  ...                             ...   \n",
       "345305              -0.324862  ...                               0   \n",
       "345306              -0.384655  ...                               0   \n",
       "345307              -0.623828  ...                               1   \n",
       "345308              -0.004153  ...                               0   \n",
       "345309              -0.205276  ...                               0   \n",
       "\n",
       "        home_ownership_status_NONE  home_ownership_status_OTHER  \\\n",
       "0                                0                            0   \n",
       "1                                0                            0   \n",
       "2                                0                            0   \n",
       "3                                0                            0   \n",
       "4                                0                            0   \n",
       "...                            ...                          ...   \n",
       "345305                           0                            0   \n",
       "345306                           0                            0   \n",
       "345307                           0                            0   \n",
       "345308                           0                            0   \n",
       "345309                           0                            0   \n",
       "\n",
       "        home_ownership_status_OWN  home_ownership_status_RENT  zipcode_bad  \\\n",
       "0                               0                           1            0   \n",
       "1                               0                           1            0   \n",
       "2                               0                           0            0   \n",
       "3                               0                           1            0   \n",
       "4                               1                           0            0   \n",
       "...                           ...                         ...          ...   \n",
       "345305                          1                           0            0   \n",
       "345306                          1                           0            0   \n",
       "345307                          0                           0            0   \n",
       "345308                          0                           1            0   \n",
       "345309                          0                           1            0   \n",
       "\n",
       "        zipcode_good  zipcode_mid  zipcode_mid_bad  zipcode_mid_good  \n",
       "0                  0            0                1                 0  \n",
       "1                  1            0                0                 0  \n",
       "2                  0            0                1                 0  \n",
       "3                  0            0                1                 0  \n",
       "4                  0            0                0                 1  \n",
       "...              ...          ...              ...               ...  \n",
       "345305             0            0                1                 0  \n",
       "345306             0            0                1                 0  \n",
       "345307             0            1                0                 0  \n",
       "345308             0            1                0                 0  \n",
       "345309             0            1                0                 0  \n",
       "\n",
       "[345310 rows x 42 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reading in the to_predict/val dataset\n",
    "val_dataset = pd.read_csv(\"data/predict_preprocessed_data.csv\")\n",
    "print(val_dataset.columns)\n",
    "val_dataset = val_dataset.drop(columns=['ID'])\n",
    "val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "81655979",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/apps/cent7/jupyterhub/lib/python3.9/site-packages/numpy/core/_methods.py:229: RuntimeWarning: invalid value encountered in subtract\n",
      "  x = asanyarray(arr - arrmean)\n",
      "/apps/cent7/jupyterhub/lib/python3.9/site-packages/scipy/stats/stats.py:5681: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  d = mean1 - mean2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Col: fico_score, t-test: statistic=200.8526, p-value=0.0000\n",
      "Col: zipcode_mid, t-test: statistic=-2.7524, p-value=0.0059\n",
      "Col: loan_duration, t-test: statistic=2.5828, p-value=0.0098\n",
      "Col: zipcode_mid_bad, t-test: statistic=2.1026, p-value=0.0355\n",
      "Col: employment_other, t-test: statistic=-1.8991, p-value=0.0575\n"
     ]
    }
   ],
   "source": [
    "# Doing t-tests to find attributes with different means with significance level of .1\n",
    "from scipy.stats import ttest_ind\n",
    "\n",
    "significance_level = .1\n",
    "mappings = []\n",
    "for col in train_dataset.columns:\n",
    "    stat, p_value = ttest_ind(train_dataset[col], val_dataset[col])\n",
    "    if (p_value < significance_level):\n",
    "        mappings.append((p_value, col, stat))\n",
    "\n",
    "mappings = sorted(mappings)\n",
    "for item in mappings:\n",
    "    print(f\"Col: {item[1]}, t-test: statistic={item[2]:.4f}, p-value={item[0]:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c355960",
   "metadata": {},
   "source": [
    "#### Attributes of interest\n",
    "Col: fico_score, t-test: statistic=200.8526, p-value=0.0000\n",
    "Col: reason_for_loan_other, t-test: statistic=-1.0533, p-value=0.2922\n",
    "Col: annual_income, t-test: statistic=0.8848, p-value=0.376\n",
    "\n",
    "#### Maybe attributes of interest\n",
    "Col: debt_to_income_ratio, t-test: statistic=-1.1229, p-value=0.2615\n",
    "Col: loan_duration, t-test: statistic=2.5828, p-value=0.0098"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bb1cc2f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train fico_score mean: -4.308556123592988e-09\n",
      "val fico_score mean: -0.39493747902189247\n"
     ]
    }
   ],
   "source": [
    "print('train fico_score mean:', train_dataset['fico_score'].mean())\n",
    "print('val fico_score mean:', val_dataset['fico_score'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a147b3fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train reason_for_loan_other mean: 0.101945\n",
      "val reason_for_loan_other mean: 0.1025744982769106\n"
     ]
    }
   ],
   "source": [
    "print('train reason_for_loan_other mean:', train_dataset['reason_for_loan_other'].mean())\n",
    "print('val reason_for_loan_other mean:', val_dataset['reason_for_loan_other'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b92034a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train annual_income mean: 2.5896502857332137e-09\n",
      "val annual_income mean: -0.0017286477643561635\n"
     ]
    }
   ],
   "source": [
    "print('train annual_income mean:', train_dataset['annual_income'].mean())\n",
    "print('val annual_income mean:', val_dataset['annual_income'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bb23ef58",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train debt_to_income_ratio mean: -7.939537009660214e-11\n",
      "val debt_to_income_ratio mean: 0.0022183783849351146\n"
     ]
    }
   ],
   "source": [
    "print('train debt_to_income_ratio mean:', train_dataset['debt_to_income_ratio'].mean())\n",
    "print('val debt_to_income_ratio mean:', val_dataset['debt_to_income_ratio'].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "79c81090",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train fico_score high mean: 693.973189\n",
      "val fico_score high mean: 680.7549216645913\n",
      "train fico_score low mean: 689.97305\n",
      "val fico_score low mean: 676.7547855550085\n",
      "train annual_income mean: 76278.9803522409\n",
      "val annual_income mean: 76156.86597987289\n",
      "Fico score range high p_value: 0.0\n",
      "Fico score range low p_value: 0.0\n",
      "Annual income p_value: 0.37628451698427645\n"
     ]
    }
   ],
   "source": [
    "# double-checking with not preprocessed data\n",
    "up_t_df = pd.read_csv('data/lending_train.csv')\n",
    "up_v_df = pd.read_csv('data/lending_topredict.csv')\n",
    "\n",
    "# print('Columns:\\n', up_t_df.columns)\n",
    "\n",
    "print('train fico_score high mean:', up_t_df['fico_score_range_high'].mean())\n",
    "print('val fico_score high mean:', up_v_df['fico_score_range_high'].mean())\n",
    "\n",
    "print('train fico_score low mean:', up_t_df['fico_score_range_low'].mean())\n",
    "print('val fico_score low mean:', up_v_df['fico_score_range_low'].mean())\n",
    "\n",
    "print('train annual_income mean:', up_t_df['annual_income'].mean())\n",
    "print('val annual_income mean:', up_v_df['annual_income'].mean())\n",
    "\n",
    "stat, p_value = ttest_ind(up_t_df['fico_score_range_high'], up_v_df['fico_score_range_high'])\n",
    "print('Fico score range high p_value:', p_value)\n",
    "\n",
    "stat, p_value = ttest_ind(up_t_df['fico_score_range_low'], up_v_df['fico_score_range_low'])\n",
    "print('Fico score range low p_value:', p_value)\n",
    "\n",
    "stat, p_value = ttest_ind(up_t_df['annual_income'], up_v_df['annual_income'])\n",
    "print('Annual income p_value:', p_value)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba4577f2",
   "metadata": {},
   "source": [
    "#### Focus on reweighting fico_score and annual_income"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7a36f7ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train debt_to_income_ratio mean: 18.27631823647009\n",
      "val debt_to_income_ratio mean: 18.30105236172407\n",
      "debt_to_income_ratio p_value: 0.2615823147626689\n"
     ]
    }
   ],
   "source": [
    "# Checking maybe attributes of interest with unprocessed data\n",
    "print('train debt_to_income_ratio mean:', up_t_df['debt_to_income_ratio'].mean())\n",
    "print('val debt_to_income_ratio mean:', up_v_df['debt_to_income_ratio'].mean())\n",
    "\n",
    "stat, p_value = ttest_ind(up_t_df['debt_to_income_ratio'], up_v_df['debt_to_income_ratio'], nan_policy='omit')\n",
    "print('debt_to_income_ratio p_value:', p_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "37ef7278",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 36 months    758182\n",
      " 60 months    241818\n",
      "Name: loan_duration, dtype: int64\n",
      "train:  3.1353414551439514\n",
      " 36 months    262561\n",
      " 60 months     82749\n",
      "Name: loan_duration, dtype: int64\n",
      "val:  3.1729809423678836\n"
     ]
    }
   ],
   "source": [
    "dist = up_t_df['loan_duration'].value_counts()\n",
    "print(dist)\n",
    "print('train: ', dist[0] / dist[1])\n",
    "\n",
    "dist = up_v_df['loan_duration'].value_counts()\n",
    "print(dist)\n",
    "print('val: ', dist[0] / dist[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9a628a9",
   "metadata": {},
   "source": [
    "#### Neither of the maybe seems that crucial"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9 (Default)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}