covariate_shift.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "5d30f62b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import pairwise_distances,silhouette_samples,silhouette_score\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "863e9238",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>requested_amnt</th>\n",
       "      <th>annual_income</th>\n",
       "      <th>debt_to_income_ratio</th>\n",
       "      <th>public_bankruptcies</th>\n",
       "      <th>state</th>\n",
       "      <th>delinquency_last_2yrs</th>\n",
       "      <th>fico_inquired_last_6mths</th>\n",
       "      <th>months_since_last_delinq</th>\n",
       "      <th>revolving_balance</th>\n",
       "      <th>total_revolving_limit</th>\n",
       "      <th>any_tax_liens</th>\n",
       "      <th>missing_vals</th>\n",
       "      <th>income_to_loan_amount_ratio</th>\n",
       "      <th>fico_score</th>\n",
       "      <th>fico_score_range</th>\n",
       "      <th>employment_length</th>\n",
       "      <th>employment_verified</th>\n",
       "      <th>type_of_application</th>\n",
       "      <th>loan_duration</th>\n",
       "      <th>reason_for_loan_credit_card</th>\n",
       "      <th>reason_for_loan_debt_consolidation</th>\n",
       "      <th>reason_for_loan_home_improvement</th>\n",
       "      <th>reason_for_loan_major_purchase</th>\n",
       "      <th>reason_for_loan_other</th>\n",
       "      <th>reason_for_loan_small_business</th>\n",
       "      <th>employment_Manager</th>\n",
       "      <th>employment_Owner</th>\n",
       "      <th>employment_RN</th>\n",
       "      <th>employment_Registered Nurse</th>\n",
       "      <th>employment_Teacher</th>\n",
       "      <th>employment_other</th>\n",
       "      <th>home_ownership_status_ANY</th>\n",
       "      <th>home_ownership_status_MORTGAGE</th>\n",
       "      <th>home_ownership_status_NONE</th>\n",
       "      <th>home_ownership_status_OTHER</th>\n",
       "      <th>home_ownership_status_OWN</th>\n",
       "      <th>home_ownership_status_RENT</th>\n",
       "      <th>zipcode_bad</th>\n",
       "      <th>zipcode_good</th>\n",
       "      <th>zipcode_mid</th>\n",
       "      <th>zipcode_mid_bad</th>\n",
       "      <th>zipcode_mid_good</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.001849</td>\n",
       "      <td>2.429058</td>\n",
       "      <td>-0.181194</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.114735</td>\n",
       "      <td>0.271535</td>\n",
       "      <td>1.984340</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.128000</td>\n",
       "      <td>2.577382</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.976558</td>\n",
       "      <td>-0.101551</td>\n",
       "      <td>-0.291395</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.171642</td>\n",
       "      <td>-0.442526</td>\n",
       "      <td>-0.481253</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>-1.196633</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1.182756</td>\n",
       "      <td>-0.565496</td>\n",
       "      <td>0.271310</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1.252866</td>\n",
       "      <td>-0.523397</td>\n",
       "      <td>-0.597631</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.113514</td>\n",
       "      <td>-0.743751</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.549844</td>\n",
       "      <td>-0.453025</td>\n",
       "      <td>1.906763</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.114735</td>\n",
       "      <td>0.237305</td>\n",
       "      <td>-0.194368</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.216111</td>\n",
       "      <td>0.463934</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.085416</td>\n",
       "      <td>0.123392</td>\n",
       "      <td>1.354786</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.446060</td>\n",
       "      <td>0.621609</td>\n",
       "      <td>1.943744</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.279070</td>\n",
       "      <td>-1.196633</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967914</th>\n",
       "      <td>-0.747450</td>\n",
       "      <td>-0.087492</td>\n",
       "      <td>0.753072</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.911427</td>\n",
       "      <td>-0.062683</td>\n",
       "      <td>-0.042805</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.112676</td>\n",
       "      <td>-0.743751</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967915</th>\n",
       "      <td>-1.205666</td>\n",
       "      <td>-0.551437</td>\n",
       "      <td>-0.475712</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.502967</td>\n",
       "      <td>-0.508778</td>\n",
       "      <td>-0.635522</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.105263</td>\n",
       "      <td>-0.139909</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967916</th>\n",
       "      <td>-0.403788</td>\n",
       "      <td>-0.277288</td>\n",
       "      <td>-0.858001</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.114735</td>\n",
       "      <td>-0.242844</td>\n",
       "      <td>-0.524557</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.191304</td>\n",
       "      <td>1.067776</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967917</th>\n",
       "      <td>-0.014304</td>\n",
       "      <td>-0.410848</td>\n",
       "      <td>-0.166566</td>\n",
       "      <td>0</td>\n",
       "      <td>43</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.114735</td>\n",
       "      <td>-0.025935</td>\n",
       "      <td>-0.318865</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.011052</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>967918</th>\n",
       "      <td>-0.060125</td>\n",
       "      <td>-0.410848</td>\n",
       "      <td>1.818018</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.795770</td>\n",
       "      <td>0.235759</td>\n",
       "      <td>-0.059044</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.291667</td>\n",
       "      <td>-0.290869</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>967919 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        requested_amnt  annual_income  debt_to_income_ratio  \\\n",
       "0             2.001849       2.429058             -0.181194   \n",
       "1            -0.976558      -0.101551             -0.291395   \n",
       "2            -1.182756      -0.565496              0.271310   \n",
       "3            -0.549844      -0.453025              1.906763   \n",
       "4             1.085416       0.123392              1.354786   \n",
       "...                ...            ...                   ...   \n",
       "967914       -0.747450      -0.087492              0.753072   \n",
       "967915       -1.205666      -0.551437             -0.475712   \n",
       "967916       -0.403788      -0.277288             -0.858001   \n",
       "967917       -0.014304      -0.410848             -0.166566   \n",
       "967918       -0.060125      -0.410848              1.818018   \n",
       "\n",
       "        public_bankruptcies  state  delinquency_last_2yrs  \\\n",
       "0                         0     15                      0   \n",
       "1                         0      3                      0   \n",
       "2                         0      7                      1   \n",
       "3                         0     22                      0   \n",
       "4                         0     27                      0   \n",
       "...                     ...    ...                    ...   \n",
       "967914                    1     29                      1   \n",
       "967915                    0     15                      0   \n",
       "967916                    0     11                      0   \n",
       "967917                    0     43                      0   \n",
       "967918                    0     25                      0   \n",
       "\n",
       "        fico_inquired_last_6mths  months_since_last_delinq  revolving_balance  \\\n",
       "0                              0                 -0.114735           0.271535   \n",
       "1                              1                 -0.171642          -0.442526   \n",
       "2                              0                 -1.252866          -0.523397   \n",
       "3                              0                 -0.114735           0.237305   \n",
       "4                              0                  2.446060           0.621609   \n",
       "...                          ...                       ...                ...   \n",
       "967914                         2                 -0.911427          -0.062683   \n",
       "967915                         0                  2.502967          -0.508778   \n",
       "967916                         1                 -0.114735          -0.242844   \n",
       "967917                         1                 -0.114735          -0.025935   \n",
       "967918                         0                  0.795770           0.235759   \n",
       "\n",
       "        total_revolving_limit  any_tax_liens  missing_vals  \\\n",
       "0                    1.984340              0             1   \n",
       "1                   -0.481253              0             0   \n",
       "2                   -0.597631              0             0   \n",
       "3                   -0.194368              0             1   \n",
       "4                    1.943744              0             0   \n",
       "...                       ...            ...           ...   \n",
       "967914              -0.042805              0             0   \n",
       "967915              -0.635522              0             0   \n",
       "967916              -0.524557              0             1   \n",
       "967917              -0.318865              0             1   \n",
       "967918              -0.059044              0             0   \n",
       "\n",
       "        income_to_loan_amount_ratio  fico_score  fico_score_range  \\\n",
       "0                          0.128000    2.577382               4.0   \n",
       "1                          0.085714   -1.196633               4.0   \n",
       "2                          0.113514   -0.743751               4.0   \n",
       "3                          0.216111    0.463934               4.0   \n",
       "4                          0.279070   -1.196633               4.0   \n",
       "...                             ...         ...               ...   \n",
       "967914                     0.112676   -0.743751               4.0   \n",
       "967915                     0.105263   -0.139909               4.0   \n",
       "967916                     0.191304    1.067776               4.0   \n",
       "967917                     0.300000    0.011052               4.0   \n",
       "967918                     0.291667   -0.290869               4.0   \n",
       "\n",
       "        employment_length  employment_verified  type_of_application  \\\n",
       "0                       4                    1                    0   \n",
       "1                      10                    2                    0   \n",
       "2                      10                    0                    0   \n",
       "3                      10                    2                    0   \n",
       "4                      10                    0                    0   \n",
       "...                   ...                  ...                  ...   \n",
       "967914                  2                    1                    0   \n",
       "967915                  6                    1                    0   \n",
       "967916                  1                    1                    0   \n",
       "967917                  1                    0                    0   \n",
       "967918                  3                    2                    0   \n",
       "\n",
       "        loan_duration  reason_for_loan_credit_card  \\\n",
       "0                   1                            0   \n",
       "1                   0                            0   \n",
       "2                   0                            0   \n",
       "3                   0                            0   \n",
       "4                   0                            0   \n",
       "...               ...                          ...   \n",
       "967914              0                            0   \n",
       "967915              0                            0   \n",
       "967916              1                            0   \n",
       "967917              0                            1   \n",
       "967918              1                            0   \n",
       "\n",
       "        reason_for_loan_debt_consolidation  reason_for_loan_home_improvement  \\\n",
       "0                                        1                                 0   \n",
       "1                                        0                                 1   \n",
       "2                                        1                                 0   \n",
       "3                                        1                                 0   \n",
       "4                                        1                                 0   \n",
       "...                                    ...                               ...   \n",
       "967914                                   0                                 1   \n",
       "967915                                   1                                 0   \n",
       "967916                                   1                                 0   \n",
       "967917                                   0                                 0   \n",
       "967918                                   1                                 0   \n",
       "\n",
       "        reason_for_loan_major_purchase  reason_for_loan_other  \\\n",
       "0                                    0                      0   \n",
       "1                                    0                      0   \n",
       "2                                    0                      0   \n",
       "3                                    0                      0   \n",
       "4                                    0                      0   \n",
       "...                                ...                    ...   \n",
       "967914                               0                      0   \n",
       "967915                               0                      0   \n",
       "967916                               0                      0   \n",
       "967917                               0                      0   \n",
       "967918                               0                      0   \n",
       "\n",
       "        reason_for_loan_small_business  employment_Manager  employment_Owner  \\\n",
       "0                                    0                   0                 0   \n",
       "1                                    0                   0                 0   \n",
       "2                                    0                   0                 0   \n",
       "3                                    0                   0                 0   \n",
       "4                                    0                   0                 0   \n",
       "...                                ...                 ...               ...   \n",
       "967914                               0                   0                 0   \n",
       "967915                               0                   0                 0   \n",
       "967916                               0                   0                 0   \n",
       "967917                               0                   0                 0   \n",
       "967918                               0                   0                 0   \n",
       "\n",
       "        employment_RN  employment_Registered Nurse  employment_Teacher  \\\n",
       "0                   0                            0                   0   \n",
       "1                   0                            0                   0   \n",
       "2                   0                            0                   0   \n",
       "3                   0                            0                   0   \n",
       "4                   0                            0                   0   \n",
       "...               ...                          ...                 ...   \n",
       "967914              0                            0                   0   \n",
       "967915              0                            0                   0   \n",
       "967916              0                            0                   0   \n",
       "967917              0                            0                   0   \n",
       "967918              0                            0                   0   \n",
       "\n",
       "        employment_other  home_ownership_status_ANY  \\\n",
       "0                      1                          0   \n",
       "1                      1                          0   \n",
       "2                      1                          0   \n",
       "3                      1                          0   \n",
       "4                      1                          0   \n",
       "...                  ...                        ...   \n",
       "967914                 1                          0   \n",
       "967915                 1                          0   \n",
       "967916                 1                          0   \n",
       "967917                 1                          0   \n",
       "967918                 1                          0   \n",
       "\n",
       "        home_ownership_status_MORTGAGE  home_ownership_status_NONE  \\\n",
       "0                                    0                           0   \n",
       "1                                    1                           0   \n",
       "2                                    0                           0   \n",
       "3                                    0                           0   \n",
       "4                                    1                           0   \n",
       "...                                ...                         ...   \n",
       "967914                               1                           0   \n",
       "967915                               0                           0   \n",
       "967916                               1                           0   \n",
       "967917                               0                           0   \n",
       "967918                               1                           0   \n",
       "\n",
       "        home_ownership_status_OTHER  home_ownership_status_OWN  \\\n",
       "0                                 0                          0   \n",
       "1                                 0                          0   \n",
       "2                                 0                          1   \n",
       "3                                 0                          1   \n",
       "4                                 0                          0   \n",
       "...                             ...                        ...   \n",
       "967914                            0                          0   \n",
       "967915                            0                          0   \n",
       "967916                            0                          0   \n",
       "967917                            0                          0   \n",
       "967918                            0                          0   \n",
       "\n",
       "        home_ownership_status_RENT  zipcode_bad  zipcode_good  zipcode_mid  \\\n",
       "0                                1            0             0            0   \n",
       "1                                0            0             0            1   \n",
       "2                                0            0             0            1   \n",
       "3                                0            0             1            0   \n",
       "4                                0            0             0            1   \n",
       "...                            ...          ...           ...          ...   \n",
       "967914                           0            0             0            1   \n",
       "967915                           1            0             0            0   \n",
       "967916                           0            0             0            0   \n",
       "967917                           1            0             1            0   \n",
       "967918                           0            0             0            1   \n",
       "\n",
       "        zipcode_mid_bad  zipcode_mid_good  \n",
       "0                     1                 0  \n",
       "1                     0                 0  \n",
       "2                     0                 0  \n",
       "3                     0                 0  \n",
       "4                     0                 0  \n",
       "...                 ...               ...  \n",
       "967914                0                 0  \n",
       "967915                1                 0  \n",
       "967916                1                 0  \n",
       "967917                0                 0  \n",
       "967918                0                 0  \n",
       "\n",
       "[967919 rows x 42 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# reading in train data\n",
    "df = pd.read_csv('data/train_preprocessed_data.csv')\n",
    "pd.set_option('max_columns', None)\n",
    "df = df.drop(columns=['loan_paid'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "ab8562ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([], dtype=int64), array([], dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "# setting up for k-means\n",
    "x = df.to_numpy()\n",
    "x[np.isinf(x)] = 0\n",
    "print(np.where(np.isinf(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdcf8496",
   "metadata": {},
   "outputs": [],
   "source": [
    "# silhouette analysis with different k-values\n",
    "k = range(3, 10)\n",
    "scores3 = []\n",
    "for cur_k in k:\n",
    "    kmeans = KMeans(n_clusters=cur_k)\n",
    "    kmeans.fit(x)\n",
    "    print(kmeans.labels_)\n",
    "    print(type(kmeans.labels_))\n",
    "    print(np.unique(kmeans.labels_, return_counts=True))\n",
    "    score = silhouette_score(x, kmeans.labels_, metric='euclidean', sample_size=25000)\n",
    "    print(score)\n",
    "    scores3.append(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed372105",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(range(3, 10), scores3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "902a8b30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "KMeans(n_clusters=10, random_state=42)"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# training the kmeans\n",
    "best_k = 10\n",
    "kmeans = KMeans(best_k, random_state=42)\n",
    "kmeans.fit(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "e5e41abc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 154831, 1: 1, 2: 180897, 3: 1, 4: 136683, 5: 1, 6: 117618, 7: 158668, 8: 84285, 9: 134934}\n"
     ]
    }
   ],
   "source": [
    "# predicting and finding the cluster frequency\n",
    "train_cluster_pred = kmeans.predict(x).astype(int)\n",
    "temp = np.unique(train_cluster_pred, return_counts=True)\n",
    "counts = dict(zip(temp[0], temp[1]))\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "c06cc88e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving model using pickles        \n",
    "with open('models/k_means.pickle', 'wb') as handle:\n",
    "        pickle.dump(kmeans, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "c7ee945b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading in val data\n",
    "df1 = pd.read_csv('data/predict_preprocessed_data.csv')\n",
    "pd.set_option('max_columns', None)\n",
    "df1 = df1.drop(columns=['ID'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "2d196feb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([], dtype=int64), array([], dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "# setting up for k-means\n",
    "x_val = df1.to_numpy()\n",
    "x_val[np.isinf(x_val)] = 0\n",
    "print(np.where(np.isinf(x_val)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "620e5d4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 53160, 2: 62558, 4: 47169, 6: 40476, 7: 54775, 8: 29149, 9: 46701, ' ': 0}\n"
     ]
    }
   ],
   "source": [
    "# getting predicted clusters of validation data points & cluster frequency\n",
    "val_cluster_pred = kmeans.predict(x_val).astype(int)\n",
    "temp = np.unique(val_cluster_pred, return_counts=True)\n",
    "counts_val = dict(zip(temp[0], temp[1]))\n",
    "counts_val.setdefault(' ', 0)\n",
    "print(counts_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "7dca2b71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0, 2, 4, 6, 7, 8, 9}\n"
     ]
    }
   ],
   "source": [
    "print(set(val_cluster_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "e8039561",
   "metadata": {},
   "outputs": [],
   "source": [
    "# generating sample weights\n",
    "counts_val[3] = 0\n",
    "counts_val[1] = 0\n",
    "counts_val[5] = 0\n",
    "weights = x.shape[0] * [0]\n",
    "for index in range (x.shape[0]):\n",
    "    weights[index] = counts_val[train_cluster_pred[index]] / counts[train_cluster_pred[index]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "c9b3090b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.11874509, 0.11932827, 0.11932827, ..., 0.11874509, 0.11933428,\n",
       "       0.11912006])"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights = np.array(weights).astype(float) * (x_val.shape[0] / x.shape[0])\n",
    "weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "c177d96a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving sample weightsusing pickles        \n",
    "with open('models/sample_weights.pickle', 'wb') as handle:\n",
    "        pickle.dump(weights, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9 (Default)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}