kNN_Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import itertools\n",
    "warnings.filterwarnings('ignore')\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import model_selection, metrics\n",
    "from sklearn.model_selection import train_test_split\n",
    "from scipy import stats\n",
    "\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, roc_auc_score, r2_score, f1_score,mean_squared_error\n",
    "\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "\n",
    "from numpy.random import seed\n",
    "seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def complete_model(trainX, trainy, testX, testy, model):\n",
    "    model.fit(trainX, trainy)\n",
    "    y_pred = model.predict(testX)\n",
    "    return calculation(testy, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculation(y_test, y_pred):  \n",
    "    mse=mean_squared_error(y_test,y_pred)\n",
    "    rmse=np.sqrt(mse)\n",
    "    r2=metrics.r2_score(y_test,y_pred)\n",
    "    pcc=stats.pearsonr(y_test,y_pred)\n",
    "    return [mse, rmse, r2, pcc[0],pcc[1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_output(model_name, val, output_val):\n",
    "    print('\\033[1mOptimized {} model {} performance: \\033[0m'.format(model_name, val))\n",
    "    print(\"MSE:         {0:.3f}\".format(output_val[0]))\n",
    "    print(\"RMSE: {0:.3f}\".format(output_val[1]))\n",
    "    print(\"R2: {0:.3f}\".format(output_val[2]))\n",
    "    print(\"PCC:    {0:.3f}\".format(output_val[3]))\n",
    "    print(\"p-value:    {0:.3f}\".format(output_val[4]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>AATS3s</th>\n",
       "      <th>AATS4s</th>\n",
       "      <th>AATSC0c</th>\n",
       "      <th>AATSC1p</th>\n",
       "      <th>MATS1p</th>\n",
       "      <th>MATS1i</th>\n",
       "      <th>VE1_Dzs</th>\n",
       "      <th>VE2_Dzs</th>\n",
       "      <th>TDB5e</th>\n",
       "      <th>...</th>\n",
       "      <th>PubchemFP497</th>\n",
       "      <th>PubchemFP609</th>\n",
       "      <th>PubchemFP622</th>\n",
       "      <th>PubchemFP674</th>\n",
       "      <th>KRFP2018</th>\n",
       "      <th>KRFP2135</th>\n",
       "      <th>KRFP4772</th>\n",
       "      <th>AD2D174</th>\n",
       "      <th>KRFPC2135</th>\n",
       "      <th>KRFPC2975</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.18</td>\n",
       "      <td>2.034846</td>\n",
       "      <td>1.948337</td>\n",
       "      <td>0.007198</td>\n",
       "      <td>-0.014797</td>\n",
       "      <td>-0.059715</td>\n",
       "      <td>-0.211553</td>\n",
       "      <td>0.493891</td>\n",
       "      <td>0.022450</td>\n",
       "      <td>34.024208</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.10</td>\n",
       "      <td>2.168422</td>\n",
       "      <td>2.021814</td>\n",
       "      <td>0.011219</td>\n",
       "      <td>-0.007268</td>\n",
       "      <td>-0.029721</td>\n",
       "      <td>-0.185476</td>\n",
       "      <td>0.457910</td>\n",
       "      <td>0.019909</td>\n",
       "      <td>32.854418</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.89</td>\n",
       "      <td>2.586843</td>\n",
       "      <td>2.484821</td>\n",
       "      <td>0.011040</td>\n",
       "      <td>0.001238</td>\n",
       "      <td>0.004958</td>\n",
       "      <td>-0.123107</td>\n",
       "      <td>0.285269</td>\n",
       "      <td>0.011411</td>\n",
       "      <td>35.089963</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.30</td>\n",
       "      <td>6.735119</td>\n",
       "      <td>5.824755</td>\n",
       "      <td>0.039992</td>\n",
       "      <td>-0.005964</td>\n",
       "      <td>-0.029783</td>\n",
       "      <td>-0.201130</td>\n",
       "      <td>0.229938</td>\n",
       "      <td>0.012774</td>\n",
       "      <td>43.154668</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.03</td>\n",
       "      <td>2.308201</td>\n",
       "      <td>2.173413</td>\n",
       "      <td>0.006154</td>\n",
       "      <td>-0.008504</td>\n",
       "      <td>-0.034845</td>\n",
       "      <td>-0.098218</td>\n",
       "      <td>0.413094</td>\n",
       "      <td>0.014753</td>\n",
       "      <td>35.676718</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Name    AATS3s    AATS4s   AATSC0c   AATSC1p    MATS1p    MATS1i   VE1_Dzs  \\\n",
       "0  5.18  2.034846  1.948337  0.007198 -0.014797 -0.059715 -0.211553  0.493891   \n",
       "1  5.10  2.168422  2.021814  0.011219 -0.007268 -0.029721 -0.185476  0.457910   \n",
       "2  5.89  2.586843  2.484821  0.011040  0.001238  0.004958 -0.123107  0.285269   \n",
       "3  5.30  6.735119  5.824755  0.039992 -0.005964 -0.029783 -0.201130  0.229938   \n",
       "4  4.03  2.308201  2.173413  0.006154 -0.008504 -0.034845 -0.098218  0.413094   \n",
       "\n",
       "    VE2_Dzs      TDB5e  ...  PubchemFP497  PubchemFP609  PubchemFP622  \\\n",
       "0  0.022450  34.024208  ...             0             0             0   \n",
       "1  0.019909  32.854418  ...             0             0             0   \n",
       "2  0.011411  35.089963  ...             0             0             0   \n",
       "3  0.012774  43.154668  ...             0             0             0   \n",
       "4  0.014753  35.676718  ...             0             0             0   \n",
       "\n",
       "   PubchemFP674  KRFP2018  KRFP2135  KRFP4772  AD2D174  KRFPC2135  KRFPC2975  \n",
       "0             0         0         0         0        0        0.0        9.0  \n",
       "1             0         0         0         0        0        0.0        9.0  \n",
       "2             0         0         0         0        0        0.0        5.0  \n",
       "3             0         0         0         0        0        0.0        9.0  \n",
       "4             0         0         0         0        0        0.0        8.0  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Read the training file\n",
    "overall_data =pd.read_csv(\"../SARS/sars_boruta_grepped_features_train.csv\")\n",
    "\n",
    "### Remove last column from the file\n",
    "overall_data = overall_data.iloc[:, :-1]\n",
    "\n",
    "## Replace the string character\n",
    "overall_data.replace('?',0,inplace=True)\n",
    "\n",
    "### Split the data\n",
    "X = overall_data.iloc[:,1:].values\n",
    "y = overall_data.loc[:,'Name'].values\n",
    "\n",
    "X[:] = np.nan_to_num(X)\n",
    "y[:] = np.nan_to_num(y)\n",
    "\n",
    "\n",
    "overall_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>AATS3s</th>\n",
       "      <th>AATS4s</th>\n",
       "      <th>AATSC0c</th>\n",
       "      <th>AATSC1p</th>\n",
       "      <th>MATS1p</th>\n",
       "      <th>MATS1i</th>\n",
       "      <th>VE1_Dzs</th>\n",
       "      <th>VE2_Dzs</th>\n",
       "      <th>TDB5e</th>\n",
       "      <th>...</th>\n",
       "      <th>PubchemFP497</th>\n",
       "      <th>PubchemFP609</th>\n",
       "      <th>PubchemFP622</th>\n",
       "      <th>PubchemFP674</th>\n",
       "      <th>KRFP2018</th>\n",
       "      <th>KRFP2135</th>\n",
       "      <th>KRFP4772</th>\n",
       "      <th>AD2D174</th>\n",
       "      <th>KRFPC2135</th>\n",
       "      <th>KRFPC2975</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.81</td>\n",
       "      <td>4.042813</td>\n",
       "      <td>4.499288</td>\n",
       "      <td>0.023375</td>\n",
       "      <td>0.014046</td>\n",
       "      <td>0.056894</td>\n",
       "      <td>-0.016811</td>\n",
       "      <td>0.585259</td>\n",
       "      <td>0.022510</td>\n",
       "      <td>39.603293</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.12</td>\n",
       "      <td>2.117682</td>\n",
       "      <td>1.930347</td>\n",
       "      <td>0.005958</td>\n",
       "      <td>-0.000610</td>\n",
       "      <td>-0.001980</td>\n",
       "      <td>-0.189903</td>\n",
       "      <td>0.291171</td>\n",
       "      <td>0.014559</td>\n",
       "      <td>33.969583</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5.18</td>\n",
       "      <td>2.702546</td>\n",
       "      <td>2.663669</td>\n",
       "      <td>0.012685</td>\n",
       "      <td>-0.000658</td>\n",
       "      <td>-0.002881</td>\n",
       "      <td>-0.074437</td>\n",
       "      <td>0.063316</td>\n",
       "      <td>0.001376</td>\n",
       "      <td>36.420713</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.30</td>\n",
       "      <td>5.116505</td>\n",
       "      <td>5.279007</td>\n",
       "      <td>0.026623</td>\n",
       "      <td>0.041073</td>\n",
       "      <td>0.183468</td>\n",
       "      <td>0.224729</td>\n",
       "      <td>0.157879</td>\n",
       "      <td>0.004934</td>\n",
       "      <td>41.440850</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.30</td>\n",
       "      <td>3.252772</td>\n",
       "      <td>3.482981</td>\n",
       "      <td>0.014990</td>\n",
       "      <td>0.006080</td>\n",
       "      <td>0.021805</td>\n",
       "      <td>0.020338</td>\n",
       "      <td>0.073163</td>\n",
       "      <td>0.002360</td>\n",
       "      <td>39.557025</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Name    AATS3s    AATS4s   AATSC0c   AATSC1p    MATS1p    MATS1i   VE1_Dzs  \\\n",
       "0  4.81  4.042813  4.499288  0.023375  0.014046  0.056894 -0.016811  0.585259   \n",
       "1  5.12  2.117682  1.930347  0.005958 -0.000610 -0.001980 -0.189903  0.291171   \n",
       "2  5.18  2.702546  2.663669  0.012685 -0.000658 -0.002881 -0.074437  0.063316   \n",
       "3  4.30  5.116505  5.279007  0.026623  0.041073  0.183468  0.224729  0.157879   \n",
       "4  4.30  3.252772  3.482981  0.014990  0.006080  0.021805  0.020338  0.073163   \n",
       "\n",
       "    VE2_Dzs      TDB5e  ...  PubchemFP497  PubchemFP609  PubchemFP622  \\\n",
       "0  0.022510  39.603293  ...             0             0             0   \n",
       "1  0.014559  33.969583  ...             0             0             0   \n",
       "2  0.001376  36.420713  ...             0             0             0   \n",
       "3  0.004934  41.440850  ...             0             0             0   \n",
       "4  0.002360  39.557025  ...             0             0             0   \n",
       "\n",
       "   PubchemFP674  KRFP2018  KRFP2135  KRFP4772  AD2D174  KRFPC2135  KRFPC2975  \n",
       "0             0         0         0       0.0      0.0        0.0        8.0  \n",
       "1             0         0         0       0.0      0.0        0.0        5.0  \n",
       "2             0         0         0       0.0      0.0        0.0       19.0  \n",
       "3             0         0         0       0.0      0.0        0.0        4.0  \n",
       "4             0         0         0       0.0      0.0        0.0        6.0  \n",
       "\n",
       "[5 rows x 36 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Read the test dataset\n",
    "\n",
    "test = pd.read_csv(\"../SARS/sars_boruta_grepped_features_test.csv\", low_memory=False)\n",
    "\n",
    "\n",
    "test = test.iloc[:, :-1]\n",
    "\n",
    "## Replace the string character from the test dataset\n",
    "test.replace('?',0,inplace=True)\n",
    "\n",
    "## Spliting the files in x and y variables [test dataset]\n",
    "X_val = test.iloc[:,1:].values\n",
    "y_val = test.loc[:,'Name'].values\n",
    "\n",
    "X_val[:] = np.nan_to_num(X_val)\n",
    "y_val[:] = np.nan_to_num(y_val)\n",
    "\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Call the defined function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_names =  ['Range_val', 'Model_info','MAE', 'RMSE','R2', 'PCC','p-val']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsRegressor(n_neighbors=3)\n",
      "KNeighborsRegressor()\n",
      "KNeighborsRegressor(n_neighbors=7)\n",
      "KNeighborsRegressor(n_neighbors=9)\n",
      "KNeighborsRegressor(n_neighbors=11)\n"
     ]
    }
   ],
   "source": [
    "### define a dataframe to save the training performance\n",
    "output  = pd.DataFrame(columns = column_names)\n",
    "\n",
    "for k in [3,5,7,9,11]:  \n",
    "    model = KNeighborsRegressor(n_neighbors=k)\n",
    "    print(model)\n",
    "    \n",
    "    for j in range(100):    \n",
    "        X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.1, random_state=j)\n",
    "        output_val = complete_model(trainX=X_train, trainy=y_train, testX=X_test, testy=y_test, model=model)\n",
    "        output.loc[len(output)] = [str(j), 'knn_k'+str(k),output_val[0], output_val[1], output_val[2], output_val[3],  output_val[4]]\n",
    "\n",
    "output.to_csv(r'''knn_training_metrics_all.csv''')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Print the output of best performing kNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### KNN\n",
    "optimized_knn = KNeighborsRegressor(n_neighbors=3)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2, random_state=51)\n",
    "### Optimized KNN model training performance\n",
    "print_result('KNN', 'training', complete_model(X_train, y_train, X_test, y_test, optimized_knn))\n",
    "### Optimized KNN model testing performance\n",
    "print_result('KNN', 'testing', complete_model(X_train, y_train, X_val, y_val, optimized_knn))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}