TitanicSurvivalPredictor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "89f18fa6",
   "metadata": {
    "papermill": {
     "duration": 0.016659,
     "end_time": "2023-11-14T13:14:04.598751",
     "exception": false,
     "start_time": "2023-11-14T13:14:04.582092",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Import Required Libraraies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "28a8a5f8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:04.632031Z",
     "iopub.status.busy": "2023-11-14T13:14:04.631612Z",
     "iopub.status.idle": "2023-11-14T13:14:05.062605Z",
     "shell.execute_reply": "2023-11-14T13:14:05.061138Z"
    },
    "papermill": {
     "duration": 0.451194,
     "end_time": "2023-11-14T13:14:05.065755",
     "exception": false,
     "start_time": "2023-11-14T13:14:04.614561",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e422bb5c",
   "metadata": {
    "papermill": {
     "duration": 0.015573,
     "end_time": "2023-11-14T13:14:05.097896",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.082323",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Read Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fe532348",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.132623Z",
     "iopub.status.busy": "2023-11-14T13:14:05.131964Z",
     "iopub.status.idle": "2023-11-14T13:14:05.169081Z",
     "shell.execute_reply": "2023-11-14T13:14:05.167997Z"
    },
    "papermill": {
     "duration": 0.057739,
     "end_time": "2023-11-14T13:14:05.171776",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.114037",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv('/kaggle/input/titanic/train.csv')\n",
    "test_data = pd.read_csv('/kaggle/input/titanic/test.csv')\n",
    "gender_sub = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "48f40020",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.206145Z",
     "iopub.status.busy": "2023-11-14T13:14:05.205268Z",
     "iopub.status.idle": "2023-11-14T13:14:05.238009Z",
     "shell.execute_reply": "2023-11-14T13:14:05.236838Z"
    },
    "papermill": {
     "duration": 0.052852,
     "end_time": "2023-11-14T13:14:05.240576",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.187724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>892</td>\n",
       "      <td>3</td>\n",
       "      <td>Kelly, Mr. James</td>\n",
       "      <td>male</td>\n",
       "      <td>34.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>330911</td>\n",
       "      <td>7.8292</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>893</td>\n",
       "      <td>3</td>\n",
       "      <td>Wilkes, Mrs. James (Ellen Needs)</td>\n",
       "      <td>female</td>\n",
       "      <td>47.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>363272</td>\n",
       "      <td>7.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>894</td>\n",
       "      <td>2</td>\n",
       "      <td>Myles, Mr. Thomas Francis</td>\n",
       "      <td>male</td>\n",
       "      <td>62.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>240276</td>\n",
       "      <td>9.6875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>895</td>\n",
       "      <td>3</td>\n",
       "      <td>Wirz, Mr. Albert</td>\n",
       "      <td>male</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>315154</td>\n",
       "      <td>8.6625</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>896</td>\n",
       "      <td>3</td>\n",
       "      <td>Hirvonen, Mrs. Alexander (Helga E Lindqvist)</td>\n",
       "      <td>female</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3101298</td>\n",
       "      <td>12.2875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Pclass                                          Name     Sex  \\\n",
       "0          892       3                              Kelly, Mr. James    male   \n",
       "1          893       3              Wilkes, Mrs. James (Ellen Needs)  female   \n",
       "2          894       2                     Myles, Mr. Thomas Francis    male   \n",
       "3          895       3                              Wirz, Mr. Albert    male   \n",
       "4          896       3  Hirvonen, Mrs. Alexander (Helga E Lindqvist)  female   \n",
       "\n",
       "    Age  SibSp  Parch   Ticket     Fare Cabin Embarked  \n",
       "0  34.5      0      0   330911   7.8292   NaN        Q  \n",
       "1  47.0      1      0   363272   7.0000   NaN        S  \n",
       "2  62.0      0      0   240276   9.6875   NaN        Q  \n",
       "3  27.0      0      0   315154   8.6625   NaN        S  \n",
       "4  22.0      1      1  3101298  12.2875   NaN        S  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7c2da3a",
   "metadata": {
    "papermill": {
     "duration": 0.016171,
     "end_time": "2023-11-14T13:14:05.273256",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.257085",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data Purification and Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c84883ad",
   "metadata": {
    "papermill": {
     "duration": 0.015652,
     "end_time": "2023-11-14T13:14:05.305226",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.289574",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Name\n",
    "### Extract titles from names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "31f53306",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.339805Z",
     "iopub.status.busy": "2023-11-14T13:14:05.339080Z",
     "iopub.status.idle": "2023-11-14T13:14:05.359219Z",
     "shell.execute_reply": "2023-11-14T13:14:05.357534Z"
    },
    "papermill": {
     "duration": 0.040616,
     "end_time": "2023-11-14T13:14:05.361867",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.321251",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All unique titles in names (Train): ['Mr' 'Mrs' 'Miss' 'Master' 'Don' 'Rev' 'Dr' 'Mme' 'Ms' 'Major' 'Lady'\n",
      " 'Sir' 'Mlle' 'Col' 'Capt' 'Countess' 'Jonkheer']\n",
      "All unique titles in names (Test): ['Mr' 'Mrs' 'Miss' 'Master' 'Ms' 'Col' 'Rev' 'Dr' 'Dona']\n"
     ]
    }
   ],
   "source": [
    "train_data['Title'] = train_data['Name'].apply(lambda x: re.search(' ([A-Za-z]+)\\.', x).group(1))\n",
    "test_data['Title'] = test_data['Name'].apply(lambda x: re.search(' ([A-Za-z]+)\\.', x).group(1))\n",
    "\n",
    "# Display the unique titles\n",
    "print(f'All unique titles in names (Train): {train_data[\"Title\"].unique()}')\n",
    "print(f'All unique titles in names (Test): {test_data[\"Title\"].unique()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "704399ca",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.398348Z",
     "iopub.status.busy": "2023-11-14T13:14:05.396963Z",
     "iopub.status.idle": "2023-11-14T13:14:05.414683Z",
     "shell.execute_reply": "2023-11-14T13:14:05.413309Z"
    },
    "papermill": {
     "duration": 0.038589,
     "end_time": "2023-11-14T13:14:05.417599",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.379010",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Train Data\n",
    "train_data['Title'] = train_data['Title'].replace(['Lady', 'Countess', 'Capt', 'Col', 'Don', 'Dr','Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Other')\n",
    "train_data['Title'] = train_data['Title'].replace(['Mlle', 'Ms'], 'Miss')\n",
    "train_data['Title'] = train_data['Title'].replace('Mme', 'Mrs')\n",
    "\n",
    "# Test Data\n",
    "test_data['Title'] = test_data['Title'].replace(['Lady', 'Countess', 'Capt', 'Col', 'Don', 'Dr','Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Other')\n",
    "test_data['Title'] = test_data['Title'].replace(['Mlle', 'Ms'], 'Miss')\n",
    "test_data['Title'] = test_data['Title'].replace('Mme', 'Mrs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5681db96",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.452407Z",
     "iopub.status.busy": "2023-11-14T13:14:05.451945Z",
     "iopub.status.idle": "2023-11-14T13:14:05.459366Z",
     "shell.execute_reply": "2023-11-14T13:14:05.458079Z"
    },
    "papermill": {
     "duration": 0.027843,
     "end_time": "2023-11-14T13:14:05.462009",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.434166",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Titles: ['Mr' 'Mrs' 'Miss' 'Master' 'Other']\n",
      "Test Titles: ['Mr' 'Mrs' 'Miss' 'Master' 'Other']\n"
     ]
    }
   ],
   "source": [
    "print(f'Train Titles: {train_data[\"Title\"].unique()}')\n",
    "print(f'Test Titles: {test_data[\"Title\"].unique()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d60d3160",
   "metadata": {
    "papermill": {
     "duration": 0.015824,
     "end_time": "2023-11-14T13:14:05.494945",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.479121",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Drop Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "724da346",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.595025Z",
     "iopub.status.busy": "2023-11-14T13:14:05.594592Z",
     "iopub.status.idle": "2023-11-14T13:14:05.608760Z",
     "shell.execute_reply": "2023-11-14T13:14:05.607792Z"
    },
    "papermill": {
     "duration": 0.100103,
     "end_time": "2023-11-14T13:14:05.611355",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.511252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data = train_data.drop('Name', axis=1)\n",
    "test_data = test_data.drop('Name', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a9d971b",
   "metadata": {
    "papermill": {
     "duration": 0.016034,
     "end_time": "2023-11-14T13:14:05.643782",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.627748",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Encode Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6f6b767c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.678528Z",
     "iopub.status.busy": "2023-11-14T13:14:05.678090Z",
     "iopub.status.idle": "2023-11-14T13:14:05.687423Z",
     "shell.execute_reply": "2023-11-14T13:14:05.686149Z"
    },
    "papermill": {
     "duration": 0.029554,
     "end_time": "2023-11-14T13:14:05.689888",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.660334",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "title_to_number = {'Mr': 0, 'Mrs': 1, 'Miss': 2, 'Master': 3, 'Other': 4}\n",
    "\n",
    "train_data['Title'] = train_data['Title'].apply(lambda x: title_to_number[x])\n",
    "test_data['Title'] = test_data['Title'].apply(lambda x: title_to_number[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efb1efd3",
   "metadata": {
    "papermill": {
     "duration": 0.016213,
     "end_time": "2023-11-14T13:14:05.722544",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.706331",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Ticket\n",
    "### Drop ticket columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4a52d1bd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.757387Z",
     "iopub.status.busy": "2023-11-14T13:14:05.756937Z",
     "iopub.status.idle": "2023-11-14T13:14:05.764665Z",
     "shell.execute_reply": "2023-11-14T13:14:05.763514Z"
    },
    "papermill": {
     "duration": 0.028429,
     "end_time": "2023-11-14T13:14:05.767390",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.738961",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Drop Ticket Column\n",
    "train_data = train_data.drop('Ticket', axis=1)\n",
    "test_data = test_data.drop('Ticket', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "017cc183",
   "metadata": {
    "papermill": {
     "duration": 0.016101,
     "end_time": "2023-11-14T13:14:05.799918",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.783817",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Sex\n",
    "### Convert sexes to 0s and 1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "65dc96ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.835254Z",
     "iopub.status.busy": "2023-11-14T13:14:05.834444Z",
     "iopub.status.idle": "2023-11-14T13:14:05.844998Z",
     "shell.execute_reply": "2023-11-14T13:14:05.843860Z"
    },
    "papermill": {
     "duration": 0.031036,
     "end_time": "2023-11-14T13:14:05.847437",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.816401",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of distinct values for sex: {0, 1}\n"
     ]
    }
   ],
   "source": [
    "# Encode Sex\n",
    "train_data['Sex'] = train_data['Sex'].replace({'male': 0, 'female': 1})\n",
    "test_data['Sex'] = test_data['Sex'].replace({'male': 0, 'female': 1})\n",
    "print(f'Number of distinct values for sex: {set(test_data[\"Sex\"])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64dfb96e",
   "metadata": {
    "papermill": {
     "duration": 0.016282,
     "end_time": "2023-11-14T13:14:05.880490",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.864208",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Cabin\n",
    "### Extract letters from Cabins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dd4616b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:05.916399Z",
     "iopub.status.busy": "2023-11-14T13:14:05.914899Z",
     "iopub.status.idle": "2023-11-14T13:14:05.929199Z",
     "shell.execute_reply": "2023-11-14T13:14:05.927983Z"
    },
    "papermill": {
     "duration": 0.035051,
     "end_time": "2023-11-14T13:14:05.931998",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.896947",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['X' 'C' 'E' 'G' 'D' 'A' 'B' 'F' 'T']\n",
      "['X' 'B' 'E' 'A' 'C' 'D' 'F' 'G']\n"
     ]
    }
   ],
   "source": [
    "train_data['Cabin'] = train_data['Cabin'].fillna(\"X\")\n",
    "print(train_data['Cabin'].str[0].unique())\n",
    "\n",
    "test_data['Cabin'] = test_data['Cabin'].fillna(\"X\")\n",
    "print(test_data['Cabin'].str[0].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17f7bc2f",
   "metadata": {
    "papermill": {
     "duration": 0.016324,
     "end_time": "2023-11-14T13:14:05.965250",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.948926",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Encode letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "93f0bae2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.001095Z",
     "iopub.status.busy": "2023-11-14T13:14:06.000451Z",
     "iopub.status.idle": "2023-11-14T13:14:06.009894Z",
     "shell.execute_reply": "2023-11-14T13:14:06.008667Z"
    },
    "papermill": {
     "duration": 0.030603,
     "end_time": "2023-11-14T13:14:06.012591",
     "exception": false,
     "start_time": "2023-11-14T13:14:05.981988",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cabin_to_number = {'X': 0, 'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7, 'T': 8}\n",
    "\n",
    "train_data['Cabin_Encoded'] = train_data['Cabin'].apply(lambda x: cabin_to_number[x[0]])\n",
    "test_data['Cabin_Encoded'] = test_data['Cabin'].apply(lambda x: cabin_to_number[x[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a500dc71",
   "metadata": {
    "papermill": {
     "duration": 0.017292,
     "end_time": "2023-11-14T13:14:06.046813",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.029521",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Drop Cabin Column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "80b6a7ea",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.083331Z",
     "iopub.status.busy": "2023-11-14T13:14:06.082683Z",
     "iopub.status.idle": "2023-11-14T13:14:06.089409Z",
     "shell.execute_reply": "2023-11-14T13:14:06.088578Z"
    },
    "papermill": {
     "duration": 0.027914,
     "end_time": "2023-11-14T13:14:06.092177",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.064263",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data = train_data.drop('Cabin', axis=1)\n",
    "test_data = test_data.drop('Cabin', axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01354d1f",
   "metadata": {
    "papermill": {
     "duration": 0.016846,
     "end_time": "2023-11-14T13:14:06.125876",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.109030",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Embarked\n",
    "### Fill Null values with 'S'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fa8e2de8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.161998Z",
     "iopub.status.busy": "2023-11-14T13:14:06.161355Z",
     "iopub.status.idle": "2023-11-14T13:14:06.174242Z",
     "shell.execute_reply": "2023-11-14T13:14:06.172841Z"
    },
    "papermill": {
     "duration": 0.034556,
     "end_time": "2023-11-14T13:14:06.177276",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.142720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data['Embarked'].describe()\n",
    "\n",
    "train_data['Embarked'] = train_data['Embarked'].fillna('S')\n",
    "test_data['Embarked'] = test_data['Embarked'].fillna('S')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4a6da15",
   "metadata": {
    "papermill": {
     "duration": 0.017151,
     "end_time": "2023-11-14T13:14:06.211980",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.194829",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Encode Embarked Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3329e395",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.247848Z",
     "iopub.status.busy": "2023-11-14T13:14:06.247412Z",
     "iopub.status.idle": "2023-11-14T13:14:06.256522Z",
     "shell.execute_reply": "2023-11-14T13:14:06.255083Z"
    },
    "papermill": {
     "duration": 0.030677,
     "end_time": "2023-11-14T13:14:06.259256",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.228579",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "embark_to_number = {'S': 0, 'C': 1, 'Q': 2}\n",
    "\n",
    "train_data['Embarked'] = train_data['Embarked'].apply(lambda x: embark_to_number[x])\n",
    "test_data['Embarked'] = test_data['Embarked'].apply(lambda x: embark_to_number[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a51a281",
   "metadata": {
    "papermill": {
     "duration": 0.01688,
     "end_time": "2023-11-14T13:14:06.293035",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.276155",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Age\n",
    "### Fill null values of age with mean age of train set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fa8340a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.329358Z",
     "iopub.status.busy": "2023-11-14T13:14:06.328928Z",
     "iopub.status.idle": "2023-11-14T13:14:06.336953Z",
     "shell.execute_reply": "2023-11-14T13:14:06.336063Z"
    },
    "papermill": {
     "duration": 0.028738,
     "end_time": "2023-11-14T13:14:06.339459",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.310721",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "mean_age = train_data['Age'].mean()\n",
    "\n",
    "train_data['Age'] = train_data['Age'].fillna(mean_age)\n",
    "test_data['Age'] = test_data['Age'].fillna(mean_age)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ade6b6a5",
   "metadata": {
    "papermill": {
     "duration": 0.017228,
     "end_time": "2023-11-14T13:14:06.373451",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.356223",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Convert ages to int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0f2b776b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.409940Z",
     "iopub.status.busy": "2023-11-14T13:14:06.408755Z",
     "iopub.status.idle": "2023-11-14T13:14:06.416555Z",
     "shell.execute_reply": "2023-11-14T13:14:06.415085Z"
    },
    "papermill": {
     "duration": 0.029171,
     "end_time": "2023-11-14T13:14:06.419530",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.390359",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data['Age'] = train_data['Age'].astype(int)\n",
    "test_data['Age'] = test_data['Age'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d318053",
   "metadata": {
    "papermill": {
     "duration": 0.016429,
     "end_time": "2023-11-14T13:14:06.453252",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.436823",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Fare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c07a59bb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.488961Z",
     "iopub.status.busy": "2023-11-14T13:14:06.488493Z",
     "iopub.status.idle": "2023-11-14T13:14:06.496168Z",
     "shell.execute_reply": "2023-11-14T13:14:06.495030Z"
    },
    "papermill": {
     "duration": 0.029156,
     "end_time": "2023-11-14T13:14:06.499379",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.470223",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of fare null values of train set: 0\n",
      "number of fare null values of test set: 1\n"
     ]
    }
   ],
   "source": [
    "print(f'number of fare null values of train set: {train_data[\"Fare\"].isna().sum()}')\n",
    "print(f'number of fare null values of test set: {test_data[\"Fare\"].isna().sum()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d03b139",
   "metadata": {
    "papermill": {
     "duration": 0.016795,
     "end_time": "2023-11-14T13:14:06.533037",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.516242",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Fill null values with mean fare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e2f8324a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.569223Z",
     "iopub.status.busy": "2023-11-14T13:14:06.568761Z",
     "iopub.status.idle": "2023-11-14T13:14:06.576046Z",
     "shell.execute_reply": "2023-11-14T13:14:06.575122Z"
    },
    "papermill": {
     "duration": 0.028383,
     "end_time": "2023-11-14T13:14:06.578477",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.550094",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "mean_fare = train_data['Fare'].mean()\n",
    "\n",
    "train_data['Fare'] = train_data['Fare'].fillna(mean_fare)\n",
    "test_data['Fare'] = test_data['Fare'].fillna(mean_fare)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a7c1d2e",
   "metadata": {
    "papermill": {
     "duration": 0.016643,
     "end_time": "2023-11-14T13:14:06.613044",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.596401",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Convert fares to ints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6a519382",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.648901Z",
     "iopub.status.busy": "2023-11-14T13:14:06.648476Z",
     "iopub.status.idle": "2023-11-14T13:14:06.655150Z",
     "shell.execute_reply": "2023-11-14T13:14:06.653859Z"
    },
    "papermill": {
     "duration": 0.027666,
     "end_time": "2023-11-14T13:14:06.657842",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.630176",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data['Fare'] = train_data['Fare'].astype(int)\n",
    "test_data['Fare'] = test_data['Fare'].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d30d2b7",
   "metadata": {
    "papermill": {
     "duration": 0.016759,
     "end_time": "2023-11-14T13:14:06.691553",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.674794",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## PassengerId (Train)\n",
    "### Drop column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "19e22fc3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.727622Z",
     "iopub.status.busy": "2023-11-14T13:14:06.727178Z",
     "iopub.status.idle": "2023-11-14T13:14:06.733997Z",
     "shell.execute_reply": "2023-11-14T13:14:06.732626Z"
    },
    "papermill": {
     "duration": 0.027996,
     "end_time": "2023-11-14T13:14:06.736443",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.708447",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data = train_data.drop(['PassengerId'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "594592b0",
   "metadata": {
    "papermill": {
     "duration": 0.016641,
     "end_time": "2023-11-14T13:14:06.769891",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.753250",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Getting data ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "06e45993",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.806626Z",
     "iopub.status.busy": "2023-11-14T13:14:06.805535Z",
     "iopub.status.idle": "2023-11-14T13:14:06.814174Z",
     "shell.execute_reply": "2023-11-14T13:14:06.813313Z"
    },
    "papermill": {
     "duration": 0.029683,
     "end_time": "2023-11-14T13:14:06.816702",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.787019",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train = train_data.drop(\"Survived\", axis=1)\n",
    "Y_train = train_data[\"Survived\"]\n",
    "X_test  = test_data.drop(\"PassengerId\", axis=1).copy()\n",
    "testPassengerIds = test_data['PassengerId']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "439beddf",
   "metadata": {
    "papermill": {
     "duration": 0.01668,
     "end_time": "2023-11-14T13:14:06.850120",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.833440",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "285378b9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.886512Z",
     "iopub.status.busy": "2023-11-14T13:14:06.885725Z",
     "iopub.status.idle": "2023-11-14T13:14:06.904562Z",
     "shell.execute_reply": "2023-11-14T13:14:06.903647Z"
    },
    "papermill": {
     "duration": 0.039916,
     "end_time": "2023-11-14T13:14:06.907175",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.867259",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for col in X_train.columns:\n",
    "    train_column = X_train[col]\n",
    "    test_column = X_test[col]\n",
    "    \n",
    "    mean_val = train_column.mean()\n",
    "    std_val = train_column.std()\n",
    "    \n",
    "    X_train[col] = (train_column - mean_val) / std_val\n",
    "    X_test[col] = (test_column - mean_val) / std_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ddec2096",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.942696Z",
     "iopub.status.busy": "2023-11-14T13:14:06.942245Z",
     "iopub.status.idle": "2023-11-14T13:14:06.959696Z",
     "shell.execute_reply": "2023-11-14T13:14:06.958371Z"
    },
    "papermill": {
     "duration": 0.038475,
     "end_time": "2023-11-14T13:14:06.962402",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.923927",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Title</th>\n",
       "      <th>Cabin_Encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>-0.737281</td>\n",
       "      <td>-0.579719</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.498667</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>-0.738132</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.565228</td>\n",
       "      <td>1.354813</td>\n",
       "      <td>0.649747</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>0.788962</td>\n",
       "      <td>1.004617</td>\n",
       "      <td>0.190788</td>\n",
       "      <td>1.397540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>1.354813</td>\n",
       "      <td>-0.272352</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.498667</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>1.119709</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1.565228</td>\n",
       "      <td>1.354813</td>\n",
       "      <td>0.419222</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>0.426816</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>0.190788</td>\n",
       "      <td>1.397540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>-0.737281</td>\n",
       "      <td>0.419222</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.478548</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>-0.738132</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass       Sex       Age     SibSp     Parch      Fare  Embarked  \\\n",
       "0  0.826913 -0.737281 -0.579719  0.432550 -0.473408 -0.498667 -0.568518   \n",
       "1 -1.565228  1.354813  0.649747  0.432550 -0.473408  0.788962  1.004617   \n",
       "2  0.826913  1.354813 -0.272352 -0.474279 -0.473408 -0.498667 -0.568518   \n",
       "3 -1.565228  1.354813  0.419222  0.432550 -0.473408  0.426816 -0.568518   \n",
       "4  0.826913 -0.737281  0.419222 -0.474279 -0.473408 -0.478548 -0.568518   \n",
       "\n",
       "      Title  Cabin_Encoded  \n",
       "0 -0.738132      -0.488186  \n",
       "1  0.190788       1.397540  \n",
       "2  1.119709      -0.488186  \n",
       "3  0.190788       1.397540  \n",
       "4 -0.738132      -0.488186  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "dad979ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:06.999177Z",
     "iopub.status.busy": "2023-11-14T13:14:06.998734Z",
     "iopub.status.idle": "2023-11-14T13:14:07.016500Z",
     "shell.execute_reply": "2023-11-14T13:14:07.015266Z"
    },
    "papermill": {
     "duration": 0.039272,
     "end_time": "2023-11-14T13:14:07.019174",
     "exception": false,
     "start_time": "2023-11-14T13:14:06.979902",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Title</th>\n",
       "      <th>Cabin_Encoded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>-0.737281</td>\n",
       "      <td>0.342381</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.498667</td>\n",
       "      <td>2.577752</td>\n",
       "      <td>-0.738132</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>1.354813</td>\n",
       "      <td>1.341322</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.498667</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>0.190788</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.369158</td>\n",
       "      <td>-0.737281</td>\n",
       "      <td>2.493947</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.458429</td>\n",
       "      <td>2.577752</td>\n",
       "      <td>-0.738132</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>-0.737281</td>\n",
       "      <td>-0.195511</td>\n",
       "      <td>-0.474279</td>\n",
       "      <td>-0.473408</td>\n",
       "      <td>-0.478548</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>-0.738132</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.826913</td>\n",
       "      <td>1.354813</td>\n",
       "      <td>-0.579719</td>\n",
       "      <td>0.432550</td>\n",
       "      <td>0.767199</td>\n",
       "      <td>-0.398071</td>\n",
       "      <td>-0.568518</td>\n",
       "      <td>0.190788</td>\n",
       "      <td>-0.488186</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass       Sex       Age     SibSp     Parch      Fare  Embarked  \\\n",
       "0  0.826913 -0.737281  0.342381 -0.474279 -0.473408 -0.498667  2.577752   \n",
       "1  0.826913  1.354813  1.341322  0.432550 -0.473408 -0.498667 -0.568518   \n",
       "2 -0.369158 -0.737281  2.493947 -0.474279 -0.473408 -0.458429  2.577752   \n",
       "3  0.826913 -0.737281 -0.195511 -0.474279 -0.473408 -0.478548 -0.568518   \n",
       "4  0.826913  1.354813 -0.579719  0.432550  0.767199 -0.398071 -0.568518   \n",
       "\n",
       "      Title  Cabin_Encoded  \n",
       "0 -0.738132      -0.488186  \n",
       "1  0.190788      -0.488186  \n",
       "2 -0.738132      -0.488186  \n",
       "3 -0.738132      -0.488186  \n",
       "4  0.190788      -0.488186  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0d4804bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:07.057468Z",
     "iopub.status.busy": "2023-11-14T13:14:07.056128Z",
     "iopub.status.idle": "2023-11-14T13:14:07.065675Z",
     "shell.execute_reply": "2023-11-14T13:14:07.064519Z"
    },
    "papermill": {
     "duration": 0.031162,
     "end_time": "2023-11-14T13:14:07.068219",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.037057",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    1\n",
       "2    1\n",
       "3    1\n",
       "4    0\n",
       "Name: Survived, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "026dc50f",
   "metadata": {
    "papermill": {
     "duration": 0.017203,
     "end_time": "2023-11-14T13:14:07.102952",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.085749",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Separate cross validation set from train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "10665643",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:07.140787Z",
     "iopub.status.busy": "2023-11-14T13:14:07.139999Z",
     "iopub.status.idle": "2023-11-14T13:14:07.148721Z",
     "shell.execute_reply": "2023-11-14T13:14:07.147675Z"
    },
    "papermill": {
     "duration": 0.03096,
     "end_time": "2023-11-14T13:14:07.151553",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.120593",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the random seed for reproducibility\n",
    "np.random.seed(42)\n",
    "\n",
    "# Shuffle the data\n",
    "shuffle_indices = np.random.permutation(len(X_train))\n",
    "X_train_shuffled = X_train.values[shuffle_indices]\n",
    "Y_train_shuffled = Y_train.values[shuffle_indices]\n",
    "\n",
    "# Define the proportion of data to be allocated for cross-validation\n",
    "val_split = 0.2\n",
    "\n",
    "# Calculate the number of examples for the cross-validation set\n",
    "num_val = int(val_split * len(X_train_shuffled))\n",
    "\n",
    "# Split the data into training and cross-validation sets\n",
    "X_train = X_train_shuffled[:-num_val]\n",
    "Y_train = Y_train_shuffled[:-num_val]\n",
    "X_val = X_train_shuffled[-num_val:]\n",
    "Y_val = Y_train_shuffled[-num_val:]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d60671f",
   "metadata": {
    "papermill": {
     "duration": 0.019919,
     "end_time": "2023-11-14T13:14:07.189372",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.169453",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b3de0063",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:07.227946Z",
     "iopub.status.busy": "2023-11-14T13:14:07.227180Z",
     "iopub.status.idle": "2023-11-14T13:14:07.241955Z",
     "shell.execute_reply": "2023-11-14T13:14:07.241031Z"
    },
    "papermill": {
     "duration": 0.037687,
     "end_time": "2023-11-14T13:14:07.244582",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.206895",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class LogisticRegression:\n",
    "    def __init__(self, max_iterations=1000, learning_rate=0.01, decay=0.9, epsilon=1e-5):\n",
    "        self.max_iterations = max_iterations\n",
    "        self.learning_rate = learning_rate\n",
    "        self.decay = decay\n",
    "        self.epsilon = epsilon\n",
    "        self.W = None\n",
    "        self.b = None\n",
    "\n",
    "        \n",
    "    def sigmoid(self, z):\n",
    "        a = 1 / (1 + np.exp(-z))\n",
    "        return a\n",
    "\n",
    "    \n",
    "    def binary_cross_entropy(self, y_pred, y):\n",
    "        epsilon = 1e-15\n",
    "        y_pred = np.clip(y_pred, epsilon, 1 - epsilon)\n",
    "        loss = -(y * np.log(y_pred) + (1 - y) * np.log(1 - y_pred)).mean()\n",
    "        return loss\n",
    "\n",
    "    \n",
    "    def gradient_descent(self, X, y):\n",
    "        m, n = X.shape\n",
    "\n",
    "        dz = self.sigmoid(X.dot(self.W) + self.b) - y\n",
    "        dw = np.dot(X.T, dz) / m\n",
    "        db = np.sum(dz) / m\n",
    "\n",
    "        return dw, db\n",
    "\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        n_samples, n_features = X.shape\n",
    "\n",
    "        self.W = np.random.randn(n_features) * 0.01\n",
    "        self.b = 0\n",
    "        previous_loss = np.inf\n",
    "        lr = self.learning_rate\n",
    "\n",
    "        for _ in range(self.max_iterations):\n",
    "            # Calculate Gradients\n",
    "            dw, db = self.gradient_descent(X, y)\n",
    "\n",
    "            # Update Parameters\n",
    "            self.W -= lr * dw\n",
    "            self.b -= lr * db\n",
    "\n",
    "            # Check for Early Stop\n",
    "            current_loss = self.binary_cross_entropy(self.sigmoid(X.dot(self.W) + self.b), y)\n",
    "            if np.abs(current_loss - previous_loss) < self.epsilon:\n",
    "                print(f'Stopping Train at Iteration No.{_}')\n",
    "                break\n",
    "\n",
    "            previous_loss = current_loss\n",
    "            lr *= decay\n",
    "\n",
    "        print(f'Training Complete with Binary Cross Entropy Loss of {current_loss}')\n",
    "\n",
    "        \n",
    "    def predict(self, X):\n",
    "        y_pred = self.sigmoid(np.dot(X, self.W) + self.b)\n",
    "        return (y_pred >= 0.5).astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ed996bb",
   "metadata": {
    "papermill": {
     "duration": 0.017421,
     "end_time": "2023-11-14T13:14:07.279532",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.262111",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Check different hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7b9adc72",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:07.317032Z",
     "iopub.status.busy": "2023-11-14T13:14:07.316569Z",
     "iopub.status.idle": "2023-11-14T13:14:12.988571Z",
     "shell.execute_reply": "2023-11-14T13:14:12.987355Z"
    },
    "papermill": {
     "duration": 5.694393,
     "end_time": "2023-11-14T13:14:12.991893",
     "exception": false,
     "start_time": "2023-11-14T13:14:07.297500",
     "status": "completed"
    },
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6212992744007927\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6209677619300583\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6001372062288594\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6017874075594064\n",
      "Stopping Train at Iteration No.63\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5693204109683926\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5698277025488624\n",
      "Stopping Train at Iteration No.77\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5545112951630043\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5549212866012389\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5207231679463089\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5195083231682462\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4559281265985801\n",
      "Training Complete with Binary Cross Entropy Loss of 0.455849842292999\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5003827546806539\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5006584145545725\n",
      "Stopping Train at Iteration No.42\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4827236454897319\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4831637691782477\n",
      "Stopping Train at Iteration No.57\n",
      "Training Complete with Binary Cross Entropy Loss of 0.46393905139515534\n",
      "Training Complete with Binary Cross Entropy Loss of 0.46389779811158277\n",
      "Stopping Train at Iteration No.68\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4559679684906452\n",
      "Training Complete with Binary Cross Entropy Loss of 0.455400225637551\n",
      "Stopping Train at Iteration No.90\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44332620567767483\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44331450123373817\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43347230529869957\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4334926839867056\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.46201865053991553\n",
      "Stopping Train at Iteration No.82\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4626561104943473\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45250566321669994\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45226330664554254\n",
      "Stopping Train at Iteration No.50\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4427882458395703\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44261125728762407\n",
      "Stopping Train at Iteration No.56\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4392155832705415\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4391463150866018\n",
      "Stopping Train at Iteration No.66\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4350874110124317\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4350179912788508\n",
      "Stopping Train at Iteration No.53\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43312266999621196\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43298149212910947\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4479298362623465\n",
      "Stopping Train at Iteration No.80\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4479006103960231\n",
      "Stopping Train at Iteration No.34\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4420023781060384\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4420862402737426\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.436906622225448\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4369059038751448\n",
      "Stopping Train at Iteration No.47\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4353504004852017\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43522382706718604\n",
      "Stopping Train at Iteration No.50\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4336880960127242\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4335536751796877\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43304662549769624\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295693708208893\n",
      "Stopping Train at Iteration No.26\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4413232623157827\n",
      "Stopping Train at Iteration No.78\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4414058443601903\n",
      "Stopping Train at Iteration No.31\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4377326552762104\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4378427149232442\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4348060402038567\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4347201207725141\n",
      "Stopping Train at Iteration No.39\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43404836764453336\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4339524762912135\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43327669924121603\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43314382080078806\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330228965649949\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4329553401386834\n",
      "Stopping Train at Iteration No.24\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4378724665376868\n",
      "Stopping Train at Iteration No.75\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4378017913101622\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43558863792788227\n",
      "Stopping Train at Iteration No.99\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4355659342221326\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4339217192330194\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4338717423946575\n",
      "Stopping Train at Iteration No.33\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4334956801166812\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4334045889195708\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4331421518559648\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43303092526700876\n",
      "Stopping Train at Iteration No.23\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43300365953056047\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295523597972185\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.622683355010724\n",
      "Stopping Train at Iteration No.83\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6130890124261846\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5947031005848855\n",
      "Stopping Train at Iteration No.114\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5964311303480692\n",
      "Stopping Train at Iteration No.63\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5709863658848607\n",
      "Stopping Train at Iteration No.172\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5693606879052464\n",
      "Stopping Train at Iteration No.77\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5535240547423309\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5558637544382133\n",
      "Stopping Train at Iteration No.115\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5198667812549347\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5190341598477988\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4467670378670714\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44660354260991153\n",
      "Stopping Train at Iteration No.33\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5025303695585599\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.501831380263101\n",
      "Stopping Train at Iteration No.42\n",
      "Training Complete with Binary Cross Entropy Loss of 0.48426120237739273\n",
      "Stopping Train at Iteration No.112\n",
      "Training Complete with Binary Cross Entropy Loss of 0.483081338296689\n",
      "Stopping Train at Iteration No.58\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4642523860761923\n",
      "Stopping Train at Iteration No.167\n",
      "Training Complete with Binary Cross Entropy Loss of 0.46386236487207766\n",
      "Stopping Train at Iteration No.67\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45552983843763073\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4554590491205619\n",
      "Stopping Train at Iteration No.90\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44340580151595776\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4430940123026416\n",
      "Stopping Train at Iteration No.102\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43346042432138504\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43310738003377913\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4617918423912261\n",
      "Stopping Train at Iteration No.82\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4620941004313246\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4522257267949948\n",
      "Stopping Train at Iteration No.109\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45191704300177066\n",
      "Stopping Train at Iteration No.50\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4426426295133558\n",
      "Stopping Train at Iteration No.159\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44264831534603616\n",
      "Stopping Train at Iteration No.56\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43925846881180597\n",
      "Stopping Train at Iteration No.194\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4390112329933019\n",
      "Stopping Train at Iteration No.66\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4351384585581293\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4349257222907822\n",
      "Stopping Train at Iteration No.54\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43311855210338224\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295882904961724\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4477345137207454\n",
      "Stopping Train at Iteration No.80\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44791953215064073\n",
      "Stopping Train at Iteration No.34\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44210942153073546\n",
      "Stopping Train at Iteration No.105\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4419408351219188\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4369430467252515\n",
      "Stopping Train at Iteration No.152\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4368606092630613\n",
      "Stopping Train at Iteration No.47\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4353320922408286\n",
      "Stopping Train at Iteration No.184\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43517588636853843\n",
      "Stopping Train at Iteration No.49\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4336648131143973\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4335340419518669\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.433045599580836\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295533189129476\n",
      "Stopping Train at Iteration No.26\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44140219041379614\n",
      "Stopping Train at Iteration No.77\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4411854077657563\n",
      "Stopping Train at Iteration No.31\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43772275488589263\n",
      "Stopping Train at Iteration No.102\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43772211439077213\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4348380998408407\n",
      "Stopping Train at Iteration No.146\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4346913315754739\n",
      "Stopping Train at Iteration No.39\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43405756496572906\n",
      "Stopping Train at Iteration No.175\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4338832974238107\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43329923625739236\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4331424937307516\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330213965153467\n",
      "Stopping Train at Iteration No.188\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4329552320641141\n",
      "Stopping Train at Iteration No.24\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43773403259415067\n",
      "Stopping Train at Iteration No.75\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43783085895165075\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43554155056903854\n",
      "Stopping Train at Iteration No.99\n",
      "Training Complete with Binary Cross Entropy Loss of 0.435594840791458\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.433986371227768\n",
      "Stopping Train at Iteration No.140\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4338413532828708\n",
      "Stopping Train at Iteration No.33\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4335064389757555\n",
      "Stopping Train at Iteration No.168\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4334000623606786\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43313815181571563\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330219310920676\n",
      "Stopping Train at Iteration No.23\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330028414853435\n",
      "Stopping Train at Iteration No.123\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295522965315814\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6195258590584725\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6156398485665713\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5980596839094412\n",
      "Stopping Train at Iteration No.114\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5977571548964139\n",
      "Stopping Train at Iteration No.63\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5721803927234118\n",
      "Stopping Train at Iteration No.172\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5706820628247533\n",
      "Stopping Train at Iteration No.77\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5531857621202615\n",
      "Stopping Train at Iteration No.216\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5553878340887309\n",
      "Stopping Train at Iteration No.115\n",
      "Training Complete with Binary Cross Entropy Loss of 0.519048944457193\n",
      "Stopping Train at Iteration No.340\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5197680466882483\n",
      "Stopping Train at Iteration No.308\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44425822528071796\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4437047319426781\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.500571198593769\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5002185110205654\n",
      "Stopping Train at Iteration No.42\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4834077569085967\n",
      "Stopping Train at Iteration No.112\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4829929180012733\n",
      "Stopping Train at Iteration No.57\n",
      "Training Complete with Binary Cross Entropy Loss of 0.46401002278773545\n",
      "Stopping Train at Iteration No.167\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4635751378279453\n",
      "Stopping Train at Iteration No.68\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4560438737922529\n",
      "Stopping Train at Iteration No.206\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45555619217555665\n",
      "Stopping Train at Iteration No.90\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44339799290053544\n",
      "Stopping Train at Iteration No.314\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44310889609366827\n",
      "Stopping Train at Iteration No.101\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43346140216114376\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330442280629872\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4621052293304421\n",
      "Stopping Train at Iteration No.82\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4621322957168719\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4523672049423274\n",
      "Stopping Train at Iteration No.109\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45204849931555885\n",
      "Stopping Train at Iteration No.50\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44273651147377574\n",
      "Stopping Train at Iteration No.159\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4425692630908196\n",
      "Stopping Train at Iteration No.56\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4392358007200975\n",
      "Stopping Train at Iteration No.194\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4391410484157233\n",
      "Stopping Train at Iteration No.67\n",
      "Training Complete with Binary Cross Entropy Loss of 0.435230909001224\n",
      "Stopping Train at Iteration No.289\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43497429173255314\n",
      "Stopping Train at Iteration No.53\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4331222150215868\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4329564784360738\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4480883949297473\n",
      "Stopping Train at Iteration No.79\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4477797452605411\n",
      "Stopping Train at Iteration No.34\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44221083980457304\n",
      "Stopping Train at Iteration No.105\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44212269124781217\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4369316539192205\n",
      "Stopping Train at Iteration No.152\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4368846515804712\n",
      "Stopping Train at Iteration No.47\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4353185596633927\n",
      "Stopping Train at Iteration No.184\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4352211493074038\n",
      "Stopping Train at Iteration No.49\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4336873541823303\n",
      "Stopping Train at Iteration No.268\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4335303733791174\n",
      "Stopping Train at Iteration No.36\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330507418636923\n",
      "Stopping Train at Iteration No.348\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295525534707635\n",
      "Stopping Train at Iteration No.26\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44126592416755966\n",
      "Stopping Train at Iteration No.77\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44121095280614275\n",
      "Stopping Train at Iteration No.31\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43766252126639243\n",
      "Stopping Train at Iteration No.102\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43773543970818496\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4348403996964196\n",
      "Stopping Train at Iteration No.146\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43470322830075414\n",
      "Stopping Train at Iteration No.39\n",
      "Training Complete with Binary Cross Entropy Loss of 0.434004357542923\n",
      "Stopping Train at Iteration No.176\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4339503503887625\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43328303745257285\n",
      "Stopping Train at Iteration No.250\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4331404634645201\n",
      "Stopping Train at Iteration No.29\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330153124343305\n",
      "Stopping Train at Iteration No.185\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295523191957835\n",
      "Stopping Train at Iteration No.24\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43783367243901805\n",
      "Stopping Train at Iteration No.75\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4378093139903431\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43556211858653426\n",
      "Stopping Train at Iteration No.99\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4355422655606168\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4339190849503596\n",
      "Stopping Train at Iteration No.140\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4338204324808735\n",
      "Stopping Train at Iteration No.33\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4335118399463666\n",
      "Stopping Train at Iteration No.168\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43342305871190323\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43314665216773235\n",
      "Stopping Train at Iteration No.235\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330302238360066\n",
      "Stopping Train at Iteration No.23\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4330022073743798\n",
      "Stopping Train at Iteration No.122\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295522965991345\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.621625776823013\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6217208487035165\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5978986908600589\n",
      "Stopping Train at Iteration No.114\n",
      "Training Complete with Binary Cross Entropy Loss of 0.6003803090590991\n",
      "Stopping Train at Iteration No.63\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5702732461139712\n",
      "Stopping Train at Iteration No.173\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5728929902420818\n",
      "Stopping Train at Iteration No.78\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5557980834017933\n",
      "Stopping Train at Iteration No.215\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5538058643764264\n",
      "Stopping Train at Iteration No.116\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5206462259653301\n",
      "Stopping Train at Iteration No.340\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5191761403065984\n",
      "Stopping Train at Iteration No.309\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44454903816495206\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4435130177861828\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5011342034806971\n",
      "Stopping Train at Iteration No.84\n",
      "Training Complete with Binary Cross Entropy Loss of 0.5003833409232853\n",
      "Stopping Train at Iteration No.42\n",
      "Training Complete with Binary Cross Entropy Loss of 0.483595256620189\n",
      "Stopping Train at Iteration No.113\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4836497412945508\n",
      "Stopping Train at Iteration No.57\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4638831528355323\n",
      "Stopping Train at Iteration No.167\n",
      "Training Complete with Binary Cross Entropy Loss of 0.46402719845220924\n",
      "Stopping Train at Iteration No.68\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45582165957851367\n",
      "Stopping Train at Iteration No.206\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45577432061693635\n",
      "Stopping Train at Iteration No.90\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4434538311970672\n",
      "Stopping Train at Iteration No.314\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44303704981846154\n",
      "Stopping Train at Iteration No.102\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4334587243186227\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43304196970667735\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4619604843991491\n",
      "Stopping Train at Iteration No.82\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4625586897623669\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4523240951725632\n",
      "Stopping Train at Iteration No.109\n",
      "Training Complete with Binary Cross Entropy Loss of 0.45216400507665766\n",
      "Stopping Train at Iteration No.50\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4426214361852995\n",
      "Stopping Train at Iteration No.159\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4426239821899084\n",
      "Stopping Train at Iteration No.56\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4392362723910986\n",
      "Stopping Train at Iteration No.194\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4392044321503541\n",
      "Stopping Train at Iteration No.66\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43509666772412914\n",
      "Stopping Train at Iteration No.289\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4350084818349904\n",
      "Stopping Train at Iteration No.54\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4331184322598678\n",
      "Stopping Train at Iteration No.690\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295650265520796\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4478800823842079\n",
      "Stopping Train at Iteration No.79\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4476356904985788\n",
      "Stopping Train at Iteration No.34\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44208295537877323\n",
      "Stopping Train at Iteration No.105\n",
      "Training Complete with Binary Cross Entropy Loss of 0.44184856824303104\n",
      "Stopping Train at Iteration No.43\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4369702067135296\n",
      "Stopping Train at Iteration No.152\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43683805272221427\n",
      "Stopping Train at Iteration No.47\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43529041252017436\n",
      "Stopping Train at Iteration No.184\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43527088367204725\n",
      "Stopping Train at Iteration No.50\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43371327311735053\n",
      "Stopping Train at Iteration No.268\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43353548380387996\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43304990707072066\n",
      "Stopping Train at Iteration No.350\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295525611746916\n",
      "Stopping Train at Iteration No.26\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4412121417142419\n",
      "Stopping Train at Iteration No.77\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4412279146870102\n",
      "Stopping Train at Iteration No.31\n",
      "Training Complete with Binary Cross Entropy Loss of 0.437645348386913\n",
      "Stopping Train at Iteration No.102\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43755690121731766\n",
      "Stopping Train at Iteration No.37\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4348069724223125\n",
      "Stopping Train at Iteration No.146\n",
      "Training Complete with Binary Cross Entropy Loss of 0.434727435750516\n",
      "Stopping Train at Iteration No.39\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4340142494990848\n",
      "Stopping Train at Iteration No.176\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4339109940152365\n",
      "Stopping Train at Iteration No.38\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4332803128564345\n",
      "Stopping Train at Iteration No.251\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4331583773620738\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43301975063348136\n",
      "Stopping Train at Iteration No.184\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4329552319801215\n",
      "Stopping Train at Iteration No.24\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4378858154713396\n",
      "Stopping Train at Iteration No.75\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4378455977171986\n",
      "Stopping Train at Iteration No.28\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4355559852283621\n",
      "Stopping Train at Iteration No.99\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43551164589299146\n",
      "Stopping Train at Iteration No.32\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4339369124223307\n",
      "Stopping Train at Iteration No.140\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4338391233602963\n",
      "Stopping Train at Iteration No.33\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43351042167956444\n",
      "Stopping Train at Iteration No.168\n",
      "Training Complete with Binary Cross Entropy Loss of 0.433398676792067\n",
      "Stopping Train at Iteration No.30\n",
      "Training Complete with Binary Cross Entropy Loss of 0.433135919190874\n",
      "Stopping Train at Iteration No.234\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43302910610121204\n",
      "Stopping Train at Iteration No.23\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43300422584430254\n",
      "Stopping Train at Iteration No.123\n",
      "Training Complete with Binary Cross Entropy Loss of 0.43295522964336186\n",
      "Best Score: 0.4259932595849838\n",
      "Best Hyperparameters: (100, 2.5, 0.8, 1e-05)\n"
     ]
    }
   ],
   "source": [
    "max_iterations_values = [100, 200, 500, 1000]\n",
    "learning_rate_values = [0.1, 0.5, 1.0, 1.5, 2.0, 2.5]\n",
    "decay_values = [0.8, 0.85, 0.9, 0.92, 0.95, 0.99]\n",
    "epsilon_values = [1e-5, 1e-10]\n",
    "\n",
    "best_score = float('inf')\n",
    "best_hyperparameters = None\n",
    "\n",
    "for max_iterations in max_iterations_values:\n",
    "    for learning_rate in learning_rate_values:\n",
    "        for decay in decay_values:\n",
    "            for epsilon in epsilon_values:\n",
    "                lr_model_test = LogisticRegression(max_iterations, learning_rate, decay, epsilon)\n",
    "                lr_model_test.fit(X_train, Y_train)\n",
    "\n",
    "                # Evaluate the model on the validation set\n",
    "                score = lr_model_test.binary_cross_entropy(lr_model_test.sigmoid(X_val.dot(lr_model_test.W) + lr_model_test.b), Y_val)\n",
    "\n",
    "                # Check if the current combination of hyperparameters achieved a better score\n",
    "                if score < best_score:\n",
    "                    best_score = score\n",
    "                    best_hyperparameters = (max_iterations, learning_rate, decay, epsilon)\n",
    "\n",
    "print(f\"Best Score: {best_score}\")\n",
    "print(f\"Best Hyperparameters: {best_hyperparameters}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae677c38",
   "metadata": {
    "papermill": {
     "duration": 0.02115,
     "end_time": "2023-11-14T13:14:13.034317",
     "exception": false,
     "start_time": "2023-11-14T13:14:13.013167",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Train model with best hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "39b3123a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:13.079657Z",
     "iopub.status.busy": "2023-11-14T13:14:13.079261Z",
     "iopub.status.idle": "2023-11-14T13:14:13.085636Z",
     "shell.execute_reply": "2023-11-14T13:14:13.084331Z"
    },
    "papermill": {
     "duration": 0.031881,
     "end_time": "2023-11-14T13:14:13.089056",
     "exception": false,
     "start_time": "2023-11-14T13:14:13.057175",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hyper Parameters: (100, 2.5, 0.8, 1e-05)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Hyper Parameters: {best_hyperparameters}\")\n",
    "max_iterations, learning_rate, decay, epsilon = best_hyperparameters\n",
    "\n",
    "model = LogisticRegression(max_iterations, learning_rate, decay, epsilon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b60a9991",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:13.133402Z",
     "iopub.status.busy": "2023-11-14T13:14:13.132960Z",
     "iopub.status.idle": "2023-11-14T13:14:13.145896Z",
     "shell.execute_reply": "2023-11-14T13:14:13.144829Z"
    },
    "papermill": {
     "duration": 0.03819,
     "end_time": "2023-11-14T13:14:13.148337",
     "exception": false,
     "start_time": "2023-11-14T13:14:13.110147",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping Train at Iteration No.24\n",
      "Training Complete with Binary Cross Entropy Loss of 0.4377901823247131\n"
     ]
    }
   ],
   "source": [
    "model.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "200ad70f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:13.192279Z",
     "iopub.status.busy": "2023-11-14T13:14:13.191581Z",
     "iopub.status.idle": "2023-11-14T13:14:13.196796Z",
     "shell.execute_reply": "2023-11-14T13:14:13.195599Z"
    },
    "papermill": {
     "duration": 0.030715,
     "end_time": "2023-11-14T13:14:13.199828",
     "exception": false,
     "start_time": "2023-11-14T13:14:13.169113",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "Y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2eb13f8",
   "metadata": {
    "papermill": {
     "duration": 0.020344,
     "end_time": "2023-11-14T13:14:13.241141",
     "exception": false,
     "start_time": "2023-11-14T13:14:13.220797",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Save Results as CSV file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "86794016",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-14T13:14:13.285036Z",
     "iopub.status.busy": "2023-11-14T13:14:13.284617Z",
     "iopub.status.idle": "2023-11-14T13:14:13.295110Z",
     "shell.execute_reply": "2023-11-14T13:14:13.293988Z"
    },
    "papermill": {
     "duration": 0.03647,
     "end_time": "2023-11-14T13:14:13.298312",
     "exception": false,
     "start_time": "2023-11-14T13:14:13.261842",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "result = pd.DataFrame({'PassengerId': testPassengerIds, 'Survived': Y_pred})\n",
    "\n",
    "result.to_csv('output.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 12.889337,
   "end_time": "2023-11-14T13:14:13.842629",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-11-14T13:14:00.953292",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}