stage_C_quiz.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "stage_C_quiz.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "oMz7z2bt3pm9",
        "1RoOROAt3pnD",
        "KKiaWvUF3pnD",
        "18ctXoAm3pnE",
        "RV2hktLm3pnK",
        "zSWVbA0m3pnK",
        "3BQLK4R03pnK",
        "1wTV-c6C3pnL",
        "uhnPGsX53pnL",
        "8QhFlvKZ3pnL",
        "K5pOgqxA3pnM",
        "0GDepYf53pnN",
        "WA62iVXy3pnN",
        "l64fNY3K3pnO"
      ]
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "adMROImh3pmt"
      },
      "source": [
        "# For data wrangling\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "# For data preprocessing\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# For modelling\n",
        "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n",
        "from xgboost import XGBClassifier\n",
        "from lightgbm import LGBMClassifier\n",
        "\n",
        "# For model evaluation\n",
        "from sklearn.metrics import accuracy_score, recall_score, f1_score"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQb7iy1A3pmu"
      },
      "source": [
        "url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00471/Data_for_UCI_named.csv'\n",
        "\n",
        "dataset_df = pd.read_csv(url)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "BzfK7zHs3pmv",
        "outputId": "7833e5d2-b49f-4e25-8030-35143b705291"
      },
      "source": [
        "dataset_df.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tau1</th>\n",
              "      <th>tau2</th>\n",
              "      <th>tau3</th>\n",
              "      <th>tau4</th>\n",
              "      <th>p1</th>\n",
              "      <th>p2</th>\n",
              "      <th>p3</th>\n",
              "      <th>p4</th>\n",
              "      <th>g1</th>\n",
              "      <th>g2</th>\n",
              "      <th>g3</th>\n",
              "      <th>g4</th>\n",
              "      <th>stab</th>\n",
              "      <th>stabf</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2.959060</td>\n",
              "      <td>3.079885</td>\n",
              "      <td>8.381025</td>\n",
              "      <td>9.780754</td>\n",
              "      <td>3.763085</td>\n",
              "      <td>-0.782604</td>\n",
              "      <td>-1.257395</td>\n",
              "      <td>-1.723086</td>\n",
              "      <td>0.650456</td>\n",
              "      <td>0.859578</td>\n",
              "      <td>0.887445</td>\n",
              "      <td>0.958034</td>\n",
              "      <td>0.055347</td>\n",
              "      <td>unstable</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9.304097</td>\n",
              "      <td>4.902524</td>\n",
              "      <td>3.047541</td>\n",
              "      <td>1.369357</td>\n",
              "      <td>5.067812</td>\n",
              "      <td>-1.940058</td>\n",
              "      <td>-1.872742</td>\n",
              "      <td>-1.255012</td>\n",
              "      <td>0.413441</td>\n",
              "      <td>0.862414</td>\n",
              "      <td>0.562139</td>\n",
              "      <td>0.781760</td>\n",
              "      <td>-0.005957</td>\n",
              "      <td>stable</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8.971707</td>\n",
              "      <td>8.848428</td>\n",
              "      <td>3.046479</td>\n",
              "      <td>1.214518</td>\n",
              "      <td>3.405158</td>\n",
              "      <td>-1.207456</td>\n",
              "      <td>-1.277210</td>\n",
              "      <td>-0.920492</td>\n",
              "      <td>0.163041</td>\n",
              "      <td>0.766689</td>\n",
              "      <td>0.839444</td>\n",
              "      <td>0.109853</td>\n",
              "      <td>0.003471</td>\n",
              "      <td>unstable</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.716415</td>\n",
              "      <td>7.669600</td>\n",
              "      <td>4.486641</td>\n",
              "      <td>2.340563</td>\n",
              "      <td>3.963791</td>\n",
              "      <td>-1.027473</td>\n",
              "      <td>-1.938944</td>\n",
              "      <td>-0.997374</td>\n",
              "      <td>0.446209</td>\n",
              "      <td>0.976744</td>\n",
              "      <td>0.929381</td>\n",
              "      <td>0.362718</td>\n",
              "      <td>0.028871</td>\n",
              "      <td>unstable</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3.134112</td>\n",
              "      <td>7.608772</td>\n",
              "      <td>4.943759</td>\n",
              "      <td>9.857573</td>\n",
              "      <td>3.525811</td>\n",
              "      <td>-1.125531</td>\n",
              "      <td>-1.845975</td>\n",
              "      <td>-0.554305</td>\n",
              "      <td>0.797110</td>\n",
              "      <td>0.455450</td>\n",
              "      <td>0.656947</td>\n",
              "      <td>0.820923</td>\n",
              "      <td>0.049860</td>\n",
              "      <td>unstable</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       tau1      tau2      tau3  ...        g4      stab     stabf\n",
              "0  2.959060  3.079885  8.381025  ...  0.958034  0.055347  unstable\n",
              "1  9.304097  4.902524  3.047541  ...  0.781760 -0.005957    stable\n",
              "2  8.971707  8.848428  3.046479  ...  0.109853  0.003471  unstable\n",
              "3  0.716415  7.669600  4.486641  ...  0.362718  0.028871  unstable\n",
              "4  3.134112  7.608772  4.943759  ...  0.820923  0.049860  unstable\n",
              "\n",
              "[5 rows x 14 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jBO6hrIJ3pmw",
        "outputId": "a539f394-266b-4724-ee82-527a246ed4f9"
      },
      "source": [
        "dataset_df.info()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 10000 entries, 0 to 9999\n",
            "Data columns (total 14 columns):\n",
            " #   Column  Non-Null Count  Dtype  \n",
            "---  ------  --------------  -----  \n",
            " 0   tau1    10000 non-null  float64\n",
            " 1   tau2    10000 non-null  float64\n",
            " 2   tau3    10000 non-null  float64\n",
            " 3   tau4    10000 non-null  float64\n",
            " 4   p1      10000 non-null  float64\n",
            " 5   p2      10000 non-null  float64\n",
            " 6   p3      10000 non-null  float64\n",
            " 7   p4      10000 non-null  float64\n",
            " 8   g1      10000 non-null  float64\n",
            " 9   g2      10000 non-null  float64\n",
            " 10  g3      10000 non-null  float64\n",
            " 11  g4      10000 non-null  float64\n",
            " 12  stab    10000 non-null  float64\n",
            " 13  stabf   10000 non-null  object \n",
            "dtypes: float64(13), object(1)\n",
            "memory usage: 1.1+ MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I62UjGE83pmw",
        "outputId": "813e19cb-2440-4c32-d07c-adad661d0be4"
      },
      "source": [
        "# Checking for missing values\n",
        "dataset_df.isnull().sum()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tau1     0\n",
              "tau2     0\n",
              "tau3     0\n",
              "tau4     0\n",
              "p1       0\n",
              "p2       0\n",
              "p3       0\n",
              "p4       0\n",
              "g1       0\n",
              "g2       0\n",
              "g3       0\n",
              "g4       0\n",
              "stab     0\n",
              "stabf    0\n",
              "dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 317
        },
        "id": "HLqHbjn03pmx",
        "outputId": "4b8597c6-6808-45f9-e599-c22539c086f5"
      },
      "source": [
        "# Checking for the summary statistics of the dataset\n",
        "dataset_df.describe()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tau1</th>\n",
              "      <th>tau2</th>\n",
              "      <th>tau3</th>\n",
              "      <th>tau4</th>\n",
              "      <th>p1</th>\n",
              "      <th>p2</th>\n",
              "      <th>p3</th>\n",
              "      <th>p4</th>\n",
              "      <th>g1</th>\n",
              "      <th>g2</th>\n",
              "      <th>g3</th>\n",
              "      <th>g4</th>\n",
              "      <th>stab</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "      <td>10000.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>5.250000</td>\n",
              "      <td>5.250001</td>\n",
              "      <td>5.250004</td>\n",
              "      <td>5.249997</td>\n",
              "      <td>3.750000</td>\n",
              "      <td>-1.250000</td>\n",
              "      <td>-1.250000</td>\n",
              "      <td>-1.250000</td>\n",
              "      <td>0.525000</td>\n",
              "      <td>0.525000</td>\n",
              "      <td>0.525000</td>\n",
              "      <td>0.525000</td>\n",
              "      <td>0.015731</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.742548</td>\n",
              "      <td>2.742549</td>\n",
              "      <td>2.742549</td>\n",
              "      <td>2.742556</td>\n",
              "      <td>0.752160</td>\n",
              "      <td>0.433035</td>\n",
              "      <td>0.433035</td>\n",
              "      <td>0.433035</td>\n",
              "      <td>0.274256</td>\n",
              "      <td>0.274255</td>\n",
              "      <td>0.274255</td>\n",
              "      <td>0.274255</td>\n",
              "      <td>0.036919</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>0.500793</td>\n",
              "      <td>0.500141</td>\n",
              "      <td>0.500788</td>\n",
              "      <td>0.500473</td>\n",
              "      <td>1.582590</td>\n",
              "      <td>-1.999891</td>\n",
              "      <td>-1.999945</td>\n",
              "      <td>-1.999926</td>\n",
              "      <td>0.050009</td>\n",
              "      <td>0.050053</td>\n",
              "      <td>0.050054</td>\n",
              "      <td>0.050028</td>\n",
              "      <td>-0.080760</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25%</th>\n",
              "      <td>2.874892</td>\n",
              "      <td>2.875140</td>\n",
              "      <td>2.875522</td>\n",
              "      <td>2.874950</td>\n",
              "      <td>3.218300</td>\n",
              "      <td>-1.624901</td>\n",
              "      <td>-1.625025</td>\n",
              "      <td>-1.624960</td>\n",
              "      <td>0.287521</td>\n",
              "      <td>0.287552</td>\n",
              "      <td>0.287514</td>\n",
              "      <td>0.287494</td>\n",
              "      <td>-0.015557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>5.250004</td>\n",
              "      <td>5.249981</td>\n",
              "      <td>5.249979</td>\n",
              "      <td>5.249734</td>\n",
              "      <td>3.751025</td>\n",
              "      <td>-1.249966</td>\n",
              "      <td>-1.249974</td>\n",
              "      <td>-1.250007</td>\n",
              "      <td>0.525009</td>\n",
              "      <td>0.525003</td>\n",
              "      <td>0.525015</td>\n",
              "      <td>0.525002</td>\n",
              "      <td>0.017142</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>75%</th>\n",
              "      <td>7.624690</td>\n",
              "      <td>7.624893</td>\n",
              "      <td>7.624948</td>\n",
              "      <td>7.624838</td>\n",
              "      <td>4.282420</td>\n",
              "      <td>-0.874977</td>\n",
              "      <td>-0.875043</td>\n",
              "      <td>-0.875065</td>\n",
              "      <td>0.762435</td>\n",
              "      <td>0.762490</td>\n",
              "      <td>0.762440</td>\n",
              "      <td>0.762433</td>\n",
              "      <td>0.044878</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>9.999469</td>\n",
              "      <td>9.999837</td>\n",
              "      <td>9.999450</td>\n",
              "      <td>9.999443</td>\n",
              "      <td>5.864418</td>\n",
              "      <td>-0.500108</td>\n",
              "      <td>-0.500072</td>\n",
              "      <td>-0.500025</td>\n",
              "      <td>0.999937</td>\n",
              "      <td>0.999944</td>\n",
              "      <td>0.999982</td>\n",
              "      <td>0.999930</td>\n",
              "      <td>0.109403</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "               tau1          tau2  ...            g4          stab\n",
              "count  10000.000000  10000.000000  ...  10000.000000  10000.000000\n",
              "mean       5.250000      5.250001  ...      0.525000      0.015731\n",
              "std        2.742548      2.742549  ...      0.274255      0.036919\n",
              "min        0.500793      0.500141  ...      0.050028     -0.080760\n",
              "25%        2.874892      2.875140  ...      0.287494     -0.015557\n",
              "50%        5.250004      5.249981  ...      0.525002      0.017142\n",
              "75%        7.624690      7.624893  ...      0.762433      0.044878\n",
              "max        9.999469      9.999837  ...      0.999930      0.109403\n",
              "\n",
              "[8 rows x 13 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GrE65m-V3pmy"
      },
      "source": [
        "### Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "osP01RfY3pm3",
        "outputId": "35bf3cc8-d4f1-4e5a-e854-95e584d32110"
      },
      "source": [
        "# Checking for imbalance in the target variable\n",
        "\n",
        "dataset_df.stabf.value_counts()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "unstable    6380\n",
              "stable      3620\n",
              "Name: stabf, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "phU3zut_3pm5"
      },
      "source": [
        "The distribution above is imbalanced. To be on a common ground for the quiz, the imbalance will not be handled. The imbalance will be handled using undersample method."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rsW2vsXZ3pm6"
      },
      "source": [
        "# Splitting the dataset into target and features\n",
        "\n",
        "features = dataset_df.drop(columns=['stab', 'stabf'])\n",
        "target = dataset_df['stabf']"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qHEpJuc93pm6"
      },
      "source": [
        "# Splitting the dataset into training and testing sets\n",
        "x_train, x_test, y_train, y_test = train_test_split(features, target, test_size=0.2, random_state=1)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "wD_ModtR3pm7",
        "outputId": "6fb80cc1-5bb0-4cae-f676-6611452ff451"
      },
      "source": [
        "x_train.head()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tau1</th>\n",
              "      <th>tau2</th>\n",
              "      <th>tau3</th>\n",
              "      <th>tau4</th>\n",
              "      <th>p1</th>\n",
              "      <th>p2</th>\n",
              "      <th>p3</th>\n",
              "      <th>p4</th>\n",
              "      <th>g1</th>\n",
              "      <th>g2</th>\n",
              "      <th>g3</th>\n",
              "      <th>g4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2694</th>\n",
              "      <td>6.255995</td>\n",
              "      <td>2.542401</td>\n",
              "      <td>7.024714</td>\n",
              "      <td>9.476518</td>\n",
              "      <td>3.529888</td>\n",
              "      <td>-1.224881</td>\n",
              "      <td>-0.688228</td>\n",
              "      <td>-1.616780</td>\n",
              "      <td>0.568221</td>\n",
              "      <td>0.618403</td>\n",
              "      <td>0.685739</td>\n",
              "      <td>0.660088</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5140</th>\n",
              "      <td>5.070581</td>\n",
              "      <td>5.490253</td>\n",
              "      <td>8.075688</td>\n",
              "      <td>0.761075</td>\n",
              "      <td>4.220888</td>\n",
              "      <td>-1.280596</td>\n",
              "      <td>-1.902185</td>\n",
              "      <td>-1.038107</td>\n",
              "      <td>0.443515</td>\n",
              "      <td>0.097244</td>\n",
              "      <td>0.916955</td>\n",
              "      <td>0.129254</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2568</th>\n",
              "      <td>1.220072</td>\n",
              "      <td>8.804028</td>\n",
              "      <td>3.874283</td>\n",
              "      <td>8.433949</td>\n",
              "      <td>3.614027</td>\n",
              "      <td>-1.039236</td>\n",
              "      <td>-0.953566</td>\n",
              "      <td>-1.621224</td>\n",
              "      <td>0.908353</td>\n",
              "      <td>0.923594</td>\n",
              "      <td>0.238881</td>\n",
              "      <td>0.660156</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3671</th>\n",
              "      <td>7.498402</td>\n",
              "      <td>6.697603</td>\n",
              "      <td>8.798626</td>\n",
              "      <td>2.126236</td>\n",
              "      <td>3.134585</td>\n",
              "      <td>-1.581906</td>\n",
              "      <td>-0.589386</td>\n",
              "      <td>-0.963293</td>\n",
              "      <td>0.260826</td>\n",
              "      <td>0.899003</td>\n",
              "      <td>0.964752</td>\n",
              "      <td>0.600598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7427</th>\n",
              "      <td>7.074006</td>\n",
              "      <td>1.337511</td>\n",
              "      <td>6.100756</td>\n",
              "      <td>7.759156</td>\n",
              "      <td>2.526922</td>\n",
              "      <td>-0.922540</td>\n",
              "      <td>-0.632600</td>\n",
              "      <td>-0.971782</td>\n",
              "      <td>0.984580</td>\n",
              "      <td>0.716082</td>\n",
              "      <td>0.836928</td>\n",
              "      <td>0.165162</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          tau1      tau2      tau3  ...        g2        g3        g4\n",
              "2694  6.255995  2.542401  7.024714  ...  0.618403  0.685739  0.660088\n",
              "5140  5.070581  5.490253  8.075688  ...  0.097244  0.916955  0.129254\n",
              "2568  1.220072  8.804028  3.874283  ...  0.923594  0.238881  0.660156\n",
              "3671  7.498402  6.697603  8.798626  ...  0.899003  0.964752  0.600598\n",
              "7427  7.074006  1.337511  6.100756  ...  0.716082  0.836928  0.165162\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Esl14qLG3pm7",
        "outputId": "55257edd-ece5-4bdb-9308-b1520fe9e975"
      },
      "source": [
        "y_train.head()"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2694    unstable\n",
              "5140    unstable\n",
              "2568    unstable\n",
              "3671    unstable\n",
              "7427    unstable\n",
              "Name: stabf, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SWvHzgwq3pm8",
        "outputId": "744f40f8-2167-4352-c46a-40e37688e1af"
      },
      "source": [
        "y_train.value_counts()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "unstable    5092\n",
              "stable      2908\n",
              "Name: stabf, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VOgrT2WK3pm8"
      },
      "source": [
        "There is still imbalance in the distribution. This can be handled by using SMOTE to balance the training data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oMz7z2bt3pm9"
      },
      "source": [
        "### Scaling the Features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3keZs-643pm9"
      },
      "source": [
        "# Instantiating the Standard Scaler\n",
        "\n",
        "scaler = StandardScaler()"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xcF3MESu3pm-"
      },
      "source": [
        "# Applying Standard Scaler on the training set\n",
        "\n",
        "x_train_scaled = pd.DataFrame(scaler.fit_transform(x_train), columns=x_train.columns)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "O8Zh1Dl23pm-",
        "outputId": "456cc421-26f4-4df0-d61c-a24ed5069370"
      },
      "source": [
        "x_train_scaled.head()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tau1</th>\n",
              "      <th>tau2</th>\n",
              "      <th>tau3</th>\n",
              "      <th>tau4</th>\n",
              "      <th>p1</th>\n",
              "      <th>p2</th>\n",
              "      <th>p3</th>\n",
              "      <th>p4</th>\n",
              "      <th>g1</th>\n",
              "      <th>g2</th>\n",
              "      <th>g3</th>\n",
              "      <th>g4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.367327</td>\n",
              "      <td>-0.986042</td>\n",
              "      <td>0.650447</td>\n",
              "      <td>1.547527</td>\n",
              "      <td>-0.291490</td>\n",
              "      <td>0.061535</td>\n",
              "      <td>1.293862</td>\n",
              "      <td>-0.845074</td>\n",
              "      <td>0.160918</td>\n",
              "      <td>0.339859</td>\n",
              "      <td>0.585568</td>\n",
              "      <td>0.492239</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>-0.064659</td>\n",
              "      <td>0.089437</td>\n",
              "      <td>1.035079</td>\n",
              "      <td>-1.641494</td>\n",
              "      <td>0.619865</td>\n",
              "      <td>-0.067235</td>\n",
              "      <td>-1.502925</td>\n",
              "      <td>0.486613</td>\n",
              "      <td>-0.293143</td>\n",
              "      <td>-1.558488</td>\n",
              "      <td>1.429649</td>\n",
              "      <td>-1.443521</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1.467850</td>\n",
              "      <td>1.298418</td>\n",
              "      <td>-0.502536</td>\n",
              "      <td>1.166046</td>\n",
              "      <td>-0.180521</td>\n",
              "      <td>0.490603</td>\n",
              "      <td>0.682560</td>\n",
              "      <td>-0.855302</td>\n",
              "      <td>1.399350</td>\n",
              "      <td>1.451534</td>\n",
              "      <td>-1.045743</td>\n",
              "      <td>0.492489</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.820081</td>\n",
              "      <td>0.529920</td>\n",
              "      <td>1.299657</td>\n",
              "      <td>-1.141975</td>\n",
              "      <td>-0.812854</td>\n",
              "      <td>-0.763632</td>\n",
              "      <td>1.521579</td>\n",
              "      <td>0.658780</td>\n",
              "      <td>-0.958319</td>\n",
              "      <td>1.361958</td>\n",
              "      <td>1.604140</td>\n",
              "      <td>0.275303</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.665424</td>\n",
              "      <td>-1.425627</td>\n",
              "      <td>0.312300</td>\n",
              "      <td>0.919137</td>\n",
              "      <td>-1.614296</td>\n",
              "      <td>0.760315</td>\n",
              "      <td>1.422019</td>\n",
              "      <td>0.639243</td>\n",
              "      <td>1.676895</td>\n",
              "      <td>0.695660</td>\n",
              "      <td>1.137504</td>\n",
              "      <td>-1.312575</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       tau1      tau2      tau3  ...        g2        g3        g4\n",
              "0  0.367327 -0.986042  0.650447  ...  0.339859  0.585568  0.492239\n",
              "1 -0.064659  0.089437  1.035079  ... -1.558488  1.429649 -1.443521\n",
              "2 -1.467850  1.298418 -0.502536  ...  1.451534 -1.045743  0.492489\n",
              "3  0.820081  0.529920  1.299657  ...  1.361958  1.604140  0.275303\n",
              "4  0.665424 -1.425627  0.312300  ...  0.695660  1.137504 -1.312575\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fMuYZ7Re3pm_"
      },
      "source": [
        "# Applying Standard Scaler on the testing set\n",
        "\n",
        "x_test_scaled = pd.DataFrame(scaler.transform(x_test), columns=x_test.columns)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "onBnqskj3pm_",
        "outputId": "28c27491-27c7-4b20-a6a7-88d278750bf7"
      },
      "source": [
        "x_test_scaled.head()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>tau1</th>\n",
              "      <th>tau2</th>\n",
              "      <th>tau3</th>\n",
              "      <th>tau4</th>\n",
              "      <th>p1</th>\n",
              "      <th>p2</th>\n",
              "      <th>p3</th>\n",
              "      <th>p4</th>\n",
              "      <th>g1</th>\n",
              "      <th>g2</th>\n",
              "      <th>g3</th>\n",
              "      <th>g4</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.593951</td>\n",
              "      <td>-0.412733</td>\n",
              "      <td>1.503924</td>\n",
              "      <td>1.116943</td>\n",
              "      <td>0.403423</td>\n",
              "      <td>-1.492971</td>\n",
              "      <td>-0.785033</td>\n",
              "      <td>1.566781</td>\n",
              "      <td>-0.901007</td>\n",
              "      <td>1.167203</td>\n",
              "      <td>-1.507330</td>\n",
              "      <td>1.084726</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.202190</td>\n",
              "      <td>0.374416</td>\n",
              "      <td>-0.188800</td>\n",
              "      <td>-0.522268</td>\n",
              "      <td>-0.225967</td>\n",
              "      <td>-1.058483</td>\n",
              "      <td>0.420047</td>\n",
              "      <td>1.028627</td>\n",
              "      <td>-1.625721</td>\n",
              "      <td>-0.395660</td>\n",
              "      <td>1.414651</td>\n",
              "      <td>1.226011</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>-1.079044</td>\n",
              "      <td>-0.313745</td>\n",
              "      <td>-0.884634</td>\n",
              "      <td>0.017080</td>\n",
              "      <td>-0.943122</td>\n",
              "      <td>0.112653</td>\n",
              "      <td>0.801335</td>\n",
              "      <td>0.733004</td>\n",
              "      <td>1.457108</td>\n",
              "      <td>-1.438495</td>\n",
              "      <td>0.651821</td>\n",
              "      <td>-1.682168</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>-0.083120</td>\n",
              "      <td>-1.107327</td>\n",
              "      <td>0.372805</td>\n",
              "      <td>-1.708152</td>\n",
              "      <td>0.753990</td>\n",
              "      <td>-1.637972</td>\n",
              "      <td>0.403805</td>\n",
              "      <td>-0.088036</td>\n",
              "      <td>0.083322</td>\n",
              "      <td>-1.672322</td>\n",
              "      <td>-0.357714</td>\n",
              "      <td>1.055865</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.873921</td>\n",
              "      <td>1.438466</td>\n",
              "      <td>0.086662</td>\n",
              "      <td>1.715037</td>\n",
              "      <td>-0.153880</td>\n",
              "      <td>-0.007015</td>\n",
              "      <td>-0.197053</td>\n",
              "      <td>0.472315</td>\n",
              "      <td>0.136549</td>\n",
              "      <td>-1.469731</td>\n",
              "      <td>0.956396</td>\n",
              "      <td>-0.819727</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       tau1      tau2      tau3  ...        g2        g3        g4\n",
              "0  0.593951 -0.412733  1.503924  ...  1.167203 -1.507330  1.084726\n",
              "1  0.202190  0.374416 -0.188800  ... -0.395660  1.414651  1.226011\n",
              "2 -1.079044 -0.313745 -0.884634  ... -1.438495  0.651821 -1.682168\n",
              "3 -0.083120 -1.107327  0.372805  ... -1.672322 -0.357714  1.055865\n",
              "4  0.873921  1.438466  0.086662  ... -1.469731  0.956396 -0.819727\n",
              "\n",
              "[5 rows x 12 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBeUZNsK3pm_"
      },
      "source": [
        "from sklearn.preprocessing import LabelEncoder"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4NkwQ62a3pnA"
      },
      "source": [
        "encoder = LabelEncoder()"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "voDr_dE93pnA"
      },
      "source": [
        "y_train = encoder.fit_transform(y_train)\n",
        "\n",
        "y_test = encoder.fit_transform(y_test)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vIyiI-J63pnB"
      },
      "source": [
        "## QUIZ QUESTIONS"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b8wLmQvW3pnB"
      },
      "source": [
        "### Question 1\n",
        "\n",
        "What is the F1 score of the case classifier?"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OMtnkR3e3pnB",
        "outputId": "e126ce8c-49d3-438a-9e08-ea1549d508a2"
      },
      "source": [
        "\"\"\"\n",
        "Information given in the confusion matrix:\n",
        "TP = 355\n",
        "FP = 1480\n",
        "FN = 45\n",
        "TN = 120\n",
        "\"\"\"\n",
        "\n",
        "# Precision formula: Precision = TP/(TP + FP)\n",
        "\n",
        "precision = 355/(355 + 1480)\n",
        "\n",
        "# Recall formula: Recall = TP/(TP + FN)\n",
        "\n",
        "recall = 355/(355 + 45)\n",
        "\n",
        "# Compute F1 score\n",
        "F1_score = 2*(precision*recall)/(precision + recall)\n",
        "\n",
        "F1_score = round(F1_score, 4)\n",
        "\n",
        "print(f'F1_score = {F1_score}')"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1_score = 0.3177\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1RoOROAt3pnD"
      },
      "source": [
        "### Question 2\n",
        "Which method can be used as best fit in Logistic Regression?\n",
        "\n",
        "### Answer 2\n",
        "Maximum Likehood"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KKiaWvUF3pnD"
      },
      "source": [
        "### Question 3\n",
        "\n",
        "Why do we use weak learners in boosting?\n",
        "\n",
        "### Answer 3\n",
        "To prevent overfitting"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "18ctXoAm3pnE"
      },
      "source": [
        "### Question 4\n",
        "\n",
        "Which confusion matrix represents the model that satisfies the stated requirements?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IwLKV4563pnE"
      },
      "source": [
        "### Answer 4"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rg7qqjwq3pnF"
      },
      "source": [
        "# Recall = TP/(TP + FN)\n",
        "# False Positive Rate (FPR) = FP/(FP + TN)\n",
        "# Cost = 5*FP + FN\n",
        "\n",
        "def value_calc(TN,FP,FN,TP):\n",
        "    recall = TP/(TP + FN)\n",
        "    fpr = FP/(FP + TN)\n",
        "    cost = 5*FP + FN\n",
        "    print(f'Recall = {recall}\\n')\n",
        "    print(f'False Positive Rate = {fpr}\\n')\n",
        "    print(f'Cost = {cost}\\n')"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UgcE6cwG3pnG",
        "outputId": "9fb66444-36fa-457b-9977-0bc446a08dc0"
      },
      "source": [
        "# For option A:\n",
        "# TN = 98, FP = 2, TP = 18, TP = 82\n",
        "\n",
        "value_calc(98,2,18,82)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Recall = 0.82\n",
            "\n",
            "False Positive Rate = 0.02\n",
            "\n",
            "Cost = 28\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tVQGEMA73pnG",
        "outputId": "81234010-a696-493c-f580-2c644213cd26"
      },
      "source": [
        "# For option B:\n",
        "# TN = 99, FP = 1, FN = 21, TP = 79\n",
        "value_calc(99,1,21,79)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Recall = 0.79\n",
            "\n",
            "False Positive Rate = 0.01\n",
            "\n",
            "Cost = 26\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UQjpYUJ13pnH",
        "outputId": "549cec70-eb2d-4cde-ed3f-5e4a6b074879"
      },
      "source": [
        "# For option C:\n",
        "# TN = 96, FP = 4, FN = 10, TP = 90\n",
        "value_calc(96,4,10,90)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Recall = 0.9\n",
            "\n",
            "False Positive Rate = 0.04\n",
            "\n",
            "Cost = 30\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1KnZNAz73pnI",
        "outputId": "99bc5e92-1d10-459b-e0ee-e195c16f9f36"
      },
      "source": [
        "# For option D:\n",
        "# TN = 91, FP = 9, FN = 22, TP = 78\n",
        "value_calc(91,9,22,78)"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Recall = 0.78\n",
            "\n",
            "False Positive Rate = 0.09\n",
            "\n",
            "Cost = 67\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "id": "kYqe-2kx3pnJ",
        "outputId": "1848de3a-c7bc-4c9f-b922-a558250991a0"
      },
      "source": [
        "answers_dict = {'Option A':[0.82,0.02,28], 'Option B':[0.79,0.01,26], 'Option C':[0.9,0.04,30], 'Option D': [0.78,0.09,67]}\n",
        "\n",
        "answers_df = pd.DataFrame(answers_dict, index=['Recall', 'False_Positive_Rate', 'Cost'])\n",
        "\n",
        "answers_df"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Option A</th>\n",
              "      <th>Option B</th>\n",
              "      <th>Option C</th>\n",
              "      <th>Option D</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>Recall</th>\n",
              "      <td>0.82</td>\n",
              "      <td>0.79</td>\n",
              "      <td>0.90</td>\n",
              "      <td>0.78</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>False_Positive_Rate</th>\n",
              "      <td>0.02</td>\n",
              "      <td>0.01</td>\n",
              "      <td>0.04</td>\n",
              "      <td>0.09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Cost</th>\n",
              "      <td>28.00</td>\n",
              "      <td>26.00</td>\n",
              "      <td>30.00</td>\n",
              "      <td>67.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                     Option A  Option B  Option C  Option D\n",
              "Recall                   0.82      0.79      0.90      0.78\n",
              "False_Positive_Rate      0.02      0.01      0.04      0.09\n",
              "Cost                    28.00     26.00     30.00     67.00"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z4fUm3xx3pnJ"
      },
      "source": [
        "From the above, option A satistied the stated conditions."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RV2hktLm3pnK"
      },
      "source": [
        "### Question 5\n",
        "Which would you use to imporove the performance of a classifier?\n",
        "\n",
        "### Answer 5\n",
        "Bagging"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zSWVbA0m3pnK"
      },
      "source": [
        "### Question 6\n",
        "Which of the following is not an Ensemble model?\n",
        "\n",
        "### Answer 6\n",
        "Decision Tree"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3BQLK4R03pnK"
      },
      "source": [
        "### Question 7\n",
        "Which metric should we use to evaluate a classifier that predicts if insurance claims are fraudulent or not?\n",
        "\n",
        "### Answer 7\n",
        "Recall"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1wTV-c6C3pnL"
      },
      "source": [
        "### Question 8\n",
        "The ROC curve was generated from a classification algorithm. What can we say about the classifier?\n",
        "\n",
        "### Answer 8\n",
        "The model has no discrimination capacity to distinguish between the positive and negative class."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uhnPGsX53pnL"
      },
      "source": [
        "### Question 9\n",
        "Based on the given matrix, which number was predicted with the least accuracy?\n",
        "\n",
        "### Answer 9\n",
        "8"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8QhFlvKZ3pnL"
      },
      "source": [
        "### Question 10\n",
        "A model has 90% accuracy and a poor recall with a training data containing 900 negative and 100 positive instances. What steps can be used to improve the model's performance?\n",
        "\n",
        "### Answer 10\n",
        "use bagging method\n",
        "\n",
        "generate synthetic samples/data using SMOTE"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K5pOgqxA3pnM"
      },
      "source": [
        "### Question 11\n",
        "How should you pre-process a label data for developing a machine learning classification algorithm?\n",
        "\n",
        "### Answer 11\n",
        "OneHotEncoding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0GDepYf53pnN"
      },
      "source": [
        "### Question 12\n",
        "What is the entropy of the target variable if its actual values are given as:\n",
        "[1,0,1,1,0,1,0]?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WA62iVXy3pnN"
      },
      "source": [
        "### Answer 12\n",
        "\n",
        "total = 7\n",
        "number of ones = 4\n",
        "number of zeros = 3\n",
        "\n",
        "entropy = -(3/7log(3/7) + 4/7log(4/7))\n",
        "\n",
        "entropy = -3/7log(3/7) - 4/7log(4/7)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l64fNY3K3pnO"
      },
      "source": [
        "### Question 13\n",
        "Which of these is not a good metric for evaluating classificatio algorithms for data with imbalanced class problems?\n",
        "\n",
        "### Answer 13\n",
        "Accuracy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r1IbQw7V3pnO"
      },
      "source": [
        "### Question 14\n",
        "What is the accuracy on the test set using the random forest classifier? In 4 decimal places\n",
        "\n",
        "### Solution 14"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wZPqgnG_3pnO"
      },
      "source": [
        "rand_forest = RandomForestClassifier(random_state = 1)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UmIQ5rMX3pnO",
        "outputId": "a56682c6-afab-4a64-e42d-d2794ac6acad"
      },
      "source": [
        "rand_forest.fit(x_train_scaled, y_train)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=None, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
              "                       n_jobs=None, oob_score=False, random_state=1, verbose=0,\n",
              "                       warm_start=False)"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K9naTSJ_3pnP"
      },
      "source": [
        "rand_forest_pred = rand_forest.predict(x_test_scaled)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9lDFunSG3pnP"
      },
      "source": [
        "accuracy = round(accuracy_score(y_test, rand_forest_pred), 4)"
      ],
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2RgxkON83pnP",
        "outputId": "894a5b19-023e-4499-fb8d-62d1a37137ca"
      },
      "source": [
        "accuracy"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.929"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q32yDrnF3pnP"
      },
      "source": [
        "### Question 15\n",
        "What is the accuracy on the test set using the xgboost classifier? In 4 decimal places\n",
        "\n",
        "### Answer 15"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NSNScZSc3pnQ"
      },
      "source": [
        "xgb_cl = XGBClassifier(random_state=1)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2_xVl7493pnQ",
        "outputId": "5e1436d2-68cb-4155-e8bf-7d1fbae47fcc"
      },
      "source": [
        "xgb_cl.fit(x_train_scaled, y_train)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
              "              colsample_bynode=1, colsample_bytree=1, gamma=0,\n",
              "              learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
              "              min_child_weight=1, missing=None, n_estimators=100, n_jobs=1,\n",
              "              nthread=None, objective='binary:logistic', random_state=1,\n",
              "              reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
              "              silent=None, subsample=1, verbosity=1)"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CIODJNf83pnQ",
        "outputId": "e34943bc-7a86-4b79-fbea-9b1f8d7a36f3"
      },
      "source": [
        "xgb_pred = xgb_cl.predict(x_test_scaled)\n",
        "\n",
        "xgb_pred"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 1, 0, ..., 0, 1, 1])"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "sFj7LIFh3pnR"
      },
      "source": [
        "acc_score = accuracy_score(y_test, xgb_pred)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VM_EjjD-3pnR",
        "outputId": "1ca918ae-2da8-49eb-a4b9-be4e14e6bb58"
      },
      "source": [
        "acc_score"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9195"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U17kcp8P3pnR"
      },
      "source": [
        "### Question 16\n",
        "What is the accuracy on the test set using the LGBM classifier? In 4 decimal places\n",
        "\n",
        "### Answer 16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SzVQlLym3pnR"
      },
      "source": [
        "lgbm_cl = LGBMClassifier(random_state=1)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XO8a2ZsJ3pnS",
        "outputId": "faa151c1-5ebb-4a79-809e-530f617f1234"
      },
      "source": [
        "lgbm_cl.fit(x_train_scaled, y_train)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,\n",
              "               importance_type='split', learning_rate=0.1, max_depth=-1,\n",
              "               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,\n",
              "               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,\n",
              "               random_state=1, reg_alpha=0.0, reg_lambda=0.0, silent=True,\n",
              "               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cdzp9pqh3pnS"
      },
      "source": [
        "lgbm_pred = lgbm_cl.predict(x_test_scaled)"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-rN9eD6a3pnS"
      },
      "source": [
        "acc_score = accuracy_score(y_test, lgbm_pred)"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O2Ohn4nZ3pnS",
        "outputId": "d4c0bf00-1889-4929-9a60-2ca09495fff1"
      },
      "source": [
        "acc_score"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9375"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9QR99jvT4aob"
      },
      "source": [
        "### Answer 17"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VcvTEAdb4i72"
      },
      "source": [
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "\n",
        "n_estimators = [50, 100, 300, 500, 1000]\n",
        "\n",
        "min_samples_split = [2, 3, 5, 7, 9]\n",
        "\n",
        "min_samples_leaf = [1, 2, 4, 6, 8]\n",
        "\n",
        "max_features = ['auto', 'sqrt', 'log2', None]"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WbA3KH7y7yR3"
      },
      "source": [
        "hyperparam_grid = {'n_estimators': n_estimators, 'min_samples_leaf': min_samples_leaf, 'min_samples_split': min_samples_split, 'max_features':max_features}"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVhKnaef8Tcx"
      },
      "source": [
        "ext_trees_cl = ExtraTreesClassifier(random_state=1)"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nywzocqZ8rsb"
      },
      "source": [
        "rand_grid_search = RandomizedSearchCV(ext_trees_cl, param_distributions=hyperparam_grid,\n",
        "                                      random_state=1,\n",
        "                                      scoring='accuracy',\n",
        "                                      n_iter=10,\n",
        "                                      cv=5,\n",
        "                                      n_jobs=-1,\n",
        "                                      verbose=1)"
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mb1trhZm9bLX",
        "outputId": "5845e596-c533-4e55-93bf-6aac3f64ba52"
      },
      "source": [
        "rand_grid_search.fit(x_train_scaled, y_train)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Fitting 5 folds for each of 10 candidates, totalling 50 fits\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
            "[Parallel(n_jobs=-1)]: Done  46 tasks      | elapsed:  1.5min\n",
            "[Parallel(n_jobs=-1)]: Done  50 out of  50 | elapsed:  1.5min finished\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomizedSearchCV(cv=5, error_score=nan,\n",
              "                   estimator=ExtraTreesClassifier(bootstrap=False,\n",
              "                                                  ccp_alpha=0.0,\n",
              "                                                  class_weight=None,\n",
              "                                                  criterion='gini',\n",
              "                                                  max_depth=None,\n",
              "                                                  max_features='auto',\n",
              "                                                  max_leaf_nodes=None,\n",
              "                                                  max_samples=None,\n",
              "                                                  min_impurity_decrease=0.0,\n",
              "                                                  min_impurity_split=None,\n",
              "                                                  min_samples_leaf=1,\n",
              "                                                  min_samples_split=2,\n",
              "                                                  min_weight_fraction_leaf=0.0,\n",
              "                                                  n_estimators=100, n_jobs=...\n",
              "                                                  oob_score=False,\n",
              "                                                  random_state=1, verbose=0,\n",
              "                                                  warm_start=False),\n",
              "                   iid='deprecated', n_iter=10, n_jobs=-1,\n",
              "                   param_distributions={'max_features': ['auto', 'sqrt', 'log2',\n",
              "                                                         None],\n",
              "                                        'min_samples_leaf': [1, 2, 4, 6, 8],\n",
              "                                        'min_samples_split': [2, 3, 5, 7, 9],\n",
              "                                        'n_estimators': [50, 100, 300, 500,\n",
              "                                                         1000]},\n",
              "                   pre_dispatch='2*n_jobs', random_state=1, refit=True,\n",
              "                   return_train_score=False, scoring='accuracy', verbose=1)"
            ]
          },
          "metadata": {},
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PecqLkxq9x5e",
        "outputId": "b50471c8-c8c1-4208-a140-cecd696bdb48"
      },
      "source": [
        "rand_grid_search.best_params_"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'max_features': None,\n",
              " 'min_samples_leaf': 8,\n",
              " 'min_samples_split': 2,\n",
              " 'n_estimators': 1000}"
            ]
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RyBRmAt7ARbU"
      },
      "source": [
        "### Question 18\n",
        "\n",
        "Train a new ExtraTreesClassifier with the new hyperparameters and compare the accuracy with the former accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SFDie2_IANh4",
        "outputId": "1bf49b27-86e0-42f0-aa0d-ae9c8832d2af"
      },
      "source": [
        "# For ExtraTreesClassifier with no hyperparameter tuning\n",
        "\n",
        "ext_trees_cl.fit(x_train_scaled, y_train)\n",
        "\n",
        "ext_trees_pred = ext_trees_cl.predict(x_test_scaled)\n",
        "\n",
        "untuned_acc_score = round(accuracy_score(y_test, ext_trees_pred), 4)\n",
        "\n",
        "untuned_acc_score"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.928"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kqA35giqIWCC",
        "outputId": "c2998167-28ed-45a2-b935-5e00959231ba"
      },
      "source": [
        "# For ExtraTreesClassifier with hyperparameter tuning\n",
        "\n",
        "ext_trees_tuned = ExtraTreesClassifier(n_estimators=1000, min_samples_split=2,\n",
        "                                       min_samples_leaf=8,\n",
        "                                       max_features=None, random_state=1)\n",
        "\n",
        "ext_trees_tuned.fit(x_train_scaled, y_train)\n",
        "\n",
        "new_pred = ext_trees_tuned.predict(x_test_scaled)\n",
        "\n",
        "tuned_acc_score = accuracy_score(y_test, ext_trees_pred)\n",
        "\n",
        "tuned_acc_score"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.928"
            ]
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QPhWlIkwMwzu"
      },
      "source": [
        "### Question 19\n",
        "What other hyperparameter optimization methods can be used apart from random search?\n",
        "\n",
        "### Answer 19\n",
        "All of the Methods given in the question."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GWancKR6Nb4E"
      },
      "source": [
        "### Question 20\n",
        "Find the feature importance using the optimal ExtraTreesClassifier model. Which features are the most and least important respectively?\n",
        "\n",
        "### Answer\n",
        "tau2, p1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gp9ZPq16NIvK",
        "outputId": "178c2944-73e3-45c5-f53f-474aa62a3a97"
      },
      "source": [
        "feature_imp = ext_trees_tuned.feature_importances_\n",
        "\n",
        "feature_imp"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.13729124, 0.14081904, 0.13413116, 0.13539377, 0.00368958,\n",
              "       0.00538171, 0.00538868, 0.00504104, 0.10295388, 0.10854578,\n",
              "       0.11218334, 0.10918078])"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cED_3hdkOm7j",
        "outputId": "e42de77f-68ed-4b94-e0ec-448c60e255db"
      },
      "source": [
        "sorted(zip(feature_imp, features), reverse=True)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.14081904109390708, 'tau2'),\n",
              " (0.13729123866104237, 'tau1'),\n",
              " (0.13539376858074692, 'tau4'),\n",
              " (0.13413115854843838, 'tau3'),\n",
              " (0.11218334415531182, 'g3'),\n",
              " (0.10918078049059556, 'g4'),\n",
              " (0.1085457837402933, 'g2'),\n",
              " (0.10295387795203104, 'g1'),\n",
              " (0.005388675009440006, 'p3'),\n",
              " (0.005381711298752081, 'p2'),\n",
              " (0.005041044094188226, 'p4'),\n",
              " (0.003689576375253394, 'p1')]"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BI-a3zn1PVWw"
      },
      "source": [
        "From the above, the feature with the highest feature importance is 'tau2' while the feature with the lowest feature importance is 'p1'"
      ]
    }
  ]
}