work/dup_ques/2 - QQP_Dimensions_Reduction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UjIFqUR0KGEA"
      },
      "source": [
        "# **Initializing And Loading Data**\n",
        "https://www.kaggle.com/datasets/quora/question-pairs-dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "zzcIyNn4e1VV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "32KcZQsFheE2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DUMP_PATH = \"/content/drive/MyDrive/Colab Notebooks/dumps/quora_question_pairs/\"\n",
        "NOTEBOOK_NAME = \"02_1\"\n",
        "N_SELECTIONS = 3\n",
        "SAMPLE_SIZE = 30_000\n",
        "TEST_SIZE = 0.2\n",
        "RANDOM_STATE = 42\n",
        "\n",
        "os.listdir(DUMP_PATH)"
      ],
      "metadata": {
        "id": "KlTAxcn30Ikq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(os.path.join(DUMP_PATH, 'final_preprocessed_data_allowStopwords1.csv'))\n",
        "\n",
        "# Shuffling the Dataset\n",
        "df = df.sample(df.shape[0], random_state=RANDOM_STATE)\n",
        "\n",
        "# Picking a sample\n",
        "if SAMPLE_SIZE == None:\n",
        "    SAMPLE_SIZE = min(df.is_duplicate.value_counts().values) * 2\n",
        "df = pd.concat([\n",
        "    df[df.is_duplicate == 0].iloc[:SAMPLE_SIZE//2],\n",
        "    df[df.is_duplicate == 1].iloc[:SAMPLE_SIZE//2]\n",
        "])\n",
        "\n",
        "# Shuffling the Dataset\n",
        "df = df.sample(df.shape[0], random_state=RANDOM_STATE)\n",
        "\n",
        "df.index = range(SAMPLE_SIZE)\n",
        "\n",
        "print(df.shape)\n",
        "df.head()"
      ],
      "metadata": {
        "id": "Aq9T-EBPbdmh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X = df.iloc[:, 6:]\n",
        "y = df.is_duplicate\n",
        "\n",
        "cols = X.columns\n",
        "\n",
        "# SCALING\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "scaler = StandardScaler()\n",
        "X = scaler.fit_transform(X)"
      ],
      "metadata": {
        "id": "6aYfx4n_044S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "i_zeros = list(df[df['is_duplicate'] == 0].index)\n",
        "i_ones = list(df[df['is_duplicate'] == 1].index)"
      ],
      "metadata": {
        "id": "g8BDRVQQ6OIx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "def model_accuracy(X, y):\n",
        "    X_train, X_test, y_train, y_test = train_test_split(\n",
        "        X,\n",
        "        y,\n",
        "        test_size=0.2,\n",
        "        random_state=RANDOM_STATE,\n",
        "        stratify=y\n",
        "    )\n",
        "    model = RandomForestClassifier()\n",
        "\n",
        "    model.fit(X_train, y_train)\n",
        "\n",
        "    pred_test = model.predict(X_test)\n",
        "\n",
        "    return accuracy_score(pred_test, y_test)"
      ],
      "metadata": {
        "id": "gSLzajaeUcgv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Visualizing**"
      ],
      "metadata": {
        "id": "BKlRt36V1RQj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def dist_plot(X, n_cols=4, cols=[]):\n",
        "    # X.shape = (30000, 22)\n",
        "    n_rows = int(np.ceil(X.shape[1]/n_cols))\n",
        "    n_rows = max(n_rows, 2)\n",
        "    # fig_size = 20 // n_cols\n",
        "    fig_size = 5\n",
        "\n",
        "    fig, axes = plt.subplots(\n",
        "        n_rows,    # no of rows\n",
        "        n_cols,    # no of cols\n",
        "        figsize = (fig_size*n_cols, fig_size*n_rows),\n",
        "    )\n",
        "\n",
        "    for i in range(X.shape[1]):\n",
        "        ax = axes[i//n_cols, i%n_cols]\n",
        "\n",
        "        sns.distplot(X[:, i][i_zeros], label='non duplicate', ax=ax)\n",
        "        sns.distplot(X[:, i][i_ones], label='duplicate', ax=ax)\n",
        "        if len(cols):\n",
        "            ax.set_title(cols[i])\n",
        "\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "\n",
        "def scatter_plot_3d(X, s=2):\n",
        "    fig = plt.figure(figsize=(8, 8))\n",
        "    ax = fig.add_subplot(111, projection='3d')\n",
        "\n",
        "    COLORS = ['tab:blue', 'tab:orange']\n",
        "\n",
        "    ax.scatter(X[:, 0], X[:, 1], X[:, 2], c=[COLORS[i] for i in y], s=s)\n",
        "    plt.show()"
      ],
      "metadata": {
        "id": "ipUL6Y0A1sW6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dist_plot(X, n_cols=4, cols=cols)"
      ],
      "metadata": {
        "id": "nSyS0av48wtZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scatter_plot_3d(X)"
      ],
      "metadata": {
        "id": "TNbCZLzFCab9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Feature Selection**"
      ],
      "metadata": {
        "id": "AwQWy0QU4Tzb"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def scores_to_index(scores):\n",
        "    return [i[0] for i in sorted(enumerate(scores), key=lambda x:x[1], reverse=True)]"
      ],
      "metadata": {
        "id": "NSjhh2d7G8KA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_on_priority(scores):\n",
        "    priority_i = scores_to_index(scores)\n",
        "\n",
        "    print(cols[priority_i])\n",
        "\n",
        "    dist_plot(\n",
        "        X[:, priority_i],\n",
        "        cols=[f\"{i}: {j}\" for i, j in zip(cols[priority_i], scores[priority_i])]\n",
        "    )\n",
        "    scatter_plot_3d(X[:, priority_i])\n",
        "\n",
        "    return cols[priority_i]"
      ],
      "metadata": {
        "id": "nFRIlaX4HBpG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.feature_selection import RFE\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.feature_selection import f_classif\n",
        "from sklearn.feature_selection import SelectKBest\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "\n",
        "# BACKWARD SELECTION\n",
        "backward_selector = RFE(\n",
        "    LogisticRegression(max_iter=150),\n",
        "    n_features_to_select=N_SELECTIONS,\n",
        "    step=1\n",
        ")\n",
        "backward_selector.fit(X, y)\n",
        "backward_score = 0 - backward_selector.ranking_\n",
        "backward_cols = scores_to_index(backward_score)\n",
        "\n",
        "\n",
        "# FORWARD SELECTION\n",
        "forward_selector = SelectKBest(f_classif, k=N_SELECTIONS)\n",
        "forward_selector.fit(X, y)\n",
        "forward_score = forward_selector.scores_\n",
        "forward_cols = scores_to_index(forward_score)\n",
        "\n",
        "\n",
        "# RANDOM FOREST SELECTION\n",
        "forest_selector = RandomForestClassifier(\n",
        "    n_estimators=100,\n",
        "    max_depth=3,\n",
        "    bootstrap=True,\n",
        "    n_jobs=-1,\n",
        "    random_state=RANDOM_STATE\n",
        ")\n",
        "forest_selector.fit(X, y)\n",
        "forest_score = forest_selector.feature_importances_\n",
        "forest_cols = scores_to_index(forest_score)"
      ],
      "metadata": {
        "id": "HtvvpAfvP498"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# FINDING BEST COLUMNS\n",
        "best = []\n",
        "for i, j, k in zip(forward_cols, backward_cols, forest_cols):\n",
        "    if i_ones not in best:\n",
        "        best.append(i)\n",
        "    if j not in best:\n",
        "        best.append(j)\n",
        "    if k not in best:\n",
        "        best.append(k)\n",
        "BEST_COLS_I = best\n",
        "BEST_COLS = list(cols[best])\n",
        "BEST_COLS"
      ],
      "metadata": {
        "id": "xYhdsdXtRicp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# PLOTTING COLUMNS BASED ON THEIR PRIORITY\n",
        "dist_plot(\n",
        "    X[:, best],\n",
        "    cols = cols[best]\n",
        ")\n",
        "scatter_plot_3d(X[:, best])"
      ],
      "metadata": {
        "id": "tlwCzDkRRlWs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Combination Using Linear Methods**"
      ],
      "metadata": {
        "id": "HNPkOOTvMXUy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.decomposition import PCA\n",
        "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
        "from sklearn.decomposition import TruncatedSVD\n",
        "from sklearn.decomposition import KernelPCA\n",
        "from sklearn.manifold import TSNE\n",
        "from sklearn.manifold import MDS\n",
        "\n",
        "pca = PCA(n_components=N_SELECTIONS)\n",
        "X_pca = pca.fit_transform(X)\n",
        "\n",
        "\n",
        "# lda = LinearDiscriminantAnalysis(n_components=2, solver='svd')\n",
        "# X_lda = lda.fit_transform(X, y)\n",
        "\n",
        "\n",
        "svd = TruncatedSVD(\n",
        "    n_components=N_SELECTIONS,\n",
        "    algorithm='randomized',\n",
        "    random_state=RANDOM_STATE\n",
        ")\n",
        "X_svd = svd.fit_transform(X)\n",
        "\n",
        "\n",
        "# kpca = KernelPCA(\n",
        "#     n_components=N_SELECTIONS,\n",
        "#     kernel='rbf',\n",
        "#     gamma=15,\n",
        "#     random_state=RANDOM_STATE\n",
        "# )\n",
        "# X_kpca = kpca.fit_transform(X)\n",
        "\n",
        "\n",
        "# tsne = TSNE(\n",
        "#     n_components = N_SELECTIONS\n",
        "# )\n",
        "# X_tsne = tsne.fit_transform(X)\n",
        "\n",
        "\n",
        "# mds = MDS(n_components=N_SELECTIONS, metric=True, random_state=RANDOM_STATE)\n",
        "# X_mds = mds.fit_transform(X)"
      ],
      "metadata": {
        "id": "sH3iU59RMr2g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "scatter_plot_3d(X_pca)\n",
        "scatter_plot_3d(X_svd)"
      ],
      "metadata": {
        "id": "fAJZj9FfPGU2"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}