baseline_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8ChnD36qHhuy"
      },
      "source": [
        "# LexiGuard: Baseline model (Bag of Words + logistic regression)\n",
        "\n",
        "[![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/CalleRosa40/lexiguard/blob/main/baseline_model.ipynb)\n",
        "\n",
        "Please run `data_preprocessing.ipynb` to create data file(s)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sLTUtsD6HupB"
      },
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nisbn3mTG0fj",
        "outputId": "c3a65a47-0bea-4de2-c594-1b5ca57dec48"
      },
      "outputs": [],
      "source": [
        "# Python standard lib\n",
        "import time; full_run_time_start = time.time() # start timing exec right away\n",
        "\n",
        "# other \"usual suspects\"\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "# scikit-learn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# imbalanced-learn\n",
        "from imblearn.over_sampling import SMOTE\n",
        "\n",
        "# display all df columns (default is 20)\n",
        "pd.options.display.max_columns = None\n",
        "\n",
        "# show all data in columns\n",
        "pd.options.display.max_colwidth = None\n",
        "\n",
        "# load utility functions\n",
        "%run functions.ipynb"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-MP847vfIJMN"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "r6YNY0NIIL4d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(9964, 6)"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Watch out which file is loaded here! Full data or just sample?\n",
        "df = pd.read_csv('data/lexiguard_data_10000.csv')\n",
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>raw</th>\n",
              "      <th>clean</th>\n",
              "      <th>clean_pp</th>\n",
              "      <th>clean_pp_lemma</th>\n",
              "      <th>clean_pp_lemma_stop</th>\n",
              "      <th>toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Naturally you can feel it in your urine. \\nTha...</td>\n",
              "      <td>Naturally you can feel it in your urine. That'...</td>\n",
              "      <td>naturally you can feel it in your urine that '...</td>\n",
              "      <td>naturally you can feel it in your urine that b...</td>\n",
              "      <td>naturally feel urine common expression germani...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Yum!  What's not to love: water+maple syrup - ...</td>\n",
              "      <td>Yum! What's not to love: water+maple syrup - t...</td>\n",
              "      <td>yum what 's not to love water+maple syrup toge...</td>\n",
              "      <td>yum what be not to love water+maple syrup toge...</td>\n",
              "      <td>yum love water+maple syrup</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Catou I will wager that mutual fund sales will...</td>\n",
              "      <td>Catou I will wager that mutual fund sales will...</td>\n",
              "      <td>catou i will wager that mutual fund sales will...</td>\n",
              "      <td>catou i will wager that mutual fund sale will ...</td>\n",
              "      <td>catou wager mutual fund sale strong rrsp seaso...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\"The shortage of priests is not a shortage of ...</td>\n",
              "      <td>\"The shortage of priests is not a shortage of ...</td>\n",
              "      <td>the shortage of priests is not a shortage of v...</td>\n",
              "      <td>the shortage of priest be not a shortage of vo...</td>\n",
              "      <td>shortage priest shortage vocation shortness si...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I dont disagree with that. It takes money to d...</td>\n",
              "      <td>I dont disagree with that. It takes money to d...</td>\n",
              "      <td>i do nt disagree with that it takes money to d...</td>\n",
              "      <td>i do not disagree with that it take money to d...</td>\n",
              "      <td>not disagree take money deal drug street level...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 raw  \\\n",
              "0  Naturally you can feel it in your urine. \\nTha...   \n",
              "1  Yum!  What's not to love: water+maple syrup - ...   \n",
              "2  Catou I will wager that mutual fund sales will...   \n",
              "3  \"The shortage of priests is not a shortage of ...   \n",
              "4  I dont disagree with that. It takes money to d...   \n",
              "\n",
              "                                               clean  \\\n",
              "0  Naturally you can feel it in your urine. That'...   \n",
              "1  Yum! What's not to love: water+maple syrup - t...   \n",
              "2  Catou I will wager that mutual fund sales will...   \n",
              "3  \"The shortage of priests is not a shortage of ...   \n",
              "4  I dont disagree with that. It takes money to d...   \n",
              "\n",
              "                                            clean_pp  \\\n",
              "0  naturally you can feel it in your urine that '...   \n",
              "1  yum what 's not to love water+maple syrup toge...   \n",
              "2  catou i will wager that mutual fund sales will...   \n",
              "3  the shortage of priests is not a shortage of v...   \n",
              "4  i do nt disagree with that it takes money to d...   \n",
              "\n",
              "                                      clean_pp_lemma  \\\n",
              "0  naturally you can feel it in your urine that b...   \n",
              "1  yum what be not to love water+maple syrup toge...   \n",
              "2  catou i will wager that mutual fund sale will ...   \n",
              "3  the shortage of priest be not a shortage of vo...   \n",
              "4  i do not disagree with that it take money to d...   \n",
              "\n",
              "                                 clean_pp_lemma_stop  toxic  \n",
              "0  naturally feel urine common expression germani...      0  \n",
              "1                         yum love water+maple syrup      0  \n",
              "2  catou wager mutual fund sale strong rrsp seaso...      0  \n",
              "3  shortage priest shortage vocation shortness si...      0  \n",
              "4  not disagree take money deal drug street level...      0  "
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 9964 entries, 0 to 9963\n",
            "Data columns (total 6 columns):\n",
            " #   Column               Non-Null Count  Dtype \n",
            "---  ------               --------------  ----- \n",
            " 0   raw                  9964 non-null   object\n",
            " 1   clean                9964 non-null   object\n",
            " 2   clean_pp             9964 non-null   object\n",
            " 3   clean_pp_lemma       9964 non-null   object\n",
            " 4   clean_pp_lemma_stop  9964 non-null   object\n",
            " 5   toxic                9964 non-null   int64 \n",
            "dtypes: int64(1), object(5)\n",
            "memory usage: 467.2+ KB\n"
          ]
        }
      ],
      "source": [
        "df.info()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Handle missing values\n",
        "\n",
        "There shouldn't be any after preprocessing, but anyway ..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Checking for NaN's ...\n",
            "No NaN's found.\n"
          ]
        }
      ],
      "source": [
        "print('Checking for NaN\\'s ...')\n",
        "\n",
        "if df.isna().sum().sum() != 0:\n",
        "    print('NaN\\'s found.')\n",
        "    rows_before = df.shape[0]\n",
        "    print('Rows before dropping:', rows_before)\n",
        "    print('Dropping rows ...')\n",
        "    df.dropna(inplace=True)\n",
        "    df.reset_index(drop=True, inplace=True)\n",
        "    rows_after = df.shape[0]\n",
        "    print('Rows after dropping:', rows_after)\n",
        "    print('Rows dropped:', rows_before - rows_after)\n",
        "\n",
        "else:\n",
        "    print('No NaN\\'s found.')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        " ## Check for imbalance in data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Nontoxic (0): 9123 (91.6 %)\n",
            "Toxic (1): 841 (8.4 %)\n"
          ]
        }
      ],
      "source": [
        "value_counts = df.toxic.value_counts()\n",
        "nontoxic_count = value_counts[0]\n",
        "toxic_count = value_counts[1]\n",
        "nontoxic_perc =\\\n",
        "    round((nontoxic_count / (nontoxic_count + toxic_count)) * 100, 1)\n",
        "toxic_perc =\\\n",
        "    round((toxic_count / (nontoxic_count + toxic_count)) * 100, 1)\n",
        "\n",
        "print(f'Nontoxic (0): {nontoxic_count} ({nontoxic_perc} %)')\n",
        "print(f'Toxic (1): {toxic_count} ({toxic_perc} %)')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Function: Create bag of words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "def bow(s):\n",
        "    vect = CountVectorizer()\n",
        "    return vect.fit_transform(s)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Run baseline model (logistic regression) on different data cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "# parameters for model\n",
        "params = {'max_iter': 2_000}\n",
        "\n",
        "# load model with parameters\n",
        "lr = LogisticRegression(**params)\n",
        "\n",
        "y = df.toxic\n",
        "\n",
        "def run_experiment(X, data_desc):\n",
        "    # Split data using 80% for train, 20% for test. Make sure test data has\n",
        "    # same toxic/nontoxic ratio as train data by using stratify arg.\n",
        "    X_train, X_test, y_train, y_test =\\\n",
        "        train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)\n",
        "\n",
        "    # use SMOTE to balance train data\n",
        "    sm = SMOTE(random_state=42)\n",
        "    X_train_sm, y_train_sm = sm.fit_resample(X_train, y_train)\n",
        "\n",
        "    # train model and time execution\n",
        "    train_time_start = time.time()\n",
        "    lr.fit(X_train_sm, y_train_sm)\n",
        "    train_time = time.time() - train_time_start\n",
        "    train_time_str = f'{int(train_time // 60)}m {round(train_time % 60)}s'\n",
        "\n",
        "    # predict test data\n",
        "    y_pred = lr.predict(X_test)\n",
        "\n",
        "    eval_model(lr.__class__.__name__,\n",
        "            params,\n",
        "            data_desc,\n",
        "            train_time_str,\n",
        "            y_test, y_pred)\n",
        "\n",
        "run_experiment(bow(df.raw), 'BOW on col \"raw\"')\n",
        "run_experiment(bow(df.clean), 'BOW on col \"clean\"')\n",
        "run_experiment(bow(df.clean_pp), 'BOW on col \"clean_pp\"')\n",
        "run_experiment(bow(df.clean_pp_lemma), 'BOW on col \"clean_pp_lemma\"')\n",
        "run_experiment(bow(df.clean_pp_lemma_stop), 'BOW on col \"clean_pp_lemma_stop\"')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Show test results + total exec time"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>model_name</th>\n",
              "      <th>model_params</th>\n",
              "      <th>data_desc</th>\n",
              "      <th>f1</th>\n",
              "      <th>acc</th>\n",
              "      <th>recall</th>\n",
              "      <th>prec</th>\n",
              "      <th>cf_matrix</th>\n",
              "      <th>exec_time</th>\n",
              "      <th>notes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LogisticRegression</td>\n",
              "      <td>{'max_iter': 2000}</td>\n",
              "      <td>BOW on col \"raw\"</td>\n",
              "      <td>0.22804</td>\n",
              "      <td>0.79277</td>\n",
              "      <td>0.36310</td>\n",
              "      <td>0.16621</td>\n",
              "      <td>[[1519, 306], [107, 61]]</td>\n",
              "      <td>0m 2s</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>LogisticRegression</td>\n",
              "      <td>{'max_iter': 2000}</td>\n",
              "      <td>BOW on col \"clean\"</td>\n",
              "      <td>0.23178</td>\n",
              "      <td>0.79378</td>\n",
              "      <td>0.36905</td>\n",
              "      <td>0.16894</td>\n",
              "      <td>[[1520, 305], [106, 62]]</td>\n",
              "      <td>0m 2s</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>LogisticRegression</td>\n",
              "      <td>{'max_iter': 2000}</td>\n",
              "      <td>BOW on col \"clean_pp\"</td>\n",
              "      <td>0.24254</td>\n",
              "      <td>0.79629</td>\n",
              "      <td>0.38690</td>\n",
              "      <td>0.17663</td>\n",
              "      <td>[[1522, 303], [103, 65]]</td>\n",
              "      <td>0m 2s</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>LogisticRegression</td>\n",
              "      <td>{'max_iter': 2000}</td>\n",
              "      <td>BOW on col \"clean_pp_lemma\"</td>\n",
              "      <td>0.23619</td>\n",
              "      <td>0.79880</td>\n",
              "      <td>0.36905</td>\n",
              "      <td>0.17367</td>\n",
              "      <td>[[1530, 295], [106, 62]]</td>\n",
              "      <td>0m 2s</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>LogisticRegression</td>\n",
              "      <td>{'max_iter': 2000}</td>\n",
              "      <td>BOW on col \"clean_pp_lemma_stop\"</td>\n",
              "      <td>0.24172</td>\n",
              "      <td>0.77020</td>\n",
              "      <td>0.43452</td>\n",
              "      <td>0.16743</td>\n",
              "      <td>[[1462, 363], [95, 73]]</td>\n",
              "      <td>0m 1s</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           model_name        model_params                         data_desc  \\\n",
              "0  LogisticRegression  {'max_iter': 2000}                  BOW on col \"raw\"   \n",
              "1  LogisticRegression  {'max_iter': 2000}                BOW on col \"clean\"   \n",
              "2  LogisticRegression  {'max_iter': 2000}             BOW on col \"clean_pp\"   \n",
              "3  LogisticRegression  {'max_iter': 2000}       BOW on col \"clean_pp_lemma\"   \n",
              "4  LogisticRegression  {'max_iter': 2000}  BOW on col \"clean_pp_lemma_stop\"   \n",
              "\n",
              "        f1      acc   recall     prec                 cf_matrix exec_time  \\\n",
              "0  0.22804  0.79277  0.36310  0.16621  [[1519, 306], [107, 61]]     0m 2s   \n",
              "1  0.23178  0.79378  0.36905  0.16894  [[1520, 305], [106, 62]]     0m 2s   \n",
              "2  0.24254  0.79629  0.38690  0.17663  [[1522, 303], [103, 65]]     0m 2s   \n",
              "3  0.23619  0.79880  0.36905  0.17367  [[1530, 295], [106, 62]]     0m 2s   \n",
              "4  0.24172  0.77020  0.43452  0.16743   [[1462, 363], [95, 73]]     0m 1s   \n",
              "\n",
              "  notes  \n",
              "0        \n",
              "1        \n",
              "2        \n",
              "3        \n",
              "4        "
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "test_results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Full run time: 0m 16s\n"
          ]
        }
      ],
      "source": [
        "full_run_time = time.time() - full_run_time_start\n",
        "print(f'Full run time: {int(full_run_time // 60)}m {round(full_run_time % 60)}s')"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}