BoWmodels.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JgRkpm6XQPnG"
   },
   "source": [
    "# **Installing the packages**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 33142,
     "status": "ok",
     "timestamp": 1710758631335,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "Dvbt_iWeNp-v",
    "outputId": "91cfe4e4-ac96-464f-a40f-f7ffd86dbfd5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (1.5.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.25.2)\n",
      "Requirement already satisfied: plotly in /usr/local/lib/python3.10/dist-packages (5.15.0)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (1.2.2)\n",
      "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.7.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.4)\n",
      "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from plotly) (8.2.3)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from plotly) (24.0)\n",
      "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.11.4)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn) (3.3.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.12.1)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.49.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (9.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.1.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "! pip install pandas numpy plotly scikit-learn matplotlib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "NG__2DQNSLPO"
   },
   "source": [
    "# **Importing libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 29857,
     "status": "ok",
     "timestamp": 1710758661175,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "vXbsY2-TRQF3",
    "outputId": "efecc915-28db-4f04-e67a-f25f01df3f10"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive/\n"
     ]
    }
   ],
   "source": [
    "# Mount the drive if not mounted\n",
    "from google.colab import drive\n",
    "drive.mount(\"/content/drive/\")\n",
    "\n",
    "from joblib import parallel_backend\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import train_test_split, KFold, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import f1_score, balanced_accuracy_score, classification_report, accuracy_score, make_scorer\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "import plotly.express as px\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "58-mD52GKupV"
   },
   "source": [
    "# **Loading and Pre-processing data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 3106,
     "status": "ok",
     "timestamp": 1710758664270,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "D89MoLaUTR-V"
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/content/drive/MyDrive/AA-Tutorial/data/Agora.csv\", encoding='ISO-8859-1')\n",
    "# Renaming all the features of the dataframe\n",
    "df = df.rename(str.strip, axis='columns')\n",
    "# Merging the Item and Item Description using a [SEP] token\n",
    "separator = ' [SEP] '\n",
    "df['TEXT'] = df.apply(lambda row: f\"{row['Item']}{separator}{row['Item Description']}\", axis=1)\n",
    "# dropping Unncessary columns\n",
    "df.drop(columns=[\"Item\", \"Item Description\", \"Category\", \"Price\", \"Origin\", \"Destination\", \"Rating\", \"Remarks\"], inplace=True)\n",
    "# Assuming that vendors Amsterdam100 and amsterdam100 are the same vendors\n",
    "df.Vendor = df.Vendor.apply(lambda x: x.lower())\n",
    "# Getting all unique vendor handles from the 'Vendor' column.\n",
    "unique_vendors = df['Vendor'].unique()\n",
    "\n",
    "# Assigning vendor IDs to vendor handles using a dictionary comprehension.\n",
    "# This approach eliminates the need for checking if a vendor already exists in the dictionary,\n",
    "# as each unique vendor will be processed once. The enumerate function provides a counter (idx),\n",
    "# which is used to assign IDs, starting from 1 for the first vendor.\n",
    "vendor_to_idx_dict = {vendor: idx + 1 for idx, vendor in enumerate(unique_vendors)}\n",
    "\n",
    "# Updating the 'Vendor' column in the DataFrame to reflect the vendor IDs.\n",
    "# The 'map' function is used to replace each vendor handle with its corresponding vendor ID\n",
    "# based on the 'vendor_to_idx_dict'. This operation is vectorized and efficient.\n",
    "df['Vendor'] = df['Vendor'].map(vendor_to_idx_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1710758664271,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "J19LzKm3VcEj",
    "outputId": "fe639459-5ac0-4e3a-8f00-07a090a636ba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(109689, 2)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7YAyU5UcVZeq"
   },
   "source": [
    "Due to the extensive time required to train on over 100K+ samples, we have decided to limit our analysis to a subset of 5K samples. To get these samples, we look into vendors that have 5+ advertisements and then allocate all the vendors that have less than 5 ads into a new class, \"others\".  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1710758664271,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "yfxNuEHqT7P5"
   },
   "outputs": [],
   "source": [
    "df = df.iloc[:5000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 310,
     "status": "ok",
     "timestamp": 1710758664577,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "1jZ9R2zNhQci"
   },
   "outputs": [],
   "source": [
    "# Assigning a vendor ID to \"others\" class\n",
    "vendor_to_idx_dict[\"others\"] = len(vendor_to_idx_dict) + 1\n",
    "# Calculate advertisement frequency for each vendor\n",
    "ad_freq = df['Vendor'].value_counts()\n",
    "# Filter vendors with ad frequency less than 5\n",
    "vendors_to_replace = ad_freq[ad_freq < 5].index\n",
    "# Update DataFrame: Replace vendor names with 'others' where ad frequency is less than 5\n",
    "df['Vendor'] = df['Vendor'].apply(lambda x: vendor_to_idx_dict['others'] if x in vendors_to_replace else x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "96TUhvSRKZ-Z"
   },
   "source": [
    "# **Visualizing data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 272,
     "status": "ok",
     "timestamp": 1710758712901,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "lhclAjAqKZDF"
   },
   "outputs": [],
   "source": [
    "def generate_freq_distribution(df, column_name):\n",
    "    \"\"\"\n",
    "    This function takes a DataFrame and a column name as inputs and generates a bar plot of the normalized frequency distribution of the values in the specified column using Plotly.\n",
    "\n",
    "    Parameters:\n",
    "    - df (pandas.DataFrame): The DataFrame containing the data.\n",
    "    - column_name (str): The name of the column to analyze for value frequencies.\n",
    "\n",
    "    The function calculates the normalized frequencies, converts them to a DataFrame, and then plots them in a bar chart. The title of the chart includes the total number of unique entries in the column.\n",
    "    \"\"\"\n",
    "\n",
    "    # Calculate the normalized frequency of the specified column\n",
    "    # and count the number of unique entries\n",
    "    freq = df[column_name].value_counts(normalize=True)\n",
    "    nr_entries = df[column_name].nunique()\n",
    "\n",
    "    # Convert the frequency Series to a DataFrame for plotting\n",
    "    # Setting the column names to the name of the analyzed column and 'frequency'\n",
    "    freq_df = freq.reset_index()\n",
    "    freq_df.columns = [column_name, 'frequency']\n",
    "\n",
    "    # Create the plot using Plotly Express\n",
    "    # The x-axis represents the unique values in the column\n",
    "    # The y-axis represents the normalized frequency of each value\n",
    "    # The plot includes a title that states the total number of unique entries\n",
    "    fig = px.bar(freq_df, x=column_name, y='frequency',\n",
    "                 labels={'frequency': 'Normalized Frequency'},\n",
    "                 title=f'Total number of Entries: {nr_entries}')\n",
    "    # Display the plot in the output\n",
    "    fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "executionInfo": {
     "elapsed": 2727,
     "status": "ok",
     "timestamp": 1710758783871,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "p0ftkmyhKmbz",
    "outputId": "f605627a-8c29-44f3-d400-f2d69861ca84"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"d5e9acc3-8ace-42c6-acab-6be690d41cae\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"d5e9acc3-8ace-42c6-acab-6be690d41cae\")) {                    Plotly.newPlot(                        \"d5e9acc3-8ace-42c6-acab-6be690d41cae\",                        [{\"alignmentgroup\":\"True\",\"hovertemplate\":\"Vendor=%{x}\\u003cbr\\u003eNormalized Frequency=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\",\"pattern\":{\"shape\":\"\"}},\"name\":\"\",\"offsetgroup\":\"\",\"orientation\":\"v\",\"showlegend\":false,\"textposition\":\"auto\",\"x\":[25,3188,149,36,167,34,48,155,209,7,5,134,40,243,93,197,174,89,165,210,82,229,94,154,179,45,96,3,160,153,39,52,206,1,15,86,95,326,196,169,11,73,198,23,27,135,212,19,42,17,2,12,56,207,171,78,159,9,13,24,4,211,189,208,22,87,190,14,225,227,61,220,143,258,161,28,272,37,201,50,142,164,104,263,214,163,106,252,180,141,55,77,80,58,117,222,120,121,213,216,100,8,46,146,199,72,110,88,246,247,118,257,168,217,260,284,288,49,327,105,170,90,184,166,287,280,274,41,259,43,255,102,250,148,116,57,114,200,60,183,29,203,145,62,132,107,267,245,262,131,230,188,172],\"xaxis\":\"x\",\"y\":[0.093,0.0742,0.0498,0.0392,0.0346,0.0308,0.0286,0.028,0.026,0.0246,0.0242,0.0212,0.0208,0.0168,0.0162,0.014,0.0136,0.0126,0.0124,0.0124,0.0114,0.0112,0.0104,0.0102,0.0096,0.0094,0.0084,0.0084,0.0082,0.008,0.008,0.0076,0.0076,0.0074,0.0074,0.0072,0.0064,0.0062,0.0056,0.0056,0.0056,0.0056,0.0054,0.0054,0.0052,0.0046,0.0046,0.0044,0.0042,0.0042,0.004,0.0038,0.0036,0.0036,0.0036,0.0034,0.0034,0.0034,0.0032,0.003,0.0028,0.0028,0.0028,0.0028,0.0026,0.0026,0.0026,0.0026,0.0026,0.0024,0.0024,0.0024,0.0024,0.0024,0.0024,0.0024,0.0024,0.0024,0.0024,0.0022,0.0022,0.0022,0.0022,0.0022,0.0022,0.0022,0.0022,0.002,0.002,0.002,0.002,0.002,0.002,0.0018,0.0018,0.0018,0.0018,0.0018,0.0018,0.0018,0.0018,0.0018,0.0018,0.0016,0.0016,0.0016,0.0016,0.0016,0.0016,0.0016,0.0016,0.0016,0.0016,0.0014,0.0014,0.0014,0.0014,0.0014,0.0014,0.0014,0.0014,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.0012,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001,0.001],\"yaxis\":\"y\",\"type\":\"bar\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Vendor\"}},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"Normalized Frequency\"}},\"legend\":{\"tracegroupgap\":0},\"title\":{\"text\":\"Total number of Entries: 153\"},\"barmode\":\"relative\"},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('d5e9acc3-8ace-42c6-acab-6be690d41cae');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                            </script>        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "generate_freq_distribution(df, \"Vendor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 341,
     "status": "ok",
     "timestamp": 1710758869769,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "DlCUPa6CLBBF"
   },
   "outputs": [],
   "source": [
    "# Calculating the sentence length by splitting tokens by ' '\n",
    "df['SENT_LEN'] = df['TEXT'].apply(lambda x: len(x.split(\" \")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "executionInfo": {
     "elapsed": 258,
     "status": "ok",
     "timestamp": 1710758882769,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "04ztdA9yLMTO",
    "outputId": "49f01705-09d7-49db-cfa1-25cf3913774a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax && window.MathJax.Hub && window.MathJax.Hub.Config) {window.MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script charset=\"utf-8\" src=\"https://cdn.plot.ly/plotly-2.24.1.min.js\"></script>                <div id=\"b147f992-54f0-4825-a192-c689a5f392ed\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"b147f992-54f0-4825-a192-c689a5f392ed\")) {                    Plotly.newPlot(                        \"b147f992-54f0-4825-a192-c689a5f392ed\",                        [{\"alignmentgroup\":\"True\",\"box\":{\"visible\":true},\"hovertemplate\":\"SENT_LEN=%{y}\\u003cextra\\u003e\\u003c\\u002fextra\\u003e\",\"legendgroup\":\"\",\"marker\":{\"color\":\"#636efa\"},\"name\":\"\",\"offsetgroup\":\"\",\"orientation\":\"v\",\"points\":\"all\",\"scalegroup\":\"True\",\"showlegend\":false,\"x0\":\" \",\"xaxis\":\"x\",\"y\":[30,62,35,20,40,40,46,38,47,39,50,13,43,42,65,48,41,28,46,51,45,7,19,47,11,43,51,51,43,11,49,38,42,52,53,52,40,18,36,47,53,45,49,37,46,44,48,50,41,48,39,43,45,49,54,34,46,44,52,41,12,52,48,23,21,49,12,44,47,55,49,48,46,38,50,23,44,21,15,46,44,6,49,25,44,55,52,32,44,39,6,33,17,15,34,51,48,48,11,44,21,13,54,49,55,54,48,46,49,28,55,27,18,49,15,45,14,50,56,40,43,37,54,36,33,40,33,30,32,32,31,33,33,31,40,43,41,32,33,30,29,30,49,50,31,43,30,31,49,33,46,33,32,31,56,34,30,35,35,32,35,30,33,33,33,33,32,34,36,32,33,33,33,35,34,36,54,33,35,35,35,35,35,30,32,31,46,33,32,31,33,31,46,31,32,31,31,31,39,40,33,32,31,33,31,42,32,31,31,32,32,31,31,31,33,35,33,33,33,35,46,35,35,34,34,35,32,35,32,32,32,36,35,35,35,32,33,34,35,30,32,33,35,34,30,31,45,33,43,30,31,32,31,31,33,40,31,33,62,5,32,31,59,32,40,30,30,46,31,33,32,33,32,33,33,34,32,30,6,30,33,41,31,23,32,31,42,6,62,47,36,31,44,41,48,32,45,19,33,43,33,51,32,44,32,31,31,29,32,46,31,32,29,30,31,32,30,32,43,33,32,32,31,34,48,40,31,31,31,30,31,32,31,33,44,52,23,40,46,23,23,23,42,37,42,49,46,17,35,42,54,23,42,41,43,42,33,30,32,34,34,36,34,35,34,34,34,35,36,32,32,37,34,35,33,35,33,35,32,35,35,36,35,33,36,33,15,36,32,39,36,32,40,32,6,43,36,39,33,33,33,31,43,40,35,33,43,35,31,47,33,32,53,48,36,32,30,29,32,33,30,31,31,30,31,30,32,30,30,32,30,30,32,30,33,31,32,31,30,32,32,32,31,30,41,32,50,35,33,39,50,32,32,35,32,35,34,32,33,35,35,34,34,34,34,35,34,46,32,34,30,32,36,35,44,30,51,40,36,35,33,25,33,31,32,44,35,21,32,39,39,33,33,31,39,32,43,42,32,42,40,36,43,32,35,34,34,35,34,33,35,34,34,34,35,34,30,35,41,32,34,35,35,32,29,43,46,29,31,38,32,43,31,53,32,45,61,30,43,31,32,31,59,32,31,31,48,31,30,32,31,30,32,31,31,31,43,31,32,44,20,58,35,30,33,35,32,47,32,39,33,33,29,39,48,38,52,52,32,39,43,43,50,43,39,31,43,30,35,32,51,33,31,31,32,30,30,39,30,40,33,31,39,31,47,30,31,45,31,32,44,31,50,10,42,30,30,31,46,30,50,54,41,41,35,43,28,43,32,32,36,30,27,32,32,32,6,38,45,45,30,32,41,43,6,30,6,33,39,39,39,32,33,32,33,27,41,45,6,30,39,39,41,34,33,33,32,45,40,36,47,32,30,33,39,35,46,27,33,46,35,32,31,27,44,41,32,33,40,41,33,27,44,30,33,33,35,45,47,35,27,33,36,39,35,33,35,47,32,35,32,33,31,43,6,33,30,43,32,32,35,32,38,14,40,30,43,33,42,42,41,31,33,27,32,47,33,32,40,35,30,43,44,49,41,38,45,50,43,43,47,52,41,47,40,54,39,39,49,42,37,40,39,39,51,54,36,40,39,41,40,37,40,46,39,40,39,48,39,40,41,42,52,48,51,50,46,46,6,49,39,48,11,46,45,45,6,49,10,6,46,52,42,41,47,40,42,42,41,37,42,38,52,39,49,39,40,40,5,47,37,40,40,9,41,39,40,40,39,44,44,46,45,48,40,45,41,6,48,49,40,42,50,48,48,37,48,48,48,38,50,41,49,49,50,39,39,48,47,48,47,47,49,37,48,48,47,47,40,41,36,39,44,43,32,39,38,38,40,40,42,38,41,43,39,39,40,37,38,40,39,39,40,37,37,40,38,37,47,41,50,47,48,47,41,47,48,50,49,39,42,49,39,48,51,38,39,48,50,38,47,39,54,40,42,50,39,38,47,47,47,48,48,48,50,57,53,49,49,34,54,38,48,39,47,38,47,38,39,48,38,48,47,38,48,55,38,35,37,38,42,33,39,55,38,38,39,47,40,48,48,54,44,48,38,48,43,40,49,50,48,49,49,47,47,47,37,40,38,48,39,35,39,48,39,39,48,48,38,36,48,37,48,48,47,48,38,50,48,48,47,48,53,40,47,47,38,40,47,40,40,47,48,50,49,35,49,15,37,46,46,11,55,52,37,28,47,48,48,45,6,50,26,48,43,41,55,48,46,51,54,48,31,40,40,10,52,57,53,24,5,42,46,44,44,41,44,52,14,66,46,42,40,48,48,44,33,5,48,34,24,40,46,46,40,49,5,51,49,50,53,47,53,46,48,51,43,57,54,43,42,25,49,20,50,38,42,43,8,44,43,46,40,46,19,6,40,47,6,43,35,63,47,43,43,48,33,51,50,49,48,44,51,48,46,48,6,46,45,25,6,48,42,37,47,39,47,44,21,10,43,45,37,40,7,32,20,45,49,47,46,23,43,47,41,42,43,44,56,44,53,32,44,48,22,24,45,12,48,51,50,9,50,55,15,41,49,46,35,43,49,47,45,111,40,45,47,47,43,40,47,45,49,47,47,54,44,44,44,52,43,37,49,46,50,45,42,52,47,25,45,6,58,58,52,31,49,33,42,50,33,38,43,41,30,44,48,44,30,32,45,38,53,63,51,46,48,50,49,53,63,24,21,54,47,63,52,50,48,44,53,6,41,51,6,40,49,42,48,52,21,17,40,43,44,41,30,41,43,35,41,43,5,6,59,48,50,45,41,36,7,49,47,44,41,45,51,47,49,44,44,42,48,48,43,39,50,49,43,55,48,55,49,55,47,42,45,49,38,70,48,51,43,10,50,50,34,52,46,37,36,23,30,40,38,45,45,40,57,54,57,44,46,43,55,54,56,56,42,40,53,48,42,45,55,57,54,45,39,40,49,41,47,53,36,42,40,23,10,40,52,34,50,33,49,42,22,40,46,51,53,40,6,48,41,39,48,47,47,37,37,27,55,41,43,44,42,42,44,40,41,34,40,49,40,54,44,39,49,49,40,38,39,35,39,50,44,43,36,42,37,54,44,54,48,36,40,54,51,54,56,54,46,52,38,40,39,35,43,40,45,58,59,41,45,48,44,40,39,40,53,40,42,52,54,41,33,21,43,54,54,39,45,47,34,43,36,27,9,38,42,41,43,21,44,35,37,52,25,43,39,41,53,44,46,20,21,31,47,38,43,34,46,47,42,43,45,38,44,18,45,34,42,17,21,41,33,44,45,42,19,44,48,37,43,44,39,21,45,39,20,20,45,43,39,45,46,46,53,33,22,29,17,45,42,18,41,39,24,40,17,17,33,42,54,43,43,42,47,47,21,37,20,20,42,45,26,38,48,36,45,44,17,37,37,20,40,45,45,21,45,47,38,33,42,21,44,42,38,46,27,46,34,63,35,47,37,51,48,47,38,47,44,36,47,17,52,5,45,37,49,44,46,45,37,47,46,29,43,46,46,44,28,43,41,47,42,44,33,46,47,43,42,45,41,33,44,46,42,45,43,48,42,47,48,27,48,40,48,26,47,45,44,27,42,44,46,37,48,47,55,46,41,45,29,47,57,50,41,49,46,55,46,35,43,53,49,45,41,37,50,41,41,44,43,46,29,41,43,41,41,43,44,46,44,46,46,55,43,44,46,45,42,43,42,41,43,44,33,42,48,43,43,43,41,43,40,44,30,43,45,40,12,45,46,19,22,41,43,41,42,38,28,45,46,44,47,42,53,41,40,43,43,49,26,47,46,21,44,47,43,43,42,35,40,39,51,41,32,43,46,30,39,50,45,44,41,56,44,44,41,46,43,41,41,34,46,29,41,45,41,42,33,38,44,47,43,32,42,35,46,42,42,42,46,41,47,39,43,30,46,42,39,48,47,48,45,26,48,43,51,46,52,43,126,26,43,50,32,49,34,22,41,23,44,41,47,41,45,41,46,41,45,40,48,39,52,53,36,57,43,14,14,43,44,28,41,8,41,51,37,41,19,12,44,50,37,48,40,22,40,26,47,55,42,37,41,40,53,51,34,39,38,41,44,41,40,46,44,6,46,39,17,32,39,41,41,51,48,42,13,49,42,33,40,41,38,13,45,39,35,42,49,44,47,40,45,55,37,47,52,34,38,52,44,47,48,43,38,37,40,44,43,28,38,47,47,51,45,52,34,39,47,50,40,42,45,14,46,49,46,42,45,60,11,42,447,54,48,43,49,48,40,45,42,19,42,44,38,43,47,56,43,31,45,52,42,42,42,54,45,43,36,41,42,22,41,40,56,41,49,47,38,39,37,42,38,39,42,45,53,39,13,49,44,46,41,38,45,43,46,45,7,31,48,42,31,48,31,39,40,45,54,39,48,47,43,36,46,32,40,9,41,50,41,43,48,39,42,42,43,44,65,39,51,34,19,45,38,39,54,51,43,42,44,36,43,9,47,27,42,13,42,43,51,34,29,42,47,43,34,28,40,45,48,28,42,53,44,41,39,44,43,41,49,40,41,39,34,54,40,47,45,52,43,48,41,18,48,41,35,53,48,45,40,39,49,49,42,15,17,44,41,47,44,43,18,11,21,39,47,36,38,11,37,45,35,45,47,18,27,40,52,42,40,18,48,41,36,6,37,38,37,15,59,37,39,11,39,43,44,42,40,19,39,44,52,49,40,46,50,37,44,50,46,43,32,38,39,49,44,45,49,40,52,45,28,53,45,43,39,45,48,42,160,57,44,29,41,49,50,41,50,50,44,48,45,52,39,39,46,41,53,40,39,47,40,44,44,29,47,38,46,21,46,39,44,43,38,41,41,43,37,38,51,39,38,47,57,41,40,18,46,47,40,47,56,38,46,32,40,48,43,13,40,41,46,48,40,31,43,48,43,43,47,48,47,47,49,30,43,37,39,36,42,37,40,43,41,55,56,43,42,45,53,39,24,45,48,47,42,43,51,42,40,51,51,41,42,48,44,39,42,35,45,41,42,49,55,49,46,38,40,47,43,42,37,41,57,46,46,45,45,51,15,53,41,52,48,49,37,50,38,46,26,12,43,18,45,54,7,47,39,53,52,38,33,53,19,51,49,41,45,35,48,5,13,13,25,5,16,45,49,38,49,45,45,49,55,49,48,48,47,50,45,48,39,25,41,41,12,52,54,40,43,50,48,33,33,40,35,47,43,50,43,37,53,46,42,43,41,9,49,44,45,46,50,26,53,44,43,42,43,42,49,44,43,49,44,42,45,27,44,46,47,44,31,42,22,27,49,54,51,43,19,46,49,53,23,54,49,31,54,45,41,45,53,53,46,47,41,42,31,45,46,38,15,49,41,41,38,38,44,36,6,50,26,36,37,44,45,45,44,44,17,29,41,48,46,41,29,31,39,33,57,19,19,6,19,11,55,49,52,45,45,45,44,45,44,45,45,44,44,34,14,41,46,48,50,46,43,44,45,48,45,47,40,45,44,46,45,44,53,46,48,45,44,45,46,47,44,47,44,47,46,45,45,45,46,54,47,45,45,26,44,45,47,45,44,58,21,6,42,47,45,44,45,43,45,45,47,43,44,45,45,49,46,44,45,45,45,45,44,50,45,45,48,58,53,15,43,41,42,48,43,49,41,56,34,44,43,45,41,71,39,27,40,42,40,42,40,53,53,34,49,38,39,39,43,37,40,45,46,43,45,48,28,38,50,47,53,39,39,41,21,46,43,40,48,42,40,38,39,46,44,22,57,30,16,41,56,42,45,11,54,39,54,35,36,45,35,43,52,41,28,39,21,25,55,38,35,42,34,36,52,34,50,41,41,44,40,11,43,42,25,31,10,45,32,40,47,43,54,41,30,41,42,36,45,27,46,42,44,31,43,47,36,44,40,37,51,42,41,43,43,43,54,41,42,46,32,44,43,53,38,48,39,43,37,41,42,52,46,41,44,47,42,46,43,46,40,44,41,35,49,43,46,45,52,49,39,45,41,40,43,49,38,41,53,44,50,42,42,48,53,27,42,36,39,32,46,40,55,16,42,21,46,54,40,53,41,42,37,57,22,53,42,36,41,8,53,14,40,42,42,52,37,20,42,39,49,38,45,44,30,47,42,42,48,43,46,39,33,35,43,39,11,13,46,42,37,53,41,45,41,11,49,48,37,45,49,48,40,40,52,43,48,40,46,42,40,47,44,30,41,10,43,48,42,47,56,40,41,40,43,40,42,39,55,43,34,47,42,49,42,47,16,29,53,23,48,18,37,52,48,57,22,25,40,55,23,34,50,53,61,15,46,12,39,22,23,28,42,39,46,54,41,39,38,45,36,43,37,44,39,38,39,46,46,27,39,39,10,47,41,49,49,52,41,44,45,46,41,42,46,52,44,47,42,44,47,55,38,41,53,42,54,33,45,40,39,39,36,42,32,43,43,47,44,42,44,43,39,39,43,39,49,46,39,40,36,51,41,35,41,43,37,36,50,43,46,28,42,35,30,44,51,48,36,35,45,37,29,47,45,41,41,41,52,40,28,52,36,52,40,37,15,52,34,39,39,41,39,39,40,47,47,42,46,54,37,11,49,11,45,49,52,46,41,46,50,50,44,55,47,43,41,46,46,40,42,33,40,45,41,41,41,43,40,41,42,41,41,43,41,41,42,43,41,41,41,42,41,19,41,42,41,43,45,40,9,40,50,41,13,40,30,47,42,42,40,53,44,43,47,52,39,35,38,27,45,44,44,47,44,50,39,45,39,60,34,43,45,36,40,40,37,45,47,41,45,42,41,12,46,35,39,10,49,44,43,11,43,45,47,30,32,43,40,46,44,44,41,49,43,32,48,41,18,46,40,46,40,43,46,41,56,39,46,38,55,45,47,40,47,39,49,58,26,42,34,46,48,39,58,31,41,53,40,42,41,41,42,46,36,15,40,60,24,23,19,60,24,41,17,25,57,37,19,66,3,16,27,10,52,53,17,3,3,42,25,37,45,17,17,19,15,17,44,41,15,57,15,44,48,44,40,41,44,45,32,13,41,46,37,56,17,41,41,40,36,40,47,47,41,10,37,39,37,47,45,45,41,44,35,37,13,7,50,38,7,15,38,49,11,40,31,19,15,13,41,13,49,9,37,45,17,51,3,38,15,46,3,46,48,24,44,50,37,38,42,46,43,38,6,36,18,46,38,42,17,38,48,44,32,41,58,42,40,40,46,31,43,40,48,39,19,15,45,35,15,15,53,45,42,56,17,15,29,44,3,41,15,12,44,33,15,43,47,53,56,3,15,7,45,44,15,41,42,49,41,45,44,46,45,42,50,58,44,50,48,44,48,40,43,42,40,41,42,44,41,42,45,4,13,3,57,30,41,40,45,43,40,31,15,3,44,49,37,61,45,19,13,43,39,30,44,47,21,18,40,48,48,41,49,58,41,29,43,36,39,24,15,32,14,13,15,46,15,51,11,6,17,13,59,17,3,45,15,44,24,39,44,30,41,45,46,43,10,39,16,42,41,37,43,16,17,41,12,43,48,32,35,33,25,39,43,47,39,43,40,47,37,17,3,52,17,15,32,42,19,13,37,3,35,19,49,28,41,25,17,19,3,15,17,25,3,40,24,3,49,15,27,46,25,13,45,3,32,17,49,23,15,45,41,3,38,52,58,17,21,15,17,49,15,17,37,15,17,13,57,47,45,19,3,52,3,50,49,50,44,48,47,50,40,19,40,45,49,42,49,41,52,3,57,17,39,33,17,53,40,42,42,58,39,40,62,53,38,58,41,38,27,62,48,40,41,41,42,41,41,33,35,41,38,41,3,15,15,52,13,45,17,48,9,3,32,41,55,15,52,43,48,44,35,15,38,3,44,72,3,35,45,15,13,17,39,37,40,46,43,41,37,17,41,32,44,32,36,49,41,38,44,33,50,24,41,45,40,50,41,13,41,44,46,35,43,32,45,42,47,45,17,42,15,52,44,45,32,40,44,42,44,46,48,38,46,41,32,45,32,35,44,51,44,46,36,26,3,15,43,41,50,3,45,46,44,35,42,31,3,13,17,17,19,54,3,43,38,37,50,50,19,29,3,37,45,47,6,41,41,46,47,46,40,51,37,40,37,20,41,39,25,35,34,42,42,36,46,43,70,39,19,36,40,34,49,42,42,40,39,41,41,42,43,51,41,46,40,36,45,44,44,42,54,48,43,37,49,43,14,22,42,21,36,41,36,42,53,39,32,47,43,47,36,32,51,41,42,8,41,42,49,40,46,51,37,17,40,25,32,51,23,43,47,46,45,39,40,41,19,54,42,44,32,47,36,38,33,47,45,47,41,42,44,43,40,36,40,46,35,33,24,58,39,40,48,43,44,41,19,41,43,36,39,44,35,37,43,42,36,43,48,41,15,38,35,38,32,13,35,52,40,44,44,36,34,42,41,41,41,52,40,46,36,46,59,35,44,41,40,17,43,44,43,42,37,39,37,45,48,43,43,19,17,53,20,29,3,15,25,25,25,52,38,18,41,36,17,49,17,38,23,17,17,47,17,16,43,19,15,53,15,17,17,49,13,36,24,12,15,40,19,60,3,17,15,39,42,41,46,12,17,3,15,3,7,21,9,3,30,3,19,51,43,9,19,19,31,45,21,22,19,15,15,23,15,37,37,41,19,38,45,19,37,17,51,52,37,47,50,60,48,18,43,19,18,18,17,20,48,48,33,43,37,48,20,48,48,18,20,11,21,39,18,40,18,44,19,22,3,40,5,48,3,8,26,57,19,19,15,53,19,9,17,13,57,3,76,45,35,17,3,17,3,17,42,41,45,37,46,32,42,43,43,52,42,46,41,42,41,40,14,43,45,36,32,41,43,49,42,51,42,41,49,41,32,36,48,41,34,34,41,38,46,47,35,46,58,47,39,39,52,41,39,45,46,32,36,46,41,43,47,32,32,3,49,43,62,15,15,46,38,8,38,3,35,56,3,27,3,19,49,44,3,69,45,28,40,17,45,44,45,39,44,51,42,51,39,38,40,39,32,51,32,39,53,43,48,41,45,47,41,35,12,16,42,32,45,32,42,39,43,48,42,42,39,44,47,14,42,42,45,40,17,35,44,42,42,32,36,38,42,37,41,40,39,42,42,45,52,45,41,27,51,52,17,44,21,47,37,26,47,3,50,13,17,51,3,47,44,17,3,45,15,43,3,49,48,17,3,52,26,17,9,17,45,50,37,15,28,17,29,37,25,27,23,3,45,11,45,11,17,3,41,49,3,48,45,60,9,13,43,17,23,56,16,22,44,29,24,27,17,12,33,11,50,22,48,36,48,23,50,48,22,40,24,47,42,39,39,41,46,37,51,48,45,45,19,55,48,14,43,40,115,64,49,42,47,59,36,51,15,9,48,42,37,31,44,19,26,44,42,41,47,42,47,42,46,28,35,47,39,19,18,12,45,23,29,46,17,37,47,12,46,17,5,42,15,38,45,41,17,40,54,26,13,53,23,9,42,15,41,21,35,8,33,45,52,43,51,26,15,16,42,11,18,43,47,15,15,23,36,44,61,39,17,13,50,15,44,44,50,48,46,49,14,52,13,40,38,52,46,6,14,52,48,59,15,24,41,20,5,14,11,48,50,10,46,24,50,54,44,21,17,51,55,44,46,43,46,61,44,25,45,23,50,14,21,48,48,42,48,52,20,47,7,46,29,31,48,15,50,38,39,43,33,26,43,45,41,19,47,45,49,49,11,47,15,12,31,88,13,58,43,41,12,44,43,43,41,45,18,39,51,41,14,17,15,55,47,52,34,19,31,17,22,7,38,44,18,13,38,26,41,18,57,50,54,13,15,13,50,47,13,15,45,17,42,20,37,51,44,37,37,39,9,13,55,45,19,11,19,18,46,45,51,26,46,37,27,15,14,39,37,44,13,23,34,39,9,13,10,17,49,16,21,53,11,14,48,14,14,45,43,13,44,19,40,51,45,13,17,21,17,51,26,53,38,17,20,39,47,48,38,42,45,42,48,18,30,49,52,15,45,51,43,35,42,29,7,23,51,35,42,50,49,46,46,24,40,48,31,37,22,43,51,50,25,32,13,13,32,47,47,31,45,48,44,57,53,44,11,49,46,48,26,47,23,46,53,44,9,49,44,49,11,47,20,13,28,42,43,39,44,34,38,19,63,52,38,43,45,12,40,43,11,30,38,51,37,43,52,13,51,44,31,31,51,22,39,19,27,50,45,43,7,31,48,16,93,45,27,21,35,11,48,48,13,41,47,30,9,47,15,12,22,52,30,14,59,46,49,41,16,21,42,39,45,11,15,13,38,10,12,12,13,16,19,21,50,32,45,20,14,32,49,32,20,9,24,21,50,20,41,16,51,39,41,21,48,51,49,9,17,43,24,11,47,42,13,43,15,47,19,20,41,44,15,23,45,41,53,56,50,12,34,17,15,46,52,42,52,33,31,47,11,39,20,40,42,29,21,11,48,11,38,15,46,33,16,25,41,29,36,14,43,42,42,13,47,44,26,36,38,28,13,15,39,19,22,15,17,42,20,56,42,45,46,46,17,23,38,17,37,34,12,20,41,41,49,42,17,17,7,44,48,46,9,10,41,41,32,34,16,47,34,16,30,9,16,30,16,46,40,34,30,24,23,22,30,33,41,16,23,49,17,13,65,7,50,44,43,7,25,52,37,36,49,13,43,39,38,51,12,37,24,47,49,16,50,55,59,41,26,16,30,56,16,34,16,33,27,9,22,30,23,18,16,16,40,16,44,32,16,16,24,6,28,49,52,17,59,31,51,45,40,41,41,44,51,40,42,51,42,41,30,28,31,43,20,30,34,33,43,16,23,42,29,16,50,16,30,16,16,26,36,23,41,33,36,474,50,11,7,26,42,32,45,45,10,47,41,42,50,50,39,37,27,27,16,55,51,23,47,45,33,48,5,49,10,28,60,48,27,37,28,6,42,49,27,44,44,11,44,32,4,8,27,4,46,38,6,44,40,90,15,40,37,40,45,39,41,43,39,46,47,37,43,16,44,42,6,44,40,44,42,18,6,40,42,41,29,41,30,44,51,42,28,41,47,36,41,41,42,40,40,43,43,38,41,37,40,39,43,42,46,48,46,38,46,50,42,46,42,40,41,6,27,51,39,41,41,38,63,9,42,41,16,40],\"y0\":\" \",\"yaxis\":\"y\",\"type\":\"violin\"}],                        {\"template\":{\"data\":{\"histogram2dcontour\":[{\"type\":\"histogram2dcontour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"choropleth\":[{\"type\":\"choropleth\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"histogram2d\":[{\"type\":\"histogram2d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmap\":[{\"type\":\"heatmap\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"heatmapgl\":[{\"type\":\"heatmapgl\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"contourcarpet\":[{\"type\":\"contourcarpet\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"contour\":[{\"type\":\"contour\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"surface\":[{\"type\":\"surface\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"},\"colorscale\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]]}],\"mesh3d\":[{\"type\":\"mesh3d\",\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}],\"scatter\":[{\"fillpattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2},\"type\":\"scatter\"}],\"parcoords\":[{\"type\":\"parcoords\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolargl\":[{\"type\":\"scatterpolargl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"bar\":[{\"error_x\":{\"color\":\"#2a3f5f\"},\"error_y\":{\"color\":\"#2a3f5f\"},\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"bar\"}],\"scattergeo\":[{\"type\":\"scattergeo\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterpolar\":[{\"type\":\"scatterpolar\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"histogram\":[{\"marker\":{\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"histogram\"}],\"scattergl\":[{\"type\":\"scattergl\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatter3d\":[{\"type\":\"scatter3d\",\"line\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattermapbox\":[{\"type\":\"scattermapbox\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scatterternary\":[{\"type\":\"scatterternary\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"scattercarpet\":[{\"type\":\"scattercarpet\",\"marker\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}}}],\"carpet\":[{\"aaxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"baxis\":{\"endlinecolor\":\"#2a3f5f\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"minorgridcolor\":\"white\",\"startlinecolor\":\"#2a3f5f\"},\"type\":\"carpet\"}],\"table\":[{\"cells\":{\"fill\":{\"color\":\"#EBF0F8\"},\"line\":{\"color\":\"white\"}},\"header\":{\"fill\":{\"color\":\"#C8D4E3\"},\"line\":{\"color\":\"white\"}},\"type\":\"table\"}],\"barpolar\":[{\"marker\":{\"line\":{\"color\":\"#E5ECF6\",\"width\":0.5},\"pattern\":{\"fillmode\":\"overlay\",\"size\":10,\"solidity\":0.2}},\"type\":\"barpolar\"}],\"pie\":[{\"automargin\":true,\"type\":\"pie\"}]},\"layout\":{\"autotypenumbers\":\"strict\",\"colorway\":[\"#636efa\",\"#EF553B\",\"#00cc96\",\"#ab63fa\",\"#FFA15A\",\"#19d3f3\",\"#FF6692\",\"#B6E880\",\"#FF97FF\",\"#FECB52\"],\"font\":{\"color\":\"#2a3f5f\"},\"hovermode\":\"closest\",\"hoverlabel\":{\"align\":\"left\"},\"paper_bgcolor\":\"white\",\"plot_bgcolor\":\"#E5ECF6\",\"polar\":{\"bgcolor\":\"#E5ECF6\",\"angularaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"radialaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"ternary\":{\"bgcolor\":\"#E5ECF6\",\"aaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"baxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"},\"caxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\"}},\"coloraxis\":{\"colorbar\":{\"outlinewidth\":0,\"ticks\":\"\"}},\"colorscale\":{\"sequential\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"sequentialminus\":[[0.0,\"#0d0887\"],[0.1111111111111111,\"#46039f\"],[0.2222222222222222,\"#7201a8\"],[0.3333333333333333,\"#9c179e\"],[0.4444444444444444,\"#bd3786\"],[0.5555555555555556,\"#d8576b\"],[0.6666666666666666,\"#ed7953\"],[0.7777777777777778,\"#fb9f3a\"],[0.8888888888888888,\"#fdca26\"],[1.0,\"#f0f921\"]],\"diverging\":[[0,\"#8e0152\"],[0.1,\"#c51b7d\"],[0.2,\"#de77ae\"],[0.3,\"#f1b6da\"],[0.4,\"#fde0ef\"],[0.5,\"#f7f7f7\"],[0.6,\"#e6f5d0\"],[0.7,\"#b8e186\"],[0.8,\"#7fbc41\"],[0.9,\"#4d9221\"],[1,\"#276419\"]]},\"xaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"yaxis\":{\"gridcolor\":\"white\",\"linecolor\":\"white\",\"ticks\":\"\",\"title\":{\"standoff\":15},\"zerolinecolor\":\"white\",\"automargin\":true,\"zerolinewidth\":2},\"scene\":{\"xaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"yaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2},\"zaxis\":{\"backgroundcolor\":\"#E5ECF6\",\"gridcolor\":\"white\",\"linecolor\":\"white\",\"showbackground\":true,\"ticks\":\"\",\"zerolinecolor\":\"white\",\"gridwidth\":2}},\"shapedefaults\":{\"line\":{\"color\":\"#2a3f5f\"}},\"annotationdefaults\":{\"arrowcolor\":\"#2a3f5f\",\"arrowhead\":0,\"arrowwidth\":1},\"geo\":{\"bgcolor\":\"white\",\"landcolor\":\"#E5ECF6\",\"subunitcolor\":\"white\",\"showland\":true,\"showlakes\":true,\"lakecolor\":\"white\"},\"title\":{\"x\":0.05},\"mapbox\":{\"style\":\"light\"}}},\"xaxis\":{\"anchor\":\"y\",\"domain\":[0.0,1.0]},\"yaxis\":{\"anchor\":\"x\",\"domain\":[0.0,1.0],\"title\":{\"text\":\"SENT_LEN\"}},\"legend\":{\"tracegroupgap\":0},\"margin\":{\"t\":60},\"violinmode\":\"group\"},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('b147f992-54f0-4825-a192-c689a5f392ed');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                            </script>        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualizing sentence length\n",
    "fig = px.violin(df, y=\"SENT_LEN\", box=True, # draw box plot inside the violin\n",
    "                points='all', # can be 'outliers', or False\n",
    "               )\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JgK_w2CKlUYM"
   },
   "source": [
    "# **Splitting data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 248,
     "status": "ok",
     "timestamp": 1710053130383,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "t-v4aPj1U9_l",
    "outputId": "5dd69f0a-0038-4994-dc4c-7bb68665a3ba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: 3750\n",
      "Test set size: 250\n",
      "Validation set size: 1000\n"
     ]
    }
   ],
   "source": [
    "train_df, temp_df = train_test_split(df, test_size=0.25, random_state=1111)\n",
    "# Calculate the proportion of test size in the temporary dataset\n",
    "test_size_in_temp = 0.20 / (0.20 + 0.05)\n",
    "# Now split the temporary set into test and validation sets\n",
    "test_df, val_df = train_test_split(temp_df, test_size=test_size_in_temp, random_state=1111)\n",
    "\n",
    "print(f\"Training set size: {len(train_df)}\")\n",
    "print(f\"Test set size: {len(test_df)}\")\n",
    "print(f\"Validation set size: {len(val_df)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0Tr7I6ecmkET"
   },
   "source": [
    "# **Model Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ND_86ViQlKuI"
   },
   "outputs": [],
   "source": [
    "# This function evaluates the performance of a trained model on a test dataset.\n",
    "def evaluateBoWModels(trained_model, test_data):\n",
    "  # Extract the 'Vendor' column from the test dataset and convert it to a list.\n",
    "  # This list contains the actual labels/classes for each test instance.\n",
    "  vendors = test_data['Vendor'].to_list()\n",
    "  # Extract the 'TEXT' column from the test dataset and convert it to a list.\n",
    "  # This list contains the input data (text) for making predictions.\n",
    "  text = test_data['TEXT'].to_list()\n",
    "\n",
    "  # Use the trained model to make predictions on the input text from the test dataset.\n",
    "  predictions = trained_model.predict(text)\n",
    "\n",
    "  # Calculate the macro-averaged F1 score, which calculates metrics for each label,\n",
    "  # and finds their unweighted mean. This does not take label imbalance into account.\n",
    "  macro_f1 = f1_score(vendors, predictions, average='macro')\n",
    "  # Calculate the micro-averaged F1 score, which aggregates the contributions of all classes\n",
    "  # to compute the average metric. This accounts for label imbalance.\n",
    "  micro_f1 = f1_score(vendors, predictions, average='micro')\n",
    "  # Calculate the weighted-averaged F1 score, which calculates metrics for each label,\n",
    "  # and finds their average weighted by support (the number of true instances for each label).\n",
    "  # This alters 'macro' to account for label imbalance.\n",
    "  weighted_f1 = f1_score(vendors, predictions, average='weighted')\n",
    "  # Calculate the balanced accuracy, which is the average of recall obtained on each class.\n",
    "  # This metric is useful for dealing with imbalanced datasets.\n",
    "  accuracy = balanced_accuracy_score(vendors, predictions)\n",
    "\n",
    "  # Return the calculated metrics: accuracy, weighted F1 score, micro F1 score, and macro F1 score.\n",
    "  return accuracy, weighted_f1, micro_f1, macro_f1\n",
    "\n",
    "# This function generates and prints a detailed classification report\n",
    "# for the predictions made by a trained model on the test dataset.\n",
    "def generate_classification_report(trained_model, test_data):\n",
    "  # Extract the 'Vendor' column from the test dataset and convert it to a list.\n",
    "  # This list contains the actual labels/classes for each test instance.\n",
    "  vendors = test_data['Vendor'].to_list()\n",
    "\n",
    "  # Extract the 'TEXT' column from the test dataset and convert it to a list.\n",
    "  # This list contains the input data (text) for making predictions.\n",
    "  text = test_data['TEXT'].to_list()\n",
    "\n",
    "  # Use the trained model to make predictions on the input text from the test dataset.\n",
    "  predictions = trained_model.predict(text)\n",
    "\n",
    "  # Generate a classification report comparing the actual labels to the predicted labels.\n",
    "  # The report includes main classification metrics: precision, recall, f1-score for each class,\n",
    "  # along with a weighted average of these metrics. The 'digits=4' parameter formats the output\n",
    "  # to display numbers with four decimal places for precision.\n",
    "  print(classification_report(np.array(vendors), np.array(predictions), digits=4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8esdVgqoqrZU"
   },
   "source": [
    "# **Model Architecture**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QTTxKe33ljYm"
   },
   "outputs": [],
   "source": [
    "class BoWModel(object):\n",
    "\n",
    "    def __init__(self, train_data, test_data, stats_model_type, n_splits=2):\n",
    "        self.train_data = train_data\n",
    "        self.test_data = test_data\n",
    "\n",
    "        # Initializing a KFold cross-validator with a specified number of splits,\n",
    "        # enabling shuffling to randomize the splits and setting a random state for reproducibility.\n",
    "        # Defining a scoring dictionary with accuracy as the metric to be used for model evaluation.\n",
    "        self.cv = KFold(n_splits=n_splits, shuffle=True, random_state=1111)\n",
    "        self.scoring = {'Accuracy': make_scorer(accuracy_score)}\n",
    "\n",
    "        # Specifying the parameters to be tuned during the hyperparameter optimization process.\n",
    "        # Includes the n-gram range for the CountVectorizer and the use of idf for the TfidfTransformer.\n",
    "        self.parameters = {'vect__ngram_range': [(1, 1), (1, 2)], 'tfidf__use_idf': (True, False)}\n",
    "        # Storing the type of statistical model to be used, as specified during instantiation.\n",
    "        self.stats_model_type = stats_model_type\n",
    "\n",
    "        # Extracting the text corpus (titles) and the target variable (vendors) from the training data.\n",
    "        self.train_corpus = list(self.train_data['TEXT'])\n",
    "        self.train_vendor = list(self.train_data['Vendor'])\n",
    "\n",
    "    def train_models(self):\n",
    "      # Conditional blocks to handle different types of statistical models.\n",
    "      # Each block initializes a model pipeline with a CountVectorizer, TfidfTransformer,\n",
    "      # and the specified classifier, followed by a GridSearchCV for hyperparameter optimization.\n",
    "      # Controlling the backend that joblib will use\n",
    "      with parallel_backend('threading', n_jobs=-1):\n",
    "          # Depending on the stats_model_type, different classifiers are used:\n",
    "          # MultinomialNB, SVC, RandomForestClassifier, LogisticRegression, or MLPClassifier.\n",
    "          # For each classifier, a GridSearchCV object is created with the specified parameters,\n",
    "          # cross-validator, scoring metrics, and verbosity. The GridSearchCV is then fitted with\n",
    "          # the training data.\n",
    "\n",
    "          # After fitting, model evaluation metrics (accuracy, weighted F1, micro F1, macro F1) are calculated\n",
    "          # using a separate function, `evaluateBoWModels`, which is not defined in this snippet but is assumed\n",
    "          # to take the fitted model and test data as input to compute the metrics.\n",
    "\n",
    "          if self.stats_model_type == 'MultinomialNB':\n",
    "            # Multinomial-NB\n",
    "            clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', MultinomialNB())])\n",
    "            gs_clf = GridSearchCV(clf, self.parameters, n_jobs=10, cv=self.cv, return_train_score=True, verbose=1, scoring=self.scoring, refit='Accuracy')\n",
    "            gs_clf = gs_clf.fit(self.train_corpus, self.train_vendor)\n",
    "            accuracy, weighted_f1, micro_f1, macro_f1 = evaluateBoWModels(gs_clf, self.test_data)\n",
    "\n",
    "          elif self.stats_model_type == 'SVC':\n",
    "            # Support Vector Machine\n",
    "            clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', SVC(class_weight='balanced', kernel=\"linear\"))])\n",
    "            gs_clf = GridSearchCV(clf, self.parameters, n_jobs=10, cv=self.cv, return_train_score=True, verbose=1, scoring=self.scoring, refit='Accuracy')\n",
    "            gs_clf = gs_clf.fit(self.train_corpus, self.train_vendor)\n",
    "            accuracy, weighted_f1, micro_f1, macro_f1 = evaluateBoWModels(gs_clf, self.test_data)\n",
    "\n",
    "          elif self.stats_model_type == 'RandomForestClassifier':\n",
    "            # Random Forest\n",
    "            clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', RandomForestClassifier(class_weight='balanced', n_estimators=100, max_depth=5, random_state=1))])\n",
    "            gs_clf = GridSearchCV(clf, self.parameters, n_jobs=10, cv=self.cv, return_train_score=True, verbose=1, scoring=self.scoring, refit='Accuracy')\n",
    "            gs_clf = gs_clf.fit(self.train_corpus, self.train_vendor)\n",
    "            accuracy, weighted_f1, micro_f1, macro_f1 = evaluateBoWModels(gs_clf, self.test_data)\n",
    "\n",
    "          elif self.stats_model_type == 'LogisticRegression':\n",
    "            # Logistic Regression\n",
    "            clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', LogisticRegression(class_weight='balanced', random_state=1))])\n",
    "            gs_clf = GridSearchCV(clf, self.parameters, n_jobs=10, cv=self.cv, return_train_score=True, verbose=1, scoring=self.scoring, refit='Accuracy')\n",
    "            gs_clf = gs_clf.fit(self.train_corpus, self.train_vendor)\n",
    "            accuracy, weighted_f1, micro_f1, macro_f1 = evaluateBoWModels(gs_clf, self.test_data)\n",
    "\n",
    "          elif self.stats_model_type == 'MLPClassifier':\n",
    "            # Multilayer Perceptron Model\n",
    "            clf = Pipeline([('vect', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', MLPClassifier(hidden_layer_sizes=(100,100,100), random_state=1))])\n",
    "            gs_clf = GridSearchCV(clf, self.parameters, n_jobs=10, cv=self.cv, return_train_score=True, verbose=1, scoring=self.scoring, refit='Accuracy')\n",
    "            gs_clf = gs_clf.fit(self.train_corpus, self.train_vendor)\n",
    "            accuracy, weighted_f1, micro_f1, macro_f1 = evaluateBoWModels(gs_clf, self.test_data)\n",
    "\n",
    "          else:\n",
    "            # If an unrecognized stats_model_type is specified, an exception is raised.\n",
    "            raise Exception(\"--stats_model_type can only be one amongst MultinomialNB, MLPClassifier, LogisticRegression, RandomForestClassifier, and SVC\")\n",
    "\n",
    "      return clf, accuracy, weighted_f1, micro_f1, macro_f1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "KOIXiHQPxcqz"
   },
   "source": [
    "# **Training Models**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "It_GI_lWxy_B"
   },
   "outputs": [],
   "source": [
    "scores_df = pd.DataFrame()\n",
    "scores_df[\"Metrics\"] = ['Accuracy','Weighted-F1','Micro-F1','Macro-F1']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fuqT48KT02Zv"
   },
   "source": [
    "[Multinomial Naive Bayes](https://en.wikipedia.org/wiki/Naive_Bayes_classifier)\n",
    "\n",
    "---\n",
    "\n",
    "- It's based on Bayes' theorem with the “naive” assumption of conditional independence between every pair of features given the value of the class variable.\n",
    "- Particularly well-suited for categorical data or problems where features describe the counts or frequency of occurrence of terms, such as text classification.\n",
    "- Assumes that the data follows a multinomial distribution, which is common in text classification problems where the features are related to word counts or frequencies.\n",
    "- Employs maximum likelihood estimation for parameter determination. It often incorporates smoothing techniques (like Laplace smoothing) to handle the issue of zero probabilities in unseen features.\n",
    "- Its simplicity and assumption of feature independence make it computationally efficient and easy to implement, offering strong baseline performance in many NLP tasks.\n",
    "- The performance of the classifier is highly dependent on how the input data is represented. For instance, using term frequency or TF-IDF for text data can significantly impact results.\n",
    "- While effective for discrete data, it's not ideal for continuous data without discretization, as it relies on feature counts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9970,
     "status": "ok",
     "timestamp": 1710053243719,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "aFc-AwMltiDR",
    "outputId": "21088fc8-050f-4788-c742-1e988208bb81"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    }
   ],
   "source": [
    "_, accuracy, weighted_f1, micro_f1, macro_f1 = BoWModel(train_df, test_df, \"MultinomialNB\").train_models()\n",
    "scores_df[\"MultinomialNB\"] = [accuracy, weighted_f1, micro_f1, macro_f1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eXUExgIQ1P4z"
   },
   "source": [
    "[Suport Vector Machine](https://en.wikipedia.org/wiki/Support_vector_machine)\n",
    "\n",
    "---\n",
    "\n",
    "- SVMs are based on the principle of finding the hyperplane that best separates different classes in the feature space with the maximum margin. The margin is defined as the distance between the hyperplane and the nearest points from each class, known as support vectors.\n",
    "- SVMs can perform linear classification but are also capable of handling nonlinear boundaries by applying kernel functions. The kernel trick maps input features into high-dimensional spaces where a linear separator is sufficient, without explicitly computing the coordinates in this higher dimension.\n",
    "- Includes a regularization parameter, which controls the trade-off between achieving a low training error and minimizing model complexity, thereby preventing overfitting.\n",
    "- SVM performance is sensitive to the scaling of data, as it relies on the calculation of distances between data points. Proper feature scaling is often necessary to improve the classifier's performance.\n",
    "- Performs well in high-dimensional spaces, even in cases where the number of dimensions exceeds the number of samples, making it suitable for applications like text classification where this is often the case.\n",
    "- The SVM training process involves solving a convex optimization problem, which guarantees finding a global minimum and thus avoids the pitfalls of local minima.\n",
    "- The training time of SVMs can be relatively high, especially for large datasets, which can be mitigated by choosing appropriate kernel functions and regularization parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 114808,
     "status": "ok",
     "timestamp": 1710053379539,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "kaQ7cVWfwy_I",
    "outputId": "60d08d75-9627-4fdb-9854-1fdacadfc6f3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    }
   ],
   "source": [
    "_, accuracy, weighted_f1, micro_f1, macro_f1 = BoWModel(train_df, test_df, \"SVC\").train_models()\n",
    "scores_df[\"SVC\"] = [accuracy, weighted_f1, micro_f1, macro_f1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "35vgL9Ot1kNm"
   },
   "source": [
    "[Random Forest](https://en.wikipedia.org/wiki/Random_forest)\n",
    "\n",
    "--- \n",
    "\n",
    "- Random Forest is an ensemble learning technique that constructs a multitude of decision trees at training time and outputs the class that is the mode of the classes (classification) or mean prediction (regression) of the individual trees.\n",
    "- Utilizes multiple decision trees to make predictions. The diversity among the trees comes from introducing randomness in the tree building process, which in turn increases the overall performance of the model.\n",
    "- Employs the bootstrap aggregating (bagging) method to promote variance reduction. Each tree in the forest is trained on a random subset of the data, sampled with replacement, making the trees decorrelated and thereby improving the model's generalization ability.\n",
    "- When building trees, each time a split is considered, a random subset of the features is chosen as candidates for the split, adding another layer of randomness that helps in making the model more robust.\n",
    "- Random Forests are known for their high accuracy, robustness, and ease of use. They are less likely to overfit than individual decision trees and work well with both categorical and numerical data.\n",
    "- Capable of handling missing values either by imputing them using the median/mode or by considering missing values as a separate category while splitting nodes in the decision trees.\n",
    "- Provides insights into feature importance, i.e., which features contribute most to the decision-making process, making it useful for feature selection in the modeling pipeline.\n",
    "- While the default settings often perform well, tuning certain hyperparameters like the number of trees in the forest, maximum depth of trees, and the number of features considered for splitting can further enhance model performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11987,
     "status": "ok",
     "timestamp": 1710053391509,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "juJzHWHiw-Hx",
    "outputId": "12cd6809-8c74-4c9c-c952-cf08d0888d72"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    }
   ],
   "source": [
    "_, accuracy, weighted_f1, micro_f1, macro_f1 = BoWModel(train_df, test_df, \"RandomForestClassifier\").train_models()\n",
    "scores_df[\"RandomForest\"] = [accuracy, weighted_f1, micro_f1, macro_f1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kw5kb4Fq2HZp"
   },
   "source": [
    "[Logistic Regression](https://en.wikipedia.org/wiki/Logistic_regression)\n",
    "\n",
    "--- \n",
    "\n",
    "- Logistic regression is a statistical method used for binary classification problems, where the outcome is dichotomous (e.g., yes/no, 0/1).\n",
    "- It estimates the probabilities of the different possible outcomes of a categorically distributed dependent variable, given a set of independent variables. The model uses the logit function (the logarithm of the odds) to model the relationship between the dependent and independent variables.\n",
    "- Utilizes the sigmoid (or logistic) function to ensure that the output probabilities range between 0 and 1. This makes it suitable for estimating the probability that a given instance belongs to a particular class.\n",
    "- Coefficients in logistic regression are estimated using maximum likelihood estimation (MLE), which seeks to maximize the likelihood of observing the sample data given the model parameters.\n",
    "- Can be used with both categorical and continuous independent variables, making it versatile for various types of data.\n",
    "- Incorporates regularization techniques (L1, L2, or both), which help to prevent overfitting by penalizing large coefficients. This is crucial for models with high dimensionality or when multicollinearity is present among the features.\n",
    "- Primarily designed for binary classification, but can be extended to multiclass classification using strategies like one-vs-rest (OvR) or multinomial logistic regression, where the latter directly models multiple classes.\n",
    "- Known for its simplicity and computational efficiency, logistic regression is often used as a baseline for binary classification problems. It provides a good starting point due to its straightforward interpretation and implementation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 315195,
     "status": "ok",
     "timestamp": 1710053706696,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "rRVY_juw1LXl",
    "outputId": "d89d175e-7bfa-4aaa-c8ca-864e307b6c33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    }
   ],
   "source": [
    "_, accuracy, weighted_f1, micro_f1, macro_f1 = BoWModel(train_df, test_df, \"LogisticRegression\").train_models()\n",
    "scores_df[\"LogisticRegression\"] = [accuracy, weighted_f1, micro_f1, macro_f1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kBqFbnE423ZU"
   },
   "source": [
    "[Multilayer perceptron](https://en.wikipedia.org/wiki/Multilayer_perceptron)\n",
    "\n",
    "---\n",
    "\n",
    "- MLP is a class of feedforward artificial neural network (ANN) that consists of at least three layers of nodes: an input layer, one or more hidden layers, and an output layer.\n",
    "- Utilizes nonlinear activation functions like sigmoid, hyperbolic tangent (tanh), or Rectified Linear Unit (ReLU) in the hidden layers. These functions help MLP to model complex relationships between the input and output variables.\n",
    "- Employs the backpropagation algorithm for training, which involves forward propagation of inputs through the network to generate an output, calculation of the error (difference between the predicted and actual output), and backward propagation of this error to adjust the weights of the connections.\n",
    "- Uses gradient-based optimization methods, such as Stochastic Gradient Descent (SGD), Adam, or RMSprop, to minimize the loss function by adjusting the network's weights and biases.\n",
    "- Implements techniques such as regularization (e.g., L2 regularization), dropout, and early stopping to prevent overfitting, especially when the network has a large number of parameters.\n",
    "- Requires input features to be scaled or normalized for optimal performance, as gradient-based optimization methods are sensitive to the scale of the input data.\n",
    "- Involves hyperparameter tuning, including the number of hidden layers, the number of neurons in each layer, the choice of activation function, and the choice of optimization algorithm, to achieve the best performance.\n",
    "- The architecture of MLP allows it to generalize from the training data to unseen data, assuming the model is well-tuned and not overfitted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1100164,
     "status": "ok",
     "timestamp": 1710054806825,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "rcud8HDr2knB",
    "outputId": "a95ebe4c-f07b-4965-94df-d56a2ca9d559"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n"
     ]
    }
   ],
   "source": [
    "_, accuracy, weighted_f1, micro_f1, macro_f1 = BoWModel(train_df, test_df, \"MLPClassifier\").train_models()\n",
    "scores_df[\"MLP\"] = [accuracy, weighted_f1, micro_f1, macro_f1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "paxfBZbaN27G"
   },
   "source": [
    "# **Checking Model Performance** \n",
    "In the context of multi-class classification problems, evaluating model performance extends beyond traditional accuracy. Metrics like balanced accuracy, weighted-F1, micro-F1, and macro-F1 offer nuanced insights into the model's effectiveness, especially when dealing with imbalanced datasets.\n",
    "\n",
    "--- \n",
    "\n",
    "### Balanced Accuracy\n",
    "- **Definition:** Balanced accuracy is the average of recall obtained on each class. It's particularly useful in dealing with imbalanced datasets where the class distribution is skewed.\n",
    "- **Utility:** Helps to overcome the limitations of regular accuracy by giving equal weight to the performance on each class, thus providing a more truthful measure of the model’s ability to classify instances across all classes.\n",
    "\n",
    "### Weighted-F1 Score\n",
    "- **Definition:** The weighted-F1 score calculates the F1 score for each class independently but then takes the average of these scores, weighted by the number of true instances for each class. It combines precision and recall into a single metric.\n",
    "- **Utility:** Offers a compromise between the precision and recall of the model, taking into account the class imbalance by weighting the score of each class by its presence in the dataset.\n",
    "\n",
    "### Micro-F1 Score\n",
    "- **Definition:** Micro-F1 score aggregates the contributions of all classes to compute the average F1 score. This involves summing the true positives, false positives, and false negatives of the system for different sets and then calculating precision, recall, and F1 score.\n",
    "- **Utility:** Particularly effective when you want to weigh each instance or prediction equally, making it suitable for datasets with class imbalances.\n",
    "\n",
    "### Macro-F1 Score\n",
    "- **Definition:** The macro-F1 score is calculated by first computing the F1 score of each class independently and then taking the average of these scores. It treats all classes equally, regardless of their frequency in the dataset.\n",
    "- **Utility:** Provides a way to assess the model's performance across all classes equally, highlighting how well the model performs on minority classes. This metric is especially relevant when the importance of each class is the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 175
    },
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1710054911092,
     "user": {
      "displayName": "Vageesh Saxena",
      "userId": "08383190130410030928"
     },
     "user_tz": -60
    },
    "id": "fAxPmEctNwAO",
    "outputId": "12a25bbe-08c2-4889-fb84-b825c96c74ab"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"scores_df\",\n  \"rows\": 4,\n  \"fields\": [\n    {\n      \"column\": \"Metrics\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Weighted-F1\",\n          \"Macro-F1\",\n          \"Accuracy\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"MultinomialNB\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.13364587599904837,\n        \"min\": 0.331296992481203,\n        \"max\": 0.596,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.5244050525636352,\n          0.33743606329620085,\n          0.331296992481203\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"SVC\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.016578898600449716,\n        \"min\": 0.6719450270183899,\n        \"max\": 0.7084807843862542,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.7084807843862542,\n          0.6719450270183899,\n          0.6823223509702638\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"RandomForest\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.052421048068768565,\n        \"min\": 0.28383238994694104,\n        \"max\": 0.4121224397199897,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.34428973097765664,\n          0.28383238994694104,\n          0.4121224397199897\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"LogisticRegression\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07654824206503479,\n        \"min\": 0.5900274131375507,\n        \"max\": 0.7757557438024769,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.6622835815165227,\n          0.5900274131375507,\n          0.7757557438024769\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"MLP\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.05481574796019103,\n        \"min\": 0.6466153885550759,\n        \"max\": 0.756,\n        \"num_unique_values\": 4,\n        \"samples\": [\n          0.7522883095118389,\n          0.6466153885550759,\n          0.6766820067409904\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe",
       "variable_name": "scores_df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-2a5da4af-dd51-4ef2-95f3-2b9f9b72dc4d\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Metrics</th>\n",
       "      <th>MultinomialNB</th>\n",
       "      <th>SVC</th>\n",
       "      <th>RandomForest</th>\n",
       "      <th>LogisticRegression</th>\n",
       "      <th>MLP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.331297</td>\n",
       "      <td>0.682322</td>\n",
       "      <td>0.412122</td>\n",
       "      <td>0.775756</td>\n",
       "      <td>0.676682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Weighted-F1</td>\n",
       "      <td>0.524405</td>\n",
       "      <td>0.708481</td>\n",
       "      <td>0.344290</td>\n",
       "      <td>0.662284</td>\n",
       "      <td>0.752288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Micro-F1</td>\n",
       "      <td>0.596000</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.344000</td>\n",
       "      <td>0.668000</td>\n",
       "      <td>0.756000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Macro-F1</td>\n",
       "      <td>0.337436</td>\n",
       "      <td>0.671945</td>\n",
       "      <td>0.283832</td>\n",
       "      <td>0.590027</td>\n",
       "      <td>0.646615</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2a5da4af-dd51-4ef2-95f3-2b9f9b72dc4d')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-2a5da4af-dd51-4ef2-95f3-2b9f9b72dc4d button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-2a5da4af-dd51-4ef2-95f3-2b9f9b72dc4d');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-a84b5b31-c8b8-4d61-8b4e-c7506aa587e6\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a84b5b31-c8b8-4d61-8b4e-c7506aa587e6')\"\n",
       "            title=\"Suggest charts\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-a84b5b31-c8b8-4d61-8b4e-c7506aa587e6 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "       Metrics  MultinomialNB       SVC  RandomForest  LogisticRegression  \\\n",
       "0     Accuracy       0.331297  0.682322      0.412122            0.775756   \n",
       "1  Weighted-F1       0.524405  0.708481      0.344290            0.662284   \n",
       "2     Micro-F1       0.596000  0.700000      0.344000            0.668000   \n",
       "3     Macro-F1       0.337436  0.671945      0.283832            0.590027   \n",
       "\n",
       "        MLP  \n",
       "0  0.676682  \n",
       "1  0.752288  \n",
       "2  0.756000  \n",
       "3  0.646615  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eg2nlxEn3WUV"
   },
   "source": [
    "# **Storing results**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZS9jyRl33UOP"
   },
   "outputs": [],
   "source": [
    "scores_df.to_csv('/content/drive/MyDrive/AA-Tutorial/data/results.csv', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pEdwRDHKCZo1"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyMruB+VAulYP8U7S9OfasP4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}