notebooks/ML_supervised2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "This is the main notebook.\n",
    "the supervised ML part of the project is all here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:04.227251Z",
     "start_time": "2023-03-31T22:17:04.175257Z"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.metrics import accuracy_score, precision_score\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import MultiLabelBinarizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.base import clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:04.308264Z",
     "start_time": "2023-03-31T22:17:04.187257Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "mails_df = pd.read_csv(\"../data/training_data.csv\", sep='~', index_col=0)\n",
    "imputer = SimpleImputer(strategy='constant', fill_value='')\n",
    "\n",
    "mails_df = pd.DataFrame(imputer.fit_transform(mails_df), columns=mails_df.columns)\n",
    "# mails_df.iloc[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:04.330267Z",
     "start_time": "2023-03-31T22:17:04.251267Z"
    }
   },
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:04.331265Z",
     "start_time": "2023-03-31T22:17:04.271274Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub('[^a-zA-Z0-9]', ' ', text)\n",
    "    tokens = text.split()\n",
    "    tokens = [token for token in tokens if token not in stop_words]\n",
    "    processed_text = ' '.join([lemmatizer.lemmatize(token) for token in tokens])\n",
    "    return processed_text\n",
    "\n",
    "\n",
    "def preprocess_sender(address):\n",
    "    address_lst = address.lower().split('@')\n",
    "    address_lst[1] = re.sub('[.]ac|[.]in|[.]com', '', address_lst[1])\n",
    "    address_lst[1] = re.sub('[.]', ' ', address_lst[1])\n",
    "    address_lst[0] = re.sub('[._]', '', address_lst[0])\n",
    "    return ' '.join(address_lst)\n",
    "\n",
    "preprocessor = ColumnTransformer(transformers=[\n",
    "    ('subject', TfidfVectorizer(preprocessor=preprocess_text, max_df=0.9, min_df=0.005), 'subject'),\n",
    "    ('body', TfidfVectorizer(preprocessor=preprocess_text, max_df=0.8, min_df=0.01), 'body'),\n",
    "    ('sender', TfidfVectorizer(preprocessor=preprocess_sender), 'sender')\n",
    "],\n",
    "    # transformer_weights={\n",
    "    #     'subject': 2,\n",
    "    #     'body': 1,\n",
    "    #     'sender': 4\n",
    "    # },\n",
    "    remainder='drop')\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('preprocessor', preprocessor)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "Loading labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:04.331265Z",
     "start_time": "2023-03-31T22:17:04.284269Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('../data/label_dict.json', 'r') as file:\n",
    "    all_labels = json.load(file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "Encoding labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:04.332265Z",
     "start_time": "2023-03-31T22:17:04.309266Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "label_list = [key for key in all_labels.keys() if re.match('Label_[0-9]', key)]\n",
    "mlb = MultiLabelBinarizer(classes=label_list)\n",
    "labels_array = [list(st.split(',')) for st in mails_df['labels']]\n",
    "\n",
    "mlb.fit(label_list)\n",
    "labels = mlb.transform(labels_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:05.541317Z",
     "start_time": "2023-03-31T22:17:04.341265Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_matrix = pipeline.fit_transform(mails_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:05.557013Z",
     "start_time": "2023-03-31T22:17:05.543319Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = feature_matrix\n",
    "y = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:05.603061Z",
     "start_time": "2023-03-31T22:17:05.560011Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(838, 2106) \n",
      "\n",
      "(360, 2106) \n",
      "\n",
      "(838, 21) \n",
      "\n",
      "(360, 21) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "lst = [X_train, X_test, y_train, y_test]\n",
    "for i in lst:\n",
    "    print(i.shape, '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "My main metric for evaluation is precision_score. It is best suited for the multi-output classification. Its details as mentioned in its documentation\n",
    "\n",
    "\n",
    "> Compute the precision.\n",
    "> The precision is the ratio tp / (tp + fp) where tp is the number of true positives and fp the number of false positives. The precision is intuitively the ability of the classifier not to label as positive a sample that is negative.\n",
    ">The best value is 1 and the worst value is 0.\n",
    "\n",
    "as sizes of all labels are different, and they are very imbalanced, so I've used average='weighted'\n",
    "\n",
    "> Calculate metrics for each label, and find their average weighted by support (the number of true instances for each label). This alters ‘macro’ to account for label imbalance; it can result in an F-score that is not between precision and recall.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:05.604055Z",
     "start_time": "2023-03-31T22:17:05.578056Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def try_different_models(models):\n",
    "    dfs = []\n",
    "    for name, model in models:\n",
    "        clf = model.fit(X_train, y_train)\n",
    "        y_pred = clf.predict(X_test)\n",
    "        y_pred_on_train = clf.predict(X_train)\n",
    "\n",
    "        score = pd.Series(\n",
    "            [\n",
    "                accuracy_score(y_test, y_pred), accuracy_score(y_train, y_pred_on_train),\n",
    "                precision_score(y_test, y_pred, average='weighted', zero_division=0),\n",
    "                precision_score(y_train, y_pred_on_train, average='weighted', zero_division=0)\n",
    "            ],\n",
    "            index=['test_accuracy', 'train_accuracy', 'precision_score', 'precision_score_train'],\n",
    "            name=name\n",
    "        )\n",
    "        dfs.append(score)\n",
    "\n",
    "    print('shape of dataset: ', X.shape, y.shape)\n",
    "    score_df = pd.DataFrame(dfs)\n",
    "    best_model_name = score_df['precision_score'].idxmax()\n",
    "    score_df.sort_values(by=['precision_score', 'test_accuracy'], ascending=False, inplace=True)\n",
    "    print(score_df)\n",
    "    return dict(models).get(best_model_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:49:30.239240Z",
     "start_time": "2023-03-31T22:49:09.721793Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of dataset:  (1198, 2106) (1198, 21)\n",
      "        test_accuracy  train_accuracy  precision_score  precision_score_train\n",
      "RF           0.755556        0.998807         0.961111               1.000000\n",
      "ETrees       0.777778        0.997613         0.936383               0.997311\n",
      "XGB          0.841667        0.997613         0.928790               1.000000\n",
      "SVM          0.838889        0.966587         0.915008               0.964353\n",
      "KNN          0.800000        0.834129         0.849182               0.887626\n",
      "DTree        0.744444        0.997613         0.737069               0.997311\n",
      "ETree        0.633333        0.997613         0.585801               0.997311\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-8 {color: black;background-color: white;}#sk-container-id-8 pre{padding: 0;}#sk-container-id-8 div.sk-toggleable {background-color: white;}#sk-container-id-8 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-8 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-8 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-8 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-8 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-8 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-8 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-8 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-8 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-8 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-8 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-8 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-8 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-8 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-8 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-8 div.sk-item {position: relative;z-index: 1;}#sk-container-id-8 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-8 div.sk-item::before, #sk-container-id-8 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-8 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-8 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-8 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-8 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-8 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-8 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-8 div.sk-label-container {text-align: center;}#sk-container-id-8 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-8 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-8\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier(class_weight=&#x27;balanced&#x27;, n_jobs=-1, random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" checked><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(class_weight=&#x27;balanced&#x27;, n_jobs=-1, random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomForestClassifier(class_weight='balanced', n_jobs=-1, random_state=42)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = [\n",
    "    ('DTree', DecisionTreeClassifier(random_state=42, class_weight='balanced')),\n",
    "    ('ETree', ExtraTreeClassifier(random_state=42, class_weight='balanced')),\n",
    "    ('ETrees', ExtraTreesClassifier(random_state=42, class_weight='balanced')),\n",
    "    ('RF', RandomForestClassifier(random_state=42, n_jobs=-1, class_weight='balanced')),\n",
    "    ('KNN', KNeighborsClassifier()),\n",
    "    ('SVM', MultiOutputClassifier(SVC(random_state=42, class_weight='balanced'), n_jobs=-1)),\n",
    "    ('XGB', XGBClassifier())\n",
    "]\n",
    "\n",
    "try_different_models(models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "SVM seems to be the best classifier among these, as other models are overfitting the training data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "tuning hyperparameters using grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:14.648294Z",
     "start_time": "2023-03-31T22:17:14.530278Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def grid_search(estimator, param_grid):\n",
    "    models = []\n",
    "    param_gird_expanded = ParameterGrid(param_grid)\n",
    "    for i, parameters in enumerate(param_gird_expanded):\n",
    "        model = clone(estimator)\n",
    "        models.append((f'{i}', model.set_params(**parameters)))\n",
    "\n",
    "    return try_different_models(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:14.650298Z",
     "start_time": "2023-03-31T22:17:14.541296Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "svc_param_grid = {\n",
    "    'estimator__C': [0.01, 1, 0.001],\n",
    "    'estimator__kernel': ['poly', 'linear'],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:39.776112Z",
     "start_time": "2023-03-31T22:17:14.566296Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of dataset:  (1198, 2106) (1198, 21)\n",
      "   test_accuracy  train_accuracy  precision_score  precision_score_train\n",
      "2       0.805556        0.982100         0.911034               0.978831\n",
      "3       0.852778        0.977327         0.896022               0.973799\n",
      "1       0.683333        0.687351         0.815450               0.824325\n",
      "0       0.000000        0.000000         0.201181               0.213948\n",
      "5       0.000000        0.000000         0.040113               0.060096\n",
      "4       0.000000        0.000000         0.021222               0.027826\n"
     ]
    }
   ],
   "source": [
    "best_model_svc = grid_search(\n",
    "    MultiOutputClassifier(SVC(class_weight='balanced', random_state=42), n_jobs=-1),\n",
    "    svc_param_grid\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:17:39.799111Z",
     "start_time": "2023-03-31T22:17:39.778110Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-6 {color: black;background-color: white;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultiOutputClassifier(estimator=SVC(C=1, class_weight=&#x27;balanced&#x27;, kernel=&#x27;poly&#x27;,\n",
       "                                    random_state=42),\n",
       "                      n_jobs=-1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultiOutputClassifier</label><div class=\"sk-toggleable__content\"><pre>MultiOutputClassifier(estimator=SVC(C=1, class_weight=&#x27;balanced&#x27;, kernel=&#x27;poly&#x27;,\n",
       "                                    random_state=42),\n",
       "                      n_jobs=-1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(C=1, class_weight=&#x27;balanced&#x27;, kernel=&#x27;poly&#x27;, random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SVC</label><div class=\"sk-toggleable__content\"><pre>SVC(C=1, class_weight=&#x27;balanced&#x27;, kernel=&#x27;poly&#x27;, random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "MultiOutputClassifier(estimator=SVC(C=1, class_weight='balanced', kernel='poly',\n",
       "                                    random_state=42),\n",
       "                      n_jobs=-1)"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model_svc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-31T22:36:13.987313Z",
     "start_time": "2023-03-31T22:36:13.951021Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rf_param_grid = {\n",
    "    'n_estimators': [50, 100, 200],\n",
    "    'max_depth': [2, 5, 10, None],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'max_features': ['sqrt', 'log2'],\n",
    "    'bootstrap': [True, False],\n",
    "    'criterion': ['gini', 'entropy']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "best_model_rf = grid_search(\n",
    "    RandomForestClassifier(random_state=42, n_jobs=-1, class_weight='balanced'),\n",
    "    rf_param_grid\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "As my custom grid search doesn't run multiple jobs in parallel, it is very slow.\n",
    "Hence, I couldn't use it to do hyperparameter tuning for all the classifiers."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gmail_organizer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}