4-classical_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classical ML Models Benchmark\n",
    "\n",
    "This notebook contains attempts to solve the problem of predicting ratings will classical ML models which support multinomial classification. The scores achieved by these models will serve as a benchmark for the deep neural network based approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General Imports\n",
    "from os.path import join\n",
    "import multiprocessing\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Classifier Imports\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "\n",
    "\n",
    "SEED = 0\n",
    "CPU_COUNT = multiprocessing.cpu_count() - 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Wrapper class for a general SKLearn classifier\n",
    "class Classifier():\n",
    "    def __init__(self, classifier_name, classifier, init_params, param_grid, seed):\n",
    "        self.classifier_name = classifier_name\n",
    "        self.seed = seed\n",
    "        self.param_grid = param_grid\n",
    "\n",
    "        #Init classifier\n",
    "        self.init_params = init_params\n",
    "        self.init_params[\"random_state\"] = seed\n",
    "        self.classifier = classifier(**self.init_params) if init_params else classifier(random_state=seed)\n",
    "\n",
    "        #Dict to explicitly store best stats\n",
    "        self.best_stats = {\"best_params\": None, \"best_score\": None}\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        print(f\"Fitting {self.classifier_name} model...\")\n",
    "        self.classifier.fit(X, y)\n",
    "\n",
    "    def predict(self, X):\n",
    "        return self.classifier.predict(X)\n",
    "    \n",
    "    def evaluate(self, X_test, y_test):\n",
    "        return self.classifier.score(X_test, y_test)\n",
    "    \n",
    "    def tune_hyperparameters(self, X, y):\n",
    "        print(f\"Tuning hyperparameters for {self.classifier_name} model...\")\n",
    "        cv = KFold(n_splits=5, random_state=self.seed, shuffle=True)\n",
    "        gscv = GridSearchCV(self.classifier, self.param_grid, scoring=\"accuracy\", cv=cv, n_jobs=-1)\n",
    "        gscv.fit(X, y)\n",
    "        self.classifier = gscv.best_estimator_\n",
    "        self.best_stats[\"best_params\"], self.best_stats[\"best_score\"] = gscv.best_params_, gscv.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: (9000, 768) | X_test: (1000, 768) | \n",
      "y_train: (9000,) | y_test: (1000,) | \n"
     ]
    }
   ],
   "source": [
    "# Loading train, val, and test data (BERT text embeddings and corresponding labels)\n",
    "data_dir = \"data/\"\n",
    "\n",
    "X_train = pickle.load(open(join(data_dir, \"downsampled_shuffled_train_embeddings.pkl\"), \"rb\")).numpy()\n",
    "y_train = pickle.load(open(join(data_dir, \"downsampled_shuffled_train_labels.pkl\"), \"rb\"))\n",
    "X_val = pickle.load(open(join(data_dir, \"downsampled_shuffled_val_embeddings.pkl\"), \"rb\")).numpy()\n",
    "y_val = pickle.load(open(join(data_dir, \"downsampled_shuffled_val_labels.pkl\"), \"rb\"))\n",
    "X_test = pickle.load(open(join(data_dir, \"downsampled_shuffled_test_embeddings.pkl\"), \"rb\")).numpy()\n",
    "y_test = pickle.load(open(join(data_dir, \"downsampled_shuffled_test_labels.pkl\"), \"rb\"))\n",
    "\n",
    "# Combine train and validation set into one as we use K-Fold cross validation\n",
    "X_train = np.concatenate([X_train, X_val])\n",
    "y_train = np.concatenate([y_train, y_val])\n",
    "\n",
    "print(f\"X_train: {X_train.shape} | X_test: {X_test.shape} | \\n\" +\n",
    "    f\"y_train: {y_train.shape} | y_test: {y_test.shape} | \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create list of classifiers\n",
    "SEED = 0\n",
    "param_grids = []\n",
    "\n",
    "# Create parameter grids for hyperparameter tuning\n",
    "rf_param_grid = {\"max_features\": [\"sqrt\", \"log2\"],\n",
    "                    \"max_depth\" : [3, 6, 8],\n",
    "                    \"criterion\" :[\"gini\", \"entropy\"]     ,\n",
    "                    \"n_jobs\": [-1]}\n",
    "\n",
    "lsvc_param_grid = {\"penalty\": [\"l2\"],\n",
    "                   \"C\": [0.0001, 0.01, 1.0, 10, 100]}\n",
    "\n",
    "lreg_param_grid = {'penalty' : ['l1', 'l2'],\n",
    "                     'C' : np.logspace(-4, 4, 20)}\n",
    "\n",
    "clf_names = [\"RandomForest\", \"LinearSVC\", \"LogisticRegression\"]\n",
    "clfs = [RandomForestClassifier, LinearSVC, LogisticRegression]\n",
    "init_params = [{'n_jobs': CPU_COUNT}, {'multi_class': 'crammer_singer'}, {'multi_class': 'multinomial', 'solver': 'lbfgs'}]\n",
    "param_grids.extend([rf_param_grid, lsvc_param_grid, lreg_param_grid])\n",
    "\n",
    "\n",
    "classifiers = [Classifier(name, model, {}, param_grid, SEED) \n",
    "               for name, model, param_grid in zip(clf_names, clfs, param_grids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting RandomForest model...\n",
      "Fitting LinearSVC model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vinay/anaconda3/envs/askoski/lib/python3.7/site-packages/sklearn/svm/_base.py:986: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting LogisticRegression model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vinay/anaconda3/envs/askoski/lib/python3.7/site-packages/sklearn/linear_model/_logistic.py:765: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit classifiers\n",
    "[clf.fit(X_train, y_train) for clf in classifiers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.19, 0.176, 0.182]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Score classifier\n",
    "[clf.evaluate(X_test, y_test) for clf in classifiers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning hyperparameters for RandomForest model...\n",
      "Tuning hyperparameters for LinearSVC model...\n",
      "Tuning hyperparameters for LogisticRegression model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vinay/anaconda3/envs/askoski/lib/python3.7/site-packages/sklearn/model_selection/_search.py:921: UserWarning: One or more of the test scores are non-finite: [       nan 0.20311111        nan 0.20055556        nan 0.20066667\n",
      "        nan 0.19933333        nan 0.19811111        nan 0.19211111\n",
      "        nan 0.19122222        nan 0.19344444        nan 0.19411111\n",
      "        nan 0.19544444        nan 0.19511111        nan 0.195\n",
      "        nan 0.19388889        nan 0.19466667        nan 0.19488889\n",
      "        nan 0.195             nan 0.19411111        nan 0.195\n",
      "        nan 0.19344444        nan 0.19455556]\n",
      "  category=UserWarning\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Tune hyperparameters\n",
    "[clf.tune_hyperparameters(X_train, y_train) for clf in classifiers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.193, 0.205, 0.208]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Score tuned clasifiers\n",
    "[clf.evaluate(X_test, y_test) for clf in classifiers]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save models\n",
    "models_dir = \"models/\"\n",
    "for clf in classifiers:\n",
    "    pickle.dump(clf, open(join(models_dir, f\"{clf.classifier_name}.pkl\"), \"wb\")) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above results, we can see that the best performing classical ML model (Logistic Regression) was able to achieve a 20.8% accuracy, showing that the data is highly non-linear and not very seperable.  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.5 64-bit ('askoski': conda)",
   "language": "python",
   "name": "python37564bitaskoskicondaf89acb10eb5d40d0a2b1becabd7f59e8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}