Stacking-k-fold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "from sklearn.metrics import f1_score, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_predictions(dataset_name, partition, fold_number=1):\n",
    "    path = './Saved_Predict_and_Proba/Folds/' + dataset_name.upper() + '/F' + str(fold_number)  + '/prob_' + partition +'.csv'\n",
    "    table_pred = pd.read_csv(path)\n",
    "    if dataset_name == 'zw':\n",
    "        label = table_pred['norm']\n",
    "        methods = table_pred.drop('norm', axis=1)\n",
    "    else: \n",
    "        label = table_pred['class']\n",
    "        methods = table_pred.drop('class', axis=1)\n",
    "    methods = methods.drop('Unnamed: 0', axis=1)\n",
    "    return label, methods\n",
    "\n",
    "def load_dataset(dataset_name, fold_number):\n",
    "    label_train, probas_train = load_predictions(dataset_name, 'train', fold_number)\n",
    "    label_val, probas_val = load_predictions(dataset_name, 'val', fold_number)\n",
    "    label_test, probas_test = load_predictions(dataset_name, 'test', fold_number)\n",
    "    return label_train, probas_train, label_test, probas_test, label_val, probas_val\n",
    "\n",
    "def filter_df_train_test(train_df, test_df, name, filter_first=True):\n",
    "    train = train_df.filter(regex=name, axis=1)\n",
    "    test = test_df.filter(regex=name, axis=1)\n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name = 'zw'\n",
    "folds = 5\n",
    "stacking = LogisticRegression(class_weight='balanced')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking group A - Varying Classification methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "algorithms_list = ['SVM', 'MLP', 'KNN', 'RF', 'EXTRA', 'CNN', 'LR', 'NB']\n",
    "results_A = np.zeros((len(algorithms_list), folds))\n",
    "\n",
    "for fold in range(folds):\n",
    "    _, _, label_test, probas_test, label_val, probas_val = load_dataset(dataset_name, fold+1)\n",
    "    for idx_alg, algorithm in enumerate(algorithms_list):\n",
    "        X_val, X_test = filter_df_train_test(probas_val, probas_test, algorithm)\n",
    "        stacking.fit(X_val, label_val)\n",
    "        y_pred = stacking.predict(X_test)\n",
    "        score = f1_score(label_test, y_pred, average='macro')\n",
    "        results_A[idx_alg, fold] = score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.78522593, 0.75878708, 0.58487545, 0.77322025, 0.75855527,\n",
       "       0.7592998 , 0.77574546, 0.71447546])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_A.mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking group B - Varying Feature Representation methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "fe_list = ['CV', 'TFIDF', 'W2V', 'GLOVE', 'FAST']\n",
    "results_B = np.zeros((len(fe_list), folds))\n",
    "for fold in range(folds):\n",
    "    _, _, label_test, probas_test, label_val, probas_val = load_dataset(dataset_name, fold+1)\n",
    "    \n",
    "    for idx_alg, fe in enumerate(fe_list):\n",
    "        X_val, X_test = filter_df_train_test(probas_val, probas_test, fe)\n",
    "        stacking.fit(X_val, label_val)\n",
    "        y_pred = stacking.predict(X_test)\n",
    "        results_B[idx_alg, fold] = f1_score(label_test, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.74545394, 0.77126745, 0.74028171, 0.66948325, 0.70326888])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_B.mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking group C -  ALL techniques"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_C = np.zeros(folds)\n",
    "\n",
    "for fold in range(folds):\n",
    "    _, _, label_test, probas_test, label_val, probas_val = load_dataset(dataset_name, fold+1)\n",
    "    stacking.fit(probas_val, label_val)\n",
    "    y_pred = stacking.predict(probas_test)\n",
    "    results_C[fold] = f1_score(label_test, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7664351699131046"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_C.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking group D - Proposed Selection scheme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "names_D = {}\n",
    "names_D['zw'] = [\n",
    "     'EXTRA-TFIDF|NB-W2V|NB-TF|KNN-GLOVE|MLP-W2V|CNN-W2V',\n",
    "     'NB-FAST|CNN-W2V|MLP-GLOVE|SVM-TF|KNN-W2V|LR-TFIDF|EXTRA-TFIDF',\n",
    "     'SVM-TF|RF-TFIDF|NB-GLOVE|KNN-FAST|NB-TFIDF|NB-W2V',\n",
    "     'LR-W2V|NB-W2V|SVM-TFIDF|LR-GLOVE|LR-W2V',\n",
    "     'LR-GLOVE|NB-W2V|SVM-TFIDF|EXTRA-TFIDF|MLP-FAST|LR-W2V',\n",
    "                 ]\n",
    "\n",
    "\n",
    "names_D['td'] = [\n",
    "     'NB-GLOVE|NB-CV|SM-W2V|NB-FAST|SVM-TFIDF',\n",
    "     'CNN-FAST|NB-FAST|LR-FAST|SVM-CV|MLP-GLOVE',\n",
    "     'MLP-W2V|LR-FAST|SVM-CV|CNN-W2V|NB-TFIDF|NB-W2V',\n",
    "     'SVM-W2V|RF-CV|SVM-TFIDF|NB-FAST|NB-CV',\n",
    "     'RF-CV|CNN-CV|MLP-FAST|CNN-FAST|NB-TFIDF',\n",
    " ]\n",
    "names_D ['union']= [\n",
    "     'MLP-W2V|NB-TFIDF|RF-TFIDF|KNN-TFIDF|SVM-CV|NN-W2V|NN-FAST|CNN-W2V|SM-FAST',\n",
    "     'MLP-TFIDF|NB-CV|SVM-CV|CNN-TF|KNN-TFIDF|CNN-FAST|XTRA-TFIDF',\n",
    "     'KNN-TFIDF|RF-CV|SVM-W2V|LR-GLOVE|NB-TFIDF|KNN-FAST|KNN-GLOVE|SVM-CV',\n",
    "     'SVM-TFIDF|KNN-TFIDF|CNN-W2V|KNN-FAST|SVM-FAST|EXTRA-CV|KNN-W2V|NB-TFIDF|SVM-GLOVE|LR-W2V|MLP-FAST',\n",
    "     'CNN-W2V|RF-TFIDF|NB-W2V|NB-TFIDF|MLP-TFIDF|MLP-W2V',\n",
    "]\n",
    "\n",
    "names = names_D[dataset_name]\n",
    "results_D = np.zeros(folds)\n",
    "\n",
    "for fold in range(folds):\n",
    "    labels_train, probas_train, label_test, probas_test, label_val, probas_val = load_dataset(dataset_name, fold+1)\n",
    "    X_val, X_test = filter_df_train_test(probas_train, probas_test, names[fold])\n",
    "    stacking.fit(X_val, labels_train)\n",
    "    y_pred = stacking.predict(X_test)\n",
    "    results_D[fold] = f1_score(label_test, y_pred, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7753397725251462"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_D.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarizing results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_stacking_names = ['Logistic Regression']\n",
    "algorithms_list_A = [alg + ' (Group A)' for alg in algorithms_list]\n",
    "group_A_df = pd.DataFrame(results_A.mean(axis=1), columns=all_stacking_names, index=algorithms_list_A)\n",
    "\n",
    "fe_list_B = [fe + ' (Group B)' for fe in fe_list]\n",
    "group_B_df = pd.DataFrame(results_B.mean(axis=1), columns=all_stacking_names, index=fe_list_B)\n",
    "\n",
    "group_C_df = pd.DataFrame(results_C.mean(axis=0).reshape(1, 1), columns=all_stacking_names, index=['Group C'])\n",
    "group_D_df = pd.DataFrame(results_D.mean(axis=0).reshape(1, 1), columns=all_stacking_names, index=['Group D'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('\\\\begin{table}\\n'\n",
      " '\\\\centering\\n'\n",
      " '\\\\caption{dataset ZW}\\n'\n",
      " '\\\\begin{tabular}{lr}\\n'\n",
      " '\\\\toprule\\n'\n",
      " '{} &  Logistic Regression \\\\\\\\\\n'\n",
      " '\\\\midrule\\n'\n",
      " 'SVM (Group A)   &               0.7852 \\\\\\\\\\n'\n",
      " 'MLP (Group A)   &               0.7588 \\\\\\\\\\n'\n",
      " 'KNN (Group A)   &               0.5849 \\\\\\\\\\n'\n",
      " 'RF (Group A)    &               0.7732 \\\\\\\\\\n'\n",
      " 'EXTRA (Group A) &               0.7586 \\\\\\\\\\n'\n",
      " 'CNN (Group A)   &               0.7593 \\\\\\\\\\n'\n",
      " 'LR (Group A)    &               0.7757 \\\\\\\\\\n'\n",
      " 'NB (Group A)    &               0.7145 \\\\\\\\\\n'\n",
      " 'CV (Group B)    &               0.7455 \\\\\\\\\\n'\n",
      " 'TFIDF (Group B) &               0.7713 \\\\\\\\\\n'\n",
      " 'W2V (Group B)   &               0.7403 \\\\\\\\\\n'\n",
      " 'GLOVE (Group B) &               0.6695 \\\\\\\\\\n'\n",
      " 'FAST (Group B)  &               0.7033 \\\\\\\\\\n'\n",
      " 'Group C         &               0.7664 \\\\\\\\\\n'\n",
      " 'Group D         &               0.7753 \\\\\\\\\\n'\n",
      " '\\\\bottomrule\\n'\n",
      " '\\\\end{tabular}\\n'\n",
      " '\\\\end{table}\\n')\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "pprint(pd.concat([group_A_df, group_B_df, group_C_df, group_D_df]).round(4).to_latex(caption='dataset '+ dataset_name.upper()))"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}