Experiment_HateVal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "B5_cgGkJif-F"
   },
   "outputs": [],
   "source": [
    "from zeugma.embeddings import EmbeddingTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.4.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "keNpadCWeDyT"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "HM2S6EsseWSr"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "SWJljGkdeghN"
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from preprocessing import *\n",
    "from function import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "9mvhZyI-ejhj"
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.calibration import CalibratedClassifierCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "qD5TrHDhVcDU"
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"./Data/Hateval/hateval2019_en_train.csv\").dropna()\n",
    "df_val = pd.read_csv(\"./Data/Hateval/hateval2019_en_dev.csv\").dropna()\n",
    "df_test = pd.read_csv(\"./Data/Hateval/hateval2019_en_test.csv\").dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "buKay1WyWHDN"
   },
   "outputs": [],
   "source": [
    "train, class_train = df_train['text'].apply(pre_processing), df_train['HS']\n",
    "val, class_val = df_val['text'].apply(pre_processing), df_val['HS']\n",
    "test, class_test = df_test['text'].apply(pre_processing), df_test['HS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uBa03X_0invh",
    "outputId": "0bf8cb63-a450-4a85-a699-8106ab1fe53f"
   },
   "outputs": [],
   "source": [
    "w2v = EmbeddingTransformer('word2vec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MOfvAe3IiqLg",
    "outputId": "269bcdec-cf8f-479d-fd03-359fc8d5ebc3"
   },
   "outputs": [],
   "source": [
    "glove = EmbeddingTransformer('glove')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oxXahEHbir7c",
    "outputId": "74320c34-e30d-44e0-9cee-b5208a1fba6b"
   },
   "outputs": [],
   "source": [
    "fasttext = EmbeddingTransformer('fasttext')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1UZHfd3Ciwv6",
    "outputId": "362d7339-c0e0-4e84-dd94-cdb12c5ad631"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<9000x16056 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 93433 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(analyzer='word', lowercase=True, stop_words='english')\n",
    "cv.fit_transform(train.values.astype('U'))\n",
    "     \n",
    "tfidf =  TfidfVectorizer(analyzer='word', lowercase=True, use_idf=True, stop_words='english')\n",
    "tfidf.fit_transform(train.values.astype('U'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction SVM base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "EHQTzhm-i_I_"
   },
   "outputs": [],
   "source": [
    "svm = {\n",
    "    'CV': {\n",
    "        'CLF': SVC(random_state=42, kernel='linear', gamma=0.1, probability=True),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': SVC(random_state=42, kernel='linear', gamma=0.1, probability=True),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': SVC(random_state=42, kernel='rbf', gamma=1, probability=True),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': SVC(random_state=42, kernel='rbf', gamma=0.5, probability=True),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': SVC(random_state=42, kernel='rbf', gamma=1, probability=True),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8qtV9P5qjNKw",
    "outputId": "e4f18dab-8b97-4690-ac6c-8a1d4a95f5f2"
   },
   "outputs": [],
   "source": [
    "svm_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in svm.items():\n",
    "    svm_clfs[ext] = get_classifier(clf['CLF'], train, class_train, clf['EXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "3_a327JBjglb"
   },
   "outputs": [],
   "source": [
    "df_pred_train = pd.DataFrame(df_train['HS'])\n",
    "df_pred_val = pd.DataFrame(df_val['HS'])\n",
    "df_pred_test = pd.DataFrame(df_test['HS'])\n",
    "\n",
    "df_prob_val = pd.DataFrame(df_val['HS'])\n",
    "df_prob_train = pd.DataFrame(df_train['HS'])\n",
    "df_prob_test = pd.DataFrame(df_test['HS'])\n",
    "\n",
    "for ext, clf in svm_clfs.items():\n",
    "  # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"SVM-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"SVM-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"SVM-{}\".format(ext)])\n",
    "  \n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "  \n",
    "  # # Probabilidades\n",
    "    cols = [\n",
    "      \"SVM-{}-{}\".format(ext,clf.classes_[0]), \n",
    "      \"SVM-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction LR model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "xCd8RcJhlAKs"
   },
   "outputs": [],
   "source": [
    "lr = {\n",
    "    'CV': {\n",
    "        'CLF': LogisticRegression(random_state=42, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': LogisticRegression(random_state=42, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': LogisticRegression(random_state=42, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': LogisticRegression(random_state=42, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': LogisticRegression(random_state=42, multi_class='auto', solver='liblinear', penalty='l1'),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gaft1kQCtTZo",
    "outputId": "4eb42ee9-82fc-4b46-b1f3-84427d51168d"
   },
   "outputs": [],
   "source": [
    "lr_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in lr.items():\n",
    "    lr_clfs[ext] = get_classifier(clf['CLF'], train, df_train['HS'], clf['EXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "Q9WSAXm7tUpU"
   },
   "outputs": [],
   "source": [
    "for ext, clf in lr_clfs.items():\n",
    "  # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"LR-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"LR-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"LR-{}\".format(ext)])\n",
    "  \n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "    # # Probabilidades\n",
    "    cols = [\n",
    "      \"LR-{}-{}\".format(ext,clf.classes_[0]), \n",
    "      \"LR-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction Random Forest\n",
    "\n",
    "With calibration for good stacking at the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "onRailt-teBu"
   },
   "outputs": [],
   "source": [
    "rf = {\n",
    "    'CV': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "                                      \n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': RandomForestClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1nCt_RN4tmKA",
    "outputId": "927c84fb-b86d-415c-fb46-760152beaa14"
   },
   "outputs": [],
   "source": [
    "rf_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in rf.items():\n",
    "    rf_clfs[ext] = get_classifier(clf['CLF'], train, df_train['HS'], clf['EXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "09ArLJnutn8T",
    "outputId": "0fc53a6d-2407-4581-f767-bc720922d7d5"
   },
   "outputs": [],
   "source": [
    "for ext, clf in rf_clfs.items():\n",
    "    # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"RF-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"RF-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"RF-{}\".format(ext)])\n",
    "  \n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "    # # Probabilidades\n",
    "    cols = [\n",
    "      \"RF-{}-{}\".format(ext,clf.classes_[0]), \n",
    "      \"RF-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PRediction Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "GHGjXwvRtu59"
   },
   "outputs": [],
   "source": [
    "nb = {\n",
    "    'CV': {\n",
    "        'CLF': MultinomialNB(alpha=1, fit_prior=False),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': MultinomialNB(alpha=0.5, fit_prior=False),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': BernoulliNB(alpha=0.5, fit_prior=True),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': BernoulliNB(alpha=0.1, fit_prior=True),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': BernoulliNB(alpha=1, fit_prior=True),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "TonmJS55uHtA"
   },
   "outputs": [],
   "source": [
    "nb_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in nb.items():\n",
    "    nb_clfs[ext] = get_classifier(clf['CLF'], train, df_train['HS'], clf['EXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "VamLfJefuJ8m"
   },
   "outputs": [],
   "source": [
    "for ext, clf in nb_clfs.items():\n",
    "    # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"NB-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"NB-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"NB-{}\".format(ext)])\n",
    "  \n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "  # # Probabilidades\n",
    "    cols = [\n",
    "      \"NB-{}-{}\".format(ext,clf.classes_[0]), \n",
    "      \"NB-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PRediction MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "id": "Q0UFePz9uj7y"
   },
   "outputs": [],
   "source": [
    "mlp = {\n",
    "    'CV': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=64, max_iter=100, activation='relu', solver='lbfgs'),\n",
    "        'EXT': cv,\n",
    "\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=64, max_iter=100, activation='logistic', solver='adam'),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=64, max_iter=100, activation='relu', solver='adam'),\n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=64, max_iter=100, activation='relu', solver='adam'),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': MLPClassifier(random_state=42, batch_size=20, max_iter=20, activation='relu', solver='adam'),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "D8-VNNZyul01",
    "outputId": "8573a536-0ca8-4399-cd3f-e7de8e9bf158"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rafael\\Anaconda3\\envs\\hate_speech\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\Rafael\\Anaconda3\\envs\\hate_speech\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\Rafael\\Anaconda3\\envs\\hate_speech\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n",
      "C:\\Users\\Rafael\\Anaconda3\\envs\\hate_speech\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py:585: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (20) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    }
   ],
   "source": [
    "mlp_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in mlp.items():\n",
    "    mlp_clfs[ext] = get_classifier(clf['CLF'], train, df_train['HS'], clf['EXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "id": "xHdgSp1UunVN"
   },
   "outputs": [],
   "source": [
    "for ext, clf in mlp_clfs.items():\n",
    "    # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"MLP-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"MLP-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"MLP-{}\".format(ext)])\n",
    "\n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "    # # Probabilidades\n",
    "    cols = [\n",
    "        \"MLP-{}-{}\".format(ext,clf.classes_[0]), \n",
    "        \"MLP-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction Extra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "KTUzrHaCuwaS"
   },
   "outputs": [],
   "source": [
    "extra = {\n",
    "    'CV': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, n_estimators=100, n_jobs=-1), \n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, n_estimators=100, n_jobs=-1), \n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': ExtraTreesClassifier(random_state=42, n_estimators=100, n_jobs=-1),\n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "X8eaNGChuzAL",
    "outputId": "5fe999b8-eb66-486c-f9fe-80d2d3b43198"
   },
   "outputs": [],
   "source": [
    "extra_clfs = {\n",
    "\n",
    "}\n",
    "for ext, clf in extra.items():\n",
    "    extra_clfs[ext] = get_classifier(clf['CLF'], train, df_train['HS'], clf['EXT'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BW3cJ3tIu01c",
    "outputId": "7935323e-3959-4307-8c61-ae20c98a305c"
   },
   "outputs": [],
   "source": [
    "for ext, clf in extra_clfs.items():\n",
    "    # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"EXTRA-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"EXTRA-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"EXTRA-{}\".format(ext)])\n",
    "\n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "    # # Probabilidades\n",
    "    cols = [\n",
    "        \"EXTRA-{}-{}\".format(ext,clf.classes_[0]), \n",
    "        \"EXTRA-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "Y0nOzkCEu7I4"
   },
   "outputs": [],
   "source": [
    "knn = {\n",
    "    'CV': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=3, algorithm='auto', n_jobs=-1), \n",
    "        'EXT': cv,\n",
    "    },\n",
    "    'TFIDF': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=3, algorithm='auto', n_jobs=-1),\n",
    "        'EXT': tfidf,\n",
    "    },\n",
    "    'W2V': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto', n_jobs=-1), \n",
    "        'EXT': w2v,\n",
    "    },\n",
    "    'GLOVE': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto', n_jobs=-1),\n",
    "        'EXT':  glove,\n",
    "    },\n",
    "    'FAST': {\n",
    "        'CLF': KNeighborsClassifier(n_neighbors=5, algorithm='auto', n_jobs=-1), \n",
    "        'EXT': fasttext,\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "BlKGAfMsu-5v"
   },
   "outputs": [],
   "source": [
    "knn_clfs = { }\n",
    "for ext, clf in knn.items():\n",
    "    knn_clfs[ext] = get_classifier(clf['CLF'], train, df_train['HS'], clf['EXT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "id": "XLZioNx2vAyJ"
   },
   "outputs": [],
   "source": [
    "for ext, clf in knn_clfs.items():\n",
    "    # Predict\n",
    "    df_train_ = pd.DataFrame(clf.predict(train), columns=[\"KNN-{}\".format(ext)])\n",
    "    df_val_ = pd.DataFrame(clf.predict(val), columns=[\"KNN-{}\".format(ext)])\n",
    "    df_test_ = pd.DataFrame(clf.predict(test), columns=[\"KNN-{}\".format(ext)])\n",
    "\n",
    "    df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "    df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "    df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "    # # Probabilidades\n",
    "    cols = [\n",
    "        \"KNN-{}-{}\".format(ext,clf.classes_[0]), \n",
    "        \"KNN-{}-{}\".format(ext,clf.classes_[1])\n",
    "    ]\n",
    "    df_train_ = pd.DataFrame(clf.predict_proba(train), columns=cols)\n",
    "    df_val_ = pd.DataFrame(clf.predict_proba(val), columns=cols)\n",
    "    df_test_ = pd.DataFrame(clf.predict_proba(test), columns=cols)\n",
    "    df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "    df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "    df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BCN40ZFTvH0G"
   },
   "source": [
    "# Prediction CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "id": "ddw95uDCvJCH"
   },
   "outputs": [],
   "source": [
    "MAX_NB_WORDS = 20000\n",
    "MAX_SEQUENCE_LENGTH=300\n",
    "\n",
    "y_train = to_categorical(df_train['HS'])\n",
    "y_val = to_categorical(df_val['HS'])\n",
    "\n",
    "tokenizer = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(train)\n",
    "\n",
    "seq_train = tokenizer.texts_to_sequences(train)\n",
    "seq_val = tokenizer.texts_to_sequences(val)\n",
    "seq_test = tokenizer.texts_to_sequences(test)\n",
    "\n",
    "data_train = pad_sequences(seq_train, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_val = pad_sequences(seq_val, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "data_test = pad_sequences(seq_test, maxlen=MAX_SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5WL49mfKvXjt",
    "outputId": "901932b3-6399-42fb-cd05-b3a739415ef9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 300, 300)          5142600   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "conv1d (Conv1D)              (None, 298, 64)           57664     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d (Global (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 514       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 5,217,418\n",
      "Trainable params: 74,818\n",
      "Non-trainable params: 5,142,600\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 300, 300)          5142600   \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_1 (Conv1D)            (None, 298, 64)           57664     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_1 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 2)                 514       \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 5,217,418\n",
      "Trainable params: 74,818\n",
      "Non-trainable params: 5,142,600\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 300, 300)          5142600   \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 298, 64)           57664     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_2 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 2)                 514       \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 5,217,418\n",
      "Trainable params: 74,818\n",
      "Non-trainable params: 5,142,600\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 300, 25)           428550    \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 300, 25)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 298, 64)           4864      \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_3 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 2)                 514       \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 450,568\n",
      "Trainable params: 22,018\n",
      "Non-trainable params: 428,550\n",
      "_________________________________________________________________\n",
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_4 (Embedding)      (None, 300, 300)          5142600   \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_4 (Conv1D)            (None, 298, 64)           57664     \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_4 (Glob (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 2)                 514       \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 2)                 0         \n",
      "=================================================================\n",
      "Total params: 5,217,418\n",
      "Trainable params: 74,818\n",
      "Non-trainable params: 5,142,600\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "cnn_cv = get_CNN(cv, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='sigmoid', dense=2)\n",
    "cnn_tfidf = get_CNN(tfidf, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='softmax', dense=2)\n",
    "cnn_w2v = get_CNN(w2v, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='sigmoid', word_embedding=True, dense=2)\n",
    "cnn_glove = get_CNN(glove, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=25, activation='sigmoid', word_embedding=True, dense=2)\n",
    "cnn_fast = get_CNN(fasttext, tokenizer, MAX_NB_WORDS, EMBEDDING_DIM=300, activation='sigmoid', word_embedding=True, dense=2)\n",
    "callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "afhb0ekxvjkn",
    "outputId": "9b5fac18-eb60-4757-f728-d1587d598add"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "450/450 [==============================] - 21s 45ms/step - loss: 0.6769 - accuracy: 0.5910 - val_loss: 0.6531 - val_accuracy: 0.6270\n",
      "Epoch 2/20\n",
      "450/450 [==============================] - 20s 44ms/step - loss: 0.6235 - accuracy: 0.6651 - val_loss: 0.6065 - val_accuracy: 0.6930\n",
      "Epoch 3/20\n",
      "450/450 [==============================] - 20s 43ms/step - loss: 0.5661 - accuracy: 0.7247 - val_loss: 0.6052 - val_accuracy: 0.6870\n",
      "Epoch 4/20\n",
      "450/450 [==============================] - 21s 47ms/step - loss: 0.5230 - accuracy: 0.7495 - val_loss: 0.5791 - val_accuracy: 0.6970\n",
      "Epoch 5/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.5093 - accuracy: 0.7617 - val_loss: 0.5709 - val_accuracy: 0.7100\n",
      "Epoch 6/20\n",
      "450/450 [==============================] - 17s 39ms/step - loss: 0.4801 - accuracy: 0.7782 - val_loss: 0.5747 - val_accuracy: 0.7020\n",
      "Epoch 7/20\n",
      "450/450 [==============================] - 17s 37ms/step - loss: 0.4617 - accuracy: 0.7855 - val_loss: 0.5630 - val_accuracy: 0.7150\n",
      "Epoch 8/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.4397 - accuracy: 0.8023 - val_loss: 0.6304 - val_accuracy: 0.6620\n",
      "Epoch 9/20\n",
      "450/450 [==============================] - 17s 37ms/step - loss: 0.4224 - accuracy: 0.8142 - val_loss: 0.5541 - val_accuracy: 0.7200\n",
      "Epoch 10/20\n",
      "450/450 [==============================] - 17s 37ms/step - loss: 0.4277 - accuracy: 0.8038 - val_loss: 0.5944 - val_accuracy: 0.7160\n",
      "Epoch 11/20\n",
      "450/450 [==============================] - 17s 37ms/step - loss: 0.3922 - accuracy: 0.8248 - val_loss: 0.5958 - val_accuracy: 0.6970\n",
      "Epoch 12/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.3863 - accuracy: 0.8289 - val_loss: 0.5880 - val_accuracy: 0.7140\n",
      "Epoch 13/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.3638 - accuracy: 0.8404 - val_loss: 0.6187 - val_accuracy: 0.7150\n",
      "Epoch 14/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.3611 - accuracy: 0.8391 - val_loss: 0.6032 - val_accuracy: 0.7150\n",
      "Epoch 15/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.3313 - accuracy: 0.8573 - val_loss: 0.6250 - val_accuracy: 0.7110\n",
      "Epoch 16/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.3184 - accuracy: 0.8633 - val_loss: 0.6196 - val_accuracy: 0.7230\n",
      "Epoch 17/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.3176 - accuracy: 0.8584 - val_loss: 0.6198 - val_accuracy: 0.7130\n",
      "Epoch 18/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.2915 - accuracy: 0.8760 - val_loss: 0.6872 - val_accuracy: 0.7120\n",
      "Epoch 19/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.2905 - accuracy: 0.8771 - val_loss: 0.6630 - val_accuracy: 0.7140\n",
      "Epoch 20/20\n",
      "450/450 [==============================] - 17s 38ms/step - loss: 0.2799 - accuracy: 0.8865 - val_loss: 0.6687 - val_accuracy: 0.7050\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x251f508eb70>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_cv.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pDRmixdNvoAv",
    "outputId": "3cff20c5-2638-4720-d8e9-e13ab2f65e4b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "45/45 [==============================] - 15s 315ms/step - loss: 0.6834 - accuracy: 0.5724 - val_loss: 0.6808 - val_accuracy: 0.5730\n",
      "Epoch 2/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.6786 - accuracy: 0.5758 - val_loss: 0.6725 - val_accuracy: 0.5730\n",
      "Epoch 3/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.6629 - accuracy: 0.5803 - val_loss: 0.6470 - val_accuracy: 0.6210\n",
      "Epoch 4/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.6316 - accuracy: 0.6553 - val_loss: 0.6298 - val_accuracy: 0.6330\n",
      "Epoch 5/20\n",
      "45/45 [==============================] - 14s 313ms/step - loss: 0.6009 - accuracy: 0.6954 - val_loss: 0.6121 - val_accuracy: 0.6980\n",
      "Epoch 6/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.5770 - accuracy: 0.7146 - val_loss: 0.6037 - val_accuracy: 0.6820\n",
      "Epoch 7/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.5595 - accuracy: 0.7244 - val_loss: 0.6013 - val_accuracy: 0.6890\n",
      "Epoch 8/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.5379 - accuracy: 0.7396 - val_loss: 0.5946 - val_accuracy: 0.6890\n",
      "Epoch 9/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.5325 - accuracy: 0.7482 - val_loss: 0.5936 - val_accuracy: 0.6790\n",
      "Epoch 10/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.5189 - accuracy: 0.7539 - val_loss: 0.5877 - val_accuracy: 0.6900\n",
      "Epoch 11/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.5099 - accuracy: 0.7553 - val_loss: 0.5856 - val_accuracy: 0.6880\n",
      "Epoch 12/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.5060 - accuracy: 0.7573 - val_loss: 0.5821 - val_accuracy: 0.6860\n",
      "Epoch 13/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4985 - accuracy: 0.7672 - val_loss: 0.5767 - val_accuracy: 0.7040\n",
      "Epoch 14/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4843 - accuracy: 0.7790 - val_loss: 0.5759 - val_accuracy: 0.7000\n",
      "Epoch 15/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4803 - accuracy: 0.7757 - val_loss: 0.5686 - val_accuracy: 0.7030\n",
      "Epoch 16/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4742 - accuracy: 0.7820 - val_loss: 0.5668 - val_accuracy: 0.7050\n",
      "Epoch 17/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4603 - accuracy: 0.7855 - val_loss: 0.5862 - val_accuracy: 0.6790\n",
      "Epoch 18/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.4680 - accuracy: 0.7825 - val_loss: 0.5603 - val_accuracy: 0.7080\n",
      "Epoch 19/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4546 - accuracy: 0.7913 - val_loss: 0.5585 - val_accuracy: 0.7080\n",
      "Epoch 20/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.4503 - accuracy: 0.7949 - val_loss: 0.5708 - val_accuracy: 0.7150\n",
      "Epoch 1/20\n",
      "45/45 [==============================] - 15s 314ms/step - loss: 0.6628 - accuracy: 0.6086 - val_loss: 0.5946 - val_accuracy: 0.6870\n",
      "Epoch 2/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.5532 - accuracy: 0.7223 - val_loss: 0.5766 - val_accuracy: 0.6860\n",
      "Epoch 3/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.5098 - accuracy: 0.7444 - val_loss: 0.5782 - val_accuracy: 0.7010\n",
      "Epoch 4/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4683 - accuracy: 0.7763 - val_loss: 0.5998 - val_accuracy: 0.6800\n",
      "Epoch 5/20\n",
      "45/45 [==============================] - 14s 314ms/step - loss: 0.4517 - accuracy: 0.7864 - val_loss: 0.6026 - val_accuracy: 0.7000\n",
      "Epoch 6/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.4334 - accuracy: 0.7997 - val_loss: 0.5675 - val_accuracy: 0.7080\n",
      "Epoch 7/20\n",
      "45/45 [==============================] - 14s 312ms/step - loss: 0.4033 - accuracy: 0.8117 - val_loss: 0.5734 - val_accuracy: 0.7020\n",
      "Epoch 8/20\n",
      "45/45 [==============================] - 14s 313ms/step - loss: 0.4121 - accuracy: 0.8142 - val_loss: 0.6316 - val_accuracy: 0.6910\n",
      "Epoch 9/20\n",
      "45/45 [==============================] - 14s 314ms/step - loss: 0.3628 - accuracy: 0.8412 - val_loss: 0.6116 - val_accuracy: 0.7060\n",
      "Epoch 10/20\n",
      "45/45 [==============================] - 14s 314ms/step - loss: 0.3416 - accuracy: 0.8492 - val_loss: 0.6313 - val_accuracy: 0.7060\n",
      "Epoch 11/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.3122 - accuracy: 0.8678 - val_loss: 0.8878 - val_accuracy: 0.6400\n",
      "Epoch 12/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.3229 - accuracy: 0.8620 - val_loss: 0.6571 - val_accuracy: 0.7030\n",
      "Epoch 13/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.3172 - accuracy: 0.8553 - val_loss: 0.6885 - val_accuracy: 0.7050\n",
      "Epoch 14/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.2676 - accuracy: 0.8912 - val_loss: 0.6728 - val_accuracy: 0.6980\n",
      "Epoch 15/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.2479 - accuracy: 0.8952 - val_loss: 0.7362 - val_accuracy: 0.6820\n",
      "Epoch 16/20\n",
      "45/45 [==============================] - 14s 316ms/step - loss: 0.2832 - accuracy: 0.8862 - val_loss: 0.7101 - val_accuracy: 0.6960\n",
      "Epoch 17/20\n",
      "45/45 [==============================] - 14s 314ms/step - loss: 0.2268 - accuracy: 0.9022 - val_loss: 0.9631 - val_accuracy: 0.6590\n",
      "Epoch 18/20\n",
      "45/45 [==============================] - 14s 312ms/step - loss: 0.2324 - accuracy: 0.8974 - val_loss: 0.7561 - val_accuracy: 0.6920\n",
      "Epoch 19/20\n",
      "45/45 [==============================] - 15s 330ms/step - loss: 0.2196 - accuracy: 0.9140 - val_loss: 0.7146 - val_accuracy: 0.7100\n",
      "Epoch 20/20\n",
      "45/45 [==============================] - 15s 335ms/step - loss: 0.1949 - accuracy: 0.9239 - val_loss: 0.8373 - val_accuracy: 0.6760\n",
      "Epoch 1/20\n",
      "45/45 [==============================] - 3s 62ms/step - loss: 0.7759 - accuracy: 0.5641 - val_loss: 0.6343 - val_accuracy: 0.6320\n",
      "Epoch 2/20\n",
      "45/45 [==============================] - 3s 59ms/step - loss: 0.6691 - accuracy: 0.6078 - val_loss: 0.6285 - val_accuracy: 0.6430\n",
      "Epoch 3/20\n",
      "45/45 [==============================] - 3s 60ms/step - loss: 0.6369 - accuracy: 0.6402 - val_loss: 0.6289 - val_accuracy: 0.6450\n",
      "Epoch 4/20\n",
      "45/45 [==============================] - 3s 58ms/step - loss: 0.6356 - accuracy: 0.6374 - val_loss: 0.6041 - val_accuracy: 0.6760\n",
      "Epoch 5/20\n",
      "45/45 [==============================] - 2s 54ms/step - loss: 0.6178 - accuracy: 0.6553 - val_loss: 0.6116 - val_accuracy: 0.6620\n",
      "Epoch 6/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.6038 - accuracy: 0.6722 - val_loss: 0.6068 - val_accuracy: 0.6660\n",
      "Epoch 7/20\n",
      "45/45 [==============================] - 2s 52ms/step - loss: 0.5898 - accuracy: 0.6807 - val_loss: 0.6102 - val_accuracy: 0.6630\n",
      "Epoch 8/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5888 - accuracy: 0.6803 - val_loss: 0.5970 - val_accuracy: 0.6890\n",
      "Epoch 9/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5781 - accuracy: 0.6991 - val_loss: 0.6026 - val_accuracy: 0.6810\n",
      "Epoch 10/20\n",
      "45/45 [==============================] - 2s 52ms/step - loss: 0.5721 - accuracy: 0.7001 - val_loss: 0.6007 - val_accuracy: 0.6780\n",
      "Epoch 11/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5629 - accuracy: 0.7038 - val_loss: 0.6422 - val_accuracy: 0.6620\n",
      "Epoch 12/20\n",
      "45/45 [==============================] - 2s 52ms/step - loss: 0.5621 - accuracy: 0.7103 - val_loss: 0.6042 - val_accuracy: 0.6740\n",
      "Epoch 13/20\n",
      "45/45 [==============================] - 2s 52ms/step - loss: 0.5603 - accuracy: 0.7057 - val_loss: 0.6026 - val_accuracy: 0.6770\n",
      "Epoch 14/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5558 - accuracy: 0.7103 - val_loss: 0.5998 - val_accuracy: 0.6830\n",
      "Epoch 15/20\n",
      "45/45 [==============================] - 2s 52ms/step - loss: 0.5611 - accuracy: 0.7080 - val_loss: 0.5993 - val_accuracy: 0.6830\n",
      "Epoch 16/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5527 - accuracy: 0.7187 - val_loss: 0.6000 - val_accuracy: 0.6840\n",
      "Epoch 17/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5386 - accuracy: 0.7216 - val_loss: 0.6131 - val_accuracy: 0.6760\n",
      "Epoch 18/20\n",
      "45/45 [==============================] - 2s 52ms/step - loss: 0.5412 - accuracy: 0.7187 - val_loss: 0.6128 - val_accuracy: 0.6800\n",
      "Epoch 19/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5276 - accuracy: 0.7372 - val_loss: 0.5925 - val_accuracy: 0.6850\n",
      "Epoch 20/20\n",
      "45/45 [==============================] - 2s 53ms/step - loss: 0.5245 - accuracy: 0.7297 - val_loss: 0.6119 - val_accuracy: 0.6780\n",
      "Epoch 1/20\n",
      "45/45 [==============================] - 14s 312ms/step - loss: 0.6685 - accuracy: 0.5898 - val_loss: 0.6266 - val_accuracy: 0.6400\n",
      "Epoch 2/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.5988 - accuracy: 0.6759 - val_loss: 0.6271 - val_accuracy: 0.6520\n",
      "Epoch 3/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.5484 - accuracy: 0.7195 - val_loss: 0.5920 - val_accuracy: 0.6920\n",
      "Epoch 4/20\n",
      "45/45 [==============================] - 14s 318ms/step - loss: 0.5140 - accuracy: 0.7533 - val_loss: 0.5724 - val_accuracy: 0.7080\n",
      "Epoch 5/20\n",
      "45/45 [==============================] - 14s 308ms/step - loss: 0.4921 - accuracy: 0.7644 - val_loss: 0.5627 - val_accuracy: 0.6960\n",
      "Epoch 6/20\n",
      "45/45 [==============================] - 14s 308ms/step - loss: 0.4641 - accuracy: 0.7807 - val_loss: 0.5788 - val_accuracy: 0.7030\n",
      "Epoch 7/20\n",
      "45/45 [==============================] - 14s 308ms/step - loss: 0.4442 - accuracy: 0.7936 - val_loss: 0.5890 - val_accuracy: 0.6920\n",
      "Epoch 8/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.4335 - accuracy: 0.8043 - val_loss: 0.6035 - val_accuracy: 0.7070\n",
      "Epoch 9/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.4263 - accuracy: 0.8003 - val_loss: 0.5910 - val_accuracy: 0.7090\n",
      "Epoch 10/20\n",
      "45/45 [==============================] - 14s 308ms/step - loss: 0.3990 - accuracy: 0.8178 - val_loss: 0.5876 - val_accuracy: 0.7160\n",
      "Epoch 11/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.3804 - accuracy: 0.8277 - val_loss: 0.6715 - val_accuracy: 0.6990\n",
      "Epoch 12/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.4038 - accuracy: 0.8136 - val_loss: 0.6524 - val_accuracy: 0.6950\n",
      "Epoch 13/20\n",
      "45/45 [==============================] - 14s 309ms/step - loss: 0.3752 - accuracy: 0.8355 - val_loss: 0.6395 - val_accuracy: 0.6870\n",
      "Epoch 14/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.3698 - accuracy: 0.8321 - val_loss: 0.6986 - val_accuracy: 0.6930\n",
      "Epoch 15/20\n",
      "45/45 [==============================] - 14s 315ms/step - loss: 0.3593 - accuracy: 0.8324 - val_loss: 0.5936 - val_accuracy: 0.7030\n",
      "Epoch 16/20\n",
      "45/45 [==============================] - 14s 315ms/step - loss: 0.3119 - accuracy: 0.8701 - val_loss: 0.7346 - val_accuracy: 0.7030\n",
      "Epoch 17/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.3183 - accuracy: 0.8640 - val_loss: 0.6568 - val_accuracy: 0.7130\n",
      "Epoch 18/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.3000 - accuracy: 0.8715 - val_loss: 0.6068 - val_accuracy: 0.7340\n",
      "Epoch 19/20\n",
      "45/45 [==============================] - 14s 310ms/step - loss: 0.3204 - accuracy: 0.8629 - val_loss: 0.7353 - val_accuracy: 0.6980\n",
      "Epoch 20/20\n",
      "45/45 [==============================] - 14s 311ms/step - loss: 0.2757 - accuracy: 0.8871 - val_loss: 0.6906 - val_accuracy: 0.6850\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x252268025c0>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn_tfidf.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)\n",
    "cnn_w2v.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)\n",
    "cnn_glove.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)\n",
    "cnn_fast.fit(data_train, y_train, validation_data=(data_val, y_val), epochs=20, batch_size=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "9meqmalXvuaJ"
   },
   "outputs": [],
   "source": [
    "# df_pred_test = df_pred_test.drop(['CNN-CV'], axis=1)\n",
    "# print(df_pred_test.columns)\n",
    "# CV\n",
    "cols = [\"CNN-CV-0\", \"CNN-CV-1\"]\n",
    "df_train_ = pd.DataFrame(np.argmax(cnn_cv.predict(data_train), axis=1), columns=[\"CNN-CV\"])\n",
    "df_val_ = pd.DataFrame(np.argmax(cnn_cv.predict(data_val), axis=1), columns=[\"CNN-CV\"])\n",
    "df_test_ = pd.DataFrame(np.argmax(cnn_cv.predict(data_test), axis=1), columns=[\"CNN-CV\"])\n",
    "  \n",
    "df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "# print(df_pred_test.columns)\n",
    "\n",
    "# # Probabilidades\n",
    "  \n",
    "df_train_ = pd.DataFrame(cnn_cv.predict(data_train), columns=cols)\n",
    "df_val_ = pd.DataFrame(cnn_cv.predict(data_val), columns=cols)\n",
    "df_test_ = pd.DataFrame(cnn_cv.predict(data_test), columns=cols)\n",
    "\n",
    "df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "\n",
    "cols = [\"CNN-TFIDF-0\", \"CNN-TFIDF-1\"]\n",
    "\n",
    "df_train_ = pd.DataFrame(np.argmax(cnn_tfidf.predict(data_train), axis=1), columns=[\"CNN-TFIDF\"])\n",
    "df_val_ = pd.DataFrame(np.argmax(cnn_tfidf.predict(data_val), axis=1), columns=[\"CNN-TFIDF\"])\n",
    "df_test_ = pd.DataFrame(np.argmax(cnn_tfidf.predict(data_test), axis=1), columns=[\"CNN-TFIDF\"])\n",
    "  \n",
    "df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "# Probabilidades\n",
    "  \n",
    "df_train_ = pd.DataFrame(cnn_tfidf.predict(data_train), columns=cols)\n",
    "df_val_ = pd.DataFrame(cnn_tfidf.predict(data_val), columns=cols)\n",
    "df_test_ = pd.DataFrame(cnn_tfidf.predict(data_test), columns=cols)\n",
    "\n",
    "df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "cols = [\"CNN-W2V-0\", \"CNN-W2V-1\"]\n",
    "\n",
    "df_train_ = pd.DataFrame(np.argmax(cnn_w2v.predict(data_train), axis=1), columns=[\"CNN-W2V\"])\n",
    "df_val_ = pd.DataFrame(np.argmax(cnn_w2v.predict(data_val), axis=1), columns=[\"CNN-W2V\"])\n",
    "df_test_ = pd.DataFrame(np.argmax(cnn_w2v.predict(data_test), axis=1), columns=[\"CNN-W2V\"])\n",
    "  \n",
    "df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "# Probabilidades\n",
    "  \n",
    "df_train_ = pd.DataFrame(cnn_w2v.predict(data_train), columns=cols)\n",
    "df_val_ = pd.DataFrame(cnn_w2v.predict(data_val), columns=cols)\n",
    "df_test_ = pd.DataFrame(cnn_w2v.predict(data_test), columns=cols)\n",
    "\n",
    "df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "\n",
    "cols = [\"CNN-GLOVE-0\", \"CNN-GLOVE-1\"]\n",
    "\n",
    "df_train_ = pd.DataFrame(np.argmax(cnn_glove.predict(data_train), axis=1), columns=[\"CNN-GLOVE\"])\n",
    "df_val_ = pd.DataFrame(np.argmax(cnn_glove.predict(data_val), axis=1), columns=[\"CNN-GLOVE\"])\n",
    "df_test_ = pd.DataFrame(np.argmax(cnn_glove.predict(data_test), axis=1), columns=[\"CNN-GLOVE\"])\n",
    "\n",
    "df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "# Probabilidades\n",
    "  \n",
    "df_train_ = pd.DataFrame(cnn_glove.predict(data_train), columns=cols)\n",
    "df_val_ = pd.DataFrame(cnn_glove.predict(data_val), columns=cols)\n",
    "df_test_ = pd.DataFrame(cnn_glove.predict(data_test), columns=cols)\n",
    "\n",
    "df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "cols = [\"CNN-FAST-0\", \"CNN-FAST-1\"]\n",
    "\n",
    "df_train_ = pd.DataFrame(np.argmax(cnn_fast.predict(data_train), axis=1), columns=[\"CNN-FAST\"])\n",
    "df_val_ = pd.DataFrame(np.argmax(cnn_fast.predict(data_val), axis=1), columns=[\"CNN-FAST\"])\n",
    "df_test_ = pd.DataFrame(np.argmax(cnn_fast.predict(data_test), axis=1), columns=[\"CNN-FAST\"])\n",
    "\n",
    "df_pred_train = pd.concat([df_pred_train, df_train_], axis=1, sort=False)\n",
    "df_pred_val = pd.concat([df_pred_val, df_val_], axis=1, sort=False)\n",
    "df_pred_test = pd.concat([df_pred_test, df_test_], axis=1, sort=False)\n",
    "\n",
    "# Probabilidades\n",
    "  \n",
    "df_train_ = pd.DataFrame(cnn_fast.predict(data_train), columns=cols)\n",
    "df_val_ = pd.DataFrame(cnn_fast.predict(data_val), columns=cols)\n",
    "df_test_ = pd.DataFrame(cnn_fast.predict(data_test), columns=cols)\n",
    "\n",
    "df_prob_train = pd.concat([df_prob_train, df_train_], axis=1, sort=False)\n",
    "df_prob_val = pd.concat([df_prob_val, df_val_], axis=1, sort=False)\n",
    "df_prob_test = pd.concat([df_prob_test, df_test_], axis=1, sort=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "B0ObWDQBEkRA"
   },
   "outputs": [],
   "source": [
    "df_pred_train.to_csv(\"./Saved_Predict_and_Proba/HATEVAL/pred_train_hateval.csv\")\n",
    "df_pred_val.to_csv(\"./Saved_Predict_and_Proba/HATEVAL/pred_val_hateval.csv\")\n",
    "df_pred_test.to_csv(\"./Saved_Predict_and_Proba/HATEVAL/pred_test_hateval.csv\")\n",
    "df_prob_train.to_csv(\"./Saved_Predict_and_Proba/HateVal/prob_train_hateval.csv\")\n",
    "df_prob_val.to_csv(\"./Saved_Predict_and_Proba/HateVal/prob_val_hateval.csv\")\n",
    "df_prob_test.to_csv(\"./Saved_Predict_and_Proba/HateVal/prob_test_hateval.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analyzing Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from pprint import pprint\n",
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "algorithms_list = ['SVM', 'MLP', 'KNN', 'RF', 'EXTRA', 'CNN', 'LR', 'NB']\n",
    "fe_list = ['CV', 'TFIDF', 'W2V', 'GLOVE', 'FAST']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_df = pd.read_csv(\"./Saved_Predict_and_Proba/HateVal/prob_val_hateval.csv\").dropna()\n",
    "probas_val, labels_val = val_df.drop(columns=[\"Unnamed: 0\", \"HS\"]), val_df[\"HS\"]\n",
    "results_f1_val = np.zeros((len(algorithms_list), len(fe_list)))\n",
    "for idx_alg, alg in enumerate(algorithms_list):\n",
    "    for idx_fe, fe in enumerate(fe_list):\n",
    "        probas = probas_val.filter(regex=alg+'-'+fe)\n",
    "        y_pred = np.argmax(probas.to_numpy(), axis=1)\n",
    "        results_f1_val[idx_alg, idx_fe] = f1_score(labels_val, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NB-FAST-0</th>\n",
       "      <th>NB-FAST-1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.610015e-01</td>\n",
       "      <td>5.389985e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.607912e-01</td>\n",
       "      <td>8.392088e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3.348711e-01</td>\n",
       "      <td>6.651289e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.994668e-01</td>\n",
       "      <td>5.331676e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9.999996e-01</td>\n",
       "      <td>3.969834e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>3.316726e-05</td>\n",
       "      <td>9.999668e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>7.580322e-08</td>\n",
       "      <td>9.999999e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>6.568345e-04</td>\n",
       "      <td>9.993432e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>2.883085e-06</td>\n",
       "      <td>9.999971e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>9.650115e-01</td>\n",
       "      <td>3.498847e-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        NB-FAST-0     NB-FAST-1\n",
       "0    4.610015e-01  5.389985e-01\n",
       "1    1.607912e-01  8.392088e-01\n",
       "2    3.348711e-01  6.651289e-01\n",
       "3    9.994668e-01  5.331676e-04\n",
       "4    9.999996e-01  3.969834e-07\n",
       "..            ...           ...\n",
       "995  3.316726e-05  9.999668e-01\n",
       "996  7.580322e-08  9.999999e-01\n",
       "997  6.568345e-04  9.993432e-01\n",
       "998  2.883085e-06  9.999971e-01\n",
       "999  9.650115e-01  3.498847e-02\n",
       "\n",
       "[1000 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(\"./Saved_Predict_and_Proba/HateVal/prob_test_hateval.csv\").dropna()\n",
    "probas_test, labels_test = test_df.drop(columns=[\"Unnamed: 0\", \"HS\"]), test_df[\"HS\"]\n",
    "results_f1_test = np.zeros((len(algorithms_list), len(fe_list)))\n",
    "for idx_alg, alg in enumerate(algorithms_list):\n",
    "    for idx_fe, fe in enumerate(fe_list):\n",
    "        probas = probas_test.filter(regex=alg+'-'+fe)\n",
    "        y_pred = np.argmax(probas.to_numpy(), axis=1)\n",
    "        results_f1_test[idx_alg, idx_fe] = f1_score(labels_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df_test = pd.DataFrame(results_f1_test, columns=fe_list, index=algorithms_list)\n",
    "results_df_val = pd.DataFrame(results_f1_val, columns=fe_list, index=algorithms_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('\\\\begin{tabular}{lrrrrr}\\n'\n",
      " '\\\\toprule\\n'\n",
      " '{} &        CV &     TFIDF &       W2V &     GLOVE &      FAST \\\\\\\\\\n'\n",
      " '\\\\midrule\\n'\n",
      " 'SVM   &  0.604611 &  0.603842 &  0.627646 &  0.561172 &  0.617781 \\\\\\\\\\n'\n",
      " 'MLP   &  0.598499 &  0.587328 &  0.615523 &  0.575249 &  0.627669 \\\\\\\\\\n'\n",
      " 'KNN   &  0.485299 &  0.113573 &  0.585816 &  0.537245 &  0.566645 \\\\\\\\\\n'\n",
      " 'RF    &  0.586481 &  0.586640 &  0.598967 &  0.558888 &  0.597227 \\\\\\\\\\n'\n",
      " 'EXTRA &  0.591253 &  0.593182 &  0.594798 &  0.565487 &  0.596252 \\\\\\\\\\n'\n",
      " 'CNN   &  0.582179 &  0.559368 &  0.604710 &  0.569472 &  0.608629 \\\\\\\\\\n'\n",
      " 'LR    &  0.607941 &  0.596645 &  0.617062 &  0.503864 &  0.589726 \\\\\\\\\\n'\n",
      " 'NB    &  0.601139 &  0.603803 &  0.528418 &  0.499833 &  0.538870 \\\\\\\\\\n'\n",
      " '\\\\bottomrule\\n'\n",
      " '\\\\end{tabular}\\n')\n"
     ]
    }
   ],
   "source": [
    "pprint(results_df_test.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df_test.to_csv('Results HatEval.csv')"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "colab": {
   "collapsed_sections": [],
   "name": "Predict_HateVal.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:deslib]",
   "language": "python",
   "name": "conda-env-deslib-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}