extrafeatures-classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1899f060",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing necessary libraries\n",
    "\n",
    "import re\n",
    "import scipy\n",
    "import pandas         as pd\n",
    "import io\n",
    "import numpy          as np\n",
    "import copy\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "\n",
    "import transformers\n",
    "from transformers                     import  RobertaModel, AutoModel, AutoTokenizer, AdamW, get_linear_schedule_with_warmup\n",
    "from transformers import XLMRobertaForSequenceClassification,AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "from transformers.modeling_outputs import SequenceClassifierOutput\n",
    "import torch\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "from sklearn.metrics                  import classification_report\n",
    "from sklearn.feature_extraction.text  import TfidfVectorizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "from torch                            import nn, optim\n",
    "from torch.utils                      import data\n",
    "from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss\n",
    "\n",
    "\n",
    "#Seeding for deterministic results\n",
    "RANDOM_SEED = 64\n",
    "np.random.seed(RANDOM_SEED)\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "   torch.cuda.manual_seed(RANDOM_SEED)\n",
    "   torch.cuda.manual_seed_all(RANDOM_SEED) \n",
    "   torch.backends.cudnn.deterministic = True  \n",
    "   torch.backends.cudnn.benchmark = False\n",
    "\n",
    "\n",
    "CLASS_NAMES = ['sadness', 'joy', 'anger', 'surprise', 'disgust', 'fear', 'others']\n",
    "MAX_LENGTH = 200\n",
    "BATCH_SIZE = 16\n",
    "EPOCHS = 5\n",
    "MODEL = \"cardiffnlp/twitter-xlm-roberta-base\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL,use_fast=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3c8602b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Converting labels to numbers\n",
    "def label_to_int(label):\n",
    "  if label   == 'sadness':\n",
    "    return 0\n",
    "  elif label == 'joy':\n",
    "    return 1\n",
    "  elif label == 'anger':\n",
    "    return 2\n",
    "  elif label == 'surprise':\n",
    "    return 3\n",
    "  elif label == 'disgust':\n",
    "    return 4\n",
    "  elif label == 'fear':\n",
    "    return 5\n",
    "  elif label == 'others':\n",
    "    return 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff9f334e",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/EmoEvalEs/train.tsv', sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "dev = pd.read_csv('data/EmoEvalEs/dev.tsv', sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "test = pd.read_csv('data/EmoEvalEs/emoevales_test.tsv', sep='\\t', quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "299065ee",
   "metadata": {},
   "source": [
    "# Dataset Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69210800",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da34b857",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.emotion.value_counts().plot(kind='bar')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "430e1c04",
   "metadata": {},
   "source": [
    "## Offensive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e458255d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train[train.offensive=='OFF'].emotion.value_counts().plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f6f458b",
   "metadata": {},
   "source": [
    "## Event"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbbd57d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for event in set(train.event.to_list()):\n",
    "    print(event)\n",
    "    train[train.event==event].emotion.value_counts().plot()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "594c3d38",
   "metadata": {},
   "source": [
    "## Sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10a568aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import pipeline\n",
    "model_path = \"cardiffnlp/twitter-xlm-roberta-base-sentiment\"\n",
    "sentiment_task = pipeline(\"sentiment-analysis\", model=model_path, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8f383d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = sentiment_task(train.tweet.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35aa50dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentiment = []\n",
    "for r in result:\n",
    "    sentiment.append(r['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29b3d3c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['sentiment'] = sentiment\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cfe886d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for sent in set(train.sentiment.to_list()):\n",
    "    print(sent)\n",
    "    train[train.sentiment==sent].emotion.value_counts().plot()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "deef54a9",
   "metadata": {},
   "source": [
    "# Dataset Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a892fe9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/EmoEvalEs/train.tsv', sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "dev = pd.read_csv('data/EmoEvalEs/dev.tsv', sep='\\t', quoting=csv.QUOTE_NONE)\n",
    "test = pd.read_csv('data/EmoEvalEs/emoevales_test.tsv', sep='\\t', quoting=csv.QUOTE_NONE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb3e8239",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.get_dummies(train, columns = ['event'])\n",
    "dev = pd.get_dummies(dev, columns = ['event'])\n",
    "test = pd.get_dummies(test, columns = ['event'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4c2bd00",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.offensive = pd.Categorical(train.offensive)\n",
    "train['offensive'] = train.offensive.cat.codes\n",
    "dev.offensive = pd.Categorical(dev.offensive)\n",
    "dev['offensive'] = dev.offensive.cat.codes\n",
    "test.offensive = pd.Categorical(test.offensive)\n",
    "test['offensive'] = test.offensive.cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b804c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "train['label'] = train.emotion.apply(label_to_int)\n",
    "dev['label'] = dev.emotion.apply(label_to_int)\n",
    "test['label'] = 1\n",
    "train.drop(columns=['emotion'],inplace=True)\n",
    "dev.drop(columns=['emotion'], inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "330f9eb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in [train,test,dev]:\n",
    "    result = sentiment_task(dataset.tweet.tolist())\n",
    "    sentiment = []\n",
    "    for r in result:\n",
    "        sentiment.append(r['label'])\n",
    "    dataset['sentiment'] = sentiment\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "507135d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.get_dummies(train, columns = ['sentiment'])\n",
    "dev = pd.get_dummies(dev, columns = ['sentiment'])\n",
    "test = pd.get_dummies(test, columns = ['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dcd1408",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "743291ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gsitk.preprocess import pprocess_twitter, Preprocessor\n",
    "\n",
    "train['tweet'] = Preprocessor(pprocess_twitter).transform(train.tweet)\n",
    "dev['tweet'] = Preprocessor(pprocess_twitter).transform(dev.tweet)\n",
    "test['tweet'] = Preprocessor(pprocess_twitter).transform(test.tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1a7b141",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11fc01f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20,20))\n",
    "sns.heatmap(train.corr(), annot=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bba0a70",
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv('preprocessed_train.csv', index=False)\n",
    "test.to_csv('preprocessed_test.csv', index=False)\n",
    "dev.to_csv('preprocessed_dev.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccc11550",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57406e7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('preprocessed_train.csv')\n",
    "test = pd.read_csv('preprocessed_test.csv')\n",
    "dev = pd.read_csv('preprocessed_dev.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d71a3e65",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6b0b592",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creates a dataset which will be used to feed to RoBERTa\n",
    "class EmotionDataset(data.Dataset):\n",
    "\n",
    "  def __init__(self, id, tweet, labelValue, extrafeats, tokenizer, max_len):\n",
    "    self.tweet    = tweet      #First input sequence that will be supplied to RoBERTa\n",
    "    self.id = id\n",
    "    self.extra_feats   = extrafeats     #Extra features\n",
    "    self.labelValue  = labelValue    #label value for each training example in the dataset\n",
    "    self.tokenizer   = tokenizer     #tokenizer that will be used to tokenize input sequences (Uses BERT-tokenizer here)\n",
    "    self.max_len     = max_len       #Maximum length of the tokens from the input sequence that BERT needs to attend to\n",
    "\n",
    "  def __len__(self):\n",
    "    return len(self.labelValue)\n",
    "\n",
    "  def __getitem__(self, item):\n",
    "    tweet    = str(self.tweet[item])\n",
    "\n",
    "    \n",
    "    #Encoding the first and the second sequence to a form accepted by RoBERTa\n",
    "    #RoBERTa does not use token_type_ids to distinguish the first sequence from the second sequnece.\n",
    "    encoding = tokenizer.encode_plus(\n",
    "        tweet,\n",
    "        max_length = self.max_len,\n",
    "        add_special_tokens= True,\n",
    "        truncation = True,\n",
    "        padding = 'max_length',\n",
    "        return_attention_mask = True,\n",
    "        return_tensors = 'pt'\n",
    "    )\n",
    "\n",
    "    return {\n",
    "        'tweet' : tweet,\n",
    "        'tweet_id': self.id[item],\n",
    "        'input_ids': encoding['input_ids'].flatten(),\n",
    "        'attention_mask': encoding['attention_mask'].flatten(),\n",
    "        'labels'  : torch.tensor(self.labelValue[item], dtype=torch.long),\n",
    "        'extra_features' : torch.tensor(self.extra_feats[item]).float()\n",
    "    }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36f42a49",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creates a data loader\n",
    "def createDataLoader(dataframe, tokenizer, max_len, batch_size):\n",
    "  ds = EmotionDataset(\n",
    "      tweet    = dataframe.tweet.to_numpy(),\n",
    "      labelValue  = dataframe.label.to_numpy(),\n",
    "      extrafeats = dataframe.drop(columns=['tweet','label','id']).to_numpy(),\n",
    "      tokenizer   = tokenizer,\n",
    "      max_len     = max_len,\n",
    "      id = dataframe.id.to_numpy()\n",
    "  )\n",
    "\n",
    "  return data.DataLoader(\n",
    "      ds,\n",
    "      batch_size  = batch_size,\n",
    "      shuffle     = False,\n",
    "      num_workers = 4\n",
    "  )\n",
    "\n",
    "#Creating data loader for test data\n",
    "trainDataLoader         = createDataLoader(train, tokenizer, MAX_LENGTH, BATCH_SIZE)\n",
    "\n",
    "#Creating data loader for test data\n",
    "devDataLoader         = createDataLoader(dev, tokenizer, MAX_LENGTH, BATCH_SIZE)\n",
    "\n",
    "#Creating data loader for test data\n",
    "testDataLoader         = createDataLoader(test, tokenizer, MAX_LENGTH, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9121ddf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Creating data loader for training data\n",
    "trainDataset        = EmotionDataset(train.id.to_numpy(), train.tweet.to_numpy(), train.label.to_numpy(),\n",
    "                                     train.drop(columns=['tweet','label','id']).to_numpy(), tokenizer, MAX_LENGTH)\n",
    "\n",
    "#Creating data loader for development data\n",
    "developmentDataset  = EmotionDataset(dev.id.to_numpy(),dev.tweet.to_numpy(), dev.label.to_numpy(),\n",
    "                                     dev.drop(columns=['tweet','label','id']).to_numpy(), tokenizer, MAX_LENGTH)\n",
    "\n",
    "#Creating data loader for test data\n",
    "testDataset         = EmotionDataset(test.id.to_numpy(),test.tweet.to_numpy(), test.label.to_numpy(),\n",
    "                                     test.drop(columns=['tweet','label','id']).to_numpy(), tokenizer, MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac8fec0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer\n",
    "class MultilabelTrainer(Trainer):\n",
    "    def compute_loss(self, model, inputs, return_outputs=False):\n",
    "        labels = inputs.pop(\"labels\")\n",
    "        outputs = model(**inputs)\n",
    "        logits = outputs.logits\n",
    "        loss_fct = torch.nn.CrossEntropyLoss()\n",
    "        loss = loss_fct(logits,labels)\n",
    "        return (loss, outputs) if return_outputs else loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad6c299a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='weighted',zero_division=0)\n",
    "    acc = accuracy_score(labels, preds)\n",
    "    return {\n",
    "        'accuracy': acc,\n",
    "        'f1': f1,\n",
    "        'precision': precision,\n",
    "        'recall': recall\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78cc1c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    output_dir='./resultsEmotion-extrafeats',                   # output directory\n",
    "    num_train_epochs=EPOCHS,                  # total number of training epochs\n",
    "    per_device_train_batch_size=BATCH_SIZE,   # batch size per device during training\n",
    "    per_device_eval_batch_size=BATCH_SIZE,    # batch size for evaluation\n",
    "    warmup_steps=100,                         # number of warmup steps for learning rate scheduler\n",
    "    weight_decay=0.01,                        # strength of weight decay\n",
    "    logging_dir='./logsEmotion-extrafeats',                     # directory for storing logs\n",
    "    logging_steps=10,                         # when to print log\n",
    "    load_best_model_at_end=True,\n",
    "    evaluation_strategy = 'epoch'\n",
    ")\n",
    "\n",
    "num_labels = len(set(train.label.tolist()))\n",
    "print(f'Num labels: {num_labels}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84f7f9fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Based on HuggingFace implementation: https://huggingface.co/transformers/_modules/transformers/models/roberta/modeling_roberta.html#RobertaForSequenceClassification\n",
    "\n",
    "class XLMRobertaForSequenceClassificationCustom(XLMRobertaForSequenceClassification):\n",
    "    def __init__(self, config):\n",
    "        super(XLMRobertaForSequenceClassification,self).__init__(config)\n",
    "        self.num_labels = config.num_labels\n",
    "        self.config = config\n",
    "\n",
    "        self.roberta = RobertaModel(config, add_pooling_layer=False)\n",
    "        self.classifier = RobertaClassificationHead(config)\n",
    "\n",
    "        self.init_weights()\n",
    "\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        labels=None,\n",
    "        output_attentions=None,\n",
    "        output_hidden_states=None,\n",
    "        return_dict=None,\n",
    "        extra_features=None\n",
    "    ):\n",
    "        r\"\"\"\n",
    "        labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):\n",
    "            Labels for computing the sequence classification/regression loss. Indices should be in :obj:`[0, ...,\n",
    "            config.num_labels - 1]`. If :obj:`config.num_labels == 1` a regression loss is computed (Mean-Square loss),\n",
    "            If :obj:`config.num_labels > 1` a classification loss is computed (Cross-Entropy).\n",
    "        \"\"\"\n",
    "        return_dict = return_dict if return_dict is not None else self.config.use_return_dict\n",
    "\n",
    "        outputs = self.roberta(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "            position_ids=position_ids,\n",
    "            head_mask=head_mask,\n",
    "            inputs_embeds=inputs_embeds,\n",
    "            output_attentions=output_attentions,\n",
    "            output_hidden_states=output_hidden_states,\n",
    "            return_dict=return_dict,\n",
    "        )\n",
    "        sequence_output = outputs[0]\n",
    "        logits = self.classifier(sequence_output, extra_features)\n",
    "\n",
    "        loss = None\n",
    "        if labels is not None:\n",
    "            if self.config.problem_type is None:\n",
    "                if self.num_labels == 1:\n",
    "                    self.config.problem_type = \"regression\"\n",
    "                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):\n",
    "                    self.config.problem_type = \"single_label_classification\"\n",
    "                else:\n",
    "                    self.config.problem_type = \"multi_label_classification\"\n",
    "\n",
    "            if self.config.problem_type == \"regression\":\n",
    "                loss_fct = MSELoss()\n",
    "                loss = loss_fct(logits.view(-1, self.num_labels), labels)\n",
    "            elif self.config.problem_type == \"single_label_classification\":\n",
    "                loss_fct = CrossEntropyLoss()\n",
    "                loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1))\n",
    "            elif self.config.problem_type == \"multi_label_classification\":\n",
    "                loss_fct = BCEWithLogitsLoss()\n",
    "                loss = loss_fct(logits, labels)\n",
    "\n",
    "        if not return_dict:\n",
    "            output = (logits,) + outputs[2:]\n",
    "            return ((loss,) + output) if loss is not None else output\n",
    "\n",
    "        return SequenceClassifierOutput(\n",
    "            loss=loss,\n",
    "            logits=logits,\n",
    "            hidden_states=outputs.hidden_states,\n",
    "            attentions=outputs.attentions,\n",
    "        )\n",
    "    \n",
    "class RobertaClassificationHead(nn.Module):\n",
    "    \"\"\"Head for sentence-level classification tasks.\"\"\"\n",
    "\n",
    "    def __init__(self, config):\n",
    "        super().__init__()\n",
    "        self.combined_feats = config.hidden_size + train.drop(columns=['tweet','label','id']).shape[1]\n",
    "        self.dense = nn.Linear(self.combined_feats, self.combined_feats)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        self.out_proj = nn.Linear(self.combined_feats, config.num_labels)\n",
    "\n",
    "    def forward(self, features, extra_features, **kwargs):\n",
    "        x = features[:, 0, :]  # take <s> token (equiv. to [CLS])\n",
    "        x = self.dropout(x)\n",
    "        x = torch.cat((x, extra_features) , dim=1)\n",
    "        x = self.dense(x)\n",
    "        x = torch.tanh(x)\n",
    "        x = self.dropout(x)\n",
    "        x = self.out_proj(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ce26ed2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = XLMRobertaForSequenceClassificationCustom.from_pretrained(MODEL, num_labels=num_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de96373",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "trainer = MultilabelTrainer(\n",
    "    model=model,                              # the instantiated 🤗 Transformers model to be trained\n",
    "    args=training_args,                       # training arguments, defined above\n",
    "    train_dataset=trainDataset,              # training dataset\n",
    "    eval_dataset=developmentDataset,                 # evaluation dataset\n",
    "    compute_metrics=compute_metrics \n",
    ")\n",
    "\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4707c97f",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.save_model(\"./resultsEmotion-extrafeats/best_model\") # save best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d889e3b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba670c00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Las métricas son bastante malas porque el test dataset tiene todas las labels a 1\n",
    "#para que entre en el trainer sin tener que hacer clases especiales\n",
    "pred = trainer.predict(testDataset)\n",
    "pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8628e13e",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = np.argmax(pred.predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "505450ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pandas as pd\n",
    "df = pd.DataFrame(testDataset[:]['tweet_id'], columns=['id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a562f85",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['emotion'] = result\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b110cc12",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Converting labels to numbers\n",
    "def int_to_label(label):\n",
    "  if label   == 0:\n",
    "    return 'sadness'\n",
    "  elif label == 1:\n",
    "    return 'joy'\n",
    "  elif label == 2:\n",
    "    return 'anger'\n",
    "  elif label == 3:\n",
    "    return 'surprise'\n",
    "  elif label == 4:\n",
    "    return 'disgust'\n",
    "  elif label == 5:\n",
    "    return 'fear'\n",
    "  elif label == 6:\n",
    "    return 'others'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db2540e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['emotion'] = df.emotion.apply(int_to_label)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d0444fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.emotion.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ed28727",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('submission-extra.tsv', header =False, sep = '\\t',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46eb5072",
   "metadata": {},
   "source": [
    "Probando el modelo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e70874e",
   "metadata": {},
   "outputs": [],
   "source": [
    "m = XLMRobertaForSequenceClassificationCustom.from_pretrained(\"./resultsEmotion-extrafeats/best_model\", num_labels=num_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b1cc65c",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\")\n",
    "m = m.to(device)\n",
    "#This function gets the predictions from the model after it is trained.\n",
    "def get_predictions(model, data_loader):\n",
    "\n",
    "  model = model.eval()\n",
    "\n",
    "  predictions = []\n",
    "  prediction_probs = []\n",
    "  real_values = []\n",
    "  ids = []\n",
    "\n",
    "  with torch.no_grad():\n",
    "    for d in tqdm(data_loader):\n",
    "\n",
    "\n",
    "      input_ids              = d[\"input_ids\"].to(device)\n",
    "      attention_mask         = d[\"attention_mask\"].to(device)\n",
    "      labels                 = d[\"labels\"].to(device)\n",
    "\n",
    "      #Getting the softmax output from model\n",
    "      outputs = model(\n",
    "        input_ids             = input_ids,\n",
    "        attention_mask        = attention_mask\n",
    "      )\n",
    "\n",
    "      _, preds = torch.max(outputs.logits, dim=1)     #Determining the model predictions\n",
    "\n",
    "\n",
    "      predictions.extend(preds)\n",
    "      prediction_probs.extend(outputs.logits)\n",
    "      real_values.extend(labels)\n",
    "      ids.extend(d['tweet_id'])\n",
    "\n",
    "  predictions = torch.stack(predictions).cpu()\n",
    "  prediction_probs = torch.stack(prediction_probs).cpu()\n",
    "  real_values = torch.stack(real_values).cpu()\n",
    "  \n",
    "  return ids, predictions, prediction_probs, real_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01fca8e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Getting model predictions on dev dataset\n",
    "ids_dev, yHat_dev, predProbs_dev, yTest_dev = get_predictions(\n",
    "  m,\n",
    "  devDataLoader\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37ae59d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "  #Printing classification report for dev dataset (Evaluating the model on Dev set)\n",
    "print(classification_report(yTest_dev, yHat_dev, target_names= CLASS_NAMES))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "049b4596",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Getting model predictions on dev dataset\n",
    "ids_test, yHat_test, predProbs_test, yTest_test = get_predictions(\n",
    "  model,\n",
    "  testDataLoader\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10cca65e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_preds = pd.DataFrame({'id': ids_dev, 'emotion': pd.Series(yHat_dev).apply(int_to_label)})\n",
    "dev_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67cfb4cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = pd.DataFrame({'id': ids_test, 'emotion': pd.Series(yHat_test).apply(int_to_label)})\n",
    "test_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef1424b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "def save_preds(obj, name, fold):\n",
    "    path = 'preds_{}/{}.pck'.format(fold, name)\n",
    "    with open(path, 'wb') as f:\n",
    "        pickle.dump(obj, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94cd76dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save_preds(dev_preds, 'xlmroberta', 'dev')\n",
    "save_preds(df, 'xlmroberta-extrafeatures', 'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab53c3e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds.to_csv('submission-roberta-extra-final.tsv',header =False, sep = '\\t',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "transformers"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}