Fake Claim Classification using BERT-base with CNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Final code with comments W10 P5 txt -> BB+emotion+SPC+SEN -> CNN  GITHUB",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qow2csNoRXmH"
      },
      "source": [
        "This is the code for the paper: https://arxiv.org/abs/2009.01047 \n",
        "\n",
        "Sentimental LIAR: Extended Corpus and Deep Learning Models for Fake Claim Classification\n",
        "\n",
        "The code is for the best performing model where the TEXT is passed into BERT-base and the output of the BERT-base is concatenated with EMO, SPC and SEN and then passed to the CNN. The model achieved the accuracy of 70% and F1 Macro Score of 0.6430\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IN-a_xrximUl"
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Up35yUrQftzU",
        "outputId": "55c28728-d186-4c56-e7db-2ba87adc2786",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "import pandas as pd\n",
        "df=pd.read_csv(\"/Users/reshmamanoj/Desktop/SentimentalLIAR-master 2/train_final-02.csv\")\n",
        "df_test=pd.read_csv(\"/Users/reshmamanoj/Desktop/SentimentalLIAR-master 2/test_final.csv\")\n",
        "df_valid=pd.read_csv(\"/Users/reshmamanoj/Desktop/SentimentalLIAR-master 2/valid_final.csv\")\n",
        "print(\"before truncating size of data is :\", df.shape, df_test.shape,df_valid.shape)\n",
        "df=df[:10232]\n",
        "df_test=df_test[:1264]\n",
        "df_valid=df_valid[:1280]\n",
        "print(\"size of data is :\", df.shape, df_test.shape, df_valid.shape)\n",
        "\n",
        "\n",
        "#check if any null values are present\n",
        "print(\"Any null in Subject? \",df['subject'].isnull().values.any())\n",
        "print(\"Any null in Speaker? \",df['speaker'].isnull().values.any())\n",
        "print(\"Any null in speaker_job? \",df['speaker_job'].isnull().values.any())\n",
        "print(\"Any null in Party? \",df['party_affiliation'].isnull().values.any())\n",
        "print(\"Any null in Context? \",df['context'].isnull().values.any())\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "before truncating size of data is : (10236, 31) (1267, 30) (1283, 31)\n",
            "size of data is : (10232, 31) (1264, 30) (1280, 31)\n",
            "Any null in Subject?  False\n",
            "Any null in Speaker?  False\n",
            "Any null in speaker_job?  True\n",
            "Any null in Party?  False\n",
            "Any null in Context?  True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XbkLPaPJNdwZ"
      },
      "source": [
        "df['comment_text']=\"\"\n",
        "df_test['comment_text']=\"\"\n",
        "df_valid[\"comment_text\"]=\"\"\n",
        "\n",
        "#combine all the columns into one for train, test and valid data\n",
        "\n",
        "df['comment_text']=df['subject'].astype(str)+ \". \"+ df[\"statement\"]+\\\n",
        "df['speaker_id'].astype(str)+\". \"+df['speaker_job'].astype(str)+\". \"+ df['party_affiliation'].astype(str)\\\n",
        "+\". \"+df['context'].astype(str)+\". \"+df['sentiment_code'].astype(str)\n",
        "\n",
        "\n",
        "df_test['comment_text']=df_test['subject'].astype(str)+ \". \"+ df_test[\"statement\"]+\". \"+\\\n",
        "df_test['speaker_id'].astype(str)+\". \"+df_test['speaker_job'].astype(str)+\". \"+ df_test['party_affiliation'].astype(str)\\\n",
        "+\". \"+df_test['context'].astype(str)+\". \"+df['sentiment_code'].astype(str)\n",
        "\n",
        "df_valid['comment_text']=df_valid['subject'].astype(str)+ \". \"+ df_valid[\"statement\"]+\\\n",
        "df_valid['speaker_id'].astype(str)+\". \"+df_valid['speaker_job'].astype(str)+\". \"+ df_valid['party_affiliation'].astype(str)\\\n",
        "+\". \"+df_valid['context'].astype(str)+\". \"+df_valid['sentiment_code'].astype(str)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eZ5OeqXDPF0C"
      },
      "source": [
        "#concatenate emotion, speakers' credit and sentiment score togehter\n",
        "df['emotion']=\"[\"+df['anger'].astype(str)+\",\"+df['disgust'].astype(str)+\",\"\\\n",
        "+df['fear'].astype(str)+\",\"+df['joy'].astype(str)+\",\"+df['sad'].astype(str)+\",\"+\\\n",
        "df[\"barely_true_counts\"].astype(str) +\",\"+ df[\"false_counts\"].astype(str)  +\",\"+\\\n",
        "df[\"half_true_counts\"].astype(str) + \",\"+df[\"mostly_true_counts\"].astype(str) +\",\"+ \\\n",
        "df[\"pants_on_fire_counts\"].astype(str)+\",\"+df[\"sentiment_score\"].astype(str)+\"]\"\n",
        "\n",
        "\n",
        "df_test['emotion']=\"[\"+df_test['anger'].astype(str)+\",\"+df_test['disgust'].astype(str)+\",\"\\\n",
        "+df_test['fear'].astype(str)+\",\"+df_test['joy'].astype(str)+\",\"+df_test['sad'].astype(str)\\\n",
        "+ \",\"+df_test[\"barely_true_counts\"].astype(str) + \",\"+ df_test[\"false_counts\"].astype(str) \\\n",
        "+\",\"+ df_test[\"half_true_counts\"].astype(str) +\",\"+ df_test[\"mostly_true_counts\"].astype(str)\\\n",
        "+\",\"+ df_test[\"pants_on_fire_counts\"].astype(str)+\",\"+df_test[\"sentiment_score\"].astype(str)+\"]\"\n",
        "\n",
        "\n",
        "df_valid['emotion']=\"[\"+df_valid['anger'].astype(str)+\",\"+df_valid['disgust'].astype(str)+\",\"\\\n",
        "+df_valid['fear'].astype(str)+\",\"+df_valid['joy'].astype(str)+\",\"+df_valid['sad'].astype(str)\\\n",
        "+ \",\"+df_valid[\"barely_true_counts\"].astype(str) + \",\"+ df_valid[\"false_counts\"].astype(str) \\\n",
        "+\",\"+ df_valid[\"half_true_counts\"].astype(str) +\",\"+ df_valid[\"mostly_true_counts\"].astype(str)\\\n",
        "+\",\"+ df_valid[\"pants_on_fire_counts\"].astype(str)+\",\"+df_valid[\"sentiment_score\"].astype(str)+\"]\"\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ahybcBEujX-m"
      },
      "source": [
        "#target should be converted from string to the list\n",
        "import ast\n",
        "def convert_to_list(text):\n",
        "  return ast.literal_eval(text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nUfBrHd2jZ1z",
        "outputId": "ecd053c9-77d7-44be-c881-c7da9be3dcf3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "for i in range(len(df[\"emotion\"])):\n",
        "  try:\n",
        "    df[\"emotion\"][i]=convert_to_list(df[\"emotion\"][i])\n",
        "  except:\n",
        "    print(i,\"====\",df[\"emotion\"][1])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9T-5ig6Pl-1G",
        "outputId": "0478de86-49ea-4bfe-8b8a-7ff82b1c83c3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "for i in range(len(df_test[\"emotion\"])):\n",
        "  try:\n",
        "    df_test[\"emotion\"][i]=convert_to_list(df_test[\"emotion\"][i])\n",
        "  except:\n",
        "    print(i,\"====\",df_test[\"emotion\"][i], type(df_test[\"emotion\"][i]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsftiolVlznH",
        "outputId": "6b3cd5b7-226a-414e-c204-908a04892305",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "for i in range(len(df_valid[\"emotion\"])):\n",
        "  try:\n",
        "    df_valid[\"emotion\"][i]=convert_to_list(df_valid[\"emotion\"][i])\n",
        "  except:\n",
        "    print(i,\"====\",df_valid[\"emotion\"][i], type(df_valid[\"emotion\"][i]))    "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QPBghNgoj_em"
      },
      "source": [
        "df['list']=df['list'].apply(convert_to_list)\n",
        "df_test['list']=df_test['list'].apply(convert_to_list)\n",
        "df_valid['list']=df_valid['list'].apply(convert_to_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E-qrs8fBNgs-"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn import metrics\n",
        "import transformers\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertTokenizer, BertModel, BertConfig\n",
        "\n",
        "from torch import cuda\n",
        "device = 'cuda' if cuda.is_available() else 'cpu'\n",
        "\n",
        "MAX_LEN = 300\n",
        "TRAIN_BATCH_SIZE = 8\n",
        "VALID_BATCH_SIZE = 8\n",
        "EPOCHS = 1\n",
        "LEARNING_RATE = 1e-05\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hiGu91VfNixm"
      },
      "source": [
        "class CustomDataset(Dataset):\n",
        "\n",
        "    def __init__(self, dataframe, tokenizer, max_len):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.data = dataframe\n",
        "        self.comment_text = dataframe.comment_text\n",
        "        self.targets = self.data.list\n",
        "        self.max_len = max_len\n",
        "        #Add emotion list from dataframe\n",
        "        self.emotion=dataframe.emotion\n",
        "        self.dfID=dataframe.ID\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.comment_text)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        comment_text = str(self.comment_text[index])\n",
        "        comment_text = \" \".join(comment_text.split())\n",
        "\n",
        "        inputs = self.tokenizer.encode_plus(\n",
        "            comment_text,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=self.max_len,\n",
        "            pad_to_max_length=True,\n",
        "            return_token_type_ids=True,\n",
        "            truncation=True\n",
        "        )\n",
        "        ids = inputs['input_ids']\n",
        "        mask = inputs['attention_mask']\n",
        "        token_type_ids = inputs[\"token_type_ids\"]\n",
        "\n",
        "\n",
        "        return {\n",
        "            'ids': torch.tensor(ids, dtype=torch.long),\n",
        "            'mask': torch.tensor(mask, dtype=torch.long),\n",
        "            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n",
        "            'targets': torch.tensor(self.targets[index], dtype=torch.float),\n",
        "            'emotion':torch.tensor(self.emotion[index], dtype=torch.float),\n",
        "            'dfID':self.dfID[index]\n",
        "        }\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BO904jDXNZaw",
        "outputId": "5aacba1b-8efe-4a58-e0f9-27a1765aa6b8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "# Creating the dataset and dataloader for the neural network\n",
        "\n",
        "train_size = 1\n",
        "train_dataset=df.sample(frac=train_size,random_state=200).reset_index(drop=True)\n",
        "test_dataset=df_test.sample(frac=train_size,random_state=200).reset_index(drop=True)\n",
        "valid_dataset=df_valid.sample(frac=1,random_state=200).reset_index(drop=True)\n",
        "\n",
        "\n",
        "print(\"FULL Dataset: {}\".format(df.shape))\n",
        "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
        "print(\"TEST Dataset: {}\".format(test_dataset.shape))\n",
        "print(\"VALID Dataset: {}\".format(valid_dataset.shape))\n",
        "\n",
        "training_set = CustomDataset(train_dataset, tokenizer, MAX_LEN)\n",
        "testing_set = CustomDataset(test_dataset, tokenizer, MAX_LEN)\n",
        "valid_set= CustomDataset(valid_dataset, tokenizer, MAX_LEN)\n",
        "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "test_params = {'batch_size': 8,\n",
        "                'shuffle': True,\n",
        "                'num_workers': 0\n",
        "                }\n",
        "\n",
        "training_loader = DataLoader(training_set, **train_params)\n",
        "testing_loader = DataLoader(testing_set, **test_params)\n",
        "valid_loader=DataLoader(valid_set,**test_params)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "FULL Dataset: (10232, 33)\n",
            "TRAIN Dataset: (10232, 33)\n",
            "TEST Dataset: (1264, 32)\n",
            "VALID Dataset: (1280, 33)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_1Lye08is-8"
      },
      "source": [
        "# Creating the customized model, by adding a drop out and a dense layer on top of BERT-base to get the final output for the model. \n",
        "from torch import nn\n",
        "\n",
        "class BERT_cnn_Class(torch.nn.Module):\n",
        "    def __init__(self):\n",
        "        super(BERT_cnn_Class, self).__init__()\n",
        "        self.l1 = transformers.BertModel.from_pretrained('bert-base-uncased')\n",
        "        self.l2 = torch.nn.Dropout(0.3)\n",
        "        \n",
        "        self.l3=torch.nn.Conv1d(1, 50, kernel_size=20,stride=1)\n",
        "        self.l4=torch.nn.Conv1d(50,100, kernel_size=20, stride=1)\n",
        "        self.max_pooling=nn.MaxPool1d(2)\n",
        "        self.l5=torch.nn.Linear(18000, 768)\n",
        "        self.l6 = torch.nn.Linear(768, 2)\n",
        "    \n",
        "    def forward(self, ids, mask, token_type_ids,emotion):\n",
        "        _, output_1= self.l1(ids, attention_mask = mask, token_type_ids = token_type_ids)\n",
        "        output_2 = self.l2(output_1) #output from bert\n",
        "        #feed into conv net\n",
        "        #first change the size to [8,1,768]        \n",
        "        output_2=torch.cat((emotion,output_2),1) # concat the output of BERT with EMO+SPC+SEN\n",
        "        output_2=output_2.unsqueeze(1)        \n",
        "        output_3=self.l3(output_2)        \n",
        "        output_3= self.max_pooling(output_3)\n",
        "        output_4=self.l4(output_3)        \n",
        "        output_4=self.max_pooling(output_4)\n",
        "        #change the shape to fit into linear function\n",
        "        output_4=output_4.view(8,-1)\n",
        "        output_5=self.l5(output_4)\n",
        "        output_6=self.l6(output_5)        \n",
        "        return output_6\n",
        "\n",
        "model = BERT_cnn_Class()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChpswINqYK9e"
      },
      "source": [
        "model.to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IgLoP_Wlivpl"
      },
      "source": [
        "import time\n",
        "import datetime\n",
        "def format_time(elapsed):\n",
        "    elapsed_rounded = int(round((elapsed)))    \n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NHTHyN8ljQx-"
      },
      "source": [
        "def loss_fn(outputs, targets):\n",
        "    return torch.nn.BCEWithLogitsLoss()(outputs, targets)\n",
        "optimizer = torch.optim.Adam(params =  model.parameters(), lr=LEARNING_RATE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iWIiFvmuixkc",
        "outputId": "a227d940-f729-40a4-871f-a9062206764c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        }
      },
      "source": [
        "#TRAIN THE MODEL\n",
        "val_losses=[]\n",
        "train_losses=[]\n",
        "accuracy_list=[]\n",
        "\n",
        "for epoch in range(2):\n",
        "  t0 = time.time()\n",
        "  model.train()\n",
        "  print(f\"\\t Epoch: {epoch}  is Started: \")\n",
        "  batch=0\n",
        "  train_loss=0\n",
        "  \n",
        "  for _,data in enumerate(training_loader, 0):\n",
        "      try:\n",
        "          ids = data['ids'].to(device, dtype = torch.long)\n",
        "          mask = data['mask'].to(device, dtype = torch.long)\n",
        "          token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "          targets = data['targets'].to(device, dtype = torch.float)\n",
        "          emotions=data['emotion'].to(device,dtype=torch.float)\n",
        "      except:\n",
        "          print(f\"some error at testing {batch}\")\n",
        "          print(data['dfID'] )\n",
        "      try:  \n",
        "        outputs = model(ids, mask, token_type_ids,emotions)\n",
        "        optimizer.zero_grad()\n",
        "        loss = loss_fn(outputs, targets)\n",
        "        train_loss+=loss.item()\n",
        "        #print(f'{count} Loss:  {loss.item()}')\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        batch+=1\n",
        "      except EOFError:\n",
        "        print(f\"{data['dfID']} error because of batch size-------->\", EOFError)\n",
        "        print(f\"some error at testing {batch}\")\n",
        "        print(data['dfID'] )\n",
        "  print(f\"   Epoch: {epoch} Train loss is :{train_loss/batch}\") \n",
        "  train_loss /=batch\n",
        "  train_losses.append(train_loss)    \n",
        "  print(f\"   Epoch {epoch} took: {format_time(time.time() - t0)} \\n\")\n",
        "\n",
        "  model.eval()\n",
        "  fin_targets=[]\n",
        "  fin_outputs=[]\n",
        "  with torch.no_grad():\n",
        "      val_loss, batch = 0, 1\n",
        "      for _, data in enumerate(testing_loader, 0):\n",
        "          ids = data['ids'].to(device, dtype = torch.long)\n",
        "          mask = data['mask'].to(device, dtype = torch.long)\n",
        "          token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "          targets = data['targets'].to(device, dtype = torch.float)\n",
        "          emotions=data['emotion'].to(device,dtype=torch.float)\n",
        "          batch+=1\n",
        "          try:\n",
        "                        outputs = model(ids, mask, token_type_ids,emotions)\n",
        "                        loss = loss_fn(outputs, targets)\n",
        "                        val_loss+=loss.item()\n",
        "                        fin_targets.extend(targets.cpu().detach().numpy().tolist())\n",
        "                        fin_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
        "          except:\n",
        "            print(f\"some error at testing {batch}\")\n",
        "            print(data['dfID'] )\n",
        "\n",
        "      val_loss/=batch\n",
        "      val_losses.append(val_loss)\n",
        "  outputs=fin_outputs\n",
        "  outputs = np.array(outputs) >= 0.5\n",
        "  targets=fin_targets\n",
        "  accuracy = metrics.accuracy_score(targets, outputs)\n",
        "  accuracy_list.append(accuracy)\n",
        "  f1_score_micro = metrics.f1_score(targets, outputs, average='micro')\n",
        "  f1_score_macro = metrics.f1_score(targets, outputs, average='macro')\n",
        "  print(f\"Epoch: {epoch} - Accuracy on Testing Data Score = {accuracy}\")\n",
        "  print(f\"Epoch: {epoch} - F1 Score on Testing Data (Micro) = {f1_score_micro}\")\n",
        "  print(f\"Epoch: {epoch} - F1 Score on Testing Data (Macro) = {f1_score_macro}\")\n",
        "  print(f\"\\n \\t Epoch {epoch} : Train Loss (Training Data):{train_loss}, Validation Loss (Testing Data): {val_loss}\")\n",
        "  print(\"_________________________________________________\\n\")\n",
        "  #if train_loss > val_loss:\n",
        "  # torch.save(model.state_dict(), \"//Users/reshmamanoj/Desktop/SentimentalLIAR-master 2/models_saved/w9p7\")\n",
        "  "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\t Epoch: 0  is Started: \n",
            "   Epoch: 0 Train loss is :0.6337657139998474\n",
            "   Epoch 0 took: 0:06:17 \n",
            "\n",
            "Epoch: 0 - Accuracy on Testing Data Score = 0.6693037974683544\n",
            "Epoch: 0 - F1 Score on Testing Data (Micro) = 0.6708860759493671\n",
            "Epoch: 0 - F1 Score on Testing Data (Macro) = 0.6011597099211216\n",
            "\n",
            " \t Epoch 0 : Train Loss (Training Data):0.6337657139998474, Validation Loss (Testing Data): 0.5962660192693554\n",
            "_________________________________________________\n",
            "\n",
            "\t Epoch: 1  is Started: \n",
            "   Epoch: 1 Train loss is :0.5931087791849804\n",
            "   Epoch 1 took: 0:06:16 \n",
            "\n",
            "Epoch: 1 - Accuracy on Testing Data Score = 0.6914556962025317\n",
            "Epoch: 1 - F1 Score on Testing Data (Micro) = 0.6958579881656805\n",
            "Epoch: 1 - F1 Score on Testing Data (Macro) = 0.6538011695906432\n",
            "\n",
            " \t Epoch 1 : Train Loss (Training Data):0.5931087791849804, Validation Loss (Testing Data): 0.5924532325387751\n",
            "_________________________________________________\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kz_2TpwBYJbm"
      },
      "source": [
        "from pandas import DataFrame\n",
        "df=DataFrame(train_losses,columns=['train_losses'])\n",
        "df=DataFrame(val_losses,columns=['val_losses'])\n",
        "df.to_csv(\"/Users/reshmamanoj/Desktop/SentimentalLIAR-master 2/Results/w10_p4.csv\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRL6AyaF4p8I",
        "outputId": "05b1006c-f8d2-4802-d3fd-3a02983afd6a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        }
      },
      "source": [
        "\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(train_losses, label=\"Training loss\")\n",
        "plt.plot(val_losses, label=\"Validation loss\")\n",
        "\n",
        "plt.xlabel('Epoch', fontsize=18)\n",
        "plt.ylabel('Losses', fontsize=16)\n",
        "plt.title('Training Loss VS Validation Loss', fontsize=15)\n",
        "\n",
        "plt.legend()\n",
        "plt.savefig('/Users/reshmamanoj/Desktop/SentimentalLIAR-master 2/Results/w10_p4-epoch1.eps')\n",
        "\n",
        "#plt.title(\"Losses\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n",
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEgCAYAAABIJS/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU5fbA8e9JBxJqAOlFCDVAIPQSUDoIUkSwIl7sUuJPxQq2q1cREUW99nJFwIYgIE1NKIIEpIbeJICU0EtIAu/vj5nguqRsks1uyvk8zz7ZmXln5szuZs++ZWbEGINSSinlCh9vB6CUUqrg0KShlFLKZZo0lFJKuUyThlJKKZdp0lBKKeUyTRpKKaVcpkkjHxMR48Kjcw63XdNev2821+tsr9c4J/vNCRHZKyITPbW/rIhIC/s1GJTB8ooikioij9vTJUTkBRHZJiIXROSwiMSIyN2Z7GOOiGzMZPnbInJSRAJdiPeq98yefiiL9fra5WpmtQ+n9R5L73Ppyj7dydP7Kyr8vB2AylRbh+fFgJ+BF4G5DvPjc7jtQ/b2t2ZzvbX2ertyuN8CzxizRkR2AEOBb9MpchPWD7Lp9vS3QATWe7cJqAB0AnoDH2Wwm6+AL0WkoTHmH++xiPgCg4HvjDEXc3gYbYE9OVw3K48BbwO/enCfykM0aeRjxpiVac9FJNh+ustxviP7y8TXGJPswrYvAuluJ4v1TudkvULoK+BREQk2xpx1WjYU+M0Ys09E6gI9gCHGmK8dyswQEclk+z8A54FhwDNOy7oAFe0YciSjz1Be8sY+lftp81QBJiKfikiciNwoIpuBJKC1iFQSkY9FZLfdHLJdRF4UkQCHda9qnkprBhKRsSKSICInRGS6iJR2KJNRU8doEfm3iBwVkSMiMtW56cRed4OIJInIahFpJSLHRGRCLl8HXxGZICJ/ishFEdksIrc4lWkkIj+JyHEROSciW0TkQYflHURkqYicth/rROSmTHb7FVbtr7/TfqoB7fj7Cz3ttfvLeQMmk8sxGGPOAXOAm9NZPBQ4AvwsIvXt92i/iJy3j32MiGT6v+3cdCOWCfZ7d0ZEPgdKprPeKyKyUUTO2p+RL0XkGofle4FywHhxakJNr7lIRB4SkR32+7ZTRMY6LZ9gf0YiRGSlfYx/iEjHzI7PFSJSXESmiMhfDp/J7k5lMv1ciEg/EVljf6ZOiMgqEYnKbWz5mSaNgq8m8CrwMtALq/ofChwHooGewGvAXcBbLmxvCHA9cA/wONAX+LcL6z0CVAZus/d3LzA6baGIVAHmYX3ZDQb+C3yJ9cWbW88DTwHvA/2A5VhNO8McyswBLtnx9cN6LULs2EoCPwK7gUF2fF/w9xf+VYwxW4F1WF/gjm4GLgNptYptwDlgsoh0F5GgbBzXV0BdEWmRNkNE/IGBwExjzCWgir2PB7Cauz4AnsN677JjFPAs1ms4GLiA9blyVgHr89AHGAPUxkpead8lA4BTWM1ube3H2vR2KCIjsd6H2cANWK/Z6yIyzqloceAzrM/MIOAi8J2IFM/mMTr7AOv/4iU77v3AXBHpYMeX6edCRK4FvsFqNr4BuNUuXzaXceVvxhh9FIAHEAwYYLjDvE/tec2yWNcPuAWrJhJgz6tpr9vXodxerL4KP4d5k4G/HKY72+s1dphngFinfc4CVjpMvwYcA4o5zBtirzshi/j3AhMzWFYW60t5vNP8ecA2+3movZ/wDLYRaS8PyeZ78hjWF1gZh3lxwEKncsOAs/Y+koFYYCQgWWw/ADgBvOYwr6+9nXbplBf7vX4S2O3Ce/aQ/dwXOAi867S9RXa5mhnE54uVtAzQyWH+sfTeU6d9+gAHgE+cyryDlXSC7OkJ9nrXOZRpZs/rmcXrd2V/6SxrgJXc73SY54PV57TAlc8FVhJJzM5npjA8tKZR8B0wxqxznGE3NYwRkXgRuQCkYP2qDwSqZ7G9X4wxqQ7T8UAF+xduZhY6TccDVR2mWwKLjDEXHObNzmKbrmiM9Uv0a6f5M4AwESmPVevaD7wnIjeLSAWnsruwvtSniUh/x+a4LEwH/LF+pab98myBU1+DMeYroAYwwl4nDOsX/bTMNm6svqnvgCEiV/o/bgb2Ab/Z+wwSkedEZCdWAkvB+uVcS0Rc7bOsBlTC6kdx9J1zQRHpJSIrROQUkAok2IvCXNxXmqpYNdP03reSQLjDvGT+2ameNjDA8fOVXS2xkuyV/Rtj0mqIHexZWX0uNgKlROQzuxZZIhfxFBiaNAq+w+nMGwNMBL7HanNvBaS132fVPHLSaToZ658rq6Gd6a3nuK9rgKOOBYwxSVj/lLlRyf7r/DqkTZe1vwy6Y/UrfAz8ZbdTR9hxnAC6YSWAmcBREZkrIrUz27Ex5k9gBX83UQ3FbjpJp2yiMeYTY8wdWF/SnwBDRaRpFsf3FVaib2s3bfUHphv7py7wH+D/sJJQb6wvwxftZa42haX1SRxxmv+PaRFpiZXoE4DbsZqe2mRzX2myfN8c5p2x30PgSjLNyT6d93/WGHM+nf0XF5HArD4XxphtWO9Hbaya7TERmWb/UCm0NGkUfOl1pt4EfGOMecoYs9AYsxqrCceb/gL+8c9kfwkGp1/cZYfsv861h4r23+Ng9UEYYwZhtUd3xfrCmZvWFm+MWWmM6WkvH4j1yznTmoDtK+A6+4tiKDDfGHMqsxWMMSnAG/Zk/Sy2/wvWF9lQrH6EEP5Zk7kJeMsY86oxZrExJg6rBpAdaZ30zq+h8/QArMR/szFmtrFGQ13Vwe8il963PHQICE6nX6QicN7YQ5mz+lwYY+YaYzpidf7fjfXZcqXvsMDSpFE4FcP6xevoVm8E4mA10E1EHDu++7lhu5uwhqY6j3QaAmw3xjjXblKMMT8Dk7B+bZZ2Wn7BGDMHq0bS0IX9pzVvjMdqKvtH05SIhDgdc5q69t/0aoqO8VzC+pV7E1a/1BZjzHqHIv94r8Uadu3cOZ+V/Vhf/v2d5g90mi4GpDjUciD9z5VzLTM9CVj9KOm9b6exmn7y0mqsH1yD02bYTYCDgWXOhbP6XBhjThljpmHV7l353BRYep5G4bQIGCUiq7DaZW8F6ng3JCZjNZHNEZE3sJpExmF94V/ObEVbmIgMdpp3zhgzX0QmA0+LSCpWR/RArKaaYQAi0gSruW4G1kiYMliji9YbY46LSB+s/oZZwJ9Ynbv3Yo2KyZQx5oiILMEavXQWa5SWo3rAbBH5GKsp6zxWR+5TWKOvrvqCSsdXwMNYv/THOy1bBDxo92kcx3qNszxL3OkYLonIq8BEETkGLMUaLdQgnX2NsV/vOVhDi29LZ5NbgT4i8hPWa7LNGHPGaZ+XxRpq/V8RSbS3HQXcDzxpN126Q7N0PjdHjTExIvIV8LaIhGD9n4zEqvndD5DV50JE7sVqovsJKwHWxUqCn7sp9vzJ2z3x+nDtQcajp+IyKPsJ1pfIceBD/h5109guU5P0R09NdNrWcLtcsD3d2XE79ryrRqlgjXo55jSvC7AB65fxOqAj1oiuMVkc+157H86PvfZyX6xhpvuxfuXGA7c6rF8Ba6jkbnt/f2H3FdjL62ENndxvx5YAvIfVH+LKe5P2Gv0vnWVlsIYErwISsZLGVqy+CJe2b29nj72POk7zK2L9uj2NVWt5FevLL1vvGVa/1QtYzU9nsAZO3ILT6CmsEWP7sZo7F2N9UTpvqwXWCaDn7GWdM/mcPAzstN+33cDYrD5HGW0rgzLpPX61lxfHako6bL/vcUAPh/Uz/VxgJYy5WAkjyX6P/gMEevv7Ii8fYh+8Uh5nj4dfijWc8hdvx6OUypomDeUxIvIf4A+sX/r1sC6PkQhEGIfRMUqp/Ev7NJQnBWKd5FcRqwlkIRCtCUOpgkNrGkoppVymQ26VUkq5rFA3T4WGhpqaNWt6OwyllCpQ1qxZc8wYk+6Z7YU6adSsWZO4uDhvh6GUUgWKiOzLaJk2TymllHKZJg2llFIu06ShlFLKZYW6T0Mp5XkpKSkkJCSQlOSuy0epvBIUFETVqlXx98/qdjl/06ShlHKrhIQEQkJCqFmzJn/fO0rlN8YYEhMTSUhIoFatWi6vp81TSim3SkpKoly5cpow8jkRoVy5ctmuEWrSUEq5nSaMgiEn75MmjXQYY/j3vC3sOprbO5EqpVThokkjHXuOnWP673/S682lvP3zDpJT9Xp6ShUUiYmJNGvWjGbNmnHNNddQpUqVK9PJycmZrhsXF8eoUaOy3Ee7du3cEuuvv/5K37593bItT9GO8HTULh/M4keieG52PBMXbufHDYd4ZVATmlUrnfXKSimvKleuHOvWrQNgwoQJBAcH83//939XlqempuLnl/5XX2RkJJGRkVnuY8WKFe4JtgDSmkYGKoQEMfXW5rx/ewtOnk9hwDvLeW7OZs5dTPV2aEqpbBo+fDj33XcfrVu35rHHHuP333+nbdu2RERE0K5dO7Zt2wb885f/hAkTGDFiBJ07d6Z27dpMmTLlyvaCg4OvlO/cuTODBw+mfv363HrrrWl3/WPevHnUr1+fFi1aMGrUqCxrFMePH+fGG2+kSZMmtGnThg0bNgAQExNzpaYUERHBmTNnOHToEJ06daJZs2Y0btyYpUuXuv01y4jWNLLQvdE1tL22HK/+tI1Plu9l4ebDvDigMV3qVfB2aErle8/N2Uz8wdNu3WbDyiUZf0OjbK+XkJDAihUr8PX15fTp0yxduhQ/Pz8WL17Mk08+ybfffnvVOlu3buWXX37hzJkz1KtXj/vvv/+qcxr++OMPNm/eTOXKlWnfvj3Lly8nMjKSe++9l9jYWGrVqsWwYcOyjG/8+PFEREQwa9Ysfv75Z+644w7WrVvHxIkTmTp1Ku3bt+fs2bMEBQXx/vvv06NHD5566ikuXbrE+fPns/165JTWNFwQEuTPCzc25uv72hLk78Ndn6xm9PQ/SDx70duhKaVcdNNNN+Hr6wvAqVOnuOmmm2jcuDFjx45l8+bN6a7Tp08fAgMDCQ0NpUKFChw+fPiqMq1ataJq1ar4+PjQrFkz9u7dy9atW6ldu/aV8x9cSRrLli3j9ttvB+C6664jMTGR06dP0759e6Kjo5kyZQonT57Ez8+Pli1b8sknnzBhwgQ2btxISEhITl+WbNOaRja0rFmWeaM78s4vu3jn153Ebj/KM30bMiCiig4xVCodOakR5JUSJUpcef7MM8/QpUsXvv/+e/bu3Uvnzp3TXScwMPDKc19fX1JTr26edqVMbowbN44+ffowb9482rdvz4IFC+jUqROxsbHMnTuX4cOHEx0dzR133OHW/WZEaxrZFOjny9huYcwd1ZFaoSWInrmeOz7+nf3HPVc9VErlzqlTp6hSpQoAn376qdu3X69ePXbv3s3evXsBmDFjRpbrdOzYkS+//BKw+kpCQ0MpWbIku3btIjw8nMcff5yWLVuydetW9u3bR8WKFRk5ciT/+te/WLt2rduPISOaNHIorGII39zXjuf7N2LtvhN0fyOWD5fuJvWSDs9VKr977LHHeOKJJ4iIiHB7zQCgWLFivPPOO/Ts2ZMWLVoQEhJCqVKlMl1nwoQJrFmzhiZNmjBu3Dg+++wzACZPnkzjxo1p0qQJ/v7+9OrVi19//ZWmTZsSERHBjBkzGD16tNuPISOF+h7hkZGRxhM3YTp48gLPzNrEkq1HCK9SilcGhdOocuYfEKUKqy1bttCgQQNvh+F1Z8+eJTg4GGMMDz74IHXr1mXs2LHeDusq6b1fIrLGGJPu2GOtabhB5dLF+PDOSN6+JYJDpy7Q7+3lvDJ/K0kpl7wdmlLKSz744AOaNWtGo0aNOHXqFPfee6+3Q3ILrWm42cnzyfx73hZmxiVQs1xx/j0gnHZ1Qj0ag1LepDWNgkVrGl5WungArw5uyrR/tcYAt3y4ise+Wc+p8yneDk0ppXJNk0YeaVcnlAVjOnFf1LV8u/YA10+K4ccNBynMNTulVOGnSSMPBfn7Mq5XfX54sD2VSgXx0LQ/GPl5HIdOXfB2aEoplSOaNDygcZVSfP9AO57q3YBlO4/RbVIsn/+2l8uXtdahlCpYPJ40RKSniGwTkZ0iMi6DMkNEJF5ENovINHteDRFZKyLr7Pn3eTby3PHz9WFkp9osHBNFRPXSPPvDZm7672/sOHzG26EpVah06dKFBQsW/GPe5MmTuf/++zNcp3PnzqQNmunduzcnT568qsyECROYOHFipvueNWsW8fHxV6afffZZFi9enJ3w05WfLqHu0aQhIr7AVKAX0BAYJiINncrUBZ4A2htjGgFj7EWHgLbGmGZAa2CciFT2WPBuUr1ccT4f0YrXb2rKrqNn6T1lKW8s2s7FVB2eq5Q7DBs2jOnTp/9j3vTp0126/hNYV6ctXTpnt0FwThrPP/88Xbt2zdG28itP1zRaATuNMbuNMcnAdKC/U5mRwFRjzAkAY8wR+2+yMSbtCoGBFOCmNRFhUIuqLI6Oond4Jd5csoM+U5YRt/e4t0NTqsAbPHgwc+fOvXLDpb1793Lw4EE6duzI/fffT2RkJI0aNWL8+PHprl+zZk2OHTsGwEsvvURYWBgdOnS4cvl0sM7BaNmyJU2bNmXQoEGcP3+eFStWMHv2bB599FGaNWvGrl27GD58ON988w0AS5YsISIigvDwcEaMGMHFixev7G/8+PE0b96c8PBwtm7dmunxefsS6p6+YGEVYL/DdAJWrcFRGICILAd8gQnGmJ/sedWAuUAd4FFjzEHnHYjIPcA9ANWrV3d3/G4VGhzIm0MjuDGiCk9/v4nB7/3G7W1q8FjPeoQE+We9AaXyu/nj4K+N7t3mNeHQ65UMF5ctW5ZWrVoxf/58+vfvz/Tp0xkyZAgiwksvvUTZsmW5dOkS119/PRs2bKBJkybpbmfNmjVMnz6ddevWkZqaSvPmzWnRogUAAwcOZOTIkQA8/fTTfPTRRzz88MP069ePvn37Mnjw4H9sKykpieHDh7NkyRLCwsK44447ePfddxkzxmpICQ0NZe3atbzzzjtMnDiRDz/8MMPj8/Yl1PPjr3U/oC7QGRgGfCAipQGMMfuNMU2wksadIlLReWVjzPvGmEhjTGT58uU9GHbOdalXgYVjO3FX+5r8b9U+uk2KZVH81ZdgVkq5xrGJyrFpaubMmTRv3pyIiAg2b978j6YkZ0uXLmXAgAEUL16ckiVL0q9fvyvLNm3aRMeOHQkPD+fLL7/M8NLqabZt20atWrUICwsD4M477yQ2NvbK8oEDBwLQokWLKxc5zIi3L6Hu6ZrGAaCaw3RVe56jBGCVMSYF2CMi27GSyOq0AsaYgyKyCegIfJO3IXtGiUA/xt/QiH5NKzPu242M/DyOPuGVGN+vIRVCgrwdnlI5k0mNIC/179+fsWPHsnbtWs6fP0+LFi3Ys2cPEydOZPXq1ZQpU4bhw4eTlJSUo+0PHz6cWbNm0bRpUz799FN+/fXXXMWbdnn13Fxa3VOXUPd0TWM1UFdEaolIADAUmO1UZhZWLQMRCcVqrtotIlVFpJg9vwzQAdhGIRNRvQxzHu7A/3UPY9GWw3R9PYYZq//UkwKVyobg4GC6dOnCiBEjrtQyTp8+TYkSJShVqhSHDx9m/vz5mW6jU6dOzJo1iwsXLnDmzBnmzJlzZdmZM2eoVKkSKSkpVy5nDhASEsKZM1ePiKxXrx579+5l586dAHzxxRdERUXl6Ni8fQl1jyYNY0wq8BCwANgCzDTGbBaR50Ukre63AEgUkXjgF6y+i0SgAbBKRNYDMcBEY4ybG0vzhwA/Hx66ri7zR3ekfqWSPP7tRm75YBV7jp3zdmhKFRjDhg1j/fr1V5JG2qXE69evzy233EL79u0zXb958+bcfPPNNG3alF69etGyZcsry1544QVat25N+/btqV+//pX5Q4cO5bXXXiMiIoJdu3ZdmR8UFMQnn3zCTTfdRHh4OD4+Ptx3X87OGvD2JdT1goX53OXLhumr9/Py/C0kp15mdNe6jOxYG3/f/NgdpZResLCg0QsWFjI+PsItrauzODqKLvUq8OpP2+j39nI2JFx98pFSSuU1TRoFRMWSQbx3ewveu60FiWcvcuPU5bz4Yzznk91/1zGllMqIJo0Cpmfja1gUHcXQVtX5cNkeur8RS8z2o94OS6l/KMzN3oVJTt4nTRoFUKli/vx7QDgz7mlDgJ8Pd378O9Ez1nH8XLK3Q1OKoKAgEhMTNXHkc8YYEhMTCQrK3pB+7Qgv4JJSLjH1l528++suShbz59m+DenfrDIi4u3QVBGVkpJCQkJCjs+BUJ4TFBRE1apV8ff/5xUoMusI16RRSGz96zTjvt3Iuv0niQorz4s3NqZa2eLeDkspVQDp6KkioP41Jfn2/naMv6Ehq/cep/sbsXy0bA+X9J4dSik30qRRiPj6CHe1r8Wi6Cja1C7LCz/GM/Cd5Ww5dNrboSmlCglNGoVQldLF+Hh4S94c2oyEExe44a1lvLZgK0kpes8OpVTuaNIopESE/s2qsDg6iv7NqjD1l130enMpK3cnejs0pVQBpkmjkCtTIoDXhzTli7tbkXr5MkPfX8kT323g1IUUb4emlCqANGkUER3rlmfhmCju7VSbGav303VSDPM3HtKx9EqpbNGkUYQUC/Dlid4N+OHBDlQICeT+L9dyzxdr+OuUjqdXSrlGk0YRFF61FD882J4netUndvtRuk2K4X8r93FZh+cqpbKgSaOI8vP14d6oa1kwphPhVUvx9KxN3Pz+b+w8cvUNZJRSKo0mjSKuZmgJvvxXa14b3ITth8/S+81lTFmyg+TUy94OTSmVD2nSUIgIN0VWY3F0FN0bVWTSou30fWspa/ad8HZoSql8RpOGuqJ8SCBv39Kcj+6M5ExSKoPfW8GE2Zs5e1Hv2aGUsmjSUFe5vkFFFkVHcUebGnz22166T4phyZbD3g5LKZUPaNJQ6QoO9OO5/o355r52lAj04+7P4nho2lqOnrno7dCUUl6kSUNlqkWNMswd1ZGxXcNYuPkwXSfFMDNuv54UqFQRpUlDZSnAz4fRXesyb3QHwioG89g3G7jto1XsSzzn7dCUUh6mSUO5rE6FEGbc05YXb2zM+v2n6DE5lvdidpF6SYfnKlVUaNJQ2eLjI9zWpgaLo6PoWLc8r8zfSv+py9l04JS3Q1NKeYAmDZUj15QK4v3bW/Durc05cuYi/acu59/ztnAhWe/ZoVRhpklD5ZiI0Cu8EovHRjEksirvx+6mx+RYlu045u3QlFJ5RJOGyrVSxf15eWATvhrZBl8f4baPVvHIzPWcOJfs7dCUUm6mSUO5TdtryzF/dEce7HItP6w7QNdJMcxef1CH5ypViGjSUG4V5O/Loz3qM+fhDlQtU4xRX/3BiE9Xc+DkBW+HppRyA00aKk80qFSS7x5ozzN9G7Jy93G6TYrh0+V7uKT37FCqQNOkofKMr49wd4daLBzbiciaZZkwJ55B765g2196zw6lCipNGirPVStbnM/uasnkm5vx5/Hz9JmylEkLt5GUosNzlSpoNGkojxARboyowuLoKPo1rcyUn3fSe8pSft9z3NuhKaWyQZOG8qiyJQKYdHMzPhvRiuTUywz57288+f1GTieleDs0pZQLNGkor4gKK8/CsZ34V4daTP/9T7q+HsNPm/7ydlhKqSxo0lBeUzzAj6f7NuT7B9pTLjiQ+/63hvu+WMPh00neDk0plQFNGsrrmlYrzeyH2vNYz3r8su0IXSfFMG3Vn1zW4blK5TuaNFS+4O/rwwOd6/DTmE40qlySJ7/fyNAPVrLr6Flvh6aUcqBJQ+UrtUJL8NXINvxnUDhbD52m15tLefvnHSSn6j07lMoPPJ40RKSniGwTkZ0iMi6DMkNEJF5ENovINHteMxH5zZ63QURu9mzkylNEhJtbVmfxI1F0a1CRiQu30+/tZfzx5wlvh6ZUkSeevJiciPgC24FuQAKwGhhmjIl3KFMXmAlcZ4w5ISIVjDFHRCQMMMaYHSJSGVgDNDDGnMxof5GRkSYuLi4vD0l5wKL4wzwzaxOHzyQxvF1N/q97PUoE+nk7LKUKLRFZY4yJTG+Zp2sarYCdxpjdxphkYDrQ36nMSGCqMeYEgDHmiP13uzFmh/38IHAEKO+xyJXXdGtYkUXRnbitdQ0+Wb6X7m/E8su2I94OS6kiydNJowqw32E6wZ7nKAwIE5HlIrJSRHo6b0REWgEBwK50lt0jInEiEnf06FE3hq68KSTInxdubMw397WlWIAvd32ymtHT/yDx7EVvh6ZUkZIfO8L9gLpAZ2AY8IGIlE5bKCKVgC+Au4wxV/WOGmPeN8ZEGmMiy5fXikhhE1mzLHNHdWD09XWZt/EQXSfF8O2aBL1nh1Ie4umkcQCo5jBd1Z7nKAGYbYxJMcbsweoDqQsgIiWBucBTxpiVHohX5UOBfr6M7RbG3FEdqRVagke+Xs8dH//O/uPnvR2aUoWep5PGaqCuiNQSkQBgKDDbqcwsrFoGIhKK1Vy12y7/PfC5MeYbz4Ws8quwiiF8c187XujfiLX7TtD9jVg+iN1N6iUdnqtUXvFo0jDGpAIPAQuALcBMY8xmEXleRPrZxRYAiSISD/wCPGqMSQSGAJ2A4SKyzn4082T8Kv/x8RFub1uTRdFRtK9TjpfmbWHAOyvYfPCUt0NTqlDy6JBbT9Mht0WLMYZ5G/9i/OxNnDifwsiOtRnTtS5B/r7eDk2pAiU/DblVKs+ICH2aVGJxdBSDmlfhvZhd9Jgcy4qdx7wdmlKFhiYNVeiULh7Aq4ObMu1frQG45cNVPPbNek6eT/ZyZEoVfJo0VKHVrk4oC8Z04r6oa/l27QG6Torhxw0HdXiuUrmgSUMVakH+vozrVZ/ZD7WnUqliPDTtD0Z+HsfBkxe8HZpSBVKuk4aIlBWRFiIS6I6AlMoLjSqX4vsH2vFU7wYs23mMbpNi+Py3vXrPDqWyKVtJQ0SeFpGXHaY7AXuB34Ed9sUGlcqX/Hx9GNmpNgvHRNG8Rhme/WEzg99bwfbDZ7wdmlIFRnZrGrcBux2m/wOsB24EDgMvuCkupV2JmOAAAB6MSURBVPJM9XLF+XxEKyYNacruY+foM2UpbyzazsXUS94OTal8L7vXl64C7AAQkfJYV6293hjzq33G9hQ3x6dUnhARBjavSlRYeV74MZ43l+xg7sZDvDIwnMiaZb0dnlL5VnZrGpewri4L1tnZScBye/oooP9tqkApFxzI5KERfHJXSy4kX2Lwe7/x9KyNnElK8XZoSuVL2U0am4HbRCQYGAHEGGPS/ruqYd3jQqkCp0u9Ciwc24kR7Wvx5ao/6TYplkXxh70dllL5TnaTxvNY14A6BVyP1aeRpjew1k1xKeVxJQL9ePaGhnx3fztKF/dn5OdxPPDlGo6cSfJ2aErlG9lKGsaYBUADrMTRyBgT47A4ln8mEaUKpIjqZZjzcAce7VGPxVuO0PX1GGas/lNPClQKvWChUpnadfQsT3y3kd/3HKdN7bK8PLAJtUJLeDsspfKUWy9YKCJVRGSSfUvV3SLS2J4/RkRa5zZYpfKTa8sHM31kG14eGM7mg6fpMTmWd37dSYres0MVUdk9ua8RsBG4HTgI1ODv0VQ1gNFujU6pfMDHRxjWqjpLoqO4vn4FXv1pG/3eXs76/Se9HZpSHpfdmsbrWDdPqgUMBMRh2QqgjZviUirfqVAyiHdva8F7t7Ug8exFBryznBd/jOd8cqq3Q1PKY7KbNDoArxhjzgLOnSGHgWvcEpVS+VjPxtew+JEohrWqzofL9tD9jVhith/1dlhKeUR2k0ZmDbmhgF46VBUJJYP8eWlAODPvbUuAnw93fvw70TPWcfyc3rNDFW7ZTRq/A3dlsGwIf58drlSR0KpWWeaN6sio6+owe/1Buk6KYdYfB3R4riq0sps0XgBuEJGFWJ3hBugqIp8BA4CX3ByfUvlekL8v0d3r8eOoDlQvW5wxM9Zx5yer2X/8vLdDU8rtsn2ehoj0ASYD1zrM3gs8aIyZ777Qck/P01Cedumy4Yvf9vLqgm0YA490D+Ou9rXw9ZEs11Uqv8jsPI0cn9wnInWACkCiMWZbLuLLM5o0lLccOHmBp7/fyC/bjtK0aileGdSEBpVKejsspVzi1pP70hhjdhpjVhhjtolIuZyHp1ThU6V0MT4e3pIpwyJIOHGBG95axqs/bSUpRe/ZoQq27J7cN1JEHnWYDheRBOCIfYa4DrlVyiYi9GtamcXRUdwYUYV3ft1FrzeXsnJ3ordDUyrHslvTeJh/DqudBJwExgClsK6Cq5RyUKZEABNvasr/7m7NpcuGoe+vZNy3Gzh1Xu/ZoQqe7CaNGsBWABEpBUQBjxlj3gLGAz3cG55ShUeHuqEsGNOJezvVZmbcfrq+EcO8jYd0eK4qULKbNHz4+wS/DlhDbn+1p/djdYwrpTJQLMCXJ3o3YPZDHagQEsgDX67lni/W8NcpvWeHKhiymzR2AH3s50OBFcaYtMHolYHj7gpMqcKscZVS/PBge57oVZ/Y7UfpNimGL1bu4/JlrXWo/C27SWMiMEZEjgG3AG85LOsCbHBXYEoVdn6+PtwbdS0Lx3aiSbVSPDNrE0P++xs7j5zxdmhKZSi7d+6bhtWP8TLQxRjzncPiw/wziSilXFCjXAn+d3drXhvchB1HztL7zWW8uXgHyal6zw6V/+id+5TKR46eucjzP8YzZ/1BwioG8/LAJrSoUcbbYakixm0n94lIOxHp6zBdTkS+EpGNIjJRRHxzG6xSRVn5kEDeGhbBx8MjOZuUyuD3VjD+h02cvaj37FD5Q3b7NF4BWjhMvwb0BrYD9wNPuikupYq06+pXZGF0FHe2rcnnK/fRbVIMS7Yc9nZYSmU7aTQA4gBExB8YDIw1xgwCnsLqHFdKuUFwoB8T+jXi2/vbERLkx92fxfHQtLUcPXPR26GpIiy7SSMYOG0/bwWUAH60p9cC1d0Ul1LK1rx6GX58uCPR3cJYuPkwXSfFMDNuv54UqLwiu0njANDUft4L2GSMOWJPlwH0BgJK5YEAPx9GXV+XeaM7ElYxmMe+2cBtH61iX+I5b4emipjsJo2vgH+LyDdANPA/h2XNsU7+U0rlkToVgplxT1tevLExG/afovsbsbwXs4vUSzo8V3lGdpPGBOA/QCBWp/gbDsuaAl+7JyylVEZ8fITb2tRgUXQUncLK88r8rfSfupxNB055OzRVBOh5GkoVYMYYftr0F8/O3kzi2Yv8q2NtxnYNo1iAjn5XOef2mzCJSGMReVBEnrH/NsrGuj1FZJuI7BSRcRmUGSIi8SKyWUSmOcz/SUROisiP6a2nVFEjIvQKr8Ti6ChublmN92N302NyLMt2HPN2aKqQylZNQ0T8gE+BYYDjTY8NMA0YbozJ8NZk9sl/24FuQAKwGhhmjIl3KFMXmAlcZ4w5ISIV0jrbReR6oDhwrzGm71U7cKI1DVXUrNydyBPfbWTPsXMMal6Vp/s0oEyJAG+HpQoYd9Y0xgNDgGeBWkAx+++zwM3238y0AnYaY3YbY5KB6UB/pzIjganGmBMADqOzMMYsAfRqbkploE3tcswf3ZEHu1zLD+sO0HVSDD+sO6DDc5XbZDdp3Aa8aIx5yRizzxhz0f77EvAicEcW61fBuu9GmgR7nqMwIExElovIShHpmZ0AReQe+9azcUePHs3OqkoVCkH+vjzaoz5zHu5A1TLFGD19HSM+Xc2BkxeyXlmpLGQ3aVQGVmSwbIW9PLf8gLpAZ6xmsA9EpLSrKxtj3jfGRBpjIsuXL++GcJQqmBpUKsl3D7Tnmb4NWbXnON0mxfDJ8j1c0nt2qFzIbtI4CLTPYFk7e3lmDgDVHKar2vMcJQCzjTEpxpg9WH0gdbMZp1IK8PUR7u5QiwVjOtGyZlmemxPPoHdXsO0vbeVVOZPdpPEl8JQ9aqq2iBQTkVoi8gTWtae+yGL91UBde50ArLv/zXYqMwurloGIhGI1V+3OZpxKKQfVyhbn07ta8ubQZvx5/Dx9pizl9YXbSErJcNyKUunKyeipz7G+7B1XFP4ePZXpNZxFpDcwGfAFPjbGvCQizwNxxpjZIiLA60BP4BLwkjFmur3uUqA+1jWwEoG7jTELMtqXjp5S6mrHzyXz4o/xfPfHAWqXL8HLA8JpXbuct8NS+Uhmo6dydHKffV5GJ6As1n3BY4FKwCRjTJNcxOpWmjSUyljs9qM8+f1GEk5c4JbW1RnXqz4lg/y9HZbKB9yeNDLYySBgpjEm35yKqklDqcydT07ljUXb+WjZHkKDA3m+f2N6Nr7G22EpL3P7GeFKqcKheIAfT/VpyKwH21MuOJD7/reG+75Yw+HTSd4OTeVTmjSUUjSpWprZD7Xn8Z71+WXbEbpOimHaqj+5rMNzlRNNGkopAPx9fbi/87X8NKYTjSuX4snvNzL0g5XsOnrW26GpfMQvqwIiUtvFbWlDqFKFQK3QEkwb2Zqv4xJ4cW48vd5cyqjr6nBPp2sJ8NPfmUVdlh3hInKZfw6vzbAoYLQjXKnC48iZJJ6bHc/cjYeoVzGEVwaFE1G9jLfDUnkss47wLGsawF1ujkcpVUBUCAli6q3NuTH+MM/M2sTAd1cwvF1N/q97PUoEuvL1oQqbLN91Y8xnnghEKZV/dWtYkTa1y/LqT9v4dMVeFm4+zIsDGtOlXgVvh6Y8TBsolVIuCQny54UbG/PNfW0pFuDLXZ+sZvT0Pzh29qK3Q1MepElDKZUtLWqUZe6oDozpWpd5Gw/RdVIM365J0Ht2FBGaNJRS2Rbo58uYrmHMG9WRa8sH88jX67nj49/5M/G8t0NTeUyThlIqx+pWDOHre9vyQv9G/PHnSbpPjuGD2N2kXrrs7dBUHtGkoZTKFR8f4fa2NVkU3YkOdUJ5ad4WBryzgk0HTnk7NJUHNGkopdyiUqlifHBHJFNvac6hU0n0n7qcV+Zv1Xt2FDKaNJRSbiMi9GlSiSXRUQxuXpX3YnbRY3IsK3Ye83Zoyk00aSil3K5UcX/+M7gJ00a2RoBbPlzFY9+s5+T5ZG+HpnJJk4ZSKs+0uzaUn8Z04v7O1/Lt2gN0nRTDjxsO6vDcAkyThlIqTwX5+/J4z/rMfqg9lUoV46Fpf/Cvz+I4ePKCt0NTOaBJQynlEY0ql+L7B9rxdJ8GrNiVSLdJMXz+2169Z0cBo0lDKeUxfr4+/KtjbRaO7UTzGmV49ofNDH5vBdsPn/F2aMpFmjSUUh5XrWxxPh/RiklDmrLn2Dn6TFnKpEXbuZiqw3PzO00aSimvEBEGNq/K4ugo+oRXYsqSHfSZsoy4vce9HZrKhCYNpZRXlQsOZPLQCD65qyUXki8x+L3feHrWRs4kpXg7NJUOTRpKqXyhS70KLBzbiRHta/Hlqj/pNimWhZv/8nZYyokmDaVUvlEi0I9nb2jI9w+0p3Rxf+75Yg0PfLmGI2eSvB2asmnSUErlO82qlWbOwx14tEc9Fm85QtfXY5j++596UmA+oElDKZUv+fv68GCXOvw0uiMNKpVk3HcbGfbBSvYcO+ft0Io0TRpKqXytdvlgvhrZhpcHhrP54Gl6TI5l6i87SdF7dniFJg2lVL7n4yMMa1WdJdFRXF+/Aq8t2MYNby1j/f6T3g6tyNGkoZQqMCqUDOLd21rw39tbcOJ8MgPeWc4LP8ZzPjnV26EVGZo0lFIFTo9G17AoOophrarz0bI9dH8jlpjtR70dVpGgSUMpVSCVDPLnpQHhzLy3LYF+Ptz58e+MnbGO4+f0nh15SZOGUqpAa1WrLHNHdWTUdXX4ccNBuk6KYdYfB3R4bh7RpKGUKvCC/H2J7l6PHx/uSI1yxRkzYx13frKa/cfPezu0QkeThlKq0Kh3TQjf3NeO5/o1Ys3e43R/I5YPl+7mkt6zw200aSilChVfH+HOdjVZGB1F22vL8eLcLQx8ZznxB097O7RCQZOGUqpQqlK6GB/dGclbwyJIOHGBfm8v49WftpKUovfsyA1NGkqpQktEuKFpZRZHR3FjRBXe+XUXvd5cym+7Er0dWoGlSUMpVeiVKRHAxJua8r+7W3PpsmHYBysZ9+0GTp3Xe3Zkl8eThoj0FJFtIrJTRMZlUGaIiMSLyGYRmeYw/04R2WE/7vRc1EqpwqBD3VAWjOnEvZ1q8/WaBK6fFMO8jYd0eG42iCdfLBHxBbYD3YAEYDUwzBgT71CmLjATuM4Yc0JEKhhjjohIWSAOiAQMsAZoYYw5kdH+IiMjTVxcXN4dkFKqwNp04BSPf7uBzQdP061hRV7o35hrSgV5O6x8QUTWGGMi01vm6ZpGK2CnMWa3MSYZmA70dyozEpialgyMMUfs+T2ARcaY4/ayRUBPD8WtlCpkGlcpxQ8PtueJXvVZuuMoXSfF8MXKfVzW4bmZ8nTSqALsd5hOsOc5CgPCRGS5iKwUkZ7ZWBcRuUdE4kQk7uhRvRaNUipjfr4+3Bt1LQvGdKJZtdI8M2sTQ/77GzuPnPF2aPlWfuwI9wPqAp2BYcAHIlLa1ZWNMe8bYyKNMZHly5fPoxCVUoVJjXIl+OLuVky8qSk7jpyl95vLeHPxDpJT9Z4dzjydNA4A1Rymq9rzHCUAs40xKcaYPVh9IHVdXFcppXJERBjcoipLHomiZ+NreGPxdvq+tZQ1+zLsNi2SPJ00VgN1RaSWiAQAQ4HZTmVmYdUyEJFQrOaq3cACoLuIlBGRMkB3e55SSrlNaHAgU4ZF8PHwSM4mpTL4vRWM/2ETZy/qPTvAw0nDGJMKPIT1Zb8FmGmM2Swiz4tIP7vYAiBRROKBX4BHjTGJxpjjwAtYiWc18Lw9Tyml3O66+hVZGB3FnW1r8vnKfXSbFMOSLYe9HZbXeXTIrafpkFullDus/fME477dwPbDZ+nbpBLjb2hE+ZBAb4eVZ/LTkFullCpwmlcvw48Pd+SRbmEs3HyYrpNimBm3v0ieFKhJQymlXBDg58PD19dl3uiO1KsYwmPfbODWD1exL/Gct0PzKE0aSimVDXUqBDP9nja8NKAxGxNO0f2NWN6L2UXqpaIxPFeThlJKZZOPj3Br6xosio4iKqw8r8zfSv+py9l04JS3Q8tzmjSUUiqHrikVxPt3RPLebc05cuYi/d5exr/nbeFCcuG9Z4cmDaWUyqWejSuxODqKm1tW5/3Y3XSfHMPSHYXzMkaaNJRSyg1KFfPn5YHhTL+nDf4+Ptz+0e88MnM9J84lezs0t9KkoZRSbtSmdjnmje7IQ13q8MO6A3SdFMMP6w4UmuG5mjSUUsrNgvx9+b8e9ZjzcAeqli3O6OnrGPHpag6cvODt0HJNk4ZSSuWRBpVK8t397Xi2b0NW7TlOt0kxfLxsD5cK8D07NGmk5/JlOHMYUpK8HYlSqoDz9RFGdKjFwrGdaFWrLM//GM/Ad1ew9a/T3g4tR/TaU+k5fxxerWU99w2AwJIQVAqCSjo9L/X386BS9jLH5/ZfXz/3HphSqkAyxjB7/UGemxPP6Qsp3Bd1LQ9dV4cgf19vh/YPmV17SpNGei6ehQ3TIekUJJ2Gi6f/fp50yp62n6e4cAkB/xKZJBanJJPe/IBg8NFKoVKFxfFzybw4N57v1h6gdmgJXh4YTuva5bwd1hWaNPLSpdS/k4pjMrnq+amM51/KakieONRsnJOMc+3HOfnY8/2CQCRvXwulVLbEbj/KU7M2sv/4BYa1qs64XvUpVczf22Fp0sj3UpKcksmp9Gs5/3juVMZkcd0bH/+sm9gybYYrCb7e/zArVdicT07ljUXb+WjZHkKDA3m+f2N6Nr7GqzFp0ijsjIHksxkkmZOZ1H4cniefzXo//sUzTixZNbEFlYSAEG1mUyoDGxJOMu7bjcQfOk2PRhV5vn9jKpYM8kosmjRU1tKa2XLUxJbWzHYxi51IFn066T13Skr+xbSZTRVaKZcu8+HSPUxevJ0AXx+e6N2AoS2r4ePj2c+8Jg3lGakXs9nE5lATSpufZTObXwZJpnQWTWwOZbSZTeVze4+d44nvNvLb7kRa1SzLy4PCubZ8sMf2r0lDFQzGQPK5DJLMqcxHsaU9Tz6T9X78irnYj5NBmcCS2sym8pwxhq/jEnhxbjxJKZcZdX0d7ul0LQF+ef/Z06Shio7Ll7JoYrNrN5mVSc3qpE6BwJCcN7EFlbT6h7SZTbngyJkknpsTz9wNh6hXMYRXBoUTUb1Mnu5Tk4ZS2ZHWzJZe81mGz51qQiaL+yn4+Ll+nk5GZfwCPPN6qHxhcfxhnvlhE3+dTuLOtjV5tEc9SgTmzYnDmSUNPVVZKWd+gRBc3nrkhDGQcj77TWzHd//93KVmtqAsztnJYjh1YAj45K8zkVXGujasSOvaZXltwTY++20vi+IP8+KNjelSv4JH49CahlL50eVLcPFMFkOlM+vfOeVCMxvWMOgs+3cyOGE0qJQ2s3nJmn3Hefzbjew8cpZ+TSvz7A0NCQ0OdNv2tXlKqaIoNTnjWo4rTWwXT8Pl1Mz3Ib45aGKzR7KlPfdz35ddUXIx9RLv/rqLqb/spESgH0/3acig5lUQNyRxTRpKqey70syWydUIsjpp9KILV3L1C8rZNdkcnxfhZrYdh88w7ruNrNl3gg51Qvn3gHCqlyueq21q0lBKeUdaM1tOm9iSTkOqCzcuCgjO2TXZ0p4HlCjQzWyXLxu+XLWP//y0jdTLl4nuFsaI9rXw883Z8FxNGkqpgsuxmS0nTWxJp1xrZgt07N9x9ZpsDvP9vXPJD0eHTl3gmVmbWbzlMC1rlmHGPW1zdDa5jp5SShVcfgHgFwolQnO2vjGQciH7TWwn9v79/OJpIIsf2L6BuWtic8O9dyqVKsYHd7Rg/qa/OHE+OU8uP6JJQylVuIlAQHHrEZLDq8devmwNg87uddlOH/w76aScz3o/AcE5P2E0qBQEBCMi9A6vlLPjdIEmDaWUyoqPz9+1BarlbBuXUq6u5WR164NzR+H4rr/nX07JfB/i83cSqdoSBn+cs1gzoUlDKaU8wdcfSpSzHjlhjHXuTWYX/HSs5ZSq6t74bZo0lFKqIBCxbg3gXyznzWxuoJfqVEop5TJNGkoppVymSUMppZTLNGkopZRymSYNpZRSLtOkoZRSymWaNJRSSrlMk4ZSSimXFeqr3IrIUWBfLjYRChxzUzgFRVE75qJ2vKDHXFTk5phrGGPSvd9xoU4auSUicRldHriwKmrHXNSOF/SYi4q8OmZtnlJKKeUyTRpKKaVcpkkjc+97OwAvKGrHXNSOF/SYi4o8OWbt01BKKeUyrWkopZRymSYNpZRSLivySUNEeorINhHZKSLj0lkeKCIz7OWrRKSm56N0LxeOOVpE4kVkg4gsEZEa3ojTnbI6Zodyg0TEiEiBH57pyjGLyBD7vd4sItM8HaO7ufDZri4iv4jIH/bnu7c34nQXEflYRI6IyKYMlouITLFfjw0i0jzXOzXGFNkH4AvsAmoDAcB6oKFTmQeA9+znQ4EZ3o7bA8fcBShuP7+/KByzXS4EiAVWApHejtsD73Nd4A+gjD1dwdtxe+CY3wfut583BPZ6O+5cHnMnoDmwKYPlvYH5gABtgFW53WdRr2m0AnYaY3YbY5KB6UB/pzL9gc/s598A14uIeDBGd8vymI0xvxhjztuTK4G8udmw57jyPgO8APwHSPJkcHnElWMeCUw1xpwAMMYc8XCM7ubKMRugpP28FHDQg/G5nTEmFjieSZH+wOfGshIoLSKVcrPPop40qgD7HaYT7HnpljHGpAKngBzeGT5fcOWYHd2N9UulIMvymO1qezVjzFxPBpaHXHmfw4AwEVkuIitFpKfHossbrhzzBOA2EUkA5gEPeyY0r8nu/3uW/HIVjirUROQ2IBKI8nYseUlEfIBJwHAvh+JpflhNVJ2xapOxIhJujDnp1ajy1jDgU2PM6yLSFvhCRBobYy57O7CCoqjXNA4A1Rymq9rz0i0jIn5YVdpEj0SXN1w5ZkSkK/AU0M8Yc9FDseWVrI45BGgM/Coie7HafmcX8M5wV97nBGC2MSbFGLMH2I6VRAoqV475bmAmgDHmNyAI68J+hZVL/+/ZUdSTxmqgrojUEpEArI7u2U5lZgN32s8HAz8bu4epgMrymEUkAvgvVsIo6O3ckMUxG2NOGWNCjTE1jTE1sfpx+hlj4rwTrlu48tmehVXLQERCsZqrdnsySDdz5Zj/BK4HEJEGWEnjqEej9KzZwB32KKo2wCljzKHcbLBIN08ZY1JF5CFgAdbIi4+NMZtF5HkgzhgzG/gIqwq7E6vDaaj3Is49F4/5NSAY+Nru8//TGNPPa0HnkovHXKi4eMwLgO4iEg9cAh41xhTYWrSLx/wI8IGIjMXqFB9ekH8EishXWIk/1O6nGQ/4Axhj3sPqt+kN7ATOA3flep8F+PVSSinlYUW9eUoppVQ2aNJQSinlMk0aSimlXKZJQymllMs0aSillHKZJg2lCjARSTshUSmP0KShlBMR6WxfHj2jR6q3Y1TKW4r0yX1KZeErrJOjnOl1ilSRpUlDqYytNcb8z9tBKJWfaPOUUjkkIjXt5qoJIjLMvjNakoj8ac+76keZiDQRke9FJNEuGy8ij4mIbzplr7HvurZbRC7ad2hbJCLd0ilbWUS+EpETInJeRBaISFheHbsqurSmoVTGitsX8nOWbIw57TDdD+tucVOBv+zp8UANHK71Y181NwZIcSh7A9aNn5oCtzqUrQksByoCnwNxQAmsK/B2BRY57L8Ef99x8EmgFjAa+MG+7PelnBy8Uuny9u0K9aGP/PbAugCcyeTxo12upj19CWjusL4A39vL2jjMXw6kAk2cys60y17vMH+ePa9HOvH5ODz/1S73mFOZRzNaXx/6yM1Dm6eUytj7QLd0Hk85lVtkjFmbNmGMMcCr9uQAABGpALTDun/FBqeyLzmVLQv0BH4yxixwDspcfcOgy8AUp3k/238L8v0xVD6kzVNKZWyHMWaxC+W2pDMv3v5b2/5by/67OYP1LzuUrYNVA/nDxTgPGmOc72uedonzgnxrYpUPaU1DqYIvsz4L8VgUqkjQpKFU7jVIZ15D+2/anfD22H8bpVO2Ptb/YlrZnVj9Ec3cFaBS7qJJQ6nc6yYizdMmxLrd4WP25CwAY902dwVwg4g0dir7hD35vV32ODAf6GXfq/0f7HWU8grt01AqY81F5LYMls1yeL4e+FlEpgKHgP5Yw2K/MMb85lBuNNaQ26V22b+AvkAPYJoxZolD2Yewksx8EfkMWAMUA1oDe4HHc3lsSuWIJg2lMjbMfqSnLtbwWYDZwDasGkM94Ajwgv24whgTJyLtgOeAB7DOr9iNlQBedyq7xz6v4xmsezzfAZzASlDv5/bAlMopvUe4Ujlkn4C3B3jOGDPBq8Eo5SHap6GUUsplmjSUUkq5TJOGUkopl2mfhlJKKZdpTUMppZTLNGkopZRymSYNpZRSLtOkoZRSymWaNJRSSrns/wEfkEGoaWs6iwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A6CHKx_ADfiZ",
        "outputId": "a3252be8-c0d2-42cc-8995-659266c94275",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "#Test on validation data\n",
        "model.eval()\n",
        "fin_targets=[]\n",
        "fin_outputs=[]\n",
        "with torch.no_grad():\n",
        "    c=0\n",
        "    for _, data in enumerate(valid_loader, 0):\n",
        "        ids = data['ids'].to(device, dtype = torch.long)\n",
        "        mask = data['mask'].to(device, dtype = torch.long)\n",
        "        token_type_ids = data['token_type_ids'].to(device, dtype = torch.long)\n",
        "        targets = data['targets'].to(device, dtype = torch.float)\n",
        "        emotions=data['emotion'].to(device,dtype=torch.float)\n",
        "        c+=1\n",
        "        try:\n",
        "                      outputs = model(ids, mask, token_type_ids,emotions)\n",
        "                      fin_targets.extend(targets.cpu().detach().numpy().tolist())\n",
        "                      fin_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
        "        except EOFError:\n",
        "          print(f\"some error at {c}\",EOFError)\n",
        "outputs=fin_outputs\n",
        "outputs = np.array(outputs) >= 0.5\n",
        "targets=fin_targets\n",
        "accuracy = metrics.accuracy_score(targets, outputs)\n",
        "\n",
        "f1_score_macro = metrics.f1_score(targets, outputs, average='macro')\n",
        "print(f\"Epoch:  {epoch}, Accuracy Score on validation data = {accuracy}\")\n",
        "\n",
        "print(f\"Epoch:  {epoch}, F1 Score on Validation Data (Macro) = {f1_score_macro}\") \n",
        "print(\"____________________________________________________________\\n________________________________________________________________\\n\\n\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch:  1, Accuracy Score on validation data = 0.69921875\n",
            "Epoch:  1, F1 Score on Validation Data (Macro) = 0.6430236972753443\n",
            "____________________________________________________________\n",
            "________________________________________________________________\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}