News_Classification.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "l9k4nvPt6ttU"
      },
      "source": [
        "<div dir=\"auto\" align=\"center\">\n",
        "    <h3>\n",
        "        بسم الله الرحمن الرحیم\n",
        "    </h3>\n",
        "    <br>\n",
        "    <h1>\n",
        "        <strong>\n",
        "            پردازش زبان‌های طبیعی\n",
        "        </strong>\n",
        "    </h1>\n",
        "    <h2>\n",
        "        <strong>\n",
        "            تمرین چهارم (دسته‌بندی اخبار)\n",
        "        </strong>\n",
        "    </h2>\n",
        "    <br>\n",
        "    <h3>\n",
        "        ارشان دلیلی - ٩٨١٠٥٧٥١\n",
        "        <br><br>\n",
        "        محمدمهدی ابوترابی - ۹۸۱۰۵۵۵۷\n",
        "        <br><br>\n",
        "        آرین احدی‌نیا - ٩٨١٠۳۸۷۸\n",
        "    </h3>\n",
        "    <br>\n",
        "</div>\n",
        "<hr>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "bDy59Vhk6ttY"
      },
      "source": [
        "<div>\n",
        "    <h3 style='direction:rtl;text-align:justify;'>\n",
        "        دسترسی به داده‌ها و مدل‌های ذخیره شده\n",
        "    </h3>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        به دلیل حجم بالای فایل‌های ذخیره شده، از قرار دادن آن‌ها در پوشه نوتبوک صرف نظر کرده و با اجرای قطعه کد زیر، از طریق گوگل درایو به آن‌ها دسترسی پیدا می‌کنیم.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gXmeQEW66ttZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b69a509e-402a-46f2-98e4-d23b30ee15ed"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n",
            "/content/drive/My Drive/NLP\n"
          ]
        }
      ],
      "source": [
        "import sys\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "address = \"NLP/\"\n",
        "sys.path.append(f\"/content/drive/My Drive/{address}\")\n",
        "%cd /content/drive/My\\ Drive/$address"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "Uhyo5mtc6tta"
      },
      "source": [
        "<div>\n",
        "    <h3 style='direction:rtl;text-align:justify;'>\n",
        "        نصب و دسترسی به کتابخانه‌های مورد نیاز\n",
        "    </h3>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        با اجرای دو قطعه کد زیر، کتابخانه‌هایی که از آن‌ها در این تمرین استفاده شده است، نصب و قابل استفاده می‌شوند.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5DJqkFOY6tta",
        "outputId": "b5088087-03a4-46d0-b185-52b96e2558ab"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (4.64.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (1.3.5)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas) (2022.7.1)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.8/dist-packages (from pandas) (1.21.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (2.25.1)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests) (4.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests) (1.26.14)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests) (2.10)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: hazm in /usr/local/lib/python3.8/dist-packages (0.7.0)\n",
            "Requirement already satisfied: nltk==3.3 in /usr/local/lib/python3.8/dist-packages (from hazm) (3.3)\n",
            "Requirement already satisfied: libwapiti>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from hazm) (0.2.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from nltk==3.3->hazm) (1.15.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: unidecode in /usr/local/lib/python3.8/dist-packages (1.3.6)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.8/dist-packages (2.9.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (2.25.1)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2023.1.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from datasets) (23.0)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.12.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.8/dist-packages (from datasets) (3.2.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.8/dist-packages (from datasets) (0.70.14)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.4.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (1.26.14)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: fasttext in /usr/local/lib/python3.8/dist-packages (0.9.2)\n",
            "Requirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from fasttext) (57.4.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from fasttext) (1.21.6)\n",
            "Requirement already satisfied: pybind11>=2.2 in /usr/local/lib/python3.8/dist-packages (from fasttext) (2.10.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.8/dist-packages (1.0.2)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.8/dist-packages (from scikit-learn) (1.2.0)\n",
            "Requirement already satisfied: numpy>=1.14.6 in /usr/local/lib/python3.8/dist-packages (from scikit-learn) (1.21.6)\n",
            "Requirement already satisfied: scipy>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn) (1.7.3)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn) (3.1.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.8/dist-packages (3.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.8/dist-packages (from nltk) (1.15.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.1+cu116)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch) (4.4.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.8/dist-packages (4.26.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.25.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.13.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.12.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.9.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.26.14)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.8/dist-packages (1.2.0)\n"
          ]
        }
      ],
      "source": [
        "! pip install tqdm\n",
        "! pip install pandas\n",
        "! pip install requests\n",
        "! pip install hazm\n",
        "! pip install unidecode\n",
        "! pip install datasets\n",
        "! pip install fasttext\n",
        "! pip install scikit-learn\n",
        "! pip install nltk\n",
        "! pip install torch\n",
        "! pip install transformers\n",
        "! pip install joblib"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bs5P-S4T6ttb"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "import csv\n",
        "import hazm\n",
        "import nltk\n",
        "import json\n",
        "import torch\n",
        "import pickle\n",
        "import zipfile\n",
        "import fasttext\n",
        "import requests\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from pathlib import Path\n",
        "from datasets import Dataset\n",
        "from string import punctuation\n",
        "from IPython.display import display\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import DataCollatorWithPadding\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score, confusion_matrix\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
        "from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer, TextClassificationPipeline\n",
        "from sklearn.model_selection import cross_validate\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import accuracy_score\n",
        "import matplotlib.pyplot as plt\n",
        "from transformers import DistilBertForSequenceClassification, Trainer, TrainingArguments\n",
        "from transformers import BigBirdModel, AutoTokenizer\n",
        "\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch.nn import functional as F\n",
        "from torch.utils.data import Dataset, DataLoader, random_split\n",
        "from torchvision import transforms, datasets\n",
        "from time import time\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "5kMts6qe6ttf"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        بعد از ذخیره شدن داده‌ها در فایل فشرده dataset.zip، آن‌ها را از این فایل استخراج کرده و وارد برنامه می‌کنیم. با اجرای قطعه کد زیر، تعداد خبرهای هر دسته و تعداد کل خبرها را می‌توان مشاهده کرد.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HuoWmM7P6ttf"
      },
      "outputs": [],
      "source": [
        "dataset = pd.read_csv('./news_5000.csv',\n",
        "                      names=['UN0', 'UN1', 'URL', 'Title', 'Category', 'UN3', 'UN4', 'UN5', 'UN6', 'UN7', 'UN8', 'UN9', 'Text', 'UN10', 'UN11', 'UN12', 'UN13', 'UN14', 'UN15', 'UN16', 'UN17', 'UN18'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 803
        },
        "id": "91nfXbVkf0PM",
        "outputId": "8f0fe5a1-93dd-4e84-acc0-1c55645650b1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       UN0                     UN1                          URL  \\\n",
              "0  4612103  MoL7VM5vRhfPHAszgyToRM  khabarfarsi.com/u/115997967   \n",
              "1  3094439  Tp4kXv3A3oB7hRV8kb5oTn  khabarfarsi.com/u/108126652   \n",
              "2   619358  ioxLEiYLABpttJPWbuhxJv   khabarfarsi.com/u/98835151   \n",
              "3  4434763  S2ktAnodxNo5rFry9WBG3Z  khabarfarsi.com/u/115005815   \n",
              "4   276713  bU8DQribGWLJtWvTrV5Sb8   khabarfarsi.com/u/97546025   \n",
              "\n",
              "                                               Title           Category  \\\n",
              "0  اجرای ویژه برنامه های گرامیداشت دهه فجر در کان...        فرهنگ و هنر   \n",
              "1  والیبال نوجوانان جهان| ستاره ای که همچنان می د...              ورزشی   \n",
              "2  شرکت های دانش بنیان ضربات اقتصادی همه گیری کرو...            اقتصادی   \n",
              "3  بزرگترین مشکل سامسونگ S21 FE 5G چیست و چرا نبا...  فناوری و ارتباطات   \n",
              "4  فیلم | بازخوانی موضع رهبر معظم انقلاب درباره ج...              سیاسی   \n",
              "\n",
              "                                                 UN3            UN4  UN5  \\\n",
              "0            http://shabestan.ir/detail/News/1141850         شبستان  NaN   \n",
              "1  https://farsnews.ir/news/14000608000920/%D9%88...  خبرگزاری فارس  NaN   \n",
              "2  https://farsnews.ir/news/13991113000098/%D8%B4...  خبرگزاری فارس  NaN   \n",
              "3  https://www.shahrsakhtafzar.com/fa/news/smartp...  شهر سخت افزار  NaN   \n",
              "4  https://www.hawzahnews.com/news/937281/%D9%81%...           حوزه  NaN   \n",
              "\n",
              "                                                 UN6  \\\n",
              "0                                                NaN   \n",
              "1  امیرمحمد گل زاده,پشت خط زن تیم والیبال نوجوانا...   \n",
              "2  کرونا,دانش,علم,دانش بنیان,شرکت های دانش بنیان,...   \n",
              "3                                                NaN   \n",
              "4                                                NaN   \n",
              "\n",
              "                                                 UN7  ...  \\\n",
              "0  http://media.shabestan.ir/Original/1400/11/07/...  ...   \n",
              "1  https://media.farsnews.ir/Uploaded/Files/Image...  ...   \n",
              "2  https://media.farsnews.ir/Uploaded/Files/Image...  ...   \n",
              "3  https://www.shahrsakhtafzar.com/fa/images/1400...  ...   \n",
              "4  https://media.hawzahnews.com/d/2020/11/28/4/10...  ...   \n",
              "\n",
              "                                                Text  UN10  UN11   UN12  UN13  \\\n",
              "0  «نصیر قمصری»، در گفت وگو با خبرنگار خبرگزاری ش...  True  True  False   NaN   \n",
              "1  به گزارش خبرنگار ورزشی خبرگزاری فارس، تیم والی...  True  True  False   NaN   \n",
              "2  از این رو، توجه به توسعه کسب وکارهای خرد و کوچ...  True  True  False   NaN   \n",
              "3  اگر به یاد داشته یا به قیمت های عنوان شده در ب...  True  True  False   NaN   \n",
              "4  انتشارات انقلاب اسلامی با همکاری صدای جمهوری ا...  True  True  False   NaN   \n",
              "\n",
              "                               UN14                              UN15 UN16  \\\n",
              "0  2022-01-29 06:38:04.225284+00:00  2022-01-29 06:45:04.225595+00:00   fa   \n",
              "1  2021-08-30 16:33:01.639304+00:00  2021-08-30 16:45:01.639629+00:00   fa   \n",
              "2  2021-02-02 07:44:59.532751+00:00  2021-02-02 07:47:59.533036+00:00   fa   \n",
              "3  2022-01-09 07:36:06.211189+00:00  2022-01-09 07:45:06.211481+00:00   fa   \n",
              "4  2021-01-04 13:08:04.787975+00:00  2021-01-04 13:41:04.788242+00:00   fa   \n",
              "\n",
              "  UN17   UN18  \n",
              "0  NaN  False  \n",
              "1  NaN  False  \n",
              "2  NaN  False  \n",
              "3  NaN  False  \n",
              "4  NaN  False  \n",
              "\n",
              "[5 rows x 22 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-830997c3-66bc-40e3-8b7a-22525a9f2b1e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>UN0</th>\n",
              "      <th>UN1</th>\n",
              "      <th>URL</th>\n",
              "      <th>Title</th>\n",
              "      <th>Category</th>\n",
              "      <th>UN3</th>\n",
              "      <th>UN4</th>\n",
              "      <th>UN5</th>\n",
              "      <th>UN6</th>\n",
              "      <th>UN7</th>\n",
              "      <th>...</th>\n",
              "      <th>Text</th>\n",
              "      <th>UN10</th>\n",
              "      <th>UN11</th>\n",
              "      <th>UN12</th>\n",
              "      <th>UN13</th>\n",
              "      <th>UN14</th>\n",
              "      <th>UN15</th>\n",
              "      <th>UN16</th>\n",
              "      <th>UN17</th>\n",
              "      <th>UN18</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4612103</td>\n",
              "      <td>MoL7VM5vRhfPHAszgyToRM</td>\n",
              "      <td>khabarfarsi.com/u/115997967</td>\n",
              "      <td>اجرای ویژه برنامه های گرامیداشت دهه فجر در کان...</td>\n",
              "      <td>فرهنگ و هنر</td>\n",
              "      <td>http://shabestan.ir/detail/News/1141850</td>\n",
              "      <td>شبستان</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>http://media.shabestan.ir/Original/1400/11/07/...</td>\n",
              "      <td>...</td>\n",
              "      <td>«نصیر قمصری»، در گفت وگو با خبرنگار خبرگزاری ش...</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2022-01-29 06:38:04.225284+00:00</td>\n",
              "      <td>2022-01-29 06:45:04.225595+00:00</td>\n",
              "      <td>fa</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3094439</td>\n",
              "      <td>Tp4kXv3A3oB7hRV8kb5oTn</td>\n",
              "      <td>khabarfarsi.com/u/108126652</td>\n",
              "      <td>والیبال نوجوانان جهان| ستاره ای که همچنان می د...</td>\n",
              "      <td>ورزشی</td>\n",
              "      <td>https://farsnews.ir/news/14000608000920/%D9%88...</td>\n",
              "      <td>خبرگزاری فارس</td>\n",
              "      <td>NaN</td>\n",
              "      <td>امیرمحمد گل زاده,پشت خط زن تیم والیبال نوجوانا...</td>\n",
              "      <td>https://media.farsnews.ir/Uploaded/Files/Image...</td>\n",
              "      <td>...</td>\n",
              "      <td>به گزارش خبرنگار ورزشی خبرگزاری فارس، تیم والی...</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2021-08-30 16:33:01.639304+00:00</td>\n",
              "      <td>2021-08-30 16:45:01.639629+00:00</td>\n",
              "      <td>fa</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>619358</td>\n",
              "      <td>ioxLEiYLABpttJPWbuhxJv</td>\n",
              "      <td>khabarfarsi.com/u/98835151</td>\n",
              "      <td>شرکت های دانش بنیان ضربات اقتصادی همه گیری کرو...</td>\n",
              "      <td>اقتصادی</td>\n",
              "      <td>https://farsnews.ir/news/13991113000098/%D8%B4...</td>\n",
              "      <td>خبرگزاری فارس</td>\n",
              "      <td>NaN</td>\n",
              "      <td>کرونا,دانش,علم,دانش بنیان,شرکت های دانش بنیان,...</td>\n",
              "      <td>https://media.farsnews.ir/Uploaded/Files/Image...</td>\n",
              "      <td>...</td>\n",
              "      <td>از این رو، توجه به توسعه کسب وکارهای خرد و کوچ...</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2021-02-02 07:44:59.532751+00:00</td>\n",
              "      <td>2021-02-02 07:47:59.533036+00:00</td>\n",
              "      <td>fa</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4434763</td>\n",
              "      <td>S2ktAnodxNo5rFry9WBG3Z</td>\n",
              "      <td>khabarfarsi.com/u/115005815</td>\n",
              "      <td>بزرگترین مشکل سامسونگ S21 FE 5G چیست و چرا نبا...</td>\n",
              "      <td>فناوری و ارتباطات</td>\n",
              "      <td>https://www.shahrsakhtafzar.com/fa/news/smartp...</td>\n",
              "      <td>شهر سخت افزار</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://www.shahrsakhtafzar.com/fa/images/1400...</td>\n",
              "      <td>...</td>\n",
              "      <td>اگر به یاد داشته یا به قیمت های عنوان شده در ب...</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2022-01-09 07:36:06.211189+00:00</td>\n",
              "      <td>2022-01-09 07:45:06.211481+00:00</td>\n",
              "      <td>fa</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>276713</td>\n",
              "      <td>bU8DQribGWLJtWvTrV5Sb8</td>\n",
              "      <td>khabarfarsi.com/u/97546025</td>\n",
              "      <td>فیلم | بازخوانی موضع رهبر معظم انقلاب درباره ج...</td>\n",
              "      <td>سیاسی</td>\n",
              "      <td>https://www.hawzahnews.com/news/937281/%D9%81%...</td>\n",
              "      <td>حوزه</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>https://media.hawzahnews.com/d/2020/11/28/4/10...</td>\n",
              "      <td>...</td>\n",
              "      <td>انتشارات انقلاب اسلامی با همکاری صدای جمهوری ا...</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2021-01-04 13:08:04.787975+00:00</td>\n",
              "      <td>2021-01-04 13:41:04.788242+00:00</td>\n",
              "      <td>fa</td>\n",
              "      <td>NaN</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 22 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-830997c3-66bc-40e3-8b7a-22525a9f2b1e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-830997c3-66bc-40e3-8b7a-22525a9f2b1e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-830997c3-66bc-40e3-8b7a-22525a9f2b1e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "dataset.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_fcbV2Q-mC-x"
      },
      "outputs": [],
      "source": [
        "data = dataset.loc[:, ['Title', 'Category', 'Text', 'URL']]\n",
        "data = data.dropna(axis='index', how='any')\n",
        "data = data.to_dict('records')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "SQOAtgLY6ttf"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "    با اجرای قطعه کد زیر، تعداد خبرهای هر دسته و تعداد کل خبرها را می‌توان مشاهده کرد.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "id": "HIkDaiJC6ttg",
        "outputId": "aa668e14-e227-4c41-b366-41f8009d5d90"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                 دسته  تعداد\n",
              "0            کل خبرها   4678\n",
              "1         فرهنگ و هنر    416\n",
              "2               ورزشی    749\n",
              "3             اقتصادی    515\n",
              "4   فناوری و ارتباطات    301\n",
              "5               سیاسی    477\n",
              "6        علمی و پزشکی    428\n",
              "7           بین الملل    678\n",
              "8               مذهبی    389\n",
              "9               حوادث    306\n",
              "10            اجتماعی    419"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8835792b-83c7-4b6c-b2ce-1aedc3ad967b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>دسته</th>\n",
              "      <th>تعداد</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>کل خبرها</td>\n",
              "      <td>4678</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>فرهنگ و هنر</td>\n",
              "      <td>416</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ورزشی</td>\n",
              "      <td>749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>اقتصادی</td>\n",
              "      <td>515</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>فناوری و ارتباطات</td>\n",
              "      <td>301</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>سیاسی</td>\n",
              "      <td>477</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>علمی و پزشکی</td>\n",
              "      <td>428</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>بین الملل</td>\n",
              "      <td>678</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>مذهبی</td>\n",
              "      <td>389</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>حوادث</td>\n",
              "      <td>306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>اجتماعی</td>\n",
              "      <td>419</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8835792b-83c7-4b6c-b2ce-1aedc3ad967b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8835792b-83c7-4b6c-b2ce-1aedc3ad967b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8835792b-83c7-4b6c-b2ce-1aedc3ad967b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "CATEGORIES = pd.unique(dataset['Category'])\n",
        "CLASSES_CATEGORIES = {k:v for k, v in zip(np.arange(len(CATEGORIES)), CATEGORIES)}\n",
        "\n",
        "CATEGORIES_CLASSES = {v: k for k, v in CLASSES_CATEGORIES.items()}\n",
        "\n",
        "length_dict = {key: 0 for key in CATEGORIES}\n",
        "for record in data:\n",
        "    length_dict[record['Category']] += 1\n",
        "\n",
        "df_dict = {\n",
        "    'دسته': CATEGORIES,\n",
        "    'تعداد': length_dict.values(),\n",
        "}\n",
        "\n",
        "df = pd.DataFrame(df_dict)\n",
        "df.index += 1\n",
        "df.loc[0] = ['کل خبرها', len(data)]\n",
        "df = df.sort_index()\n",
        "display(df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "YuwgMYpB6ttg"
      },
      "source": [
        "<div>\n",
        "    <h3 style='direction:rtl;text-align:justify;'>\n",
        "        ٢. پیش پردازش اولیه‌ی متن\n",
        "    </h3>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        ابزار مورد استفاده برای پیش‌پردازش متن ورودی به صورت ماژولار طراحی شده است؛ به طوری که با صدا زدن تابع preprocess از آن، متن داده شده با عبور از یک خط لوله به صورت مرحله به مرحله تغییر می‌کند تا به یک ساختار استاندارد برسد. این مراحل عبارتند از:\n",
        "    </p>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <li style='direction:rtl;text-align:justify;'>\n",
        "        نرمال سازی داده‌ها (normalize)\n",
        "    </li>\n",
        "    <li style='direction:rtl;text-align:justify;'>\n",
        "        حذف لینک‌ها (remove_links)\n",
        "    </li>\n",
        "    <li style='direction:rtl;text-align:justify;'>\n",
        "        حذف نشانه‌های نگارشی (remove_punctuations)\n",
        "    </li>\n",
        "    <li style='direction:rtl;text-align:justify;'>\n",
        "        واحد سازی داده‌ها (word_tokenize)\n",
        "    </li>\n",
        "    <li style='direction:rtl;text-align:justify;'>\n",
        "        حذف کلمات نامعتبر (remove_invalid_words)\n",
        "    </li>\n",
        "    <li style='direction:rtl;text-align:justify;'>\n",
        "        حذف ایست‌واژه‌ها (remove_stopwords)\n",
        "    </li>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "phcZ2s706ttg"
      },
      "outputs": [],
      "source": [
        "class Preprocessor:\n",
        "\n",
        "    def __init__(self, stopwords_path):\n",
        "        self.stopwords = []\n",
        "        with open(stopwords_path, encoding='utf-8') as file:\n",
        "            self.stopwords = file.read().split()\n",
        "\n",
        "    def preprocess(self, text):\n",
        "        text = self.normalize(text)\n",
        "        text = self.remove_links(text)\n",
        "        text = self.remove_punctuations(text)\n",
        "        words = self.word_tokenize(text)\n",
        "        words = self.remove_invalid_words(words)\n",
        "        words = self.remove_stopwords(words)\n",
        "        return words\n",
        "\n",
        "    def normalize(self, text):\n",
        "        return hazm.Normalizer().normalize(text)\n",
        "\n",
        "    def remove_links(self, text):\n",
        "        patterns = ['\\S*http\\S*', '\\S*www\\S*', '\\S+\\.ir\\S*', '\\S+\\.com\\S*', '\\S+\\.org\\S*', '\\S*@\\S*']\n",
        "        for pattern in patterns:\n",
        "            text = re.sub(pattern, ' ', text)\n",
        "        return text\n",
        "\n",
        "    def remove_punctuations(self, text):\n",
        "        return re.sub(f'[{punctuation}؟،٪×÷»«]+', '', text)\n",
        "\n",
        "    def word_tokenize(self, text):\n",
        "        return hazm.word_tokenize(text)\n",
        "\n",
        "    def remove_invalid_words(self, words):\n",
        "        return [word for word in words if len(word) > 3 or re.match('^[\\u0600-\\u06FF]{2,3}$', word)]\n",
        "\n",
        "    def remove_stopwords(self, words):\n",
        "        return [word for word in words if word not in self.stopwords]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2tqXc9WR6tth"
      },
      "outputs": [],
      "source": [
        "def get_preprocessed_texts(data, preprocessor):\n",
        "    preprocessed_title = []\n",
        "    preprocessed_texts = []\n",
        "    preprocessed_total = []\n",
        "    titles = [doc['Title'] for doc in data]\n",
        "    preprocessed_title = [preprocessor.preprocess(title) for title in titles]\n",
        "    texts = [doc['Text'] for doc in data]\n",
        "    preprocessed_texts = [preprocessor.preprocess(text) for text in tqdm(texts)]\n",
        "    preprocessed_total = [preprocessor.preprocess(' '.join([doc['Title'], doc['Text']])) for doc in tqdm(data)]\n",
        "    return preprocessed_title, preprocessed_texts, preprocessed_total"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "E3M4tMlL6tth"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        با اجرای قطعه کد زیر، یک instance از ماژول Preprocessor ایجاد کرده و شروع به پیش پردازش داده‌ها می‌کنیم، یا داده‌های پیش‌پردازش‌شده‌ی ذخیره‌شده را از فایل مربوطه بازیابی می‌کنیم.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Izni3XhQ6tth"
      },
      "outputs": [],
      "source": [
        "preprocessor = Preprocessor(stopwords_path='./stopwords.txt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3MYNCBkS8CTe",
        "outputId": "cd918bfa-c418-4910-96a3-aeb9c4989e37"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4678/4678 [00:06<00:00, 707.35it/s]\n",
            "100%|██████████| 4678/4678 [00:10<00:00, 454.93it/s]\n"
          ]
        }
      ],
      "source": [
        "preprocessed_title, preprocessed_texts, preprocessed_total = get_preprocessed_texts(data, preprocessor)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "7GgSPBt36tti"
      },
      "source": [
        "<div>\n",
        "    <h3 style='direction:rtl;text-align:justify;'>\n",
        "        ٣. دسته‌بندی اخبار بر اساس برچسب خبر\n",
        "    </h3>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این مرحله با استفاده از دو مدل متفاوت زبانی، یکی مبتنی بر روش سنتی رگرسیون لجستیک و دیگری بر پایه ترنسفرمر، شروع به یادگیری بر روی خبرهای استخراج شده می‌کنیم و در انتها، نتیجه اجرای ده کوئری را بر روی هر یک مشاهده می‌کنیم. در نهایت، ارزیابی‌های مورد بحث در داک تمرین را بر روی داده‌های تست انجام می‌دهیم.\n",
        "    </p>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        کوئری‌های مورد استفاده در این بخش در زیر آمده‌اند.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qASkPvTs6tti"
      },
      "outputs": [],
      "source": [
        "QUERIES = [\n",
        "    \"نتیجه توافق برجام\",\n",
        "    \"حمله ارتش روسیه به اوکراین\",\n",
        "    \"افزایش نرخ تورم در کشور\",\n",
        "    \"آمار فوتی‌های کرونا\",\n",
        "    \"اقدامات شهرداری تهران\",\n",
        "    \"صعود ایران به جام جهانی\",\n",
        "    \"بیماری آبله میمونی\",\n",
        "    \"جشنواره فیلم فجر\",\n",
        "    \"رونمایی از گوشی جدید اپل\",\n",
        "    \"روش پخت غذا\",\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l8bXllAK6ttj"
      },
      "outputs": [],
      "source": [
        "labels = [CATEGORIES_CLASSES[d['Category']] for d in data]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B75i0xR2yjH9",
        "outputId": "037a57c9-f5b8-40bd-ff50-743920a8fa89"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['فرهنگ و هنر', 'ورزشی', 'اقتصادی', 'فناوری و ارتباطات', 'سیاسی', 'علمی و پزشکی', 'بین الملل', 'مذهبی', 'حوادث', 'اجتماعی'])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "CATEGORIES_CLASSES.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hUCIx2PM2SfH",
        "outputId": "d80c97b1-30c6-4c7e-e825-4afac04c7735"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "CLASSES_CATEGORIES.keys()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "ABtH0P3n6tti"
      },
      "source": [
        "<div>\n",
        "    <h4 style='direction:rtl;text-align:justify;'>\n",
        "        ١. مدل مبتنی بر روش Logistic Regression\n",
        "    </h4>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این بخش از کلاس TfidfVectorizer از کتاب‌خانه‌ی sklearn استفاده کرده و مجموعه کلمات یکتای موجود در عناوین خبرها را به عنوان ورودی به آن می‌دهیم. این کلمات یکتا را از روی فایل ذخیره شده‌ی vocabulary.pickle می‌خوانیم. در ادامه، یک مدل ایجاد کرده و بردار embedding عنوان هر خبر را به روش tf-idf به دست می‌آوریم. سپس، داده‌هایمان را به دو مجموعه train و test افراز می‌کنیم. حال، با استفاده از کلاس LogisticRegression کتاب‌خانه‌ی sklearn شروع به یادگیری داده‌های train می‌کنیم.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pPQFUkyu6ttj"
      },
      "outputs": [],
      "source": [
        "class TF_IDF:\n",
        "\n",
        "    def __init__(self):\n",
        "        self.vectorizer = TfidfVectorizer()\n",
        "\n",
        "    def fit_vectorizer(self, data):\n",
        "        self.vectorizer.fit(list(map(lambda doc: ' '.join(doc), data)))\n",
        "    \n",
        "    def transform_vectorizer(self, preprocessed_title, preprocessed_texts):\n",
        "        embeddings = []\n",
        "        for title, text in zip(preprocessed_title, preprocessed_texts):\n",
        "          emb_title = self.vectorizer.transform([' '.join(title)]).toarray()[0]\n",
        "          emb_text = self.vectorizer.transform([' '.join(text)]).toarray()[0]\n",
        "          embeddings.append(((emb_text * 19) + emb_title) / 20)\n",
        "        return embeddings\n",
        "\n",
        "    def save_TF_IDF_model(self, path=\"TF_IDF_model.pickle\"):\n",
        "        with open(path, \"wb\") as file:\n",
        "            pickle.dump(self.vectors.todense().tolist(), file)\n",
        "\n",
        "    def load_TF_IDF_model(self, path=\"TF_IDF_model.pickle\"):\n",
        "        with open(path, \"rb\") as file:\n",
        "            return pickle.load(file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aYf1ZnBD6ttj"
      },
      "outputs": [],
      "source": [
        "TF_IDF_model = TF_IDF()\n",
        "TF_IDF_model.fit_vectorizer(preprocessed_total)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_2O3Noj9r_rf"
      },
      "outputs": [],
      "source": [
        "embeddings = TF_IDF_model.transform_vectorizer(preprocessed_title, preprocessed_texts)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fBhxqbi76ttj"
      },
      "outputs": [],
      "source": [
        "train_vectors, test_vectors, train_labels, test_labels = train_test_split(embeddings, labels, test_size=0.15, shuffle=True, random_state=11)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yUElni9L6ttj"
      },
      "outputs": [],
      "source": [
        "import joblib\n",
        "classifier = joblib.load('logistic_regression.joblib')\n",
        "\n",
        "# classifier = LogisticRegression(max_iter=1000, random_state=0)\n",
        "# classifier.fit(train_vectors, train_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "X4KAwZxJ6ttj"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این قسمت، دسته‌ی هر یک از کوئری‌های مشخص شده را با استفاده از مدل یادگیری پیش‌بینی می‌کنیم.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JjwNgR-VUyBR"
      },
      "outputs": [],
      "source": [
        "def predict_logistic(x):\n",
        "    preprocessed_query = ' '.join(preprocessor.preprocess(x))\n",
        "    query_embed = TF_IDF_model.vectorizer.transform([preprocessed_query])\n",
        "    predicted_class_code = classifier.predict(query_embed)[0]\n",
        "    return CLASSES_CATEGORIES[predicted_class_code]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "VAcm8KhsVW4F",
        "outputId": "f959d99b-452a-4e31-e5f9-e6bf351f7cc9"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ورزشی'"
            ]
          },
          "execution_count": 80,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "predict_logistic('بایرن مونیخ قهرمان لیگ قهرمانان اروپا شد.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "PCCkdl6wuID6",
        "outputId": "81c7058a-e113-439f-eee6-2ed29b534899"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-469eb91e-0555-4f68-9cc6-897cc9d8ff5f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Query</th>\n",
              "      <th>Predicted Category</th>\n",
              "      <th>Probability</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>نتیجه توافق برجام</td>\n",
              "      <td>بین الملل</td>\n",
              "      <td>0.359573</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>حمله ارتش روسیه به اوکراین</td>\n",
              "      <td>بین الملل</td>\n",
              "      <td>0.923956</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>افزایش نرخ تورم در کشور</td>\n",
              "      <td>اقتصادی</td>\n",
              "      <td>0.567293</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>آمار فوتی‌های کرونا</td>\n",
              "      <td>اجتماعی</td>\n",
              "      <td>0.289385</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>اقدامات شهرداری تهران</td>\n",
              "      <td>ورزشی</td>\n",
              "      <td>0.158217</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>صعود ایران به جام جهانی</td>\n",
              "      <td>ورزشی</td>\n",
              "      <td>0.477349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>بیماری آبله میمونی</td>\n",
              "      <td>علمی و پزشکی</td>\n",
              "      <td>0.324221</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>جشنواره فیلم فجر</td>\n",
              "      <td>فرهنگ و هنر</td>\n",
              "      <td>0.978701</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>رونمایی از گوشی جدید اپل</td>\n",
              "      <td>فناوری و ارتباطات</td>\n",
              "      <td>0.733749</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>روش پخت غذا</td>\n",
              "      <td>علمی و پزشکی</td>\n",
              "      <td>0.340093</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-469eb91e-0555-4f68-9cc6-897cc9d8ff5f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-469eb91e-0555-4f68-9cc6-897cc9d8ff5f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-469eb91e-0555-4f68-9cc6-897cc9d8ff5f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                        Query Predicted Category  Probability\n",
              "0           نتیجه توافق برجام          بین الملل     0.359573\n",
              "1  حمله ارتش روسیه به اوکراین          بین الملل     0.923956\n",
              "2     افزایش نرخ تورم در کشور            اقتصادی     0.567293\n",
              "3         آمار فوتی‌های کرونا            اجتماعی     0.289385\n",
              "4       اقدامات شهرداری تهران              ورزشی     0.158217\n",
              "5     صعود ایران به جام جهانی              ورزشی     0.477349\n",
              "6          بیماری آبله میمونی       علمی و پزشکی     0.324221\n",
              "7            جشنواره فیلم فجر        فرهنگ و هنر     0.978701\n",
              "8    رونمایی از گوشی جدید اپل  فناوری و ارتباطات     0.733749\n",
              "9                 روش پخت غذا       علمی و پزشکی     0.340093"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "queries_class_prediction = []\n",
        "\n",
        "for query in QUERIES:\n",
        "    preprocessed_query = ' '.join(preprocessor.preprocess(query))\n",
        "    query_embed = TF_IDF_model.vectorizer.transform([preprocessed_query])\n",
        "    predicted_class_code = classifier.predict(query_embed)[0]\n",
        "    probability = classifier.predict_proba(query_embed)[0][predicted_class_code]\n",
        "    queries_class_prediction.append({\n",
        "        'Query': query,\n",
        "        'Predicted Category': CLASSES_CATEGORIES[predicted_class_code],\n",
        "        'Probability': probability\n",
        "    })\n",
        "\n",
        "display(pd.DataFrame(queries_class_prediction))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "5AjPHN6Q6ttk"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این قسمت، بررسی می‌کنیم که مدل یادگیری‌مان بر روی داده‌های test چه عملکردی دارد. با اجرای قطعه کد زیر، معیار ارزیابی F1-Macro و نیز accuracy و ماتریس درهم‌ریختگی محاسبه شده و نمایش داده می‌شوند. (در این ماتریس، ستون‌ها ground truth و سطرها پیش‌بینی ما هستند.)\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "id": "utqZ_WynuPo_",
        "outputId": "43fd6a38-2b3b-493a-b5af-2ed743a0fd6e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accurracy: 0.789\n",
            "\n",
            "F1-Macro: 0.776\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-a01f6936-0062-4c5b-b620-3d3419453f14\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>فرهنگ و هنر</th>\n",
              "      <th>ورزشی</th>\n",
              "      <th>اقتصادی</th>\n",
              "      <th>فناوری و ارتباطات</th>\n",
              "      <th>سیاسی</th>\n",
              "      <th>علمی و پزشکی</th>\n",
              "      <th>بین الملل</th>\n",
              "      <th>مذهبی</th>\n",
              "      <th>حوادث</th>\n",
              "      <th>اجتماعی</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>فرهنگ و هنر</th>\n",
              "      <td>53</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ورزشی</th>\n",
              "      <td>2</td>\n",
              "      <td>109</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>اقتصادی</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>61</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>فناوری و ارتباطات</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>سیاسی</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>52</td>\n",
              "      <td>1</td>\n",
              "      <td>15</td>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>علمی و پزشکی</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>53</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>بین الملل</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>91</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>مذهبی</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>45</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>حوادث</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>39</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>اجتماعی</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a01f6936-0062-4c5b-b620-3d3419453f14')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a01f6936-0062-4c5b-b620-3d3419453f14 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a01f6936-0062-4c5b-b620-3d3419453f14');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                   فرهنگ و هنر  ورزشی  اقتصادی  فناوری و ارتباطات  سیاسی  \\\n",
              "فرهنگ و هنر                 53      3        1                  1      3   \n",
              "ورزشی                        2    109        0                  0      0   \n",
              "اقتصادی                      3      0       61                  1      6   \n",
              "فناوری و ارتباطات            1      1        2                 24      0   \n",
              "سیاسی                        1      0        4                  1     52   \n",
              "علمی و پزشکی                 1      1        1                  1      0   \n",
              "بین الملل                    0      1        1                  0      1   \n",
              "مذهبی                        4      0        2                  0      6   \n",
              "حوادث                        0      0        1                  0      0   \n",
              "اجتماعی                      2      0        6                  0      5   \n",
              "\n",
              "                   علمی و پزشکی  بین الملل  مذهبی  حوادث  اجتماعی  \n",
              "فرهنگ و هنر                   3          2      3      0        3  \n",
              "ورزشی                         0          0      0      0        3  \n",
              "اقتصادی                       2          6      0      0        2  \n",
              "فناوری و ارتباطات             1          1      0      0        2  \n",
              "سیاسی                         1         15      8      1        2  \n",
              "علمی و پزشکی                 53          2      0      0        6  \n",
              "بین الملل                     1         91      0      0        0  \n",
              "مذهبی                         4          1     45      0        4  \n",
              "حوادث                         0          4      1     39        1  \n",
              "اجتماعی                       4          2      0      0       27  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "mean_acc = classifier.score(test_vectors, test_labels)\n",
        "print(f\"Accurracy: {round(mean_acc, 3)}\\n\")\n",
        "\n",
        "predicted_labels = classifier.predict(test_vectors)\n",
        "f1_macro = f1_score(test_labels, predicted_labels, labels=np.arange(10), average='macro')\n",
        "print(f\"F1-Macro: {round(f1_macro, 3)}\\n\")\n",
        "\n",
        "confusion_mat = confusion_matrix(test_labels, predicted_labels, labels=np.arange(10))\n",
        "confusion_mat_df = pd.DataFrame(confusion_mat).rename(columns=CLASSES_CATEGORIES, index=CLASSES_CATEGORIES)\n",
        "display(confusion_mat_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_lfL40LKv4It"
      },
      "outputs": [],
      "source": [
        "classifier = LogisticRegression(max_iter=1000, random_state=0)\n",
        "scoring = ['accuracy', 'precision_macro', 'recall_macro','precision_micro', 'recall_micro', 'f1_micro', 'f1_macro']\n",
        "scores = cross_validate(classifier, embeddings, labels, scoring=scoring)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "in this section we calculate f1_score accuracy precision and recall based on macro and micro, and report them with std by using cross validation"
      ],
      "metadata": {
        "id": "DH-QlGgqJoSh"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eWzYFM_XzUjE",
        "outputId": "dad1dc9e-a7eb-4f3f-fdff-0e0040fb4f77"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.788 accuracy with a standard deviation of 0.008\n",
            "0.788 precision_macro with a standard deviation of 0.007\n",
            "0.765 recall_macro with a standard deviation of 0.010\n",
            "0.788 precision_micro with a standard deviation of 0.008\n",
            "0.788 recall_micro with a standard deviation of 0.008\n",
            "0.788 f1_micro with a standard deviation of 0.008\n",
            "0.772 f1_macro with a standard deviation of 0.009\n"
          ]
        }
      ],
      "source": [
        "for item in scoring:\n",
        "  print(f\"%0.3f {item} with a standard deviation of %0.3f\" % (scores[f'test_{item}'].mean(), scores[f'test_{item}'].std()))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L2P8yTqeK13T",
        "outputId": "7ec740a4-9a0f-4619-c823-e8f0112d2d5c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['logistic_regression.joblib']"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import joblib\n",
        "joblib.dump(classifier, 'logistic_regression.joblib')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "s5VKEYiI6ttk"
      },
      "source": [
        "<div>\n",
        "    <h4 style='direction:rtl;text-align:justify;'>\n",
        "        ٢. مدل مبتنی بر پایه Transformer\n",
        "    </h4>\n",
        "</div>\n",
        "\n",
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این بخش، با استفاده از کلاس AutoModelForSequenceClassification از کتاب‌خانه‌ی transformers نوشته‌های موجود را به دسته‌ی متناظرشان مرتبط می‌کنیم. برای این کار، از مدل BigBird که از پیش تمرین داده شده است به عنوان زیربنا استفاده می‌کنیم و سپس با افراز داده‌های موجود به سه بخش train و validation و test، با استفاده از دو بخش train و validation آن را بیش‌تر در راستای هدفمان که تشخیص دسته‌ی خبرهاست تمرین می‌دهیم.\n",
        "        همچنین در طول تمرین دادن از یک AutoTokenizer مبتنی بر ParsBERT استفاده می‌کنیم.\n",
        "    </p>\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        برای تمرین دادن این مدل از کلاس Trainer از همان کتاب‌خانه‌ی transformers استفاده می‌کنیم و پارامترهای تمرین دادن را هم مطابق آنچه در ادامه مشهود است با استفاده از متغیر training_args به آن می‌دهیم. در نهایت پس از پایان فرایند تمرین دادن، مدل را ذخیره می‌کنیم تا در دفعات بعد بتوانیم به صورت مستقیم آن را از فایل بازیابی کنیم و از آن استفاده کنیم.\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "szAs8eXz7OlW"
      },
      "outputs": [],
      "source": [
        "torch.cuda.ipc_collect()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dKD7cWIS3QGX"
      },
      "outputs": [],
      "source": [
        "import gc\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 376,
          "referenced_widgets": [
            "06ebb679792947aaa8e21e50ec78cb91",
            "699e4f2ffad2411283ab0bdbbd0300bd",
            "eabb08084a194ffba888eadba4efc8ca",
            "21ed63c2557b4d6da1ed92da82e90f5d",
            "c3e3d2d2de014c97b9c668046b23e1fd",
            "9b3f82cb26b945969341fcd6cc0ef824",
            "6327641838f34de5b344071f66a35328",
            "73bc48efacc84e34bcf24e948d7e3236",
            "47d4c66991b74716a55f5680e815c433",
            "f82099530f43421d80e93add7d6f3a37",
            "ea8816c60bd249d0b6daf2192a2eebc2",
            "ca1e80e36b2945afa7ee544c39c838e1",
            "102b88213bbf45648d0be29344b1b341",
            "2434da5a50824baa861220fe906f3a60",
            "694671cba5af41298e88ae57d4d0cec4",
            "a9d4794bb7924cb4935b6430a5abb48e",
            "558737ec88ea45b486ac279dd115ffb4",
            "fd0212f1be874e20a1fe20f9aedb3c84",
            "8acb86b9ac124875a56225d42be1da6f",
            "d56e6cac9499463aa36a1e05181b59cb",
            "baaef6c112444f0a9afc1c0235a6c2db",
            "a2533228d23d4751b7d28dcd1a1f1503",
            "19ad38ddedc1478aa48f2ec8c038883c",
            "f8568441bf89474998aab4f9a24d7063",
            "89b14867d50a44898aeea85530ce8550",
            "c528c549c0a14eeeb12a6f357102658e",
            "fb30eea46b544282bd6395c481525b4d",
            "3e8557f582154ccf8f11479e15e8b878",
            "dbea1b121f004f03969e6f0bb9ab2cef",
            "7f48fe4877f14cd488378a8093912153",
            "2ce66d1d6942441b8c907f72c3e1b585",
            "f91c768b10ba479da4f1033f1ed01a40",
            "aa6d2faa563b466f87e958ebaae92003",
            "5bc29eaf3e3b45949ffa7e9c69bfbe72",
            "dc6d9746a32c4f9ca99192dea9422901",
            "7f834102ef184bf5aa1e16bba7ca331c",
            "ee9809d10207494284fd27feae6219d2",
            "fb4924e359184cb1bd400f74b34e59e0",
            "c69ef06ef1f44ebbad798d1195150bb5",
            "1c5d8e171ba14dcab4cee6b93cd564d0",
            "8f0a8a6e6a844ee291019d207d3f48c6",
            "c2f5f67c64c9451fb743c7a0c85cb278",
            "c1b37f02ab4f40dbb3d2db2303e7990e",
            "e062ecafae4340ca8f58c617780e03c9",
            "fefa4dac2ca440359cc976c837021a10",
            "c430b8f4ead84b7daebdb29f991071b1",
            "0197fcae4e7b4774b24be12d3db61b52",
            "099209362d1c4da29139f55266b243ae",
            "94ce0d1685064d54abec49757f25871d",
            "f2c7202269a146fab714ea063b6330f2",
            "fbfcb20e89794c229e0648cbf35af849",
            "694c9eef216147b980eba1a63797c07a",
            "e214603f88fc43a88ea6b76a6ac2ba50",
            "7aac5aab9d2346b2ad30b5ea0ea5654e",
            "b9e56b14a0d546e298e1355eba110bc4"
          ]
        },
        "id": "T5KegbUIoeLe",
        "outputId": "7a735c60-8734-43a6-9e4e-be94021c4ed5"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/837 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "06ebb679792947aaa8e21e50ec78cb91"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)\"pytorch_model.bin\";:   0%|          | 0.00/314M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ca1e80e36b2945afa7ee544c39c838e1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at SajjadAyoubi/distil-bigbird-fa-zwnj were not used when initializing BigBirdModel: ['cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BigBirdModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BigBirdModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BigBirdModel were not initialized from the model checkpoint at SajjadAyoubi/distil-bigbird-fa-zwnj and are newly initialized: ['bert.pooler.bias', 'bert.pooler.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "Some weights of the model checkpoint at SajjadAyoubi/distil-bigbird-fa-zwnj were not used when initializing BigBirdModel: ['cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BigBirdModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BigBirdModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BigBirdModel were not initialized from the model checkpoint at SajjadAyoubi/distil-bigbird-fa-zwnj and are newly initialized: ['bert.pooler.bias', 'bert.pooler.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/365 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "19ad38ddedc1478aa48f2ec8c038883c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/426k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5bc29eaf3e3b45949ffa7e9c69bfbe72"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fefa4dac2ca440359cc976c837021a10"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "\n",
        "MODEL_NAME = \"SajjadAyoubi/distil-bigbird-fa-zwnj\"\n",
        "model = BigBirdModel.from_pretrained(MODEL_NAME, block_size=32)\n",
        "model = BigBirdModel.from_pretrained(MODEL_NAME, attention_type=\"original_full\")\n",
        "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h0WIA4s0GJWH"
      },
      "outputs": [],
      "source": [
        "dataset = dataset.loc[:, ['Title', 'Category', 'Text', 'URL']]\n",
        "dataset = dataset.dropna(axis='index', how='any')\n",
        "\n",
        "dataset['category_code'] = dataset['Category'].apply(lambda x: CATEGORIES_CLASSES[x])\n",
        "labels = dataset['category_code'].to_numpy()\n",
        "X_text = [' '.join(x) for x in preprocessed_total]\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X_text, labels, test_size=0.2, random_state=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "w8OnXqwdEiCB"
      },
      "outputs": [],
      "source": [
        "train_encodings = tokenizer(X_train, truncation=True, padding=True)\n",
        "test_encodings = tokenizer(X_test, truncation=True, padding=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GyS6QNepCGwn"
      },
      "outputs": [],
      "source": [
        "class NewsDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "\n",
        "train_dataset = NewsDataset(train_encodings, y_train)\n",
        "test_dataset = NewsDataset(test_encodings, y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N94N078vof41"
      },
      "outputs": [],
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "cl_model = AutoModelForSequenceClassification.from_pretrained('transformer_model').to(device)\n",
        "MODEL_NAME = \"SajjadAyoubi/distil-bigbird-fa-zwnj\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zMOCExKVFM3E",
        "outputId": "7e871eb4-32fb-4abb-da29-2f0362e5b9d1"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at SajjadAyoubi/distil-bigbird-fa-zwnj were not used when initializing BigBirdForSequenceClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.bias']\n",
            "- This IS expected if you are initializing BigBirdForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BigBirdForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BigBirdForSequenceClassification were not initialized from the model checkpoint at SajjadAyoubi/distil-bigbird-fa-zwnj and are newly initialized: ['bert.pooler.weight', 'classifier.out_proj.bias', 'classifier.dense.weight', 'bert.pooler.bias', 'classifier.out_proj.weight', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "# cl_model = AutoModelForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=10).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "TzGzeB0AFV-3",
        "outputId": "a4f94c65-082a-4788-9eee-9030c9e2a2c3"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "***** Running training *****\n",
            "  Num examples = 3742\n",
            "  Num Epochs = 5\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1170\n",
            "  Number of trainable parameters = 79120906\n",
            "Attention type 'block_sparse' is not possible if sequence_length: 338 <= num global tokens: 2 * config.block_size + min. num sliding tokens: 3 * config.block_size + config.num_random_blocks * config.block_size + additional buffer: config.num_random_blocks * config.block_size = 352 with config.block_size = 32, config.num_random_blocks = 3. Changing attention type to 'original_full'...\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1170' max='1170' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1170/1170 11:08, Epoch 5/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>2.326200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>2.292800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>30</td>\n",
              "      <td>2.293700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>40</td>\n",
              "      <td>2.261300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>50</td>\n",
              "      <td>2.222100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>60</td>\n",
              "      <td>2.159800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>70</td>\n",
              "      <td>2.131600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>80</td>\n",
              "      <td>2.053600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>90</td>\n",
              "      <td>1.861300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>1.839700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>110</td>\n",
              "      <td>1.672800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>120</td>\n",
              "      <td>1.512900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>130</td>\n",
              "      <td>1.255600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>140</td>\n",
              "      <td>1.228600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>150</td>\n",
              "      <td>1.168100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>160</td>\n",
              "      <td>0.993200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>170</td>\n",
              "      <td>1.052400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>180</td>\n",
              "      <td>0.924500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>190</td>\n",
              "      <td>0.788800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>0.751300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>210</td>\n",
              "      <td>0.767900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>220</td>\n",
              "      <td>0.670600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>230</td>\n",
              "      <td>0.812300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>240</td>\n",
              "      <td>0.585000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>250</td>\n",
              "      <td>0.706300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>260</td>\n",
              "      <td>0.643400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>270</td>\n",
              "      <td>0.862200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>280</td>\n",
              "      <td>0.699500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>290</td>\n",
              "      <td>0.496900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>0.667900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>310</td>\n",
              "      <td>0.567800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>320</td>\n",
              "      <td>0.543600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>330</td>\n",
              "      <td>0.578400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>340</td>\n",
              "      <td>0.629000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>350</td>\n",
              "      <td>0.660200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>360</td>\n",
              "      <td>0.581100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>370</td>\n",
              "      <td>0.594100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>380</td>\n",
              "      <td>0.619900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>390</td>\n",
              "      <td>0.773100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>0.436200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>410</td>\n",
              "      <td>0.563400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>420</td>\n",
              "      <td>0.566300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>430</td>\n",
              "      <td>0.666600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>440</td>\n",
              "      <td>0.455100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>450</td>\n",
              "      <td>0.577200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>460</td>\n",
              "      <td>0.569100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>470</td>\n",
              "      <td>0.577400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>480</td>\n",
              "      <td>0.500500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>490</td>\n",
              "      <td>0.481500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.376500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>510</td>\n",
              "      <td>0.529100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>520</td>\n",
              "      <td>0.419200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>530</td>\n",
              "      <td>0.457200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>540</td>\n",
              "      <td>0.401400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>550</td>\n",
              "      <td>0.311000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>560</td>\n",
              "      <td>0.477200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>570</td>\n",
              "      <td>0.325800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>580</td>\n",
              "      <td>0.577300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>590</td>\n",
              "      <td>0.436500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>600</td>\n",
              "      <td>0.435300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>610</td>\n",
              "      <td>0.485100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>620</td>\n",
              "      <td>0.468200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>630</td>\n",
              "      <td>0.446100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>640</td>\n",
              "      <td>0.461900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>650</td>\n",
              "      <td>0.534300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>660</td>\n",
              "      <td>0.267300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>670</td>\n",
              "      <td>0.409300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>680</td>\n",
              "      <td>0.353000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>690</td>\n",
              "      <td>0.399500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>700</td>\n",
              "      <td>0.601800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>710</td>\n",
              "      <td>0.375500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>720</td>\n",
              "      <td>0.236000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>730</td>\n",
              "      <td>0.238700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>740</td>\n",
              "      <td>0.248400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>750</td>\n",
              "      <td>0.284700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>760</td>\n",
              "      <td>0.173100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>770</td>\n",
              "      <td>0.326300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>780</td>\n",
              "      <td>0.247400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>790</td>\n",
              "      <td>0.245500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>0.220900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>810</td>\n",
              "      <td>0.160100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>820</td>\n",
              "      <td>0.164800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>830</td>\n",
              "      <td>0.229000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>840</td>\n",
              "      <td>0.271300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>850</td>\n",
              "      <td>0.313300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>860</td>\n",
              "      <td>0.300400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>870</td>\n",
              "      <td>0.273000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>880</td>\n",
              "      <td>0.159500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>890</td>\n",
              "      <td>0.186300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>900</td>\n",
              "      <td>0.207900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>910</td>\n",
              "      <td>0.094400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>920</td>\n",
              "      <td>0.110100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>930</td>\n",
              "      <td>0.123900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>940</td>\n",
              "      <td>0.262200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>950</td>\n",
              "      <td>0.103200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>960</td>\n",
              "      <td>0.128200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>970</td>\n",
              "      <td>0.091100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>980</td>\n",
              "      <td>0.054200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>990</td>\n",
              "      <td>0.087100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.110800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1010</td>\n",
              "      <td>0.091000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1020</td>\n",
              "      <td>0.100400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1030</td>\n",
              "      <td>0.074600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1040</td>\n",
              "      <td>0.122800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1050</td>\n",
              "      <td>0.099800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1060</td>\n",
              "      <td>0.055000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1070</td>\n",
              "      <td>0.097100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1080</td>\n",
              "      <td>0.090800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1090</td>\n",
              "      <td>0.075900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1100</td>\n",
              "      <td>0.104000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1110</td>\n",
              "      <td>0.128700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1120</td>\n",
              "      <td>0.105200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1130</td>\n",
              "      <td>0.068300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1140</td>\n",
              "      <td>0.094400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1150</td>\n",
              "      <td>0.155900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1160</td>\n",
              "      <td>0.128800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1170</td>\n",
              "      <td>0.069200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/checkpoint-500\n",
            "Configuration saved in ./results/checkpoint-500/config.json\n",
            "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
            "Saving model checkpoint to ./results/checkpoint-1000\n",
            "Configuration saved in ./results/checkpoint-1000/config.json\n",
            "Model weights saved in ./results/checkpoint-1000/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1170, training_loss=0.5769668171548435, metrics={'train_runtime': 670.3803, 'train_samples_per_second': 27.91, 'train_steps_per_second': 1.745, 'total_flos': 1658816961930960.0, 'train_loss': 0.5769668171548435, 'epoch': 5.0})"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./results',          # output directory\n",
        "    num_train_epochs=5,            # total number of training epochs\n",
        "    per_device_train_batch_size=16,  # batch size per device during training\n",
        "    per_device_eval_batch_size=16,   # batch size for evaluation\n",
        "    warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
        "    weight_decay=0.01,               # strength of weight decay\n",
        "    logging_dir='./logs',            # directory for storing logs\n",
        "    logging_steps=10,\n",
        ")\n",
        "\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=cl_model,                      # the instantiated 🤗 Transformers model to be trained\n",
        "    args=training_args,                  # training arguments, defined above\n",
        "    train_dataset=train_dataset,         # training dataset\n",
        ")\n",
        "\n",
        "trainer.train()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "E_aia_eq6ttl"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این قسمت، دسته‌ی هر یک از کوئری‌های مشخص شده را با استفاده از مدل یادگیری پیش‌بینی می‌کنیم.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jd89VSgZTM-R",
        "outputId": "fa374f07-90c8-48e7-9f69-764966669478"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "BigBirdModel(\n",
              "  (embeddings): BigBirdEmbeddings(\n",
              "    (word_embeddings): Embedding(42000, 768, padding_idx=0)\n",
              "    (position_embeddings): Embedding(4096, 768)\n",
              "    (token_type_embeddings): Embedding(2, 768)\n",
              "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (encoder): BigBirdEncoder(\n",
              "    (layer): ModuleList(\n",
              "      (0): BigBirdLayer(\n",
              "        (attention): BigBirdAttention(\n",
              "          (self): BigBirdSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BigBirdSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BigBirdIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (intermediate_act_fn): NewGELUActivation()\n",
              "        )\n",
              "        (output): BigBirdOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (1): BigBirdLayer(\n",
              "        (attention): BigBirdAttention(\n",
              "          (self): BigBirdSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BigBirdSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BigBirdIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (intermediate_act_fn): NewGELUActivation()\n",
              "        )\n",
              "        (output): BigBirdOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (2): BigBirdLayer(\n",
              "        (attention): BigBirdAttention(\n",
              "          (self): BigBirdSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BigBirdSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BigBirdIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (intermediate_act_fn): NewGELUActivation()\n",
              "        )\n",
              "        (output): BigBirdOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (3): BigBirdLayer(\n",
              "        (attention): BigBirdAttention(\n",
              "          (self): BigBirdSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BigBirdSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BigBirdIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (intermediate_act_fn): NewGELUActivation()\n",
              "        )\n",
              "        (output): BigBirdOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (4): BigBirdLayer(\n",
              "        (attention): BigBirdAttention(\n",
              "          (self): BigBirdSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BigBirdSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BigBirdIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (intermediate_act_fn): NewGELUActivation()\n",
              "        )\n",
              "        (output): BigBirdOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "      (5): BigBirdLayer(\n",
              "        (attention): BigBirdAttention(\n",
              "          (self): BigBirdSelfAttention(\n",
              "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "          (output): BigBirdSelfOutput(\n",
              "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (intermediate): BigBirdIntermediate(\n",
              "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (intermediate_act_fn): NewGELUActivation()\n",
              "        )\n",
              "        (output): BigBirdOutput(\n",
              "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (pooler): Linear(in_features=768, out_features=768, bias=True)\n",
              "  (activation): Tanh()\n",
              ")"
            ]
          },
          "execution_count": 63,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "cl_model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SMMb3QHkUQJf"
      },
      "outputs": [],
      "source": [
        "def predict(x):\n",
        "  preprocessed_query = ' '.join(preprocessor.preprocess(x))\n",
        "  _input = tokenizer(preprocessed_query, truncation=True, padding=True, return_tensors='pt').to(device)\n",
        "  output = cl_model(**_input)\n",
        "  predicted_label = output[0].softmax(1).argmax().item()\n",
        "  predicted_class_code = CLASSES_CATEGORIES[predicted_label]\n",
        "  return predicted_class_code"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "id": "Bid1vsm8UpPQ",
        "outputId": "39a6ecd8-e4ad-4fb1-ad91-9d3bfa80abd8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Attention type 'block_sparse' is not possible if sequence_length: 8 <= num global tokens: 2 * config.block_size + min. num sliding tokens: 3 * config.block_size + config.num_random_blocks * config.block_size + additional buffer: config.num_random_blocks * config.block_size = 352 with config.block_size = 32, config.num_random_blocks = 3. Changing attention type to 'original_full'...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'ورزشی'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ],
      "source": [
        "predict('بایرن مونیخ قهرمان لیگ قهرمانان اروپا شد.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "-jxegNVH6ttl",
        "outputId": "cef49b0e-8bc9-4446-ebf0-1f089cd95b91"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                        Query Predicted Category\n",
              "0           نتیجه توافق برجام          بین الملل\n",
              "1  حمله ارتش روسیه به اوکراین          بین الملل\n",
              "2     افزایش نرخ تورم در کشور            اقتصادی\n",
              "3         آمار فوتی‌های کرونا            اجتماعی\n",
              "4       اقدامات شهرداری تهران              سیاسی\n",
              "5     صعود ایران به جام جهانی              ورزشی\n",
              "6          بیماری آبله میمونی       علمی و پزشکی\n",
              "7            جشنواره فیلم فجر        فرهنگ و هنر\n",
              "8    رونمایی از گوشی جدید اپل  فناوری و ارتباطات\n",
              "9                 روش پخت غذا       علمی و پزشکی"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3e68847d-98c1-41b9-8ea4-e05c907dd8c5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Query</th>\n",
              "      <th>Predicted Category</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>نتیجه توافق برجام</td>\n",
              "      <td>بین الملل</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>حمله ارتش روسیه به اوکراین</td>\n",
              "      <td>بین الملل</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>افزایش نرخ تورم در کشور</td>\n",
              "      <td>اقتصادی</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>آمار فوتی‌های کرونا</td>\n",
              "      <td>اجتماعی</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>اقدامات شهرداری تهران</td>\n",
              "      <td>سیاسی</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>صعود ایران به جام جهانی</td>\n",
              "      <td>ورزشی</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>بیماری آبله میمونی</td>\n",
              "      <td>علمی و پزشکی</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>جشنواره فیلم فجر</td>\n",
              "      <td>فرهنگ و هنر</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>رونمایی از گوشی جدید اپل</td>\n",
              "      <td>فناوری و ارتباطات</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>روش پخت غذا</td>\n",
              "      <td>علمی و پزشکی</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3e68847d-98c1-41b9-8ea4-e05c907dd8c5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3e68847d-98c1-41b9-8ea4-e05c907dd8c5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3e68847d-98c1-41b9-8ea4-e05c907dd8c5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "queries_class_prediction = []\n",
        "\n",
        "for query in QUERIES:\n",
        "    predicted_class_code = predict(query)\n",
        "    queries_class_prediction.append({\n",
        "        'Query': query,\n",
        "        'Predicted Category': predicted_class_code,\n",
        "    })\n",
        "\n",
        "display(pd.DataFrame(queries_class_prediction))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false,
        "id": "4b-doPSG6ttm"
      },
      "source": [
        "<div dir=\"auto\" align=\"justify\">\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "        در این قسمت، بررسی می‌کنیم که مدل یادگیری‌مان بر روی داده‌های test چه عملکردی دارد. با اجرای قطعه کد زیر، معیار ارزیابی F1-Macro و نیز accuracy و ماتریس درهم‌ریختگی محاسبه شده و نمایش داده می‌شوند. (در این ماتریس، ستون‌ها ground truth و سطرها پیش‌بینی ما هستند.)\n",
        "    </p>\n",
        "    <p style='direction:rtl;text-align:justify;'>\n",
        "         مشاهده می‌کنیم که دسته‌هایی که مدل برای درخواست‌هایمان در بخش قبلی پیشنهاد داده همگی منطقی هستند و معیارهای F1-Macro و دقت میانگین و ماتریس درهم‌ریختگی نیز نشان‌دهنده‌ی کارکرد خوب این مدل هستند.\n",
        "    </p>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_vMgBX7qFWej",
        "outputId": "d9cafbb2-7bb2-4b2b-fb18-4f920ac8a9d5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 936/936 [03:30<00:00,  4.45it/s]\n"
          ]
        }
      ],
      "source": [
        "y_pred_transformers = []\n",
        "for x in tqdm(X_test):\n",
        "    _input = tokenizer(x, truncation=True, padding=True, return_tensors='pt').to(device)\n",
        "    output = cl_model(**_input)\n",
        "    y_pred_transformers.append(output[0].softmax(1).argmax().item())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470
        },
        "id": "mxopEVLC5f0U",
        "outputId": "1a662295-1388-4294-98c0-73b09fb71390"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accurracy: 0.793\n",
            "\n",
            "F1-Macro: 0.779\n",
            "\n",
            "F1-Micro: 0.793\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   فرهنگ و هنر  ورزشی  اقتصادی  فناوری و ارتباطات  سیاسی  \\\n",
              "فرهنگ و هنر                 62      2        1                  1      0   \n",
              "ورزشی                        1    142        0                  1      0   \n",
              "اقتصادی                      1      0       80                  2      5   \n",
              "فناوری و ارتباطات            1      0        3                 45      1   \n",
              "سیاسی                        2      0        4                  2     63   \n",
              "علمی و پزشکی                 1      0        3                  1      0   \n",
              "بین الملل                    1      0        6                  1      7   \n",
              "مذهبی                        5      1        0                  0      4   \n",
              "حوادث                        1      0        1                  1      0   \n",
              "اجتماعی                      4      1        7                  2      2   \n",
              "\n",
              "                   علمی و پزشکی  بین الملل  مذهبی  حوادث  اجتماعی  \n",
              "فرهنگ و هنر                   5          2      8      1        1  \n",
              "ورزشی                         1          0      2      0        4  \n",
              "اقتصادی                       2          6      0      0       15  \n",
              "فناوری و ارتباطات             7          1      2      2        3  \n",
              "سیاسی                         0          9     11      0        6  \n",
              "علمی و پزشکی                 69          0      0      0        7  \n",
              "بین الملل                     2        119      0      3        0  \n",
              "مذهبی                         1          0     64      0        7  \n",
              "حوادث                         1          2      0     48        3  \n",
              "اجتماعی                       1          1      2      0       50  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8ec58abf-6424-4f80-9997-88e201c38115\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>فرهنگ و هنر</th>\n",
              "      <th>ورزشی</th>\n",
              "      <th>اقتصادی</th>\n",
              "      <th>فناوری و ارتباطات</th>\n",
              "      <th>سیاسی</th>\n",
              "      <th>علمی و پزشکی</th>\n",
              "      <th>بین الملل</th>\n",
              "      <th>مذهبی</th>\n",
              "      <th>حوادث</th>\n",
              "      <th>اجتماعی</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>فرهنگ و هنر</th>\n",
              "      <td>62</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ورزشی</th>\n",
              "      <td>1</td>\n",
              "      <td>142</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>اقتصادی</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>80</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>فناوری و ارتباطات</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>45</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>سیاسی</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>63</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>علمی و پزشکی</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>69</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>بین الملل</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>119</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>مذهبی</th>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>64</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>حوادث</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>48</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>اجتماعی</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>50</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8ec58abf-6424-4f80-9997-88e201c38115')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8ec58abf-6424-4f80-9997-88e201c38115 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8ec58abf-6424-4f80-9997-88e201c38115');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "print(f\"Accurracy: {round(accuracy_score(y_test, y_pred_transformers), 3)}\\n\")\n",
        "\n",
        "f1_macro = f1_score(y_test, y_pred_transformers, average='macro')\n",
        "print(f\"F1-Macro: {round(f1_macro, 3)}\\n\")\n",
        "\n",
        "f1_micro = f1_score(y_test, y_pred_transformers, average='micro')\n",
        "print(f\"F1-Micro: {round(f1_micro, 3)}\\n\")\n",
        "\n",
        "\n",
        "confusion_mat = confusion_matrix(y_test, y_pred_transformers, labels=np.arange(10))\n",
        "confusion_mat_df = pd.DataFrame(confusion_mat).rename(columns=CLASSES_CATEGORIES, index=CLASSES_CATEGORIES)\n",
        "display(confusion_mat_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XHMZtD_HB-Aa",
        "outputId": "ad090a2c-8f96-42e0-f645-4ef1f92afc74"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Macro-average precision: 0.79\n",
            "Macro-average recall: 0.78\n",
            "Micro-average precision: 0.79\n",
            "Micro-average recall: 0.79\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "precision, recall, f1, support = precision_recall_fscore_support(y_test, y_pred_transformers, average='macro')\n",
        "print(\"Macro-average precision: {:.2f}\".format(precision))\n",
        "print(\"Macro-average recall: {:.2f}\".format(recall))\n",
        "precision, recall, f1, support = precision_recall_fscore_support(y_test, y_pred_transformers, average='micro')\n",
        "print(\"Micro-average precision: {:.2f}\".format(precision))\n",
        "print(\"Micro-average recall: {:.2f}\".format(recall))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PAFV9DVj6ttm",
        "outputId": "7c95ab73-fecc-4bac-f476-319574fc59f1"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Configuration saved in transformer_model/config.json\n",
            "Model weights saved in transformer_model/pytorch_model.bin\n"
          ]
        }
      ],
      "source": [
        "cl_model.save_pretrained('transformer_model')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yTzgdzqtAkUQ"
      },
      "outputs": [],
      "source": [
        "!zip -r /content/sample_data.zip /content/transformer_model\n",
        "from google.colab import files\n",
        "files.download('/content/sample_data.zip')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zapaeGcIVsm-"
      },
      "source": [
        "# 3- NN with fasttext"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this sectin we will use fasttext to classify our news. Fasttext is a library for learning of word embeddings and text classification. It is a library for efficient learning of word representations and sentence classification. It works on standard, generic hardware. Models can later be reduced in size to even fit on mobile devices. we will transform input tokens into fast text word embeddings and then get weighted average of these embeddings based on tdf-idf of them to get a vector representation of our input. Then we will use a simple neural network to classify our news."
      ],
      "metadata": {
        "id": "ZVrKND--HAzF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mFognPY9dOtL"
      },
      "outputs": [],
      "source": [
        "import fasttext\n",
        "\n",
        "class FastText:\n",
        "\n",
        "    def __init__(self, preprocessor=None, method='skipgram'):\n",
        "        self.method = method\n",
        "        self.model = None\n",
        "        self.preprocessor = preprocessor\n",
        "\n",
        "    def train(self, texts):\n",
        "        with open('FastText_train.txt', 'w', encoding='utf-8') as file:\n",
        "            file.write('\\n'.join(list(map(lambda doc: ' '.join(doc), texts))))\n",
        "        self.model = fasttext.train_unsupervised('FastText_train.txt', self.method, minn=2, maxn=6, wordNgrams=10,)\n",
        "        os.remove('FastText_train.txt')\n",
        "\n",
        "    def get_query_embedding(self, query, tf_idf_vectorizer):\n",
        "        if self.preprocessor:\n",
        "            query = self.preprocessor.preprocess(query)\n",
        "        if type(query) == str:\n",
        "            query = query.split()\n",
        "        tf_idf_embed = tf_idf_vectorizer.transform([' '.join(query)]).toarray()[0]\n",
        "        weights = []\n",
        "        vocab_weights_list = [tf_idf_embed[tf_idf_vectorizer.vocabulary_[word]] for word in query if word in tf_idf_vectorizer.vocabulary_]\n",
        "        min_weight = min(vocab_weights_list) if len(vocab_weights_list) > 0 else 1\n",
        "        for word in query:\n",
        "          if word in tf_idf_vectorizer.vocabulary_:\n",
        "            weight = tf_idf_embed[tf_idf_vectorizer.vocabulary_[word]]\n",
        "          else:\n",
        "            weight = min_weight\n",
        "          weights.append(weight)  \n",
        "        weights = np.array(weights)\n",
        "        query_embed = np.mean([self.model.get_word_vector(query[i])* weights[i] for i in range(len(query))], axis=0)\n",
        "        return query_embed\n",
        "\n",
        "    def save_FastText_model(self, path='FastText_model.bin'):\n",
        "        self.model.save_model(path)\n",
        "  \n",
        "    def load_FastText_model(self, path=\"FastText_model.bin\"):\n",
        "        self.model = fasttext.load_model(path)\n",
        "\n",
        "    def prepare(self, dataset, mode, save=False):\n",
        "        if mode == 'train':\n",
        "            self.train(dataset)\n",
        "        if mode == 'load':\n",
        "            self.load_FastText_model()\n",
        "        if save:\n",
        "            self.save_FastText_model()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CD3uLm5heSmw",
        "outputId": "3b5a5dce-ac43-4b09-e848-d5f1cc7a603b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
          ]
        }
      ],
      "source": [
        "FastText_model = FastText(preprocessor=preprocessor)\n",
        "FastText_model.prepare(preprocessed_total, mode='load', save=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u8YNcTTtiIOz",
        "outputId": "92c55260-8314-42e3-ce5a-6d6c851b0083"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0.9410250782966614, 'کرونای'),\n",
              " (0.916727602481842, 'کرونایی'),\n",
              " (0.898478090763092, 'کروناویروس'),\n",
              " (0.8946961164474487, 'کرون'),\n",
              " (0.8778902292251587, 'شیوع'),\n",
              " (0.8608316779136658, 'ابتلا'),\n",
              " (0.8591848611831665, 'مبتلا'),\n",
              " (0.8578081727027893, 'امیکرون'),\n",
              " (0.8418070673942566, 'واکسیناسیون'),\n",
              " (0.8371744751930237, 'اومیکرون')]"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ],
      "source": [
        "text = 'کرونا'\n",
        "\n",
        "FastText_model.model.get_nearest_neighbors(text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "REJu55PnikHU",
        "outputId": "fc1f04eb-55fa-48fd-c474-26abf411bf48"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<1x24767 sparse matrix of type '<class 'numpy.float64'>'\n",
              "\twith 0 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ],
      "source": [
        "text =  \"اقدامات شهرداری تهران\"\n",
        "TF_IDF_model.vectorizer.transform([' '.join(text)])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "up3Q-5u55kIm",
        "outputId": "998ee217-7ebe-4e80-e759-d720eedadc19"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.19476603, -0.05470406,  0.26639822, -0.11887681,  0.00698298,\n",
              "        0.08140592,  0.03982836, -0.12047374, -0.01527921,  0.17961872,\n",
              "        0.07180371, -0.1990444 ,  0.04150448, -0.13858917,  0.28285876,\n",
              "       -0.02802586, -0.06896676,  0.28099856, -0.01300358, -0.02638977,\n",
              "        0.22719795,  0.0039195 , -0.13312572, -0.02441638,  0.03303911,\n",
              "        0.12361485, -0.04417124,  0.14267294,  0.04583634, -0.15697455,\n",
              "       -0.00375187, -0.06187701, -0.0521847 , -0.1545234 ,  0.18178572,\n",
              "       -0.2687355 ,  0.19002302,  0.12841092,  0.01574793,  0.03527972,\n",
              "       -0.01417547,  0.00664668,  0.07076176, -0.06302169, -0.09014467,\n",
              "       -0.04865832,  0.15475081,  0.10616121, -0.09342459, -0.19020575,\n",
              "        0.16507177,  0.1298375 ,  0.34082544,  0.06601405,  0.12385812,\n",
              "       -0.06427272, -0.10237358, -0.07235847, -0.15280043, -0.12795481,\n",
              "       -0.15126085, -0.13417037,  0.03775533, -0.01011365, -0.01213493,\n",
              "       -0.07887498,  0.0649427 , -0.17023367, -0.30656424, -0.11922104,\n",
              "        0.14192787, -0.17609583, -0.02462482, -0.11290904, -0.19397537,\n",
              "        0.21364653, -0.04263808, -0.19089313, -0.0420551 ,  0.26963538,\n",
              "       -0.00523264,  0.01581393, -0.09760012, -0.0740027 , -0.04365213,\n",
              "        0.04423971, -0.01917148, -0.0190117 ,  0.11300626,  0.1517597 ,\n",
              "        0.13950819, -0.23070307, -0.05730475,  0.19471525, -0.13175528,\n",
              "        0.14404874, -0.08950099,  0.06424808,  0.00211167,  0.04981725],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ],
      "source": [
        "FastText_model.get_query_embedding(text, TF_IDF_model.vectorizer)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "In this section you can see out nueral network"
      ],
      "metadata": {
        "id": "YqXFyFpZIHRd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Drmr670Di6oO"
      },
      "outputs": [],
      "source": [
        "class ClassifierModel(nn.Module):\n",
        "    def __init__(self, in_features=100, num_classes=10):\n",
        "        super().__init__()\n",
        "        self.seq = nn.Sequential(\n",
        "            nn.Linear(in_features, 50),\n",
        "            nn.BatchNorm1d(50),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(50, 30),\n",
        "            nn.BatchNorm1d(30),\n",
        "            nn.ReLU(),\n",
        "            nn.Dropout(p=0.15),\n",
        "            nn.Linear(30, 15),\n",
        "            nn.BatchNorm1d(15),\n",
        "            nn.Tanh(), \n",
        "            nn.Dropout(p=0.05),\n",
        "            nn.Linear(15, num_classes),\n",
        "            nn.LogSoftmax(dim=1)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.seq(x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0mgllW13sTRz",
        "outputId": "1b86e2b6-2e8f-4148-d574-949e99d00b6a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4678/4678 [00:21<00:00, 216.40it/s]\n"
          ]
        }
      ],
      "source": [
        "embeddings = [FastText_model.get_query_embedding(' '.join(text), TF_IDF_model.vectorizer) for text in tqdm(preprocessed_total)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uhBxRC8bQPKA"
      },
      "outputs": [],
      "source": [
        "class NewsDataSet(Dataset):\n",
        "    def __init__(self, embeddings: list, labels: list):\n",
        "        self.y = labels\n",
        "        self.embeddings = embeddings\n",
        "        \n",
        "    def __len__(self):\n",
        "        return len(self.y)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        return self.embeddings[i], self.y[i]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yc92n7PjsAAz"
      },
      "outputs": [],
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(embeddings, labels, test_size=0.2, random_state=42)\n",
        "X_val, X_test, y_val, y_test = train_test_split(X_test, y_test, test_size=0.5, random_state=42)\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset= NewsDataSet(X_train, y_train), batch_size=32, shuffle=True)\n",
        "val_loader = DataLoader(dataset= NewsDataSet(X_val, y_val), batch_size=64, shuffle=False)\n",
        "test_loader = DataLoader(dataset= NewsDataSet(X_test, y_test), batch_size=64, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tdSLx_cjrgM0"
      },
      "outputs": [],
      "source": [
        "# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "# model = ClassifierModel().to(device)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load\n",
        "PATH = \"nn_fasttext_model.pt\"\n",
        "model = torch.load(PATH)\n",
        "model.eval()\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model = model.to(device)"
      ],
      "metadata": {
        "id": "yPRQ2Ags3Mi5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e7vS5NApQpiW"
      },
      "outputs": [],
      "source": [
        "criterion = nn.CrossEntropyLoss()\n",
        "learning_rate = 1e-4\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7VkPX36UQ5So"
      },
      "outputs": [],
      "source": [
        "def eval_epoch(model: nn.Module, criterion: nn.Module, dataloader: torch.utils.data.DataLoader, test_mode=False):\n",
        "    epoch_true = 0\n",
        "    epoch_all = 0\n",
        "    eval_loss = 0\n",
        "    i = 0\n",
        "    predicted_labels = []\n",
        "    true_labels = []\n",
        "\n",
        "    \n",
        "\n",
        "    with torch.no_grad(), tqdm(enumerate(dataloader), total=len(dataloader)) as pbar:\n",
        "        for i, (x, label) in pbar:\n",
        "            x = x.to(device)\n",
        "            label = label.to(device)\n",
        "            predicted = model(x)\n",
        "\n",
        "            loss = criterion(predicted, label)\n",
        "\n",
        "            eval_loss += loss.item()\n",
        "\n",
        "            predictions = predicted.argmax(-1)\n",
        "            epoch_all += len(predictions)\n",
        "            epoch_true += (predictions == label).sum()\n",
        "            true_labels.extend(label)\n",
        "            predicted_labels.extend(predictions)\n",
        "            discription = 'Validation' if not test_mode else 'Test'\n",
        "            pbar.set_description(f'{discription} Acc: {epoch_true * 100. / epoch_all:.2f}%')\n",
        "\n",
        "    predicted_labels = [_.item() for _ in predicted_labels]\n",
        "    true_labels = [_.item() for _ in true_labels]\n",
        "    f1_score_macro = f1_score(true_labels, predicted_labels, average='macro')\n",
        "    return eval_loss, predicted_labels, true_labels, f1_score_macro"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R1hb651GRszl",
        "outputId": "29a9af89-c32e-4b8d-9912-21f48f7f2a47"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 25.09%: 100%|██████████| 117/117 [00:01<00:00, 81.24it/s]\n",
            "Validation Acc: 46.15%: 100%|██████████| 8/8 [00:00<00:00, 231.86it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1 finished in 1.51s\n",
            "[Epoch 1]\tTrain Loss: 0.0677\tValidation Loss: 0.0336\t F1 score macro: 0.3492277985524248\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 46.69%: 100%|██████████| 117/117 [00:03<00:00, 32.13it/s]\n",
            "Validation Acc: 54.70%: 100%|██████████| 8/8 [00:00<00:00, 28.41it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 2 finished in 4.00s\n",
            "[Epoch 2]\tTrain Loss: 0.0596\tValidation Loss: 0.0308\t F1 score macro: 0.43536904555194117\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 53.02%: 100%|██████████| 117/117 [00:01<00:00, 64.56it/s]\n",
            "Validation Acc: 58.55%: 100%|██████████| 8/8 [00:00<00:00, 164.46it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 3 finished in 1.91s\n",
            "[Epoch 3]\tTrain Loss: 0.0553\tValidation Loss: 0.0287\t F1 score macro: 0.48017337868483895\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 57.46%: 100%|██████████| 117/117 [00:01<00:00, 114.15it/s]\n",
            "Validation Acc: 60.47%: 100%|██████████| 8/8 [00:00<00:00, 205.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 4 finished in 1.08s\n",
            "[Epoch 4]\tTrain Loss: 0.0521\tValidation Loss: 0.0273\t F1 score macro: 0.5104525566260416\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 60.13%: 100%|██████████| 117/117 [00:01<00:00, 113.07it/s]\n",
            "Validation Acc: 61.97%: 100%|██████████| 8/8 [00:00<00:00, 222.71it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 5 finished in 1.09s\n",
            "[Epoch 5]\tTrain Loss: 0.0497\tValidation Loss: 0.0261\t F1 score macro: 0.5422911208209873\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 60.56%: 100%|██████████| 117/117 [00:01<00:00, 104.15it/s]\n",
            "Validation Acc: 63.25%: 100%|██████████| 8/8 [00:00<00:00, 204.85it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 6 finished in 1.19s\n",
            "[Epoch 6]\tTrain Loss: 0.0476\tValidation Loss: 0.0251\t F1 score macro: 0.5563464101540608\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 61.97%: 100%|██████████| 117/117 [00:00<00:00, 119.27it/s]\n",
            "Validation Acc: 64.53%: 100%|██████████| 8/8 [00:00<00:00, 143.53it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 7 finished in 1.07s\n",
            "[Epoch 7]\tTrain Loss: 0.0460\tValidation Loss: 0.0244\t F1 score macro: 0.5751841153094809\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 63.34%: 100%|██████████| 117/117 [00:01<00:00, 111.53it/s]\n",
            "Validation Acc: 65.38%: 100%|██████████| 8/8 [00:00<00:00, 214.30it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 8 finished in 1.11s\n",
            "[Epoch 8]\tTrain Loss: 0.0444\tValidation Loss: 0.0237\t F1 score macro: 0.5915050939590445\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 64.51%: 100%|██████████| 117/117 [00:01<00:00, 101.73it/s]\n",
            "Validation Acc: 64.74%: 100%|██████████| 8/8 [00:00<00:00, 192.40it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 9 finished in 1.22s\n",
            "[Epoch 9]\tTrain Loss: 0.0430\tValidation Loss: 0.0230\t F1 score macro: 0.5840846358614478\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 65.15%: 100%|██████████| 117/117 [00:01<00:00, 102.40it/s]\n",
            "Validation Acc: 65.81%: 100%|██████████| 8/8 [00:00<00:00, 174.74it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 10 finished in 1.21s\n",
            "[Epoch 10]\tTrain Loss: 0.0418\tValidation Loss: 0.0224\t F1 score macro: 0.60614466543104\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 66.19%: 100%|██████████| 117/117 [00:01<00:00, 103.22it/s]\n",
            "Validation Acc: 67.95%: 100%|██████████| 8/8 [00:00<00:00, 173.78it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 11 finished in 1.20s\n",
            "[Epoch 11]\tTrain Loss: 0.0406\tValidation Loss: 0.0219\t F1 score macro: 0.6331548994559689\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 66.94%: 100%|██████████| 117/117 [00:01<00:00, 86.81it/s]\n",
            "Validation Acc: 67.31%: 100%|██████████| 8/8 [00:00<00:00, 170.93it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 12 finished in 1.42s\n",
            "[Epoch 12]\tTrain Loss: 0.0396\tValidation Loss: 0.0213\t F1 score macro: 0.622948008948396\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 67.90%: 100%|██████████| 117/117 [00:01<00:00, 111.47it/s]\n",
            "Validation Acc: 68.80%: 100%|██████████| 8/8 [00:00<00:00, 220.13it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 13 finished in 1.12s\n",
            "[Epoch 13]\tTrain Loss: 0.0385\tValidation Loss: 0.0208\t F1 score macro: 0.64595250793867\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 68.04%: 100%|██████████| 117/117 [00:00<00:00, 131.95it/s]\n",
            "Validation Acc: 70.30%: 100%|██████████| 8/8 [00:00<00:00, 197.86it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 14 finished in 0.95s\n",
            "[Epoch 14]\tTrain Loss: 0.0379\tValidation Loss: 0.0204\t F1 score macro: 0.6658965410597872\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 68.49%: 100%|██████████| 117/117 [00:00<00:00, 139.55it/s]\n",
            "Validation Acc: 70.30%: 100%|██████████| 8/8 [00:00<00:00, 172.82it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 15 finished in 0.91s\n",
            "[Epoch 15]\tTrain Loss: 0.0368\tValidation Loss: 0.0202\t F1 score macro: 0.673195301665751\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 69.00%: 100%|██████████| 117/117 [00:00<00:00, 142.86it/s]\n",
            "Validation Acc: 70.73%: 100%|██████████| 8/8 [00:00<00:00, 196.94it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 16 finished in 0.88s\n",
            "[Epoch 16]\tTrain Loss: 0.0362\tValidation Loss: 0.0196\t F1 score macro: 0.6769655540622732\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 69.88%: 100%|██████████| 117/117 [00:01<00:00, 115.20it/s]\n",
            "Validation Acc: 69.87%: 100%|██████████| 8/8 [00:00<00:00, 192.02it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 17 finished in 1.08s\n",
            "[Epoch 17]\tTrain Loss: 0.0354\tValidation Loss: 0.0195\t F1 score macro: 0.6667456001080813\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 69.54%: 100%|██████████| 117/117 [00:01<00:00, 116.03it/s]\n",
            "Validation Acc: 71.15%: 100%|██████████| 8/8 [00:00<00:00, 243.26it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 18 finished in 1.06s\n",
            "[Epoch 18]\tTrain Loss: 0.0349\tValidation Loss: 0.0189\t F1 score macro: 0.6813496949349661\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 70.44%: 100%|██████████| 117/117 [00:00<00:00, 137.11it/s]\n",
            "Validation Acc: 70.73%: 100%|██████████| 8/8 [00:00<00:00, 184.29it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 19 finished in 0.91s\n",
            "[Epoch 19]\tTrain Loss: 0.0341\tValidation Loss: 0.0186\t F1 score macro: 0.6790996286887269\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 71.27%: 100%|██████████| 117/117 [00:00<00:00, 136.88it/s]\n",
            "Validation Acc: 70.51%: 100%|██████████| 8/8 [00:00<00:00, 182.13it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 20 finished in 0.92s\n",
            "[Epoch 20]\tTrain Loss: 0.0333\tValidation Loss: 0.0183\t F1 score macro: 0.6713396431669396\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 70.52%: 100%|██████████| 117/117 [00:00<00:00, 132.91it/s]\n",
            "Validation Acc: 70.94%: 100%|██████████| 8/8 [00:00<00:00, 223.01it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 21 finished in 0.94s\n",
            "[Epoch 21]\tTrain Loss: 0.0331\tValidation Loss: 0.0182\t F1 score macro: 0.6826216948123673\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 71.43%: 100%|██████████| 117/117 [00:00<00:00, 124.73it/s]\n",
            "Validation Acc: 70.94%: 100%|██████████| 8/8 [00:00<00:00, 197.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 22 finished in 1.00s\n",
            "[Epoch 22]\tTrain Loss: 0.0326\tValidation Loss: 0.0177\t F1 score macro: 0.6785695774688485\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.10%: 100%|██████████| 117/117 [00:00<00:00, 134.26it/s]\n",
            "Validation Acc: 70.73%: 100%|██████████| 8/8 [00:00<00:00, 168.23it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 23 finished in 0.95s\n",
            "[Epoch 23]\tTrain Loss: 0.0320\tValidation Loss: 0.0177\t F1 score macro: 0.6825353873903466\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.15%: 100%|██████████| 117/117 [00:01<00:00, 102.99it/s]\n",
            "Validation Acc: 70.94%: 100%|██████████| 8/8 [00:00<00:00, 209.47it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 24 finished in 1.20s\n",
            "[Epoch 24]\tTrain Loss: 0.0316\tValidation Loss: 0.0174\t F1 score macro: 0.6826959037515079\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.98%: 100%|██████████| 117/117 [00:00<00:00, 120.03it/s]\n",
            "Validation Acc: 71.58%: 100%|██████████| 8/8 [00:00<00:00, 209.35it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 25 finished in 1.03s\n",
            "[Epoch 25]\tTrain Loss: 0.0309\tValidation Loss: 0.0173\t F1 score macro: 0.691925913912092\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 73.44%: 100%|██████████| 117/117 [00:01<00:00, 115.42it/s]\n",
            "Validation Acc: 71.37%: 100%|██████████| 8/8 [00:00<00:00, 184.22it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 26 finished in 1.09s\n",
            "[Epoch 26]\tTrain Loss: 0.0307\tValidation Loss: 0.0170\t F1 score macro: 0.6870590720399697\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.69%: 100%|██████████| 117/117 [00:01<00:00, 101.99it/s]\n",
            "Validation Acc: 72.01%: 100%|██████████| 8/8 [00:00<00:00, 172.00it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 27 finished in 1.22s\n",
            "[Epoch 27]\tTrain Loss: 0.0305\tValidation Loss: 0.0169\t F1 score macro: 0.6965287744713847\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.45%: 100%|██████████| 117/117 [00:00<00:00, 118.75it/s]\n",
            "Validation Acc: 71.15%: 100%|██████████| 8/8 [00:00<00:00, 222.54it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 28 finished in 1.05s\n",
            "[Epoch 28]\tTrain Loss: 0.0299\tValidation Loss: 0.0168\t F1 score macro: 0.6835259718606947\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.96%: 100%|██████████| 117/117 [00:01<00:00, 113.60it/s]\n",
            "Validation Acc: 72.01%: 100%|██████████| 8/8 [00:00<00:00, 180.28it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 29 finished in 1.10s\n",
            "[Epoch 29]\tTrain Loss: 0.0296\tValidation Loss: 0.0167\t F1 score macro: 0.7025783562333611\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 73.84%: 100%|██████████| 117/117 [00:01<00:00, 112.40it/s]\n",
            "Validation Acc: 71.58%: 100%|██████████| 8/8 [00:00<00:00, 214.21it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 30 finished in 1.10s\n",
            "[Epoch 30]\tTrain Loss: 0.0294\tValidation Loss: 0.0166\t F1 score macro: 0.6928273728977955\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 72.88%: 100%|██████████| 117/117 [00:01<00:00, 103.42it/s]\n",
            "Validation Acc: 73.29%: 100%|██████████| 8/8 [00:00<00:00, 170.59it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 31 finished in 1.21s\n",
            "[Epoch 31]\tTrain Loss: 0.0291\tValidation Loss: 0.0162\t F1 score macro: 0.7138897418081884\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 73.60%: 100%|██████████| 117/117 [00:01<00:00, 103.93it/s]\n",
            "Validation Acc: 72.86%: 100%|██████████| 8/8 [00:00<00:00, 151.44it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 32 finished in 1.20s\n",
            "[Epoch 32]\tTrain Loss: 0.0285\tValidation Loss: 0.0163\t F1 score macro: 0.7091694573667187\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 73.76%: 100%|██████████| 117/117 [00:01<00:00, 98.23it/s]\n",
            "Validation Acc: 72.22%: 100%|██████████| 8/8 [00:00<00:00, 161.17it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 33 finished in 1.27s\n",
            "[Epoch 33]\tTrain Loss: 0.0285\tValidation Loss: 0.0161\t F1 score macro: 0.7020489047897028\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 73.92%: 100%|██████████| 117/117 [00:01<00:00, 97.78it/s]\n",
            "Validation Acc: 73.72%: 100%|██████████| 8/8 [00:00<00:00, 175.51it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 34 finished in 1.27s\n",
            "[Epoch 34]\tTrain Loss: 0.0282\tValidation Loss: 0.0160\t F1 score macro: 0.7202817541786322\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.83%: 100%|██████████| 117/117 [00:01<00:00, 93.00it/s]\n",
            "Validation Acc: 72.65%: 100%|██████████| 8/8 [00:00<00:00, 178.34it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 35 finished in 1.33s\n",
            "[Epoch 35]\tTrain Loss: 0.0273\tValidation Loss: 0.0160\t F1 score macro: 0.7058895808806316\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.13%: 100%|██████████| 117/117 [00:01<00:00, 95.74it/s]\n",
            "Validation Acc: 73.50%: 100%|██████████| 8/8 [00:00<00:00, 152.52it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 36 finished in 1.31s\n",
            "[Epoch 36]\tTrain Loss: 0.0274\tValidation Loss: 0.0157\t F1 score macro: 0.7138161094724726\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.91%: 100%|██████████| 117/117 [00:00<00:00, 129.60it/s]\n",
            "Validation Acc: 73.08%: 100%|██████████| 8/8 [00:00<00:00, 208.06it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 37 finished in 0.97s\n",
            "[Epoch 37]\tTrain Loss: 0.0271\tValidation Loss: 0.0157\t F1 score macro: 0.7093630036278504\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.61%: 100%|██████████| 117/117 [00:00<00:00, 136.27it/s]\n",
            "Validation Acc: 73.08%: 100%|██████████| 8/8 [00:00<00:00, 211.20it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 38 finished in 0.92s\n",
            "[Epoch 38]\tTrain Loss: 0.0269\tValidation Loss: 0.0156\t F1 score macro: 0.7130542019791127\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 75.15%: 100%|██████████| 117/117 [00:00<00:00, 121.55it/s]\n",
            "Validation Acc: 73.29%: 100%|██████████| 8/8 [00:00<00:00, 154.87it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 39 finished in 1.04s\n",
            "[Epoch 39]\tTrain Loss: 0.0270\tValidation Loss: 0.0155\t F1 score macro: 0.7105459865607245\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.80%: 100%|██████████| 117/117 [00:00<00:00, 119.34it/s]\n",
            "Validation Acc: 72.22%: 100%|██████████| 8/8 [00:00<00:00, 140.40it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 40 finished in 1.07s\n",
            "[Epoch 40]\tTrain Loss: 0.0267\tValidation Loss: 0.0154\t F1 score macro: 0.6984819525518765\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.88%: 100%|██████████| 117/117 [00:00<00:00, 130.29it/s]\n",
            "Validation Acc: 73.29%: 100%|██████████| 8/8 [00:00<00:00, 206.53it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 41 finished in 0.96s\n",
            "[Epoch 41]\tTrain Loss: 0.0264\tValidation Loss: 0.0153\t F1 score macro: 0.7162897039159719\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.77%: 100%|██████████| 117/117 [00:00<00:00, 136.87it/s]\n",
            "Validation Acc: 73.08%: 100%|██████████| 8/8 [00:00<00:00, 219.95it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 42 finished in 0.91s\n",
            "[Epoch 42]\tTrain Loss: 0.0263\tValidation Loss: 0.0153\t F1 score macro: 0.7140970605340934\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.77%: 100%|██████████| 117/117 [00:00<00:00, 123.18it/s]\n",
            "Validation Acc: 73.72%: 100%|██████████| 8/8 [00:00<00:00, 203.85it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 43 finished in 1.02s\n",
            "[Epoch 43]\tTrain Loss: 0.0263\tValidation Loss: 0.0152\t F1 score macro: 0.7190563862445863\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.40%: 100%|██████████| 117/117 [00:00<00:00, 127.42it/s]\n",
            "Validation Acc: 73.29%: 100%|██████████| 8/8 [00:00<00:00, 178.93it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 44 finished in 0.98s\n",
            "[Epoch 44]\tTrain Loss: 0.0258\tValidation Loss: 0.0153\t F1 score macro: 0.7131940597834033\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.48%: 100%|██████████| 117/117 [00:00<00:00, 135.44it/s]\n",
            "Validation Acc: 72.44%: 100%|██████████| 8/8 [00:00<00:00, 178.24it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 45 finished in 0.93s\n",
            "[Epoch 45]\tTrain Loss: 0.0257\tValidation Loss: 0.0152\t F1 score macro: 0.7047048182861462\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 75.60%: 100%|██████████| 117/117 [00:00<00:00, 132.46it/s]\n",
            "Validation Acc: 73.08%: 100%|██████████| 8/8 [00:00<00:00, 194.03it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 46 finished in 0.96s\n",
            "[Epoch 46]\tTrain Loss: 0.0257\tValidation Loss: 0.0151\t F1 score macro: 0.7140636654270471\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.43%: 100%|██████████| 117/117 [00:00<00:00, 123.50it/s]\n",
            "Validation Acc: 73.08%: 100%|██████████| 8/8 [00:00<00:00, 155.80it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 47 finished in 1.02s\n",
            "[Epoch 47]\tTrain Loss: 0.0257\tValidation Loss: 0.0152\t F1 score macro: 0.7129080213664561\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.77%: 100%|██████████| 117/117 [00:01<00:00, 114.82it/s]\n",
            "Validation Acc: 73.93%: 100%|██████████| 8/8 [00:00<00:00, 249.95it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 48 finished in 1.07s\n",
            "[Epoch 48]\tTrain Loss: 0.0254\tValidation Loss: 0.0151\t F1 score macro: 0.7231771855757637\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 74.99%: 100%|██████████| 117/117 [00:01<00:00, 111.98it/s]\n",
            "Validation Acc: 74.36%: 100%|██████████| 8/8 [00:00<00:00, 177.15it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 49 finished in 1.11s\n",
            "[Epoch 49]\tTrain Loss: 0.0252\tValidation Loss: 0.0150\t F1 score macro: 0.7282849757404979\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Training Acc: 75.52%: 100%|██████████| 117/117 [00:01<00:00, 115.72it/s]\n",
            "Validation Acc: 74.15%: 100%|██████████| 8/8 [00:00<00:00, 241.79it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 50 finished in 1.07s\n",
            "[Epoch 50]\tTrain Loss: 0.0249\tValidation Loss: 0.0150\t F1 score macro: 0.7196611768952205\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "num_epochs = 50\n",
        "\n",
        "train_loss_arr, val_loss_arr = [], []\n",
        "f1_macro_scores = []\n",
        "for epoch in range(num_epochs):\n",
        "    start_time = time()\n",
        "\n",
        "    train_loss, val_loss = 0, 0\n",
        "    epoch_all = 0\n",
        "    epoch_true = 0\n",
        "    i = 0\n",
        "\n",
        "    model.train()\n",
        "    \n",
        "    with tqdm(enumerate(train_loader), total=len(train_loader)) as pbar:\n",
        "        for i, (x, label) in pbar:\n",
        "            x = x.to(device)\n",
        "            label = label.to(device)\n",
        "            predicted = model(x)\n",
        "\n",
        "            loss = criterion(predicted, label)\n",
        "            \n",
        "            train_loss += loss.item()\n",
        "\n",
        "            predictions = predicted.argmax(-1)\n",
        "            \n",
        "            epoch_all += len(predictions)\n",
        "            epoch_true += (predictions == label).sum()\n",
        "\n",
        "            pbar.set_description(f'Training Acc: {epoch_true * 100. / epoch_all:.2f}%')\n",
        "            loss.backward()\n",
        "\n",
        "            optimizer.step()\n",
        "            optimizer.zero_grad()  \n",
        "  \n",
        "    model.eval()\n",
        "    \n",
        "    val_loss, predicted_labels, true_labels, f1_score_macro = eval_epoch(model, criterion, val_loader)\n",
        "    \n",
        "    train_loss /= len(train_loader.dataset)\n",
        "    val_loss /= len(val_loader.dataset)\n",
        "    train_loss_arr.append(train_loss)\n",
        "    val_loss_arr.append(val_loss)\n",
        "    f1_macro_scores.append(f1_score_macro)\n",
        "    \n",
        "    end_time = time()\n",
        "    \n",
        "    print(f'Epoch {epoch + 1} finished in {end_time - start_time:.2f}s')\n",
        "\n",
        "    print(f\"[Epoch {epoch + 1}]\\t\"\n",
        "        f\"Train Loss: {train_loss:.4f}\\t\"\n",
        "        f\"Validation Loss: {val_loss:.4f}\\t F1 score macro: {f1_score_macro}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "Wa2YlUGQWHUx",
        "outputId": "1ba94ccb-934b-48f5-df6f-9d70a67c25eb"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgV1fnA8e+bhWwkAUIIS1gFlJ1AEEVQRKyKKIIg4AZitWppa6u1at2q1Z/Wal1b2wpFLYoWxVI3Ki6odWORHWVHE5BAgCSEJGR5f3+cSQjxAgFyM0nu+3meee6dmTN33sFr3nvmzDlHVBVjjDGmqjC/AzDGGFM3WYIwxhgTkCUIY4wxAVmCMMYYE5AlCGOMMQFZgjDGGBOQJQhj6iARURHp7HccJrRZgjAhTUQ2i8hwv+M4ViIyWUQ+8TsO0zBZgjDGGBOQJQhjAhCRKBF5TES2estjIhLl7WsuIm+IyB4R2SUiH4tImLfvNyKSKSJ5IvKNiJx1iM+fISLPiMi7XtkFItL+EGUTReR5EdkhIltE5A4RCRORbsAzwKkisldE9gTr38OEJksQxgT2W+AUoC/QBzgZuMPbdxOQASQDKcDtgIrIicBUYICqxgPnAJsPc47LgPuA5sBSYOYhyj0JJAKdgDOAK4GrVHUNcB3wmao2VtUmx3SlxhyCJQhjArsMuFdVs1R1B/A74ApvXzHQCmivqsWq+rG6Qc1KgSigu4hEqupmVd1wmHO8qaofqWoRLiGdKiJtKxcQkXBgAnCbquap6mbgkUqxGBM0liCMCaw1sKXS+hZvG8DDwHrgvyKyUURuBVDV9cCNwD1AlojMEpHWHNp35W9UdS+wq9I5yjUHIgPE0uZoL8iYo2UJwpjAtgKV2wTaedvwfsnfpKqdgAuBX5W3Najqi6o62DtWgYcOc46K2oKINAaalZ+jkp24GkvVWDK99zYcswkaSxDGQKSIRFdaIoCXgDtEJFlEmgN3Af8EEJGRItJZRATIwd1aKhORE0VkmNeYXQgUAGWHOe8IERksIo1wbRGfq+p3lQuoainwCnC/iMR7Ddm/Ko8F2A6kep9hTI2yBGEMvIX7Y16+3AP8HlgELAdWAEu8bQBdgPnAXuAz4M+q+gGu/eFB3K/+74EWwG2HOe+LwN24W0v9gcsPUe5nQD6wEfjEO266t+99YBXwvYjsrP4lG3NkYhMGGVP7RGQGkKGqdxyprDF+sRqEMcaYgCxBGGOMCSioCUJEzvV6k64vfxSwyv4oEXnZ2/+FiHTwtl8mIksrLWUi0jeYsRpTm1R1st1eMnVd0NogvA4+a4Gzcb1OFwITVXV1pTI3AL1V9ToRmQCMVtXxVT6nF/C6qp4QlECNMcYEFBHEzz4ZWK+qGwFEZBYwClhdqcwo3BMjALOBp0RE9OCsNRGYdaSTNW/eXDt06FADYRtjTOhYvHjxTlVNDrQvmAmiDZV6iuJqEQMPVUZVS0QkB0jCPSZYbjwukfyAiFwLXAvQrl07Fi1aVDORG2NMiBCRLYfaV6cbqUVkILBPVVcG2q+qf1PVdFVNT04OmACNMcYco2AmiEwqDSUApHJgeIAflPF6ryYC2ZX2T8D1aDXGGFPLgpkgFgJdRKSjNwzABGBulTJzgUne+7HA++XtD974+pdQjfYHY4wxNS9obRBem8JUYB4QDkxX1VUici+wSFXnAtOAF0RkPW64gQmVPuJ04LvyRm5jTGgpLi4mIyODwsJCv0NpEKKjo0lNTSUyMrLaxzSYoTbS09PVGqmNaTg2bdpEfHw8SUlJuHERzbFSVbKzs8nLy6Njx44H7RORxaqaHui4Ot1IbYwJXYWFhZYcaoiIkJSUdNS1MUsQxpg6y5JDzTmWf0tLEFu2wB13uFdjjDEVLEHk5sL998PHH/sdiTGmjsjOzqZv37707duXli1b0qZNm4r1/fv3H/bYRYsW8fOf/7yWIg2uYPakrh+6dYPYWFi4EC4/1HwtxphQkpSUxNKlSwG45557aNy4MTfffHPF/pKSEiIiAv/5TE9PJz09YJtvvWM1iIgI6NfPJQhjjDmEyZMnc9111zFw4EBuueUWvvzyS0499VTS0tIYNGgQ33zzDQAffvghI0eOBFxymTJlCkOHDqVTp0488cQTfl7CUbMaBMCAAfCXv0BJiUsYxpi65cYbwftFX2P69oXHHjuqQzIyMvj0008JDw8nNzeXjz/+mIiICObPn8/tt9/Oq6+++oNjvv76az744APy8vI48cQTuf7664+qL4Kf7K8hQHo6FBbCqlXQp4/f0Rhj6qhx48YRHh4OQE5ODpMmTWLdunWICMXFxQGPOf/884mKiiIqKooWLVqwfft2UlNTazPsY2YJAlwNAtxtJksQxtQ9R/lLP1ji4uIq3t95552ceeaZzJkzh82bNzN06NCAx0RFRVW8Dw8Pp6SkJNhh1hhrgwDo3BmaNLF2CGNMteXk5NCmTRsAZsyY4W8wQWIJAkDE3WayoTqMMdV0yy23cNttt5GWllavagVHw8ZiKnf77fDww5CXB9HRNReYMeaYrFmzhm7duvkdRoMS6N/UxmKqjvR09xTTsmV+R2KMMXWCJYhylRuqjTHGWIKokJoKKSmWIIwxxmMJopyIq0VYQ7UxxgCWIA6Wng5r1riGamOMCXGWICobMABUYckSvyMxxhjfWYKozBqqjTGeM888k3nz5h207bHHHuP6668PWH7o0KGUP2o/YsQI9uzZ84My99xzD3/84x8Pe97XX3+d1atXV6zfddddzJ8//2jDrxGWICpLTob27a0dwhjDxIkTmTVr1kHbZs2axcSJE4947FtvvUWTJk2O6bxVE8S9997L8OHDj+mzjpcliKrS060GYYxh7NixvPnmmxUTBG3evJmtW7fy0ksvkZ6eTo8ePbj77rsDHtuhQwd27twJwP3330/Xrl0ZPHhwxZDgAH//+98ZMGAAffr04eKLL2bfvn18+umnzJ07l1//+tf07duXDRs2MHnyZGbPng3Ae++9R1paGr169WLKlCkUFRVVnO/uu++mX79+9OrVi6+//rpG/g1ssL6qBgyAV1+F7GxISvI7GmMMcOM7N7L0+5od7rtvy748du6hBwFs1qwZJ598Mm+//TajRo1i1qxZXHLJJdx+++00a9aM0tJSzjrrLJYvX07v3r0DfsbixYuZNWsWS5cupaSkhH79+tG/f38AxowZwzXXXAPAHXfcwbRp0/jZz37GhRdeyMiRIxk7duxBn1VYWMjkyZN577336Nq1K1deeSV/+ctfuPHGGwFo3rw5S5Ys4c9//jN//OMfefbZZ4/738hqEFWVt0PYbSZjQl7l20zlt5deeeUV+vXrR1paGqtWrTrodlBVH3/8MaNHjyY2NpaEhAQuvPDCin0rV65kyJAh9OrVi5kzZ7Jq1arDxvLNN9/QsWNHunbtCsCkSZP46KOPKvaPGTMGgP79+7N58+ZjveSDWA2iKi+7s2gRnHOOv7EYYwAO+0s/mEaNGsUvf/lLlixZwr59+2jWrBl//OMfWbhwIU2bNmXy5MkUFhYe02dPnjyZ119/nT59+jBjxgw+/PDD44q1fFjxmhxS3GoQVSUmQteu1g5hjKFx48aceeaZTJkyhYkTJ5Kbm0tcXByJiYls376dt99++7DHn3766bz++usUFBSQl5fHf/7zn4p9eXl5tGrViuLiYmbOnFmxPT4+nrwAfbFOPPFENm/ezPr16wF44YUXOOOMM2roSgOzBBHIgAGWIIwxgLvNtGzZMiZOnEifPn1IS0vjpJNO4tJLL+W000477LH9+vVj/Pjx9OnTh/POO48B5bewgfvuu4+BAwdy2mmncdJJJ1VsnzBhAg8//DBpaWls2LChYnt0dDT/+Mc/GDduHL169SIsLIzrrruu5i+4EhvuO5DHH3dz4GZmQuvWNfOZxpijYsN91zwb7rsmWIc5Y4yxBBFQ374QHm5PMhljQlpQE4SInCsi34jIehG5NcD+KBF52dv/hYh0qLSvt4h8JiKrRGSFiNTeNG+xsdCjh9UgjPFZQ7kFXhccy79l0BKEiIQDTwPnAd2BiSLSvUqxq4HdqtoZ+BPwkHdsBPBP4DpV7QEMBYqDFWtA5Q3V9gU1xhfR0dFkZ2dbkqgBqkp2djbRRzmdcjD7QZwMrFfVjQAiMgsYBVTuVTIKuMd7Pxt4SkQE+BGwXFWXAahqdhDjDGzAAJg2DTZtgk6dav30xoS61NRUMjIy2LFjh9+hNAjR0dGkpqYe1THBTBBtgO8qrWcAAw9VRlVLRCQHSAK6Aioi84BkYJaq/qHqCUTkWuBagHbt2tVs9JV7VFuCMKbWRUZG0rFjR7/DCGl1tZE6AhgMXOa9jhaRs6oWUtW/qWq6qqYnJyfXbAQ9e0JcHLz7bs1+rjHG1BPBTBCZQNtK66netoBlvHaHRCAbV9v4SFV3quo+4C2gXxBj/aFGjWDcOHj5ZcjPr9VTG2NMXRDMBLEQ6CIiHUWkETABmFulzFxgkvd+LPC+uhapeUAvEYn1EscZHNx2UTuuuspNP/rqq7V+amOM8VvQEoSqlgBTcX/s1wCvqOoqEblXRMqHNJwGJInIeuBXwK3esbuBR3FJZimwRFXfDFashzRkCHTuDNOn1/qpjTHGbzbUxpE88AD89rewbp1LFsYY04DYUBvH48orISwMZszwOxJjjKlVliCOJDXVzQsxYwaUlvodjTHG1BpLENUxZYob2XX+fL8jMcaYWmMJojouuMDNT22N1caYEGIJojqiouDyy+H11yG79kf9MMYYP1iCqK6rroL9++HFF/2OxBhjaoUliOrq0wf697fbTMaYkGEJ4mhMmQJLl8JXX/kdiTHGBJ0liKMxcaJrj7BahDEmBFiCOBpNm8KYMTBzJhQW+h2NMcYElSWIozVlCuzeDf/+t9+RGGNMUFmCOFrDhkH79vDUUzYdqTGmQbMEcbTCwuDmm+GTT2wyIWNMg2YJ4lhccw20a+dGebVahDGmgbIEcSyiouDuu9181dYWYYxpoCxBHKsrr4SuXeHOO22UV2NMg2QJ4lhFRMDvfgcrV7p5q40xpoGxBHE8LrkEevd2t5uKi/2OxhhjapQliOMRFga//z2sXw/PPed3NMYYU6MsQRyvkSNh4EB3u8l6VxtjGhBLEMdLBO6/HzIy4K9/9TsaY4ypMZYgasJZZ8GZZ8IDD0B+vt/RGGNMjbAEUVPuvx+ysuCJJ/yOxBhjaoQliJpy6qmuPeLBB93tJmOMqecsQdSkP/0JSkrcUBw2BIcxpp6zBFGTOneGhx6Cd96BadP8jsYYY46LJYiadsMNrsH6V7+CLVv8jsYYY46ZJYiaFhbmpiRVdZMLlZX5HZExxhyToCYIETlXRL4RkfUicmuA/VEi8rK3/wsR6eBt7yAiBSKy1FueCWacNa5DB3jkEXj/fXimfoVujDHlgpYgRCQceBo4D+gOTBSR7lWKXQ3sVtXOwJ+Ahyrt26Cqfb3lumDFGTTXXAPnnAO//jVs3Oh3NMYYc9SCWYM4GVivqhtVdT8wCxhVpcwooHwQo9nAWSIiQYyp9ojAs89CZCRcdZXdajLG1DvBTBBtgO8qrWd42wKWUdUSIAdI8vZ1FJGvRGSBiAwJYpzBk5oKjz8OH31kHeiMMfVOXW2k3ga0U9U04FfAiyKSULWQiFwrIotEZNGOHTtqPchqufJK14Hutttg+XK/ozHGmGoLZoLIBNpWWk/1tgUsIyIRQCKQrapFqpoNoKqLgQ1A16onUNW/qWq6qqYnJycH4RJqQPmtpmbNYPRo2LXL74iMMaZagpkgFgJdRKSjiDQCJgBzq5SZC0zy3o8F3ldVFZFkr5EbEekEdAHqb0tvSgq8+qobgmPiRJui1BhTLwQtQXhtClOBecAa4BVVXSUi94rIhV6xaUCSiKzH3UoqfxT2dGC5iCzFNV5fp6r1+6f3KafA00/Df/8Ld9zhdzTGGHNEog1kzKD09HRdtGiR32Ec2XXXuXkjXnkFxo3zOxpjTIgTkcWqmh5oX11tpG64nngCBg2CyZNhxQq/ozHGmEOyBFHbGjWC2bMhMdE1Wu/e7XdExhgTkCUIP7Rq5Rqtv/3WGq2NMXWWJQi/nHoqPPUUzJvnBvUrKfE7ImOMOUiE3wGEtGuvhe+/h7vvhsJC+Oc/3dAcxhhTB1iC8Ntdd0FcHNx8MxQUuKeboqP9jsoYY+wWU51w002uj8R//gMXXAD5+X5HZIwxliDqjBtugBkz3BwS554Lubl+R2SMCXGWIOqSSZPgpZfg889h+HAbt8kY4ytLEHXNJZfAa6/BsmVw1lmQne13RMaYEGUJoi664AKYOxfWrHFJYudOvyMyxoQgSxB11TnnuCTxzTeWJIwxvrAEUZf96EcuSaxdC8OGQV2dFMkY0yBZgqjrzj7bPf66bp1LEllZfkdkjAkRliDqg+HD4Y03YMMGSxLGmFpjCaK+OOsslyQ2boTBg13bhDHGBJEliPpk2DA3I93u3XDyyfD2235HZIxpwCxB1DeDB8OiRdCxI5x/Pjz8MDSQWQGNMXWLJYj6qH17+N//YOxYuOUWuPJKNxqsMcbUoGolCBGJE5Ew731XEblQRGxcaj/FxcHLL8N997lhws84A7Zu9TsqY0wDUt0axEdAtIi0Af4LXAHMCFZQpppE4I47YM4cWL0a+veHBQv8jsoY00BUN0GIqu4DxgB/VtVxQI/ghWWOykUXwWefuXmuhw2DBx6AsjK/ozLG1HPVThAicipwGfCmty08OCGZY9KzJyxcCOPHw29/6xqwbXgOY8xxqG6CuBG4DZijqqtEpBPwQfDCMsckPh5mzoRnnoEPPoC+fV1jtjHGHINqJQhVXaCqF6rqQ15j9U5V/XmQYzPHQgR+8hN3yykmxjVe/+EPdsvJGHPUqvsU04sikiAiccBKYLWI/Dq4oZnjkpbm+kuMHg2/+Y275bR9u99RGWPqkereYuquqrnARcDbQEfck0ymLktMhFdegb/8BT78EPr0gXff9TsqY0w9Ud0EEen1e7gImKuqxYB1360PROC661wDdlKSG0L8N7+B4mK/IzPG1HHVTRB/BTYDccBHItIeyA1WUCYIyp9y+slPXJvE4MFu4D9jjDmE6jZSP6GqbVR1hDpbgDOPdJyInCsi34jIehG5NcD+KBF52dv/hYh0qLK/nYjsFZGbq3k95nBiY90TTv/6l5uEqG9ft15a6ndkxpg6qLqN1Iki8qiILPKWR3C1icMdEw48DZwHdAcmikj3KsWuBnaramfgT8BDVfY/imvzMDVp7FhYuhTS0+H6693IsJ995ndUxpg6prq3mKYDecAl3pIL/OMIx5wMrFfVjaq6H5gFjKpSZhTwnPd+NnCWiAiAiFwEbAJWVTNGczTat4f33oOXXoLvv4dBg2DKFJuMyBhToboJ4gRVvdv7Y79RVX8HdDrCMW2A7yqtZ3jbApZR1RIgB0gSkcbAb4DfHe4EInJtea1mh83XfPREYMIEN/nQLbe4Qf+6doUnn4SSEr+jM8b4rLoJokBEBpeviMhpQEFwQgLgHuBPqrr3cIVU9W+qmq6q6cnJyUEMp4Fr3BgeegiWL3e3m37+c/f61Vd+R2aM8VF1E8R1wNMisllENgNPAT85wjGZQNtK66netoBlRCQCSASygYHAH7xz3QjcLiJTqxmrOVYnnQTz5rlG7K1bYcAAuO02KAjmbwFjTF1V3aeYlqlqH6A30FtV04BhRzhsIdBFRDqKSCNgAjC3Spm5wCTv/Vjgfe8pqSGq2kFVOwCPAQ+o6lPVuyRzXERcI/aaNTBpEjz4oOtg99FHfkdmjKllRzWjnKrmej2qAX51hLIlwFRgHrAGeMUb6O9eEbnQKzYN1+aw3vu8HzwKa3zStClMm+Z6XpeUuDGdrr8ecq37izGhQvQY5zMWke9Ute2RS9aO9PR0XbRokd9hNEz5+XDnnfD445CQ4Bq2J0927RTuoTNjTD0lIotVNT3QvuOZk9qG2ggVcXHw6KPwxRcwciQ89xyccgp07+4atzOrNi0ZYxqCwyYIEckTkdwASx7QupZiNHVFejq88ILrN/Hss9C8Odx6K7RrBxdcAOvX+x2hMaYGHTZBqGq8qiYEWOJVNaK2gjR1TEICXH01fPwxrFsHt9/u3vfuDY88YkN3GNNAHM8tJmOgc2e47z5YvRqGD4ebb4bTToNV1gHemPrOEoSpGa1bw7//DS++6G419esHv/+9DStuTD1mCcLUHBGYONHVJkaPdk8+pafDjBmQk+N3dMaYo2QJwtS8Fi1g1iyYM8c9InvVVZCSAhdfDK++aj2zjaknLEGY4LnoIteI/fnnbqKi//3P9dJOSXH9KL780u8IjTGHYQnCBJcIDBzoOtllZLie2ePGudrFwIFw9tmwYAEcY4dNY0zwWIIwtSciwj3pNG2aSxZ/+AOsWAFDh8KQIfD225YojKlDLEEYf8THw69/DZs2wVNPwXffwYgRrlH7uedgzx6/IzQm5FmCMP6KiYGf/tS1VUybBnl5rn2iRQs4/3z4xz9g1y6/ozQmJFmCMHVDo0ZuytOvv3aN2r/4hXtcdsoU16h97rnw/PNQVOR3pMaEDEsQpm4JC3ON1w8/DBs3wsKFcNNNroYxaRJ07OgGCLRbUMYEnSUIU3eJuDaJBx90vbPnzYOePd0AgW3bwq9+Bd9+63eUxjRYliBM/SACP/oR/Pe/bq7sUaPgiSegUye4/HJYu9bvCI1pcCxBAGVa5ncI5mj07Qv//Ke7BfWLX7g+Fd27wzXXuKehjDE1IuQTxJJtS+j2dDdWZdnoo/VOu3ZuePGNG2HqVNeI3bkz/PKXkJXld3TG1HshnyDaJbYjKz+LX877Jcc6/arxWUoKPPaYa8i+4ooDt55uv93dklq7FgoL/Y7SmHon5BNE89jm3HPGPby78V3eWPuG3+GY49GunZvpbvVqNzXq//0fnHMOnHii62/RqhWceqobcfaRR2DJEpvcyJjDkIbyqzk9PV0XLVp0TMcWlxbT+5nelJaVsvKGlTQKb1TD0RlfbNvmnn7avPngZeNG9wrQtCmccQYMGwZnngk9ergGcWNChIgsVtX0QPts2lAgMjySR3/0KCNeHMGTXzzJTYNu8jskUxNatXLLkCE/3Ld1K3zwAbz/vnt9/XW3vX17Nzz5VVe5GokxIcxqEJWMmDmC/333P9b9bB0t4lrUUGSmXti82SWLl16C+fNdLWL4cDf39qhREB3td4TGBMXhahAh3wZR2aPnPMq+4n3c+f6dfodialuHDm5Yj3ffdQMI3nUXfPMNTJjgplP9yU9g5ky3r4H8qDLmSKwGUcWN79zIk18+yZJrl9CnZZ8aiMzUW2VlrlYxbRq8+aYbSBCgZUs47TQYNAhOPx3697d2C1NvHa4GYQmiit0Fu+nyZBd6pfTi/SvfR+x/fAPuaadVq9yseJ9+6paNG92+fv3ceFHjxkFkpL9xGnOU7BbTUWga05R7z7yXDzd/yJyv5/gdjqkrwsOhd2+4/np44QXYsME1dD/zjJt3+7LL4IQT3OOzubl+R2tMjbAaRAAlZSWk/TWN/P35rP7paqIjrIHSHEZZGbz1lksOH37oJkP68Y/d01OdOrkRaBMS/I7SmIB8u8UkIucCjwPhwLOq+mCV/VHA80B/IBsYr6qbReRk4G/lxYB7VPWwP+drMkEAzN84n7NfOJvbBt/GA2c9UGOfaxq4xYtdonjllYM74TVr5hJFx46uQbxdu4OXZs2sHcP4wpcEISLhwFrgbCADWAhMVNXVlcrcAPRW1etEZAIwWlXHi0gssF9VS0SkFbAMaK2qJYc6X00nCIAfz/0x076axstjX+aSHpfU6GebBm7PHncbatMm11axadOB5dtvfzj0R1ycG4TwkktcW0arVv7EbUKOXwniVNwv/3O89dsAVPX/KpWZ55X5TEQigO+BZK0UlIh0BD4H2tR2gigqKWLY88P4attX/G/K/0hrlVajn29ClCrs2OESRfmyZYvrsLdsmatJDB3qHrEdMwaaN/c7YtOA+dWTug1QeezlDGDgocp4tYUcIAnYKSIDgelAe+CKwyWHYImKiOK1S14j/e/pjJo1ioXXLCSlcUpth2EaGhE353aLFm5CpMrWrIGXX3Yd9n7yEzdf9xlnwCmnwIABbmnd2p+4TcgJZg1iLHCuqv7YW78CGKiqUyuVWemVyfDWN3hldlYq0w14DjhdVQurnONa4FqAdu3a9d+yZUtQrmXJtiUMnj6Yfq368f6k922sJhN8qq42MWsWvPMOrFx5oE2jdWuXKPr3hzZt3Gi2KSku4aSkQFSUv7GbesWvGkQm0LbSeqq3LVCZDO8WUyKusbqCqq4Rkb1AT2BRlX1/w2vMTk9PD1pre79W/fjHqH8w4dUJ3PDmDfz9gr9b/wgTXCKuTaJvXzfl6r59sHQpfPmlm6d74UL4978DH9ukiZtAacAAV0NJT4euXd1838YchWAmiIVAF68NIROYAFxapcxcYBLwGTAWeF9V1TvmO++2U3vgJGBzEGM9ovE9x7MiawX3f3w/fVL68LOBP/MzHBNqYmNdz+1Bgw5s27cPtm93kyNVft26FZYvh7//HR5/3JWNj3cd+oYPh/HjoUsXf67D1CvBfsx1BPAY7jHX6ap6v4jcCyxS1bkiEg28AKQBu4AJqrrRux11K1AMlAH3qurrhztXMBqpqyrTMka/PJo3177J25e9zdknnB3U8xlzXEpK4OuvYdEit3zxhXsFd3tqwgT31JSNWhvSbKiNGpRXlMeg6YNYl72OGRfNYELPCUE/pzE1JiPD9dGYNcvdpgI3rtSQIQduQVX+mxAT43qQ9+sHqanWV6MBsgRRw3bu28mYl8fw8bcfc88Z93DXGXdZm4SpfzZsOJAsVq06+I9/+fvi4gPbkpNdoujXD04+Gc4/38aeagAsQQRBUUkR175xLc8ve55Le13KtAun2ZAcpuHJz3ftGYsXuylalyxxyaSkxLVj/OEPbr4M+4FUb9lgfUEQFRHFjFEzeGDYA7y44kWGPTeMrPwsv8MypmbFxbl5vDGJ2AYAABPMSURBVKdOhenT3ZNUeXnuCaqICBg92vXT+PJLvyM1QWA1iBowe/VsrphzBS0bt+SNiW/Qo0UPX+IwplaVlLi5Mu66yz1BNXEiPPCAG2sqL8+1d1ReEhNh4ED36K711agz7BZTLViYuZALZ11IXlEeT494miv7XGntEiY05OW5W02PPOKSRnT0gcmVAmnUyCWJgQNdW0ZaGrRtayPe+sQSRC3JyM3g8tcuZ8GWBUzoOYFnzn+GxOhEX2MyptZkZMBjj7mG7dTUA0ubNq73986d7lHb8mXRIteXo1zjxgcfl5QEe/fC7t1u8MPyZe9eV0vp0cMtPXu615QUaws5BpYgalFpWSkPfvIgd394N20T2zJzzEwGtR105AONCTUlJa7Be/VqyMx0Cab8NSMDsrNdraJJk4OXmBg3Qu7KlbBr14HPS0pyT1ideqobu2rgQDeMujksSxA++Dzjcy599VK+zfmWu864i98O+S3hYeF+h2VMw6Hqeo6vWuWWlStdY/mKFW4SJ4ATT3QJ44QTXLtHVJS7xVX1tXwpX4+Pd8eGQI3EEoRPcgpzuOGtG3hxxYsMaTeE6aOm07lZZ7/DMqZhy8tzt68++ww+/9y97tx55OOqat0aRoyAkSPdECVxcTUfax1gCcJnLyx7galvT2V/6X7uO/M+bjzlRiLCgjkMljGmgqprF9m/H4qKfvhadd/+/a5m8s47MG+em2M8KsrN0TFihJtGtmnTg5fo+tsHyhJEHZCZm8lP3/op//7m3/Rv1Z9pF06jT8s+fodljDmc4mL45BN44w23rF0buFx0tGsbiYhwS2Tkgfci7pZX+VJa6l5jYlzjeu/eB5YTTqj1UXctQdQRqsrs1bOZ+vZUdhXs4pZBt3DnGXdaD2xj6otvv4Xvv3dPVlVdymsjJSUHL2VlEB7u/vCXL+HhrmayYgWsW3egzSQ21iWNVq0OTCqVnHzgfZcubnDFGmwbsQRRx+wq2MVN/72JGUtncGLSidwz9B4u7nYxkeE2ro0xIWffPvck1/Llblm9+sDw7Tt2HJgoqlxCgnu0t1cvV+vo1cstTZoc0+ktQdRR7254l6lvT2Vt9lpax7fm+vTrubb/tbSIa+F3aMaYuqCszPX9yMpyNZc1a1yto3zJyXHlRo2C1w87I8IhWYKow8q0jHfWv8MTXzzBvA3zaBTeiIk9J/Kzk39G/9b9/Q7PGFNXqbr+IitWuMdyhww5po+xBFFPfL3za5768ilmLJ1BfnE+g9oO4hcDf8Hok0bb7SdjTFBYgqhncgpzmP7VdJ5a+BQbd28kNSGVG9Jv4Jr+19A8trnf4RljGhBLEPVUaVkpb617i8e/eJz3Nr1HVHgUl/W6jKknTyWtVZrf4RljGgBLEA3AqqxVPPnlkzy/7HkKSgpIa5nG1WlXc2mvS2ka09Tv8Iwx9ZQliAZkd8FuZq6YybNLnmXZ9mVER0QzptsYrk67mqEdhhImNgeUMab6LEE0QKrKkm1LmP7VdGaumElOUQ4dm3Tk8t6Xc0XvK+iS1MXvEI0x9YAliAauoLiA19a8xoxlM3hv43soyimpp3BF7ysY32M8SbFJfodojKmjLEGEkMzcTF5c8SIvLH+BFVkriAyLZESXEYzvMZ6RXUcSHxXvd4jGmDrEEkSIWvb9Ml5Y/gIvrXyJrXlbiY6I5rzO5zGu+zhLFsYYwBJEyCvTMj797lP+tepfzF4zm615W4kKj+K8Lucx5qQxjOw60p6EMiZEWYIwFQIli4iwCIZ2GMrok0Zz0UkX0Tq+td9hGmNqiSUIE1CZlrEwcyFzvp7DnK/nsDbbjXU/sM1AxnQbw8XdLuaEZif4HKUxJpgsQZgjUlXW7FzDnDUuWSzethiAvi37MrbbWC7ufjEnNT/J5yiNMTXNEoQ5apv3bOa1Na8xe/VsPsv4DIAeyT246KSLOLvT2ZySegpREVE+R2mMOV6+JQgRORd4HAgHnlXVB6vsjwKeB/oD2cB4Vd0sImcDDwKNgP3Ar1X1/cOdyxJE8GTmZvLamtd4dc2rfPLtJ5RqKbGRsZze/nSGdxzO8E7D6ZXSy3pxG1MP+ZIgRCQcWAucDWQAC4GJqrq6UpkbgN6qep2ITABGq+p4EUkDtqvqVhHpCcxT1TaHO58liNqRU5jDgi0LmL9xPvM3zmfNzjUAJEQlkJqQSqvGrWgV38q9Nm5Fu8R2nH3C2SREJfgcuTEmEL8SxKnAPap6jrd+G4Cq/l+lMvO8Mp+JSATwPZCslYISEcHVLlqpatGhzmcJwh+ZuZnM3zifhVsXsm3vNrblbWNr3la27d3G/tL9AMRGxjKu+zimpE1hSLshSA3Op2uMOT6HSxARQTxvG+C7SusZwMBDlVHVEhHJAZKAnZXKXAwsCZQcRORa4FqAdu3a1VzkptraJLRhUt9JTOo76aDtqsruwt2s3rGaF5a5znrPLXuOzs06M6XvFK7scyVtEg5bKTTG+CyYNYixwLmq+mNv/QpgoKpOrVRmpVcmw1vf4JXZ6a33AOYCP1LVDYc7n9Ug6rb8/fm8uuZVpn81nQVbFiAIzWObkxCVQEJUAvFR8RXve7fozfie4+nQpIPfYRvT4NXLW0wikgq8D1ylqv870vksQdQf63etZ9bKWWTmZpK7P5e8ojxyi3LJLcplT+EeNu3ZBMCpqacysedELulxCSmNU3yO2piGya8EEYFrpD4LyMQ1Ul+qqqsqlfkp0KtSI/UYVb1ERJoAC4Dfqepr1TmfJYiGY9PuTcxaOYtZq2axfPtywiSMYR2HcWHXC+nTsg89W/SkWUwzv8M0pkHw8zHXEcBjuMdcp6vq/SJyL7BIVeeKSDTwApAG7AImqOpGEbkDuA1YV+njfqSqWYc6lyWIhmn1jtW8tOIlXlr5Eht2H7jL2KpxK3q26Fmx9EjuQffk7jYAoTFHyTrKmXpPVcnIzWDVjlWszFpZsazesZqCkoKKcm0T2tKjRQ+6N+9Ot+RupMSl0DSmKc1imtE0uilNY5oSHRHt45UYU7f49RSTMTVGRGib2Ja2iW05t/O5FdtLy0rZtGcTq3esZlXWKlbtWMXqHav5cPOHFJYUBvysmIgY+rfuz/ldzuf8LufTs0VPe/TWmACsBmEapNKyUr7N+Zad+3ayu3A3uwt2V7xm5Wfx0bcfsWTbEsDVOkZ0GcH5Xc6nd0pvIsIiCA8LJ1zCK97HRMQQGR7p81UZU/PsFpMxAWzN28pb697izXVv8u6Gd8kvzj9k2YiwCE5MOpFeKb3o1cJbUnrRPrG91T5MvWYJwpgjKCop4uNvP2bLni2UaimlZaWUaiklZSWUlpWSXZDNiqwVrNi+gi05WyqOi4mIoWlMU+IbxR/UnyMxKpGuSV3pk9KHPi370Ca+jSUSUydZG4QxRxAVEcXwTsOrVTanMIeVWStZkbWCtdlryS3KJW+/68uRV5THxn0b2VWwi+eWPVdxTFJMEr1TetMnpQ+dm3WmTUIbUhNSaRPfhhZxLQgPCw/WpRlzzKwGYUyQ5Bblsnz7cpZ9v4xl292yYvuKg566AgiXcFrFtyI5NpnGjRr/YImLjCOuURxxkXHERsYS18i9Nm7UmOTYZFIap5Acm2xtJOaYWA3CGB8kRCUwuN1gBrcbXLGtTMvIys8iMzeTzLxMMnMzycjNIDMvk+yCbPbu38uOfTvYtGcTe/fvrVhKykqOeL5mMc1IiUuhZeOWdGvejX6t+pHWKo0eyT1s7g5zTKwGYUw9UFxaTH5xPvuK95G/373m7c8jKz+LrPwstu/dzvZ8t2zL28bKrJXk7c8DIDIskh4tetCvZT+axzavaGMpKSupaGeJjoimZeOWBw/XHt+K5rHNbZ6PBs5qEMbUc5HhkTQJb0KT6CbVKl+mZWzcvZEl25awZNsSvvr+K/6z9j/kFuUGfIy3oLiAnKKcH3xOmISRGJVIYnQiTaLd+ROjEkmISkDRHySaMi2jXUI7erboSa+UXvRs0bPaMZu6x2oQxhgACooL2LZ3G9/v/Z5tedvYtncb2/duJ6cohz2Fe9hTuKfifW5RLmESRriEEx7mJRoJR0TYtHvTQckmNSGVni160rpxa6IioogKj6p4bRTeiLhGcTSLaVaxJMUkuZ7vMU2JCLPfsMFmNQhjzBHFRMbQqWknOjXtdFyfUz4sSvmTXuWvK7avoKi0iP2l+ykqKaKo9JDzfx2IKSLmB0PCxzeKJyYyhuiIaKLDo92rtzSPbe6eDvOeEkuJS/nBE2JlWsa+4n3s3b+X0rJSmsc2P2Ibjaqyd/9esguyaRrdlISohJB4bNkShDGmRlUeFuW8LucdspyqUlxWTP7+fHYV7DpoyS7IZlfBrgNDwVcaFv7bnG8pLCmsWIpKiyreV1X+hFh0RHRFg3/+/nyUg++cJEQlkBybTIu4FiTHJZMYlciugl2ufSd/O1n5WQd9flxk3EGPKreJb0OT6CbERMYQE+GSV3kSE4TCkkIKSgooKC6goKSAwpJCSstKDxonrLzW1CS6CRFhEYRJGGEShiAV7yPCImo1MVmCMMb4QkRoFN6IRjGNaBrTlBM44bg+r0zL2Llv50FPhmXkZpCRm0FRaRHxjeKJbxRf8fhwfFQ8grBj3w525O9gx74dZOVnsXnPZnIKc0iKTaJFXAu6JXejRWwLWsS1ICk2iV0Fuw48hZaXyYItC9iat7VaT5rVhOiIaKLCo9yrd6vugq4X8Mg5j9T4uSxBGGMahDAJo0Wc+0Oe1iqtVs9dpmWullCphlD+XlUrahbltYqYiBhEhD2Fe9hdsJtdBbvYXehecwpzKNVSyrTsB0v57bnKNaei0iLaJrYNynVZgjDGmOMUJmHERsYSGxl7VMfFRsbSOr51kKI6fvaAszHGmIAsQRhjjAnIEoQxxpiALEEYY4wJyBKEMcaYgCxBGGOMCcgShDHGmIAsQRhjjAmowYzmKiI7gC1HKNYc2FkL4dQ1oXrdELrXbtcdWo7nuturanKgHQ0mQVSHiCw61LC2DVmoXjeE7rXbdYeWYF233WIyxhgTkCUIY4wxAYVagvib3wH4JFSvG0L32u26Q0tQrjuk2iCMMcZUX6jVIIwxxlSTJQhjjDEBhUyCEJFzReQbEVkvIrf6HU+wiMh0EckSkZWVtjUTkXdFZJ332tTPGINBRNqKyAcislpEVonIL7ztDfraRSRaRL4UkWXedf/O295RRL7wvu8vi0gjv2MNBhEJF5GvROQNb73BX7eIbBaRFSKyVEQWeduC8j0PiQQhIuHA08B5QHdgooh09zeqoJkBnFtl263Ae6raBXjPW29oSoCbVLU7cArwU++/cUO/9iJgmKr2AfoC54rIKcBDwJ9UtTOwG7jaxxiD6RfAmkrroXLdZ6pq30p9H4LyPQ+JBAGcDKxX1Y2quh+YBYzyOaagUNWPgF1VNo8CnvPePwdcVKtB1QJV3aaqS7z3ebg/Gm1o4Neuzl5vNdJbFBgGzPa2N7jrBhCRVOB84FlvXQiB6z6EoHzPQyVBtAG+q7Se4W0LFSmqus17/z2Q4mcwwSYiHYA04AtC4Nq92yxLgSzgXWADsEdVS7wiDfX7/hhwC1DmrScRGtetwH9FZLGIXOttC8r3PKImPsTUH6qqItJgn20WkcbAq8CNqprrflQ6DfXaVbUU6CsiTYA5wEk+hxR0IjISyFLVxSIy1O94atlgVc0UkRbAuyLydeWdNfk9D5UaRCbQttJ6qrctVGwXkVYA3muWz/EEhYhE4pLDTFV9zdscEtcOoKp7gA+AU4EmIlL+A7Ahft9PAy4Ukc24W8bDgMdp+NeNqmZ6r1m4HwQnE6TveagkiIVAF+8Jh0bABGCuzzHVprnAJO/9JODfPsYSFN7952nAGlV9tNKuBn3tIpLs1RwQkRjgbFz7ywfAWK9Yg7tuVb1NVVNVtQPu/+f3VfUyGvh1i0iciMSXvwd+BKwkSN/zkOlJLSIjcPcsw4Hpqnq/zyEFhYi8BAzFDf+7HbgbeB14BWiHGxL9ElWt2pBdr4nIYOBjYAUH7knfjmuHaLDXLiK9cY2S4bgffK+o6r0i0gn3y7oZ8BVwuaoW+Rdp8Hi3mG5W1ZEN/bq965vjrUYAL6rq/SKSRBC+5yGTIIwxxhydULnFZIwx5ihZgjDGGBOQJQhjjDEBWYIwxhgTkCUIY4wxAVmCMKYOEJGh5SOSGlNXWIIwxhgTkCUIY46CiFzuzb+wVET+6g2Ut1dE/uTNx/CeiCR7ZfuKyOcislxE5pSP0S8inUVkvjeHwxIROcH7+MYiMltEvhaRmVJ5ICljfGAJwphqEpFuwHjgNFXtC5QClwFxwCJV7QEswPVeB3ge+I2q9sb18C7fPhN42pvDYRBQPgpnGnAjbs6STrjxhozxjY3makz1nQX0BxZ6P+5jcIOilQEve2X+CbwmIolAE1Vd4G1/DviXN45OG1WdA6CqhQDe532pqhne+lKgA/BJ8C/LmMAsQRhTfQI8p6q3HbRR5M4q5Y51/JrKYwaVYv9/Gp/ZLSZjqu89YKw3Dn/5PMDtcf8flY8geinwiarmALtFZIi3/QpggTfbXYaIXOR9RpSIxNbqVRhTTfYLxZhqUtXVInIHbjavMKAY+CmQD5zs7cvCtVOAG3b5GS8BbASu8rZfAfxVRO71PmNcLV6GMdVmo7kac5xEZK+qNvY7DmNqmt1iMsYYE5DVIIwxxgRkNQhjjDEBWYIwxhgTkCUIY4wxAVmCMMYYE5AlCGOMMQH9P7/yMacFIJejAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "ls_epoch = [_ + 1 for _ in range(num_epochs)]\n",
        "plt.plot(ls_epoch, train_loss_arr, color='r', label='Train')\n",
        "plt.plot(ls_epoch, val_loss_arr, color='g', label='Validation')\n",
        "plt.title('Loss plot')\n",
        "plt.ylabel('Loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend()\n",
        "plt.show()   "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "KTujRC6vWzSH",
        "outputId": "18b73d9a-3052-4ba0-c0c9-884831dccf8f"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de5xN9f748dd7LsZdCBn3+61CJpfKKSSKUAqdOocupKLLKV1O+eaQ6tT5RacQUii3XJNERig6mMG432YmlxEZ18y4zGW/f3/sRRt7xqbZs83s9/Px2I/Z67PWZ633MmO/9+fzWeuzRFUxxhhjLhQS6ACMMcZcnSxBGGOM8coShDHGGK8sQRhjjPHKEoQxxhivLEEYY4zxyhKEMQ4RqSMicSJyQkSeDXQ8xgSaJQhj/vAysERVi6nqf0WklYgsEZHjIrIr0MEZk9ssQRjzhyrAZo/lVOAzYEBgwvmDiIQFOgYTfCxBGAOIyA9AK+BjEUkRkdqqulpVvwASfahfUES+FJHDInJMRGJEpJyzrpSIfC4iv4rIURGZ41Gvt4jEi8gREZkrIpEe61REnhGRncBOp6yj0w12TER+FpEbc/rfwpizLEEYA6hqa+AnoJ+qFlXVHZe5i55ACaASUBroC5xy1n0BFAYaAGWBYQAi0hp4B+gGlAd2A1Mv2G8XoBlQX0Qa427RPOkcYzQwV0QiLjNWY3xiCcKYnJGO+0O7pqpmquoaVf1dRMoDdwN9VfWoqqar6jKnzsPAZ6q6VlXPAK8BLUSkqsd+31HVI6p6CugDjFbVVc4xJgBngOa5dI4myFiCMCZnfAEsBKY6XUnviUg47hbFEVU96qVOJO5WAwCqmgIcBip4bLPX430V4EWne+mYiBxz9h+JMX5gCcKYHOC0DP6lqvWBW4COwN9xf8CXEpFrvFT7FfeHPgAiUgR3K2Sf56493u8FhqrqNR6vwqo6JafPxxiwBGFMlkQkREQKAuHuRSkoIgWy2LaViNwgIqHA77i7nFyquh/4DhgpIiVFJFxE/uJUmwI8KiKNnHGEt4FVqrori5DGAn1FpJm4FRGRDiJSLOfO2pg/WIIwJmt/wT3QPB+o7Lz/PottrwNm4E4OW4FluLudAP6GO2FsAw4CzwOoajQwEJgJ7AdqAD2yCkZVY4HewMfAUSAe6HWF52bMJYk9MMgYY4w31oIwxhjjlSUIY4wxXlmCMMYY45UlCGOMMV7lmwnArr32Wq1atWqgwzDGmDxlzZo1h1S1jLd1+SZBVK1aldjY2ECHYYwxeYqI7M5qnXUxGWOM8coShDHGGK8sQRhjjPEq34xBeJOenk5SUhKnT58OdCjmChUsWJCKFSsSHh4e6FCMCTr5OkEkJSVRrFgxqlatiogEOhxzmVSVw4cPk5SURLVq1QIdjjFBJ193MZ0+fZrSpUtbcsijRITSpUtbC9CYAMnXCQKw5JDH2e/PmMDJ9wnCGGPyuuOnjzM+bjzr9q/L1ePm6zEIY4zJy7Yd2sZHqz5iwvoJpKanEhYSxpBWQ3j51pcJEf9/v7cWRC7473//S7169ejatSstWrQgIiKC//znP4EOyxhzFXKpi293fEu7L9tRb0Q9Pl33KQ/Uf4BlvZZxX937eG3xa9z1xV38euJXv8diLYhcMHLkSKKjoylQoAC7d+9mzpw5uR5DRkYGYWFXz69bVVFVQkLsO4oxZ60/sJ5uM7qx4/AOIotFMqTVEPo06UPZImUBaFm5Je3WtePZBc9y46gbGd9lPB1rd/RbPFfPJ4afPb/geeIOxOXoPhtd14jh7Ydnu03fvn1JTEzk7rvv5rHHHuOFF17g22+/veS+U1NT6datG0lJSWRmZjJw4EC6d+9OTEwMzz33HKmpqURERLB48WLCw8N56qmniI2NJSwsjA8++IBWrVoxfvx4Zs2aRUpKCpmZmcyfP5/+/fuzadMm0tPTGTRoEJ07d/Z6/PHjxzNnzhxSU1PZuXMnL730EmlpaXzxxRdEREQwf/58SpUqxdixYxkzZgxpaWnUrFmTL774gsKFC/Pbb7+dO3eAUaNGERkZSbt27WjWrBlr1qxh/vz5fPzxx3z33XeICG+88Qbdu3e//F+EyXVr96+lQZkGRIRFBDqUfOO7nd/RbUY3SkSUYErXKXSt15Xw0PPv/xERHr/pcW6tfCsPzXyIe6fcS/+m/Xmv7XsUDCuY4zHZ1zc/++STT4iMjGTJkiW88MILPtdbsGABkZGRrF+/nk2bNtG+fXvS0tLo3r07H374IevXryc6OppChQoxYsQIRISNGzcyZcoUevbsee7S0LVr1zJjxgyWLVvG0KFDad26NatXr2bJkiUMGDCA1NTULGPYtGkTs2bNIiYmhtdff53ChQuzbt06WrRowcSJEwG4//77iYmJYf369dSrV49x48YB8Oyzz3L77bezfv161q5dS4MGDQDYuXMnTz/9NJs3byY2Npa4uLhz5zJgwAD2799/pf/UJpfM3T6XJmOa0H1Gd1zqCnQ4OSrDlcHMLTN5/OvHmb11dq6d3yexn3DvlHupVaoWq55YRY/re1yUHDzVvbYuKx9fyfPNnuej1R9xx/g7/BJr0LQgLvVN/2pzww038OKLL/LKK6/QsWNHWrZsycaNGylfvjw333wzAMWLFwdg+fLl9O/fH4C6detSpUoVduzYAUDbtm0pVaoUAN9//z1z5849N/5x+vRp9uzZQ7169bzG0KpVK4oVK0axYsUoUaIE995777nYNmzYALiTyBtvvMGxY8dISUmhXbt2APzwww/nkkhoaCglSpTg6NGjVKlShebNm5+L+6GHHiI0NJRy5cpx++23ExMTQ6dOnXL2H9PkmGOnj9F3Xl9KFSrF19u/ZuAPAxnaZmigw8rW8dPH+eXYL9S9tm6W37L3n9jP2LVjGbNmDPtO7CMiNILP4j7jxnI3MvAvA7m/3v1+GRR2qYuXF73M//vf/6NDrQ5MfWAqRQsU9aluRFgEw9oPo22NtiSnJvslvqBJEHlN7dq1Wbt2LfPnz+eNN96gTZs23HfffZe9nyJFipx7r6rMnDmTOnXq+FQ3IuKP7oOQkJBzyyEhIWRkZADQq1cv5syZQ8OGDRk/fjxLly71OR6T97y48EUOph5k1ROr+CT2E95e/jYNyjbgrzf8NdCheZXpyuSeyffw896fCZVQ6lxbh4blGrpf1zUkPCScsWvHMnPrTDJcGdxV4y5GdhhJuxrtmLFlBm/99BYPTn+Q+mXqM/AvA3mw/oOEhoT6fHxVJSUthaIFil50T8/J9JP8bfbfmLV1Fs/c/AzD2w8nLOTyP5LvqXXPZdfxlXUxXaV+/fVXChcuzCOPPMKAAQNYu3YtderUYf/+/cTExABw4sQJMjIyaNmyJZMmTQJgx44d7Nmzx2sSaNeuHR999BGqCsC6dX/+muoTJ05Qvnx50tPTz8UA0KZNG0aNGgVAZmYmx48fv6huy5YtmTZtGpmZmSQnJ/Pjjz/StGnTPx2T8a7HjB7U+bgO7y5/lwMpBy67/qKERXwW9xkDbhlAk8gmjOgwgpaVW/LY14+xet/qLOvtPrabjpM7cv3I6xn641D2HN+T7XFOZ5xm3o55vL74dSbETWBr8tYr7j4ZvnI4P+/9mX/e9k9eu+01apSswfI9y3l18avcPelu7vziThYmLKR/0/5s77edhY8spFOdTkSERfDwjQ+z6alNTO06FUF4aOZDNBjZgO92fufTsQ+mHuTOL+6k+LvFKTS0ENU+rEaLcS24f9r9PPPtM9w+/nZmb53NsHbD+Ojuj64oOfjb1RdRPnbgwAGioqL4/fffCQkJYfjw4WzZsuVcV5GnjRs3MmDAAEJCQggPD2fUqFEUKFCAadOm0b9/f06dOkWhQoWIjo7m6aef5qmnnuKGG24gLCyM8ePHn/ft/6yBAwfy/PPPc+ONN+JyuahWrRrz5s37U+c0ZMgQmjVrRpkyZWjWrBknTpwA4MMPP6RPnz6MGzeO0NBQRo0aRfny5c+re9999/G///2Phg0bIiK89957XHfddX8qHuPdwviFTNs8jVqlavHa4tcYuGQgnep0os9NfWhbo+0luydS0lLo/U1v6pSuw5t3vAlAgdACzOw2k6afNqXL1C7E9I6hQvEK5+qoKqPXjGbAogEANCzXkDeWvMHAJQNpU70NvRr24r5691E4vDBHTh3h2x3fMmf7HBbGLyQ1/fyxseIRxYmKjKJpZFOaVWzG3TXvvuQA+fZD23ljyRt0qtOJt1q/dd43+COnjrDhtw0cPnmYu2vdTeHwwl73ERoSSvfru/NggweZvXU2/7f0/7hn8j30b9qff9/5bwqFF/Jab1XSKh6Y/gCHTh7in7f9k3RXOvtT9rP/xH62H97Ost3LUFVmdZ9Fl7pdsj2PgDp7uWFefzVp0kQvtGXLlovKTN5jv8c/Jz0zXRuMaKA1Pqyhp9NP67bkbfrSwpf02veuVQahVYZV0Xd+ekdPpp3Mch/95/dXGSS6fPfyi9ZtOLBBi75dVKPGRGlqWqqqqv5y9BdtPaG1Mgi9c+KduuvoLlVVTTySqIOWDNJqw6spg9BibxfT5p8219B/hSqD0Mj/F6l9v+mrC3Yu0JNpJ3Xzwc36+brP9al5T2mT0U00bHDYuX1mF29GZoa2+LSFlvp3Kd1/Yv+f/Bf8w6n0U/r8d88rg9DrR16vGw5sOG+9y+XST2I+0QJDCmi14dV03f51We7L5XLlWFx/BhCrWXyuBvyDPadeliDyL/s9/jmjYkYpg9BZW2adV346/bRO2zRN20xoowxCqw2vpl9v+/qiD66fdv+kDEL7z++f5TG+3va1yiDR7tO768jVI7XI0CJa9O2iOjp2tNcPwkxXpi7btUwfnfOo3jzmZv1n9D91ddJqzXRlZnsup9JP6Scxn6gMEr3ri7v0VPopr9u9v+J9ZRA6acOkbPd3pb7b+Z2We7+cRgyJ0OH/G64ul0tPpZ/Sx+Y8pgxC23/ZXg+fPOyXY+c0SxBXsUOHDmnDhg0veh06dChXjr9gwYKLjt2lS5dcObav8sLv8Wp17NQxLfNeGb3989uz/cb6Q+IPWn9EfWUQes+ke3Tn4Z2qqnoy7aTW/qi2Vh1eVU+cOZHtsd756R1lEMogtO3EtudaDf4wbu04ZRB695d36+n00+et25q8VSOGRGiXqV38+i39YMpB7Ti547mE0GR0E2UQOvCHgZqRmeG34+a0gCUIoD2wHYgHXvWyfhgQ57x2AMc81vUEdjqvnpc6VlYJ4mppxpkr43K5LEFk4UzGmUtuM+D7ASqDRNf8uuaS26ZlpOl/VvxHi75dVAsMKaCvL379XHfKooRFl6zvcrl06I9Dddzacbny/27smrHKILTDpA7nkkRGZoY2G9ssx7uWsuJyuXTk6pFa8K2CWuKdEjp321y/HzOnBSRBAKFAAlAdKACsB+pns31/4DPnfSkg0flZ0nlfMrvjeUsQiYmJmpycbEkij3K5XJqcnKyJiYmBDuWqs3LvSi30ViHt920/TctI87pNwpEELTCkgPaa0+uy9r3v933615l/PdcaePzrx3MiZL/4JOYTZRDaaUonPZNxRv+9/N/KIHTyhsm5Gseuo7t03+/7cvWYOSW7BOHPq5iaAvGqmgggIlOBzsCWLLZ/CHjTed8OWKSqR5y6i3C3RqZcTgAVK1YkKSmJ5OTkKwjfXA3OPnLU/CE9M50+8/oQIiF8HPMxm5M3M/3B6ZQuXPq87V6JfoXwkHCGtr68G9kii0Uy6f5J9LmpDzO2zGBI6yE5GX6OejLqSTI1k2fmP0OHyR34afdP3F/vfnpc3yNX46hyTZVcPV5u8WeCqADs9VhOApp521BEqgDVgB+yqVvBS70+QB+AypUrX7Tf8PBwe1SlyXc+XPUhG37bwKxusziRdoLe3/Tm5rE3M/ehuVxf9noAftr9EzO2zGDwHYOJLBZ5Rce5vert3F719pwM3S+evvlpMl2ZPLvgWUoXKs3Ie0bag6ZyyNVyo1wPYIaqZl5OJVUdo6pRqhpVpkwZP4VmjH+dSj9Fn2/6cOtnt3L45OFst911bBdvLn2TTnU60aVuF/7e8O8s67WMUxmnaDGuBV9v+xqXunhh4QtULF6RF295MZfOIrD6N+vP7O6zmf/wfMoVLRfocPINfyaIfUAlj+WKTpk3PTi/++hy6hqTZ+0+tpvbPr+NsWvHErMvho5TOpKa5n0CRVXlmfnPIAgf3f3RuW/JzSs2J7Z3LPWurUeXaV3oPLUza/av4Z0272R5A1h+1KVuF5pWsDvxc5I/E0QMUEtEqolIAdxJYO6FG4lIXdwD0f/zKF4I3CUiJUWkJHCXU2bMVS89M52xa8Yyb8c8Ml1ZN4p/+OUHosZGEX8knrk95jLtgWms3rearl91JS0z7aLtZ26dyfyd8xnSagiVS5zfpVqheAWW9VrGIzc+wrwd87g58uardn4kk3f4bQxCVTNEpB/uD/ZQ3FcobRaRwbhHzc8mix7AVGc0/WzdIyIyBHeSARh8dsDamKvZryd+pdv0bqzYuwKAyiUq0+emPjx+0+NcV9Q9jYiqMmzlMAYsGkCd0nWY02MOtUvXBmB0x9H0/qY3veb04sv7vzw3Bcbx08d59rtnuan8TfRv1t/rsQuFF2Jil4l0rtOZqMioXHkkpcnfxONzOU+LiorS2NjYQIdhgtjSXUvpPqM7KWkpjOk4hoJhBRkVO4rFvywmLCSM++rexxM3PcHncZ8zddNUutbryuedP6dYRLHz9vPu8nd5bfFr9G/anw/bf4iI0G9+P0bFjmL1E6tpEtkkQGdo8iMRWaOqUd7W2WR9xvxJqsp/fv4Pry1+jZqlavLD33+gQVn3A5K61u/KjsM7GB07ms/jPmf6lukIwjtt3uGVW1/xerXNK7e+QnJqMh+s/ICyRcrStnpbRsaM5Nlmz1pyMLnKWhDG/AnHTx+n19e9mLNtDg/Wf5BxncZd1CI461T6KeZun0uVa6rQvGLzbPfrUhe95vTiiw1fULZIWcJDwtn6zNYs923MlbIWhMkXVPVPX9+e6cpk8LLBnMk8Q7cG3Wh8XeNs93kg5QAzt8zk56SfSctMI9OVSYYrg0x1/9yavJX9KfsZ1m4YzzV7Ltt9FQovRPfrfXvmdoiEMK7TOPc02Du/ZVa3WZYcTK6zFoTJEwYvG8x7K96jZqmaNC7fmEblGtHoOverRMESPu0jPTOdnnN6MmXTFEIkBJe6qFWqFt0bdKf79d3P3WR2Nil8teUrftr9E4pSqXglihYoSmhIKGEhYYSK+2fRAkUZdMcgbqt8m1/O+0zGGTYd3GRdS8ZvsmtBWIIwV70pG6fw11l/5c7qdxIWEkbcgbjznohWq1Qt3mr9Fg/WfzDLb/CnM07TfUZ35m6fyztt3qH3Tb2ZtXUW0zZPY8muJbjURf0y9SldqDTL9yxHUeqXqc+D9R/kwfoPnhtTMCa/sQRhrhpHTx1l3o55LExYSOtqrXms8WPZbr/m1zXc9vlt3Bx5M9F/j6ZAaAHA/S0/7kAccQfimL5lOmv3r6VL3S6MvGck5Yud/+S6lLQUukztwuJfFvPx3R/zTNNnzlv/W8pvzNw6k2mbp3H89HG61O1iScEEDUsQJqB+PfErX2/7mtnbZrNk1xIyXBkUCS9Canoqzzd7nv/c9R+vD4L/LeU3osa6r+eP6R1D2SJlve4/w5XB8JXDGbhkIBGhEXzQ7gMebfQoIsKx08foMLkDK5NW8lmnz+jZqKe/T9eYPCW7BGF30hi/+Xnvz7T8vCUVPqjA0/OfZvfx3bzY4kVWPr6SY68e47lmzzF81XDu/+r+i6aXOJNxhq5fdeXwycN83ePrLJMDQFhIGC/d8hIb+m6g4XUNeXzu49z15V3E7Iuh9YTWxOyL4asHvrLkYMxlshaEyXGHTx7m1ehX+XTdp1QsXpEnmzzJ/fXup9619S4aI/h49cc8t+A5Gl3XiG8e+obIYpGoKn2+6cOn6z5l2gPT6Nagm8/HdqmL0bGjeTn6ZVLSUigUVohZ3WfRvmb7nD5NY/IF62IyucKlLibETWDAogEcO32MF5q/wJt3vEnRAkWzrfftjm/pPqM7JQuV5Nu/fsuPu3+k/3f9eb3l67zV+q0rimXP8T28u/xd/nrDX/12hZEx+YElCON3mw5u4qlvn2L5nuXcWulWRnUYxQ3lbvC5ftyBODpM7sDvZ37nVPopOtTuwOzus20+IWP8zG6UM371fcL3dJjcgRIRJc4NBF/uB3uj6xqx+onVdJraiUxXJl/c94UlB2MCzFoQ5k85nXGaBiMbEB4SzorHVlz02MvLpapkaiZhIfbdxZjcYC0I4zfvr3ifxKOJLPrboj+dHABEhDCxP0tjrgbWhjdXbNexXby9/G0eqP8Ad1a/M9DhGGNymCUIc8VeWPgCIRLCB3d9EOhQjDF+YAnCXJEF8QuYs20Ob7R8g0olKl26gjEmz/FrghCR9iKyXUTiReTVLLbpJiJbRGSziEz2KM8UkTjnddGzrE3gnMk4Q//v+lOrVC3+0eIfgQ7HGOMnfhsNFJFQYATQFkgCYkRkrqpu8dimFvAacKuqHhURz/kUTqlqI3/FZ67cB//7gPgj8Sx4eAERYRGBDscY4yf+bEE0BeJVNVFV04CpQOcLtukNjFDVowCqetCP8ZgcsOf4Ht766S3uq3sf7Wq2C3Q4xhg/8meCqADs9VhOcso81QZqi8gKEVkpIp4T5hQUkVinvIu3A4hIH2eb2OTk5JyN3nj14vcvoqoMazcs0KEYY/ws0BechwG1gDuAisCPInKDqh4DqqjqPhGpDvwgIhtVNcGzsqqOAcaA+0a53A09+EQnRjNjywyGtBpClWuqBDocY4yf+bMFsQ/wvLylolPmKQmYq6rpqvoLsAN3wkBV9zk/E4GlQGM/xmouwaUuXl70MlWvqcpLt7wU6HCMMbnAnwkiBqglItVEpADQA7jwaqQ5uFsPiMi1uLucEkWkpIhEeJTfCmzBBMzsrbNZd2Ad/7rjXxQMKxjocIwxucBvXUyqmiEi/YCFQCjwmapuFpHBQKyqznXW3SUiW4BMYICqHhaRW4DRIuLCncTe9bz6yeSuTFcmA5cMpO61dXn4hocDHY4xJpf4dQxCVecD8y8o+z+P9wr8w3l5bvMz4Ptc0cavpmyawtZDW/nqga+8PhrUGJM/2Z3UJlvpmem8ufRNGl3XiK71uwY6HGNMLgr0VUzmKjc+bjyJRxP55qFv7PkMxgQZ+x8fpDJdmdw75V4GLR2ES11etzmdcZrBPw6mecXmdKjVIZcjNMYEmrUggtScbXOYt2Me83bMY+uhrYzvPJ5C4YXO22bMmjEk/Z7EhC4TEJEARWqMCRRrQQSpYSuHUe2aavz7zn8zffN0Wk9szcHUP2Y6SU1LZehPQ2lVtRWtq7UOYKTGmECxBBGEYvbFsGLvCp5r9hwv3/oyM7rNYP2B9TT/tDlbk7cC8NHqjziYepC3Wr8V4GiNMYFiCSIIDVs5jGIFivFo40cBuL/e/SzttZST6SdpMa4Fs7bO4r0V73FPrXu4pdItAY7WGBMoliCCTNLvSUzfMp0nbnqC4hHFz5U3rdCUVU+somLxinT9qitHTx/lrVbWejAmmNkgdZD5ePXHuNTFs82evWhdlWuqsOKxFTzxzRNEFo2kcXmb/sqYYGYJIoikpqUyZs0Y7qt7H1Wvqep1mxIFSzD9wem5G5gx5qpkXUxBZOL6iRw9fZQXmr8Q6FCMMXmAJYgg4VIXw1cN5+bIm23g2RjjE+tiChLf7fyOHYd3MPn+yXbTmzHGJ9aCCBLDVg6jQrEKPFD/gUCHYozJIyxBBIENv21g8S+L6de0H+Gh4YEOxxiTR1iCCALDVw6ncHhh+jTpE+hQjDF5iCWIfO5AygEmbZxEz4Y9KVWoVKDDMcbkIT4lCBHpJCL/cV73+rpzEWkvIttFJF5EXs1im24iskVENovIZI/yniKy03n19PWY5nyDlw3GpS7+0eIfl97YGGM8XPIqJhF5B2gKTHKKnhWRFqr6z0vUCwVGAG2BJCBGROZ6PltaRGoBrwG3qupRESnrlJcC3gSiAAXWOHWPXvYZBrEdh3cwZs0Y+kb1pWapmoEOxxiTx/jSgugAtFXVz1T1M6A90NGHek2BeFVNVNU0YCrQ+YJtegMjzn7wq+rZ+abbAYtU9YizbpFzXHMZ3vjhDQqGFWTgXwYGOhRjTB7k6xjENR7vS/hYpwKw12M5ySnzVBuoLSIrRGSliLS/jLqISB8RiRWR2OTkZB/DCg6r961m+pbpvHTLS5QrWi7Q4Rhj8iBfbpR7G1gnIksAAf4CeB1PuMLj1wLuACoCP4rIDb5WVtUxwBiAqKgozaGY8jxV5eVFL1O2SFlebPFioMMxxuRR2SYIEQkBXEBz4Gan+BVVPeDDvvcBlTyWKzplnpKAVaqaDvwiIjtwJ4x9uJOGZ92lPhzTAAviF7Bs9zI+vvtjikUUC3Q4xpg8KtsuJlV1AS+r6n5Vneu8fEkOADFALRGpJiIFgB7A3Au2mYOTCETkWtxdTonAQuAuESkpIiWBu5yyoJeWmcbRU1mP1We6Mnkl+hVqlKxB7ya9czEyY0x+48sYRLSIvCQilUSk1NnXpSqpagbQD/cH+1bgK1XdLCKDRaSTs9lC4LCIbAGWAANU9bCqHgGG4E4yMcBgpyyoudRFpymdiPwgkneXv0t6ZvpF20zaOImNBzcytPVQCoQWCECUxpj8QlSz77oXkV+8FKuqVvdPSFcmKipKY2NjAx2GX7234j1eiX6FqMgoYn+NpUGZBozuOJpbK98KwOmM09T5uA5lCpdhde/VhIjdB2mMyZ6IrFHVKG/rLvkJoqrVvLyuquQQDFYlreL1H16na72urH5iNXN7zOVE2glu+/w2+nzThyOnjjBi9Qj2HN/De23fs+RgjPnTfGlBPANMUtVjznJJ4CFVHZkL8fksP7cgjp8+TuPRjcnUTOKejKNkoZIApKSl8K+l/2LYymGUKlSKdFc6zSo0Y8EjCwIcsTEmr/hTLQig99nkAODcuD+IhMEAABfGSURBVGajn7lEVen7bV/2HN/DlK5TziUHgKIFivL+Xe+zps8aqpesTkpaCu/e+W4AozXG5Ce+3AcRKiKiTlPDmULDRj9zyfi48UzdNJW3Wr2V5ZPgGl7XkBWPreDQyUN2U5wxJsf4kiAWANNEZLSz/KRTZvxs26Ft9PuuH62qtuLV27K/NzE0JNSSgzEmR/mSIF7BnRSecpYXAZ/6LSIDuK9I6jGjB4XCCvHl/V8SGhIa6JCMMUHmkgnCuVlulPMyueTV6FdZ/9t6vnnoGyKLRQY6HGNMEPJluu9awDtAfaDg2XK71NV/ft77M/9d9V/63dyPjrV9mTjXGGNyni9XMX2Ou/WQAbQCJgJf+jOoYJaWmUafb/pQqUQl3rnznUCHY4wJYr4kiEKquhj3PRO7VXUQ7mdEGD94f8X7bE7ezMh7RlK0QNFAh2OMCWK+DFKfcWZ13Ski/XDPtGqfXH6w4/AOhvw4hAfrP0iH2paDjTGB5UsL4jmgMPAs0AT4G2DPiM5hqsqT856kYFhBPmz/YaDDMcYYn65iinHepgCP+jec4DU+bjxLdy1ldMfRlC9WPtDhGGOMT1cxRQGvA1U8t1fVG/0YV1A5mHqQlxa9xG2Vb+OJm54IdDjGGAP4NgYxCRgAbMT9dDmTw/6x8B+cOHOCMR3H2Cysxpirhi8JIllVL3wSnMkhC+MXMmnjJN68/U3qlakX6HCMMeYcXxLEmyLyKbAYOHO2UFVn+S2qIHHk1BH6ftuXOqXr8NptrwU6HGOMOY8vCeJRoC4Qzh9dTApcMkGISHvgQyAU+FRV371gfS/gfdyXzgJ8rKqfOusycXdrAexR1U7kI2mZaTzw1QP8euJXlvZcSkRYRKBDMsaY8/iSIG5W1TqXu2NnWvARQFsgCYgRkbmquuWCTaepaj8vuzilqo0u97h5gary1LynWLJrCV/e9yUtKrUIdEjGGHMRX0ZEfxaR+lew76ZAvKomqmoaMBXofAX7yXfe//l9Pov7jIF/GcjDNz4c6HCMMcYrXxJEcyBORLaLyAYR2SgiG3yoVwHY67Gc5JRdqKuz3xkiUsmjvKCIxIrIShHp4u0AItLH2SY2OTnZh5ACb/bW2bwa/SrdGnRj0B2DAh2OMcZkyZcupvZ+PP43wBRVPSMiTwITgNbOuiqquk9EqgM/iMhGVU3wrKyqY4Ax4H4mtR/jzBFrfl3Dw7MepmmFpozvPN4uaTXGXNV8uZN69xXuex/g2SKoyB+D0Wf3fdhj8VPgPY91+5yfiSKyFGgMnJcg8pKk35O4d8q9lClShq97fE2h8EKBDskYY7Llz6+wMUAtEakmIgWAHsB591OIiOecEp2ArU55SRGJcN5fC9wKXDi4nWecSj/FvVPuJSUthXkPzbNHgxpj8gRfupiuiKpmOLO/LsR9metnqrpZRAYDsc7Nd8+KSCfcz5o4AvRyqtcDRouIC3cSe9fL1U95xuSNk4k7EMec7nO4odwNgQ7HGGN8IqqX7roXkSpALVWNFpFCQJiqnvB7dJchKipKY2NjAx2GVy0/b8mhk4fY8vQWRCTQ4RhjzDkiskZVo7ytu2QXk4j0BmYAo52iisCcnAsvf4s/Es/yPcvp2bCnJQdjTJ7iyxjEM7jHAH4HUNWdQFl/BpWfTIibQIiE8Lcb/xboUIwx5rL4kiDOODe6ASAiYbin2jCX4FIXEzdMpG31tlQo7u0WEGOMuXr5kiCWicg/gUIi0haYjvv+BXMJS3ctZc/xPfRq1CvQoRhjzGXzJUG8AiTjnjjvSWA+8IY/g8ovxseNp0RECTrXsRlGjDF5T7aXuToT7m1W1brA2NwJKX84ceYEM7fO5JEbHrGb4owxeVK2LQhVzQS2i0jlXIon35ixZQYn009a95IxJs/y5Ua5ksBmEVkNpJ4tzG/PZ8hp49ePp3bp2jSv2DzQoRhjzBXxJUEM9HsU+UzCkQR+3P0jQ1sPtXsfjDF5li+T9S0TkXLAzU7RalU96N+w8raJ6yciiN37YIzJ03y5k7obsBp4EOgGrBKRB/wdWF7lUhcT1k/gzup3UqlEpUtXMMaYq5QvXUyv437s6EEAESkDROOefsNc4MfdP7L7+G7ebvN2oEMxxpg/xZf7IEIu6FI67GO9oDQ+bjzFChSjS12vD8Ezxpg8w5cWxAIRWQhMcZa7A9/5L6S8KyUthRlbZvDQ9Q9ROLxwoMMxxpg/xZdB6gEicj9wm1M0RlVn+zesvGn65umkpqfavQ/GmHzhkglCRKoB81V1lrNcSESqquoufweX14xbN47apWtzS6VbAh2KMcb8ab6MJUwHXB7LmU6Z8bAleQsr9q7gicZP2L0Pxph8wZcEEeY53bfzvoAvOxeR9iKyXUTiReRVL+t7iUiyiMQ5ryc81vUUkZ3Oq6cvxwukT9d+SnhIOD0bXfWhGmOMT3wZpE4WkU7OM6QRkc7AoUtVcib6GwG0BZKAGBGZ6+XZ0tNUtd8FdUsBbwJRuJ89scape9SHeHPdmYwzTFw/kc51O1O2iD1LyRiTP/jSgugL/FNE9ojIXtzTfz/pQ72mQLyqJjqtjqmAr/NetwMWqeoRJyksAtr7WDfXzdk2h8OnDvNE4ycuvbExxuQRvlzFlAA0F5GiznKKj/uuAOz1WE4CmnnZrquI/AXYAbygqnuzqHvRI9lEpA/QB6By5cBNODt27ViqlKhC2xptAxaDMcbkNF+m2nhORIrjnsl1uIisFZG7cuj43wBVVfVG3K2ECZdTWVXHqGqUqkaVKVMmh0K6PIlHE1n8y2Ieb/w4IWL3Dxpj8g9fPtEeU9XfgbuA0sDfgHd9qLcP8JyMqKJTdo6qHlbVM87ip0ATX+teLcatHUeIhPBo40cDHYoxxuQoXxLE2Ws27wEmqupmj7LsxAC1RKSaiBQAegBzz9uxSHmPxU7AVuf9QuAuESkpIiVxJ6eFPhwzV2W4Mvg87nPurnk3FYtXDHQ4xhiTo3y5immNiHwPVANeE5FinH9fhFeqmiEi/XB/sIcCn6nqZhEZDMQ6V0U9KyKdgAzgCNDLqXtERIbgTjIAg1X1yGWem9/N3zmf/Sn76X1T70CHYowxOU5UNfsNREKARkCiqh4TkdJABVXdkBsB+ioqKkpjY2Nz9Zj3TrmXNb+uYc8LewgL8SXXGmPM1UVE1qhqlLd1l+xiUlWXqq5V1WPO8uGrLTkEQtLvSczfOZ9HGz1qycEYky/ZZTdXaHzceFzq4rHGjwU6FGOM8QtLEFfApS7GrRtHm2ptqFGqRqDDMcYYv7iiBHH2prlgFZ0Yza5ju2xw2hiTr11pC+LC+ZSCysT1EyldqLQ9Nc4Yk69lOboqIv/IahUQtC0Il7r4PuF77ql1DxFhEYEOxxhj/Ca7FsTbQEmg2AWvopeol69t/G0jySeTubP6nYEOxRhj/Cq76zPXAnNUdc2FKzyf2xBsohOjAWhTrU2AIzHGGP/KLkE8ChzOYp3XmyqCQfQv0dS7th4Vil80uawxxuQr2XUVvaGqh0TkuQtXqOpvfozpqnUm4ww/7v7RupeMMUEhuwTRREQigcecSfNKeb5yK8CrycqklZxMP2kJwhgTFLLrYvoEWAxUB9Zw/gyu6pQHlejEaEIllNur3B7oUIwxxu+ybEGo6n9VtR7uWVirq2o1j1fQJQdwjz80rdCUEgVLBDoUY4zxO18m63sqNwK52h0/fZzV+1Zb95IxJmgE7f0Ml2vprqW41GUJwhgTNCxB+Cg6MZrC4YVpXrF5oEMxxphcYQnCR9G/RHN7ldspEFog0KEYY0yu8GuCEJH2IrJdROJF5NVstusqIioiUc5yVRE5JSJxzusTf8Z5KUm/J7Ht0DbrXjLGBBW/PQpNREKBEUBbIAmIEZG5qrrlgu2KAc8Bqy7YRYKqNvJXfJfj7PQaliCMMcHEny2IpkC8qiaqahowFejsZbshwL+B036M5U+JToymbJGyXF/2+kCHYowxucafCaICsNdjOckpO0dEbgIqqeq3XupXE5F1IrJMRFp6O4CI9BGRWBGJTU5OzrHAPakq0YnRtKnWhhCxIRtjTPAI2CeeiIQAHwAvelm9H6isqo2BfwCTRaT4hRup6hhVjVLVqDJlyvglzs3Jm/kt9TfrXjLGBB1/Joh9QCWP5YpO2VnFgOuBpSKyC2gOzBWRKFU9o6qHAZzpxhOA2n6MNUs2/mCMCVb+TBAxQC0RqSYiBYAewNyzK1X1uKpeq6pVVbUqsBLopKqxIlLGGeRGRKoDtYBEP8aapejEaGqVqkXlEpUDcXhjjAkYvyUIVc0A+gELga3AV6q6WUQGi0inS1T/C7BBROKAGUBfVT3ir1izkp6ZztJdS2lbvW1uH9oYYwLOb5e5AqjqfGD+BWX/l8W2d3i8nwnM9Gdsvli1bxWp6anWvWSMCUp2WU42ohOjCZEQ7qh6R6BDMcaYXGcJIhvRidFERUZRslDJQIdijDG5zhJEFlSV9b+tp1mFZoEOxRhjAsISRBYOph4kJS2FmqVqBjoUY4wJCEsQWUg4mgBgCcIYE7QsQWQh/kg8ADVK1ghwJMYYExiWILKQcCQBQah6TdVAh2KMMQFhCSILCUcTqFyiMhFhEYEOxRhjAsISRBbij8RTo5R1LxljgpcliCwkHE2gZkkboDbGBC9LEF4cP32cQycPWQvCGBPULEF4cfYSV7uCyRgTzCxBeJFwxO6BMMYYSxBenL0HonrJ6gGOxBhjAscShBcJRxMoV6QcxSKKBToUY4wJGEsQXiQcTbABamNM0LME4UX8kXgboDbGBD2/JggRaS8i20UkXkRezWa7riKiIhLlUfaaU2+7iLTzZ5yeTqWfIun3JBugNsYEPb89clREQoERQFsgCYgRkbmquuWC7YoBzwGrPMrqAz2ABkAkEC0itVU101/xnvXLsV8Au8TVGGP82YJoCsSraqKqpgFTgc5ethsC/Bs47VHWGZiqqmdU9Rcg3tmf39klrsYY4+bPBFEB2OuxnOSUnSMiNwGVVPXby63r1O8jIrEiEpucnJwjQZ+7Sc4GqY0xQS5gg9QiEgJ8ALx4pftQ1TGqGqWqUWXKlMmRuOKPxFM8ojilC5XOkf0ZY0xe5bcxCGAfUMljuaJTdlYx4HpgqYgAXAfMFZFOPtT1m4SjCdQsVRMnJmOMCVr+bEHEALVEpJqIFMA96Dz37EpVPa6q16pqVVWtCqwEOqlqrLNdDxGJEJFqQC1gtR9jPccucTXGGDe/JQhVzQD6AQuBrcBXqrpZRAY7rYTs6m4GvgK2AAuAZ3LjCqYMVwa7ju2yAWpjjMG/XUyo6nxg/gVl/5fFtndcsDwUGOq34LzYe3wvGa4Ma0EYYwx2J/V5zk7SZ1cwGWOMJYjznL3E1bqYjDHGEsR5Eo4kEBEaQWSxyECHYowxAWcJwkP80XhqlKpBiNg/izHG2Cehh4QjCTZAbYwxDksQDlV1PwfCEoQxxgCWIM45kHKAk+knbYDaGGMcliAcNkmfMcaczxKE4+w9ENaCMMYYN0sQjoQjCYRKKFVKVAl0KMYYc1WwBOGIPxpP5RKVCQ8ND3QoxhhzVbAE4Ug4kmDdS8YY48EShMMucTXGmPNZggCOnjrKkVNHrAVhjDEeLEFgl7gaY4w3liCwS1yNMcYbSxC4B6gBqpesHuBIjDHm6uHXBCEi7UVku4jEi8irXtb3FZGNIhInIstFpL5TXlVETjnlcSLyiT/jTDiaQPmi5SkcXtifhzHGmDzFb48cFZFQYATQFkgCYkRkrqpu8dhssqp+4mzfCfgAaO+sS1DVRv6Kz1P8kXjrXjLGmAv4swXRFIhX1URVTQOmAp09N1DV3z0WiwDqx3iylHA0wQaojTHmAv5MEBWAvR7LSU7ZeUTkGRFJAN4DnvVYVU1E1onIMhFp6a8gT6af5NcTv1KzpLUgjDHGU8AHqVV1hKrWAF4B3nCK9wOVVbUx8A9gsogUv7CuiPQRkVgRiU1OTr6i46empfLQ9Q/RtELTKzwDY4zJn/w2BgHsAyp5LFd0yrIyFRgFoKpngDPO+zVOC6M2EOtZQVXHAGMAoqKirqh7qkyRMkzuOvlKqhpjTL7mzxZEDFBLRKqJSAGgBzDXcwMRqeWx2AHY6ZSXcQa5EZHqQC0g0Y+xGmOMuYDfWhCqmiEi/YCFQCjwmapuFpHBQKyqzgX6icidQDpwFOjpVP8LMFhE0gEX0FdVj/grVmOMMRcT1YBcOJTjoqKiNDY29tIbGmOMOUdE1qhqlLd1AR+kNsYYc3WyBGGMMcYrSxDGGGO8sgRhjDHGK0sQxhhjvMo3VzGJSDKw+xKbXQscyoVwrkbBeu523sHFzvvyVVHVMt5W5JsE4QsRic3qcq78LljP3c47uNh55yzrYjLGGOOVJQhjjDFeBVuCGBPoAAIoWM/dzju42HnnoKAagzDGGOO7YGtBGGOM8ZElCGOMMV4FTYIQkfYisl1E4kXk1UDH4y8i8pmIHBSRTR5lpURkkYjsdH6WDGSM/iAilURkiYhsEZHNIvKcU56vz11ECorIahFZ75z3v5zyaiKyyvl7n+Y8kyXfEZFQ59HE85zlYDnvXSKyUUTiRCTWKcvxv/WgSBDOw4dGAHcD9YGHRKR+YKPym/FA+wvKXgUWq2otYLGznN9kAC+qan2gOfCM8zvO7+d+Bmitqg2BRkB7EWkO/BsYpqo1cT9r5fEAxuhPzwFbPZaD5bwBWqlqI4/7H3L8bz0oEgTQFIhX1URVTcP9eNPOAY7JL1T1R+DChyt1BiY47ycAXXI1qFygqvtVda3z/gTuD40K5PNzV7cUZzHceSnQGpjhlOe78wYQkYq4n0T5qbMsBMF5ZyPH/9aDJUFUAPZ6LCc5ZcGinKrud94fAMoFMhh/E5GqQGNgFUFw7k43SxxwEFgEJADHVDXD2SS//r0PB17G/dRJgNIEx3mD+0vA9yKyRkT6OGU5/rfut0eOmquTqqqI5Ntrm0WkKDATeF5Vf3d/qXTLr+euqplAIxG5BpgN1A1wSH4nIh2Bg6q6RkTuCHQ8AXCbqu4TkbLAIhHZ5rkyp/7Wg6UFsQ+o5LFc0SkLFr+JSHkA5+fBAMfjFyISjjs5TFLVWU5xUJw7gKoeA5YALYBrROTsF8D8+Pd+K9BJRHbh7jJuDXxI/j9vAFR1n/PzIO4vBU3xw996sCSIGKCWc4VDAaAHMDfAMeWmuUBP531P4OsAxuIXTv/zOGCrqn7gsSpfn7uIlHFaDohIIaAt7vGXJcADzmb57rxV9TVVraiqVXH/f/5BVR8mn583gIgUEZFiZ98DdwGb8MPfetDcSS0i9+DuswwFPlPVoQEOyS9EZApwB+7pf38D3gTmAF8BlXFPid5NVS8cyM7TROQ24CdgI3/0Sf8T9zhEvj13EbkR94BkKO4vfF+p6mARqY77m3UpYB3wiKqeCVyk/uN0Mb2kqh2D4bydc5ztLIYBk1V1qIiUJof/1oMmQRhjjLk8wdLFZIwx5jJZgjDGGOOVJQhjjDFeWYIwxhjjlSUIY4wxXlmCMOYqICJ3nJ2R1JirhSUIY4wxXlmCMOYyiMgjzvMX4kRktDNRXoqIDHOex7BYRMo42zYSkZUiskFEZp+dn19EaopItPMMh7UiUsPZfVERmSEi20RkknhOJGVMAFiCMMZHIlIP6A7cqqqNgEzgYaAIEKuqDYBluO9eB5gIvKKqN+K+w/ts+SRghPMMh1uAszNwNgaex/3Mkuq45xsyJmBsNldjfNcGaALEOF/uC+GeEM0FTHO2+RKYJSIlgGtUdZlTPgGY7syhU0FVZwOo6mkAZ3+rVTXJWY4DqgLL/X9axnhnCcIY3wkwQVVfO69QZOAF213p/DWecwZlYv8/TYBZF5MxvlsMPODMwX/2GcBVcP8/OjuD6F+B5ap6HDgqIi2d8r8By5yn3SWJSBdnHxEiUjhXz8IYH9k3FGN8pKpbROQN3E/yCgHSgWeAVKCps+4g7nEKcE+5/ImTABKBR53yvwGjRWSws48Hc/E0jPGZzeZqzJ8kIimqWjTQcRiT06yLyRhjjFfWgjDGGOOVtSCMMcZ4ZQnCGGOMV5YgjDHGeGUJwhhjjFeWIIwxxnj1/wEZ62Mjhl+nCQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "ls_epoch = [_ + 1 for _ in range(num_epochs)]\n",
        "plt.plot(ls_epoch, f1_macro_scores, color='g', label='f1_score_macro')\n",
        "plt.title('f1 score')\n",
        "plt.ylabel('f1 score macro')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend()\n",
        "plt.show() "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DbBG7ZRwW27A",
        "outputId": "89427fb8-e0ae-4804-e16c-9416e78d488c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Test Acc: 75.64%: 100%|██████████| 8/8 [00:00<00:00, 217.76it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loss: 0.0131768198349537\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "eval_loss, predicted_labels, true_labels, f1_score_macro = eval_epoch(model, criterion, test_loader, test_mode=True)\n",
        "print(f'Loss: {eval_loss / len(test_loader.dataset)}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ps11XU1XXqqF",
        "outputId": "61c1049b-0e09-415c-d757-4b954fa105c7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[37,  1,  0,  0,  1,  2,  1,  8,  0,  0],\n",
              "       [ 0, 66,  0,  0,  1,  0,  0,  0,  0,  1],\n",
              "       [ 0,  0, 32,  4,  1,  0,  2,  0,  0,  4],\n",
              "       [ 1,  0,  1, 22,  1,  2,  0,  1,  1,  2],\n",
              "       [ 3,  0,  3,  1, 21,  0,  9,  7,  0,  1],\n",
              "       [ 1,  0,  3,  2,  0, 29,  0,  1,  0,  6],\n",
              "       [ 0,  0,  2,  1,  2,  2, 55,  1,  1,  0],\n",
              "       [ 2,  0,  0,  0,  3,  2,  1, 39,  0,  1],\n",
              "       [ 0,  0,  1,  1,  0,  2,  3,  1, 26,  2],\n",
              "       [ 0,  1,  2,  1,  2,  1,  1,  3,  3, 27]])"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\n",
        "\n",
        "conf_mx = confusion_matrix(true_labels, predicted_labels)\n",
        "\n",
        "conf_mx"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 470
        },
        "id": "WDd1TVjrXvbx",
        "outputId": "17880ac5-b709-4653-cf01-25756ecda50c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accurracy: 0.756\n",
            "\n",
            "F1-Macro: 0.738\n",
            "\n",
            "F1-Micro: 0.756\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "                   فرهنگ و هنر  ورزشی  اقتصادی  فناوری و ارتباطات  سیاسی  \\\n",
              "فرهنگ و هنر                 37      1        0                  0      1   \n",
              "ورزشی                        0     66        0                  0      1   \n",
              "اقتصادی                      0      0       32                  4      1   \n",
              "فناوری و ارتباطات            1      0        1                 22      1   \n",
              "سیاسی                        3      0        3                  1     21   \n",
              "علمی و پزشکی                 1      0        3                  2      0   \n",
              "بین الملل                    0      0        2                  1      2   \n",
              "مذهبی                        2      0        0                  0      3   \n",
              "حوادث                        0      0        1                  1      0   \n",
              "اجتماعی                      0      1        2                  1      2   \n",
              "\n",
              "                   علمی و پزشکی  بین الملل  مذهبی  حوادث  اجتماعی  \n",
              "فرهنگ و هنر                   2          1      8      0        0  \n",
              "ورزشی                         0          0      0      0        1  \n",
              "اقتصادی                       0          2      0      0        4  \n",
              "فناوری و ارتباطات             2          0      1      1        2  \n",
              "سیاسی                         0          9      7      0        1  \n",
              "علمی و پزشکی                 29          0      1      0        6  \n",
              "بین الملل                     2         55      1      1        0  \n",
              "مذهبی                         2          1     39      0        1  \n",
              "حوادث                         2          3      1     26        2  \n",
              "اجتماعی                       1          1      3      3       27  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-90d07b6f-c310-4ae0-824c-86bb8890a7c2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>فرهنگ و هنر</th>\n",
              "      <th>ورزشی</th>\n",
              "      <th>اقتصادی</th>\n",
              "      <th>فناوری و ارتباطات</th>\n",
              "      <th>سیاسی</th>\n",
              "      <th>علمی و پزشکی</th>\n",
              "      <th>بین الملل</th>\n",
              "      <th>مذهبی</th>\n",
              "      <th>حوادث</th>\n",
              "      <th>اجتماعی</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>فرهنگ و هنر</th>\n",
              "      <td>37</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ورزشی</th>\n",
              "      <td>0</td>\n",
              "      <td>66</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>اقتصادی</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>32</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>فناوری و ارتباطات</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>سیاسی</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>21</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>علمی و پزشکی</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>بین الملل</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>55</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>مذهبی</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>39</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>حوادث</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>26</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>اجتماعی</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-90d07b6f-c310-4ae0-824c-86bb8890a7c2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-90d07b6f-c310-4ae0-824c-86bb8890a7c2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-90d07b6f-c310-4ae0-824c-86bb8890a7c2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {}
        }
      ],
      "source": [
        "print(f\"Accurracy: {round(accuracy_score(true_labels, predicted_labels), 3)}\\n\")\n",
        "\n",
        "f1_macro = f1_score(true_labels, predicted_labels, average='macro')\n",
        "print(f\"F1-Macro: {round(f1_macro, 3)}\\n\")\n",
        "\n",
        "f1_micro = f1_score(true_labels, predicted_labels, average='micro')\n",
        "print(f\"F1-Micro: {round(f1_micro, 3)}\\n\")\n",
        "\n",
        "\n",
        "confusion_mat = confusion_matrix(true_labels, predicted_labels, labels=np.arange(10))\n",
        "confusion_mat_df = pd.DataFrame(confusion_mat).rename(columns=CLASSES_CATEGORIES, index=CLASSES_CATEGORIES)\n",
        "display(confusion_mat_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pngYGOs4ZqGN",
        "outputId": "a92f95dd-6841-4f1d-dcfb-a9254f301462"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro-average precision: 0.75\n",
            "Macro-average recall: 0.74\n",
            "Micro-average precision: 0.76\n",
            "Micro-average recall: 0.76\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "precision, recall, f1, support = precision_recall_fscore_support(true_labels, predicted_labels, average='macro')\n",
        "print(\"Macro-average precision: {:.2f}\".format(precision))\n",
        "print(\"Macro-average recall: {:.2f}\".format(recall))\n",
        "precision, recall, f1, support = precision_recall_fscore_support(true_labels, predicted_labels, average='micro')\n",
        "print(\"Micro-average precision: {:.2f}\".format(precision))\n",
        "print(\"Micro-average recall: {:.2f}\".format(recall))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "function to predict:"
      ],
      "metadata": {
        "id": "XtE8l7loJKQN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZQt9OxqDZx3P"
      },
      "outputs": [],
      "source": [
        "def predict(x):\n",
        "  embedding = FastText_model.get_query_embedding(x, TF_IDF_model.vectorizer)\n",
        "  output = model(torch.from_numpy(embedding).unsqueeze(0))\n",
        "  predicted_label = output.argmax(-1).item()\n",
        "  predicted_class_code = CLASSES_CATEGORIES[predicted_label]\n",
        "  return predicted_class_code"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "-XZMJXy-Z5m3",
        "outputId": "46c6b10d-2825-413a-ad57-3bbe7363d064"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'ورزشی'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 63
        }
      ],
      "source": [
        "predict('بایرن مونیخ قهرمان لیگ قهرمانان اروپا شد.')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7E6396zGaYJD"
      },
      "outputs": [],
      "source": [
        "PATH = \"nn_fasttext_model.pt\"\n",
        "\n",
        "# Save\n",
        "torch.save(model, PATH)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "interpreter": {
      "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
    },
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.0"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "06ebb679792947aaa8e21e50ec78cb91": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_699e4f2ffad2411283ab0bdbbd0300bd",
              "IPY_MODEL_eabb08084a194ffba888eadba4efc8ca",
              "IPY_MODEL_21ed63c2557b4d6da1ed92da82e90f5d"
            ],
            "layout": "IPY_MODEL_c3e3d2d2de014c97b9c668046b23e1fd"
          }
        },
        "699e4f2ffad2411283ab0bdbbd0300bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b3f82cb26b945969341fcd6cc0ef824",
            "placeholder": "​",
            "style": "IPY_MODEL_6327641838f34de5b344071f66a35328",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "eabb08084a194ffba888eadba4efc8ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73bc48efacc84e34bcf24e948d7e3236",
            "max": 837,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_47d4c66991b74716a55f5680e815c433",
            "value": 837
          }
        },
        "21ed63c2557b4d6da1ed92da82e90f5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f82099530f43421d80e93add7d6f3a37",
            "placeholder": "​",
            "style": "IPY_MODEL_ea8816c60bd249d0b6daf2192a2eebc2",
            "value": " 837/837 [00:00&lt;00:00, 17.6kB/s]"
          }
        },
        "c3e3d2d2de014c97b9c668046b23e1fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9b3f82cb26b945969341fcd6cc0ef824": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6327641838f34de5b344071f66a35328": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73bc48efacc84e34bcf24e948d7e3236": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47d4c66991b74716a55f5680e815c433": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f82099530f43421d80e93add7d6f3a37": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea8816c60bd249d0b6daf2192a2eebc2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca1e80e36b2945afa7ee544c39c838e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_102b88213bbf45648d0be29344b1b341",
              "IPY_MODEL_2434da5a50824baa861220fe906f3a60",
              "IPY_MODEL_694671cba5af41298e88ae57d4d0cec4"
            ],
            "layout": "IPY_MODEL_a9d4794bb7924cb4935b6430a5abb48e"
          }
        },
        "102b88213bbf45648d0be29344b1b341": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_558737ec88ea45b486ac279dd115ffb4",
            "placeholder": "​",
            "style": "IPY_MODEL_fd0212f1be874e20a1fe20f9aedb3c84",
            "value": "Downloading (…)&quot;pytorch_model.bin&quot;;: 100%"
          }
        },
        "2434da5a50824baa861220fe906f3a60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8acb86b9ac124875a56225d42be1da6f",
            "max": 314339179,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d56e6cac9499463aa36a1e05181b59cb",
            "value": 314339179
          }
        },
        "694671cba5af41298e88ae57d4d0cec4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_baaef6c112444f0a9afc1c0235a6c2db",
            "placeholder": "​",
            "style": "IPY_MODEL_a2533228d23d4751b7d28dcd1a1f1503",
            "value": " 314M/314M [00:06&lt;00:00, 62.0MB/s]"
          }
        },
        "a9d4794bb7924cb4935b6430a5abb48e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "558737ec88ea45b486ac279dd115ffb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd0212f1be874e20a1fe20f9aedb3c84": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8acb86b9ac124875a56225d42be1da6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d56e6cac9499463aa36a1e05181b59cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "baaef6c112444f0a9afc1c0235a6c2db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a2533228d23d4751b7d28dcd1a1f1503": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "19ad38ddedc1478aa48f2ec8c038883c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f8568441bf89474998aab4f9a24d7063",
              "IPY_MODEL_89b14867d50a44898aeea85530ce8550",
              "IPY_MODEL_c528c549c0a14eeeb12a6f357102658e"
            ],
            "layout": "IPY_MODEL_fb30eea46b544282bd6395c481525b4d"
          }
        },
        "f8568441bf89474998aab4f9a24d7063": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e8557f582154ccf8f11479e15e8b878",
            "placeholder": "​",
            "style": "IPY_MODEL_dbea1b121f004f03969e6f0bb9ab2cef",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "89b14867d50a44898aeea85530ce8550": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f48fe4877f14cd488378a8093912153",
            "max": 365,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2ce66d1d6942441b8c907f72c3e1b585",
            "value": 365
          }
        },
        "c528c549c0a14eeeb12a6f357102658e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f91c768b10ba479da4f1033f1ed01a40",
            "placeholder": "​",
            "style": "IPY_MODEL_aa6d2faa563b466f87e958ebaae92003",
            "value": " 365/365 [00:00&lt;00:00, 11.1kB/s]"
          }
        },
        "fb30eea46b544282bd6395c481525b4d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e8557f582154ccf8f11479e15e8b878": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbea1b121f004f03969e6f0bb9ab2cef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f48fe4877f14cd488378a8093912153": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ce66d1d6942441b8c907f72c3e1b585": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f91c768b10ba479da4f1033f1ed01a40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa6d2faa563b466f87e958ebaae92003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5bc29eaf3e3b45949ffa7e9c69bfbe72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dc6d9746a32c4f9ca99192dea9422901",
              "IPY_MODEL_7f834102ef184bf5aa1e16bba7ca331c",
              "IPY_MODEL_ee9809d10207494284fd27feae6219d2"
            ],
            "layout": "IPY_MODEL_fb4924e359184cb1bd400f74b34e59e0"
          }
        },
        "dc6d9746a32c4f9ca99192dea9422901": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c69ef06ef1f44ebbad798d1195150bb5",
            "placeholder": "​",
            "style": "IPY_MODEL_1c5d8e171ba14dcab4cee6b93cd564d0",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "7f834102ef184bf5aa1e16bba7ca331c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f0a8a6e6a844ee291019d207d3f48c6",
            "max": 426422,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c2f5f67c64c9451fb743c7a0c85cb278",
            "value": 426422
          }
        },
        "ee9809d10207494284fd27feae6219d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c1b37f02ab4f40dbb3d2db2303e7990e",
            "placeholder": "​",
            "style": "IPY_MODEL_e062ecafae4340ca8f58c617780e03c9",
            "value": " 426k/426k [00:00&lt;00:00, 4.08MB/s]"
          }
        },
        "fb4924e359184cb1bd400f74b34e59e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c69ef06ef1f44ebbad798d1195150bb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c5d8e171ba14dcab4cee6b93cd564d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f0a8a6e6a844ee291019d207d3f48c6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c2f5f67c64c9451fb743c7a0c85cb278": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c1b37f02ab4f40dbb3d2db2303e7990e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e062ecafae4340ca8f58c617780e03c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fefa4dac2ca440359cc976c837021a10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c430b8f4ead84b7daebdb29f991071b1",
              "IPY_MODEL_0197fcae4e7b4774b24be12d3db61b52",
              "IPY_MODEL_099209362d1c4da29139f55266b243ae"
            ],
            "layout": "IPY_MODEL_94ce0d1685064d54abec49757f25871d"
          }
        },
        "c430b8f4ead84b7daebdb29f991071b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f2c7202269a146fab714ea063b6330f2",
            "placeholder": "​",
            "style": "IPY_MODEL_fbfcb20e89794c229e0648cbf35af849",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "0197fcae4e7b4774b24be12d3db61b52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_694c9eef216147b980eba1a63797c07a",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e214603f88fc43a88ea6b76a6ac2ba50",
            "value": 112
          }
        },
        "099209362d1c4da29139f55266b243ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7aac5aab9d2346b2ad30b5ea0ea5654e",
            "placeholder": "​",
            "style": "IPY_MODEL_b9e56b14a0d546e298e1355eba110bc4",
            "value": " 112/112 [00:00&lt;00:00, 4.69kB/s]"
          }
        },
        "94ce0d1685064d54abec49757f25871d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f2c7202269a146fab714ea063b6330f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbfcb20e89794c229e0648cbf35af849": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "694c9eef216147b980eba1a63797c07a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e214603f88fc43a88ea6b76a6ac2ba50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7aac5aab9d2346b2ad30b5ea0ea5654e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b9e56b14a0d546e298e1355eba110bc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}