Entropy_based_allocation_agnews.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m5tUoHe9FRhs"
      },
      "source": [
        "## Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dr7BCHS-nIRW",
        "outputId": "cbe8eddf-5eaa-4d6c-fbb7-449cce313355",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m68.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.0)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m26.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m77.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m76.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import torch\n",
        "from torch.nn import BCEWithLogitsLoss, BCELoss\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import MultiLabelBinarizer\n",
        "from sklearn.metrics import classification_report, confusion_matrix, multilabel_confusion_matrix, f1_score, accuracy_score\n",
        "import pickle\n",
        "#from transformers import *\n",
        "from tqdm import tqdm, trange\n",
        "from ast import literal_eval"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zB5Dij6JuItX"
      },
      "outputs": [],
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UhjnJEwKnISB",
        "outputId": "6ebdb654-69f0-4d67-c021-45516e0d3bb7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ]
        }
      ],
      "source": [
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "  raise SystemError('GPU device not found')\n",
        "print('Found GPU at: {}'.format(device_name))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "uorMX_zrnISM",
        "outputId": "592f2787-4b02-442a-e5d6-0ba80908248a",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 699
        },
        "id": "62QeEJcKtWm3",
        "outputId": "cfa31f43-420e-465e-85a4-a425d481f3cb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            3  Title: Wall St. Bears Claw Back Into the Black...   \n",
              "1            3  Title: Carlyle Looks Toward Commercial Aerospa...   \n",
              "2            3  Title: Oil and Economy Cloud Stocks' Outlook (...   \n",
              "3            3  Title: Iraq Halts Oil Exports from Main Southe...   \n",
              "4            3  Title: Oil prices soar to all-time record, pos...   \n",
              "\n",
              "            label1    label2                                         label3  \\\n",
              "0  topic: business  business                                topic: business   \n",
              "1         business  business                                topic: business   \n",
              "2  topic: business  business  the topic of this news article is \"business\".   \n",
              "3         business  business                                       business   \n",
              "4         business  business                                topic: business   \n",
              "\n",
              "  label4                                             label5 label6  \\\n",
              "0  true.                                              FALSE  true.   \n",
              "1  true.                                             false.   TRUE   \n",
              "2   TRUE                                             false.  true.   \n",
              "3   TRUE  false. the news title and description belong t...  true.   \n",
              "4  true.                                              FALSE   TRUE   \n",
              "\n",
              "                                              label7  \\\n",
              "0  false. this news title and description belongs...   \n",
              "1                                              FALSE   \n",
              "2                                              FALSE   \n",
              "3                                               TRUE   \n",
              "4                                               TRUE   \n",
              "\n",
              "                     label8  ... sci6 world7 sports7 business7 sci7  \\\n",
              "0                  business  ...    0      1       0         1    0   \n",
              "1                  business  ...    0      1       0         1    0   \n",
              "2  the topic is \"business\".  ...    0      1       0         1    0   \n",
              "3  the topic is \"business\".  ...    0      1       0         1    0   \n",
              "4                  business  ...    0      1       0         1    0   \n",
              "\n",
              "   entropy_world  entropy_sports  entropy_business  entropy_sci   entropy  \n",
              "0       0.863121             0.0               0.0     0.000000  0.215780  \n",
              "1       0.863121             0.0               0.0     0.000000  0.215780  \n",
              "2       0.863121             0.0               0.0     0.000000  0.215780  \n",
              "3       0.863121             0.0               0.0     0.591673  0.363698  \n",
              "4       0.863121             0.0               0.0     0.591673  0.363698  \n",
              "\n",
              "[5 rows x 54 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2f53cf07-0a68-4332-a0a8-2755d0d798ab\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>sci6</th>\n",
              "      <th>world7</th>\n",
              "      <th>sports7</th>\n",
              "      <th>business7</th>\n",
              "      <th>sci7</th>\n",
              "      <th>entropy_world</th>\n",
              "      <th>entropy_sports</th>\n",
              "      <th>entropy_business</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Wall St. Bears Claw Back Into the Black...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>true.</td>\n",
              "      <td>false. this news title and description belongs...</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Carlyle Looks Toward Commercial Aerospa...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil and Economy Cloud Stocks' Outlook (...</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>business</td>\n",
              "      <td>the topic of this news article is \"business\".</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.215780</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Iraq Halts Oil Exports from Main Southe...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false. the news title and description belong t...</td>\n",
              "      <td>true.</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>the topic is \"business\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>Title: Oil prices soar to all-time record, pos...</td>\n",
              "      <td>business</td>\n",
              "      <td>business</td>\n",
              "      <td>topic: business</td>\n",
              "      <td>true.</td>\n",
              "      <td>FALSE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>business</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.863121</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.591673</td>\n",
              "      <td>0.363698</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 54 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2f53cf07-0a68-4332-a0a8-2755d0d798ab')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2f53cf07-0a68-4332-a0a8-2755d0d798ab button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2f53cf07-0a68-4332-a0a8-2755d0d798ab');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "import pandas as pd\n",
        "data=pd.read_csv('agnews_train(entropy).csv')\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1E11xVRKU1Ie",
        "outputId": "bf698576-9d38-4881-8882-ebde02012eee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1154 entries, 0 to 1153\n",
            "Data columns (total 54 columns):\n",
            " #   Column            Non-Null Count  Dtype  \n",
            "---  ------            --------------  -----  \n",
            " 0   Class Index       1154 non-null   int64  \n",
            " 1   text              1154 non-null   object \n",
            " 2   label1            1154 non-null   object \n",
            " 3   label2            1154 non-null   object \n",
            " 4   label3            1154 non-null   object \n",
            " 5   label4            1154 non-null   object \n",
            " 6   label5            1154 non-null   object \n",
            " 7   label6            1154 non-null   object \n",
            " 8   label7            1154 non-null   object \n",
            " 9   label8            1154 non-null   object \n",
            " 10  label9            1154 non-null   object \n",
            " 11  label10           1154 non-null   object \n",
            " 12  label11           1154 non-null   object \n",
            " 13  label12           1154 non-null   object \n",
            " 14  label13           1154 non-null   object \n",
            " 15  4label1           1154 non-null   float64\n",
            " 16  5label1           1154 non-null   float64\n",
            " 17  MV_world          1154 non-null   int64  \n",
            " 18  MV_sports         1154 non-null   int64  \n",
            " 19  MV_business       1154 non-null   int64  \n",
            " 20  MV_sci            1154 non-null   int64  \n",
            " 21  world1            1154 non-null   int64  \n",
            " 22  sports1           1154 non-null   int64  \n",
            " 23  business1         1154 non-null   int64  \n",
            " 24  sci1              1154 non-null   int64  \n",
            " 25  world2            1154 non-null   int64  \n",
            " 26  sports2           1154 non-null   int64  \n",
            " 27  business2         1154 non-null   int64  \n",
            " 28  sci2              1154 non-null   int64  \n",
            " 29  world3            1154 non-null   int64  \n",
            " 30  sports3           1154 non-null   int64  \n",
            " 31  business3         1154 non-null   int64  \n",
            " 32  sci3              1154 non-null   int64  \n",
            " 33  world4            1154 non-null   int64  \n",
            " 34  sports4           1154 non-null   int64  \n",
            " 35  business4         1154 non-null   int64  \n",
            " 36  sci4              1154 non-null   int64  \n",
            " 37  world5            1154 non-null   int64  \n",
            " 38  sports5           1154 non-null   int64  \n",
            " 39  business5         1154 non-null   int64  \n",
            " 40  sci5              1154 non-null   int64  \n",
            " 41  world6            1154 non-null   int64  \n",
            " 42  sports6           1154 non-null   int64  \n",
            " 43  business6         1154 non-null   int64  \n",
            " 44  sci6              1154 non-null   int64  \n",
            " 45  world7            1154 non-null   int64  \n",
            " 46  sports7           1154 non-null   int64  \n",
            " 47  business7         1154 non-null   int64  \n",
            " 48  sci7              1154 non-null   int64  \n",
            " 49  entropy_world     1154 non-null   float64\n",
            " 50  entropy_sports    1154 non-null   float64\n",
            " 51  entropy_business  1154 non-null   float64\n",
            " 52  entropy_sci       1154 non-null   float64\n",
            " 53  entropy           1154 non-null   float64\n",
            "dtypes: float64(7), int64(33), object(14)\n",
            "memory usage: 487.0+ KB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gl5C4Deh34Pc"
      },
      "outputs": [],
      "source": [
        "data['gold_label']=data['Class Index']\n",
        "def calculate_alignment(threshold, df_input):\n",
        "  df_low=df_input[df_input['entropy']<=threshold]\n",
        "  print(df_low.shape[0])\n",
        "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
        "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
        "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
        "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
        "  df_low_correct1=df_low[df_low['MV_world']==df_low['world_gold']]\n",
        "  df_low_correct2=df_low[df_low['MV_sports']==df_low['gold_sports']]\n",
        "  df_low_correct3=df_low[df_low['MV_business']==df_low['gold_business']]\n",
        "  df_low_correct4=df_low[df_low['MV_sci']==df_low['gold_sci']]\n",
        "  return (df_low_correct1.shape[0]+df_low_correct2.shape[0]+df_low_correct3.shape[0]+df_low_correct4.shape[0])/4/df_low.shape[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "jMmv4_a235Zw",
        "outputId": "5619f03e-62be-48ed-8e72-3d4c18eb775b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "141\n",
            "266\n",
            "832\n",
            "1064\n",
            "1146\n",
            "1154\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-5-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:5: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['world_gold']=df_low['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sports']=df_low['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_business']=df_low['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
            "<ipython-input-5-cc88c7172cb6>:8: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df_low['gold_sci']=df_low['gold_label'].apply(lambda x:1 if x==4 else 0)\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAy0ElEQVR4nO3df1xUdd7//yc/HIYWgVwQhCiDLLISSpQPWbtXLUVSLZlbbpkSW7Yo2FVcu16QJGa70XYVlyySmpW5WqttIldbhhHbL1vSQtztCi0LEyMB3avAKH7O+f4x38YmQRkUOIyP++12btw483qfeZ2TOU/PeZ8zHoZhGAIAADAxz6FuAAAA4HgILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPQILAAAwPS8h7qBk8Vms+mLL77QyJEj5eHhMdTtAACAPjAMQ4cPH1ZYWJg8PXs/j+I2geWLL75QRETEULcBAAD6Yf/+/TrjjDN6fd1tAsvIkSMl2XfY399/iLsBAAB90dLSooiICMfneG/cJrB8dxnI39+fwAIAwDBzvOkcTLoFAACmR2ABAACmR2ABAACmR2ABAACmR2ABAACmR2ABAACm5za3NZ90hiG9+6704YdSV5cUHi5dfbXk4zPUnQEAcMohsPyQYUirV0tLl0offGBf5+UldXdLwcHSnXdK2dkSz3oBAGDQcEno+2w2ac4c6Y47pMhI6dVXpc5O+/Lhh9Itt0h//KN0+eVSU9NQdwsAwCmDwPJ9DzwgPf20tGaNVFoqXXWV5O0teXhI48dLhYXStm1SY6OUkmI/6wIAAAYcgeU7X30lPfqo/XLP7Nm9111wgfT88/b5La+8MmjtAQBwKiOwfOdPf5I6OqT5849f+5OfSHFxUnHxwPcFAAAILA7l5dKVV0pjxvSt/tZbpddes0/SBQAAA4rA8p3Dh6WgoL7XBwXZb3duaxu4ngAAgCQCyxEBAfbJtH3V1CSNGCFZrQPXEwAAkERgOeKaa6TXX5c+//z4tYZhn/Mydar9DiIAADCgCCzfue026bTTpP/+7+PXvvaa9M9/SvPmDXxfAACAwOIwcqR0331SQYG0YkXvdVVV9gfI/fSn9ue0AACAAUdg+b7sbOnuu6W5c6XERGnTJqmlRfr2W+m99+xPwJ0yRYqKkkpKJE8OHwAAg4HvEvo+Dw/7dwhNmWL/eeONzq9HREiLFkn33GO/fAQAAAYFgeWHPDykm2+2L//8p1RTY/8uofBw+2UgL6+h7hAAgFMOgeVYJkywLwAAYEgxCQMAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJge39aMoWcY0ldfSd9+K51+uuTrO9QdAQBMpl9nWIqLizV27FhZrVbFx8dr+/btvdZ2dnZqyZIlioqKktVqVUxMjMrKypxquru7df/99+vss8+Wr6+voqKi9OCDD8owjP60h+Hiq6+kwkLp/POlUaOk8HDpRz+Srr9e2rxZstmGukMAgEm4HFg2bNigrKws5eXlaceOHYqJiVFSUpKampp6rM/NzdXKlStVVFSkmpoapaena9q0aaqurnbU/OEPf9Dy5cu1bNky7dq1S3/4wx/0yCOPqKioqP97BnPbscMeVH7zGyk2Vlq/XnrpJamoSPr8c+naa6Wf/1xqbR3qTgEAJuBhuHgaIz4+XpMmTdKyZcskSTabTREREZo/f76ys7OPqg8LC9PChQuVkZHhWDd9+nT5+vpq3bp1kqTrrrtOISEheuqpp3qtOZ6WlhYFBASoublZ/v7+ruwSBttHH0kJCdI550glJdIZZzi/bhj28HLrrdJll0l//avkzdVLAHBHff38dukMS0dHh6qqqpSYmHhkA56eSkxMVGVlZY9j2tvbZbVandb5+vpq69atjt8vvfRSVVRU6OOPP5Yk/eMf/9DWrVs1derUXntpb29XS0uL04Jh4j/+QwoKkl599eiwIkkeHvbLQiUlUlmZ9Pzzg98jAMBUXAoshw4dUnd3t0JCQpzWh4SEqKGhoccxSUlJKigo0J49e2Sz2VReXq6SkhIdOHDAUZOdna1f/vKXio6O1ogRI3TxxRfrnnvu0cyZM3vtJT8/XwEBAY4lIiLClV3BUNm71z4/JTtbCgw8du1VV0lXXiktXz4orQEAzGvAb2suLCzUuHHjFB0dLYvFoszMTKWlpcnT88hbP//883r22Wf13HPPaceOHVqzZo0effRRrVmzptft5uTkqLm52bHs379/oHcFJ8OGDfaJtb/8Zd/q77pL2rpVqqsb2L4AAKbm0sSAoKAgeXl5qbGx0Wl9Y2OjQkNDexwTHBys0tJStbW16V//+pfCwsKUnZ2tyMhIR81vf/tbx1kWSbrooou0b98+5efnKzU1tcft+vj4yMfHx5X2YQYNDdKZZ0qnnda3+uho53EAgFOSS2dYLBaLJk6cqIqKCsc6m82miooKJSQkHHOs1WpVeHi4urq6tHHjRqWkpDhe++abb5zOuEiSl5eXbNzW6n4sFqm9ve/1bW1HxgEATlkuXxLKysrSqlWrtGbNGu3atUtz585Va2ur0tLSJEmzZ89WTk6Oo37btm0qKSlRbW2t3n77bV1zzTWy2WxasGCBo+b666/X73//e7388sv67LPPtGnTJhUUFGjatGknYRdhKrGx0qef2u8U6otXXrFfQjrnnAFtCwBgbi7fKzpjxgwdPHhQixYtUkNDg2JjY1VWVuaYiFtXV+d0tqStrU25ubmqra2Vn5+fkpOTtXbtWgV+b8JlUVGR7r//fs2bN09NTU0KCwvTr3/9ay1atOjE9xDmcuON9juEli+Xli49dm1Hh/TEE9KsWZKf36C0BwAwJ5efw2JWPIdlGHngAenBB6UXX5SSk3uusdmkO++U1q6Vdu6ULrhgUFsEAAyOAXkOC3BSLFwoXXedlJIiLVokffHFkdcMQ/r73+2vr1kjrV5NWAEA8OWHGALe3tILL0j33ScVFEgPPST9v/9nv+yzb5+0e7cUFSWVltofIAcAOOVxSQhDq7nZftmnstL+bc0//rH0i1/YHxrnyQlAAHB3ff38JrAAAIAhwxwWAADgNggsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9AgsAADA9PoVWIqLizV27FhZrVbFx8dr+/btvdZ2dnZqyZIlioqKktVqVUxMjMrKypxqxo4dKw8Pj6OWjIyM/rQHAADcjMuBZcOGDcrKylJeXp527NihmJgYJSUlqampqcf63NxcrVy5UkVFRaqpqVF6erqmTZum6upqR817772nAwcOOJby8nJJ0k033dTP3QIAAO7EwzAMw5UB8fHxmjRpkpYtWyZJstlsioiI0Pz585WdnX1UfVhYmBYuXOh0tmT69Ony9fXVunXrenyPe+65Ry+99JL27NkjDw+PPvXV0tKigIAANTc3y9/f35VdAgAAQ6Svn98unWHp6OhQVVWVEhMTj2zA01OJiYmqrKzscUx7e7usVqvTOl9fX23durXX91i3bp1+9atfHTOstLe3q6WlxWkBAADuyaXAcujQIXV3dyskJMRpfUhIiBoaGnock5SUpIKCAu3Zs0c2m03l5eUqKSnRgQMHeqwvLS3VV199pdtvv/2YveTn5ysgIMCxREREuLIrAABgGBnwu4QKCws1btw4RUdHy2KxKDMzU2lpafL07Pmtn3rqKU2dOlVhYWHH3G5OTo6am5sdy/79+weifQAAYAIuBZagoCB5eXmpsbHRaX1jY6NCQ0N7HBMcHKzS0lK1trZq37592r17t/z8/BQZGXlU7b59+/Taa6/pzjvvPG4vPj4+8vf3d1oAAIB7cimwWCwWTZw4URUVFY51NptNFRUVSkhIOOZYq9Wq8PBwdXV1aePGjUpJSTmqZvXq1Ro9erSuvfZaV9oCAABuztvVAVlZWUpNTVVcXJwmT56spUuXqrW1VWlpaZKk2bNnKzw8XPn5+ZKkbdu2qb6+XrGxsaqvr9fixYtls9m0YMECp+3abDatXr1aqamp8vZ2uS0AAODGXE4GM2bM0MGDB7Vo0SI1NDQoNjZWZWVljom4dXV1TvNT2tralJubq9raWvn5+Sk5OVlr165VYGCg03Zfe+011dXV6Ve/+tWJ7REAAHA7Lj+Hxax4DgsAAMPPgDyHBQAAYCgQWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOkRWAAAgOn1K7AUFxdr7Nixslqtio+P1/bt23ut7ezs1JIlSxQVFSWr1aqYmBiVlZUdVVdfX6/bbrtNP/7xj+Xr66uLLrpI77//fn/aAwAAbsblwLJhwwZlZWUpLy9PO3bsUExMjJKSktTU1NRjfW5urlauXKmioiLV1NQoPT1d06ZNU3V1taPmyy+/1JQpUzRixAi98sorqqmp0WOPPabTTz+9/3sGAADchodhGIYrA+Lj4zVp0iQtW7ZMkmSz2RQREaH58+crOzv7qPqwsDAtXLhQGRkZjnXTp0+Xr6+v1q1bJ0nKzs7WO++8o7fffrvfO9LS0qKAgAA1NzfL39+/39sBAACDp6+f3y6dYeno6FBVVZUSExOPbMDTU4mJiaqsrOxxTHt7u6xWq9M6X19fbd261fH7iy++qLi4ON10000aPXq0Lr74Yq1ateqYvbS3t6ulpcVpAQAA7smlwHLo0CF1d3crJCTEaX1ISIgaGhp6HJOUlKSCggLt2bNHNptN5eXlKikp0YEDBxw1tbW1Wr58ucaNG6ctW7Zo7ty5uvvuu7VmzZpee8nPz1dAQIBjiYiIcGVXAADAMDLgdwkVFhZq3Lhxio6OlsViUWZmptLS0uTpeeStbTabLrnkEj300EO6+OKLddddd2nOnDlasWJFr9vNyclRc3OzY9m/f/9A7woAABgiLgWWoKAgeXl5qbGx0Wl9Y2OjQkNDexwTHBys0tJStba2at++fdq9e7f8/PwUGRnpqBkzZozGjx/vNO78889XXV1dr734+PjI39/faQEAAO7JpcBisVg0ceJEVVRUONbZbDZVVFQoISHhmGOtVqvCw8PV1dWljRs3KiUlxfHalClT9NFHHznVf/zxxzrrrLNcaQ8AALgpb1cHZGVlKTU1VXFxcZo8ebKWLl2q1tZWpaWlSZJmz56t8PBw5efnS5K2bdum+vp6xcbGqr6+XosXL5bNZtOCBQsc27z33nt16aWX6qGHHtLNN9+s7du364knntATTzxxknYTAAAMZy4HlhkzZujgwYNatGiRGhoaFBsbq7KyMsdE3Lq6Oqf5KW1tbcrNzVVtba38/PyUnJystWvXKjAw0FEzadIkbdq0STk5OVqyZInOPvtsLV26VDNnzjzxPQQAAMOey89hMSuewwIAwPAzIM9hAQAAGAoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHoEFgAAYHr9CizFxcUaO3asrFar4uPjtX379l5rOzs7tWTJEkVFRclqtSomJkZlZWVONYsXL5aHh4fTEh0d3Z/WAACAG3I5sGzYsEFZWVnKy8vTjh07FBMTo6SkJDU1NfVYn5ubq5UrV6qoqEg1NTVKT0/XtGnTVF1d7VR3wQUX6MCBA45l69at/dsjAADgdlwOLAUFBZozZ47S0tI0fvx4rVixQqeddpqefvrpHuvXrl2r++67T8nJyYqMjNTcuXOVnJysxx57zKnO29tboaGhjiUoKKh/ewQAANyOS4Glo6NDVVVVSkxMPLIBT08lJiaqsrKyxzHt7e2yWq1O63x9fY86g7Jnzx6FhYUpMjJSM2fOVF1d3TF7aW9vV0tLi9MCAADck0uB5dChQ+ru7lZISIjT+pCQEDU0NPQ4JikpSQUFBdqzZ49sNpvKy8tVUlKiAwcOOGri4+P1zDPPqKysTMuXL9fevXt1+eWX6/Dhw732kp+fr4CAAMcSERHhyq4AAIBhZMDvEiosLNS4ceMUHR0ti8WizMxMpaWlydPzyFtPnTpVN910kyZMmKCkpCRt3rxZX331lZ5//vlet5uTk6Pm5mbHsn///oHeFQAAMERcCixBQUHy8vJSY2Oj0/rGxkaFhob2OCY4OFilpaVqbW3Vvn37tHv3bvn5+SkyMrLX9wkMDNS5556rTz75pNcaHx8f+fv7Oy0AAMA9uRRYLBaLJk6cqIqKCsc6m82miooKJSQkHHOs1WpVeHi4urq6tHHjRqWkpPRa+/XXX+vTTz/VmDFjXGkPAAC4KZcvCWVlZWnVqlVas2aNdu3apblz56q1tVVpaWmSpNmzZysnJ8dRv23bNpWUlKi2tlZvv/22rrnmGtlsNi1YsMBR85vf/EZvvvmmPvvsM/3973/XtGnT5OXlpVtuueUk7CIAABjuvF0dMGPGDB08eFCLFi1SQ0ODYmNjVVZW5piIW1dX5zQ/pa2tTbm5uaqtrZWfn5+Sk5O1du1aBQYGOmo+//xz3XLLLfrXv/6l4OBgXXbZZXr33XcVHBx84nsIAACGPQ/DMIyhbuJkaGlpUUBAgJqbm5nPAgDAMNHXz2++SwgAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQU4FezaJd19t3TuuVJoqP3n/PlSTc1QdwYAfUJgAdxZW5t0223S+PHShg1SUpKUkSFdc430l79IF1wg3Xqr9O23Q90pAByTy0+6BTBMdHVJN94ovf669OST0qxZksVy5PVHH5WefdYeYP7v/6S//lUaMWLo+gWAY+AMC+CuioulV1+VXnxRuuMO57Ai2X9PS7MHlYoKqahoaPoEgD4gsADuyGazB5abb5auuurYtT/7mTRjhvT44/ZxAGBCBBbAHb3zjrRnjzR3bt/q586VPv1Ueuutge0LAPqJwAK4o9pa+8/Jk/tWHx9v/7l378D0AwAniMACQHKPL20H4MYILIA7ioqy/6ys7Fv9u+86jwMAkyGwAO5oyhTpvPOk5cv7Vv/449K4cdJllw1sXwDQTwQWwB15eEiZmdILL0ibNx+7dssW+0PkMjIkT/5KAGBO/O0EuKu5c6XrrpOmTbOfafnh02zb2qQVK6SUFGnqVHtgAQCT8jAM95ht19LSooCAADU3N8vf33+o2wHMob1dmjdPevppadQo6Re/kIKDpYMHpY0bpX/9S7r9dntw8fEZ6m4BnIL6+vlNYAFOBZ98Yg8lW7ZILS2Sv7909dVSerp97goADBECCwAAML2+fn4zhwUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJgegQUAAJhevwJLcXGxxo4dK6vVqvj4eG3fvr3X2s7OTi1ZskRRUVGyWq2KiYlRWVlZr/UPP/ywPDw8dM899/SnNQAA4IZcDiwbNmxQVlaW8vLytGPHDsXExCgpKUlNTU091ufm5mrlypUqKipSTU2N0tPTNW3aNFVXVx9V+95772nlypWaMGGC63sCAADclsuBpaCgQHPmzFFaWprGjx+vFStW6LTTTtPTTz/dY/3atWt13333KTk5WZGRkZo7d66Sk5P12GOPOdV9/fXXmjlzplatWqXTTz+9f3sDAADckkuBpaOjQ1VVVUpMTDyyAU9PJSYmqrKysscx7e3tslqtTut8fX21detWp3UZGRm69tprnbZ9LO3t7WppaXFaAACAe3IpsBw6dEjd3d0KCQlxWh8SEqKGhoYexyQlJamgoEB79uyRzWZTeXm5SkpKdODAAUfN+vXrtWPHDuXn5/e5l/z8fAUEBDiWiIgIV3YFAAAMIwN+l1BhYaHGjRun6OhoWSwWZWZmKi0tTZ6e9rfev3+//v3f/13PPvvsUWdijiUnJ0fNzc2OZf/+/QO1CwAAYIi5FFiCgoLk5eWlxsZGp/WNjY0KDQ3tcUxwcLBKS0vV2tqqffv2affu3fLz81NkZKQkqaqqSk1NTbrkkkvk7e0tb29vvfnmm/rjH/8ob29vdXd397hdHx8f+fv7Oy0AAMA9uRRYLBaLJk6cqIqKCsc6m82miooKJSQkHHOs1WpVeHi4urq6tHHjRqWkpEiSfvazn+mDDz7Qzp07HUtcXJxmzpypnTt3ysvLqx+7BQAA3Im3qwOysrKUmpqquLg4TZ48WUuXLlVra6vS0tIkSbNnz1Z4eLhjPsq2bdtUX1+v2NhY1dfXa/HixbLZbFqwYIEkaeTIkbrwwgud3uNHP/qRfvzjHx+1HgAAnJpcDiwzZszQwYMHtWjRIjU0NCg2NlZlZWWOibh1dXWO+SmS1NbWptzcXNXW1srPz0/Jyclau3atAgMDT9pOAAAA9+ZhGIYx1E2cDC0tLQoICFBzczPzWQAAGCb6+vnNdwkBAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADTI7AAAADT61dgKS4u1tixY2W1WhUfH6/t27f3WtvZ2aklS5YoKipKVqtVMTExKisrc6pZvny5JkyYIH9/f/n7+yshIUGvvPJKf1oDAABuyOXAsmHDBmVlZSkvL087duxQTEyMkpKS1NTU1GN9bm6uVq5cqaKiItXU1Cg9PV3Tpk1TdXW1o+aMM87Qww8/rKqqKr3//vu68sorlZKSog8//LD/ewYAANyGh2EYhisD4uPjNWnSJC1btkySZLPZFBERofnz5ys7O/uo+rCwMC1cuFAZGRmOddOnT5evr6/WrVvX6/uMGjVK//Vf/6U77rijT321tLQoICBAzc3N8vf3d2WXAADAEOnr57dLZ1g6OjpUVVWlxMTEIxvw9FRiYqIqKyt7HNPe3i6r1eq0ztfXV1u3bu2xvru7W+vXr1dra6sSEhJ67aW9vV0tLS1OCwAAcE8uBZZDhw6pu7tbISEhTutDQkLU0NDQ45ikpCQVFBRoz549stlsKi8vV0lJiQ4cOOBU98EHH8jPz08+Pj5KT0/Xpk2bNH78+F57yc/PV0BAgGOJiIhwZVcAAMAwMuB3CRUWFmrcuHGKjo6WxWJRZmam0tLS5Onp/NbnnXeedu7cqW3btmnu3LlKTU1VTU1Nr9vNyclRc3OzY9m/f/9A7woAABgiLgWWoKAgeXl5qbGx0Wl9Y2OjQkNDexwTHBys0tJStba2at++fdq9e7f8/PwUGRnpVGexWHTOOedo4sSJys/PV0xMjAoLC3vtxcfHx3FX0XcLAABwTy4FFovFookTJ6qiosKxzmazqaKi4pjzTSTJarUqPDxcXV1d2rhxo1JSUo5Zb7PZ1N7e7kp7AADATXm7OiArK0upqamKi4vT5MmTtXTpUrW2tiotLU2SNHv2bIWHhys/P1+StG3bNtXX1ys2Nlb19fVavHixbDabFixY4NhmTk6Opk6dqjPPPFOHDx/Wc889pzfeeENbtmw5SbsJAACGM5cDy4wZM3Tw4EEtWrRIDQ0Nio2NVVlZmWMibl1dndP8lLa2NuXm5qq2tlZ+fn5KTk7W2rVrFRgY6KhpamrS7NmzdeDAAQUEBGjChAnasmWLrrrqqhPfQwAAMOy5/BwWs+I5LAAADD8D8hwWAACAoUBgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApkdgAQAApuc91A0AgFtpbZUqKqRDhyRfX2nyZCkqaqi7AoY9AgsAnAwNDdLDD0vPPCM1Nzu/ds010m9/K1155ZC0BrgDLgkBwIn66CP7mZR166S5c6U9e6TOTunLL6XVq+1nWxITpWXLhrpTYNjiDAsAnIgvv7SfQRk5UqqslMLDj7wWGCjdfrs0e7b0m99I8+dLYWHSjTcOVbfAsMUZFgA4EU8+KX3xhbR5s3NY+T5PT+mxx6SpU6X775cMY3B7BNwAgQUA+stmk5Yvl2bMkM4669i1Hh7Sf/yHVFMjvfXW4PQHuBECCwD01yefSHv3Srfd1rf6K6+UxoyRysoGti/ADRFYAKC/WlrsP0eP7lu9h4cUHHxkHIA+I7AAQH+NHGn/efBg3+oNw37HkL//wPUEuCkCCwD01znnSGPHSs8+27f6N9+0T9C96qoBbQtwRwQWAOgvLy8pPV1av17av//YtYZhv1Po/POlK64YnP4AN0JgAYATMWeOfQ5LcrL9abc9MQwpO1t66SVpyRL7XBYALiGwAMCJGDXKftfP//2fNGGC/Tkrn31mDymtrfan3156qfTII9J//7f0i18MdcfAsERgAYATNX68tH27dPPNUmGhdPbZ9stFfn7SrFnSj35kDzX33DPUnQLDVr8CS3FxscaOHSur1ar4+Hht376919rOzk4tWbJEUVFRslqtiomJUdkPnkGQn5+vSZMmaeTIkRo9erRuuOEGffTRR/1pDQCGRni4/buC6uuljRulFSukP/1J2r1beu01KSlpqDsEhjWXA8uGDRuUlZWlvLw87dixQzExMUpKSlJTU1OP9bm5uVq5cqWKiopUU1Oj9PR0TZs2TdXV1Y6aN998UxkZGXr33XdVXl6uzs5OXX311Wptbe3/ngHAUBg50v5dQXfdZT+7ct55Q90R4BY8DMO1L7WIj4/XpEmTtOz//9ZRm82miIgIzZ8/X9nZ2UfVh4WFaeHChcrIyHCsmz59unx9fbVu3boe3+PgwYMaPXq03nzzTf3kJz/pU18tLS0KCAhQc3Oz/HnGAQAAw0JfP79dOsPS0dGhqqoqJSYmHtmAp6cSExNVWVnZ45j29nZZrVandb6+vtq6dWuv79Pc3CxJGjVqlCvtAQAAN+VSYDl06JC6u7sVEhLitD4kJEQNvdzOl5SUpIKCAu3Zs0c2m03l5eUqKSnRgQMHeqy32Wy65557NGXKFF144YW99tLe3q6WlhanBQAAuKcBv0uosLBQ48aNU3R0tCwWizIzM5WWliZPz57fOiMjQ//7v/+r9evXH3O7+fn5CggIcCwRERED0T4AADABlwJLUFCQvLy81NjY6LS+sbFRoaGhPY4JDg5WaWmpWltbtW/fPu3evVt+fn6KjIw8qjYzM1MvvfSSXn/9dZ1xxhnH7CUnJ0fNzc2OZf/xnjIJAACGLZcCi8Vi0cSJE1VRUeFYZ7PZVFFRoYSEhGOOtVqtCg8PV1dXlzZu3KiUlBTHa4ZhKDMzU5s2bdLf/vY3nX322cftxcfHR/7+/k4LAABwT96uDsjKylJqaqri4uI0efJkLV26VK2trUpLS5MkzZ49W+Hh4crPz5ckbdu2TfX19YqNjVV9fb0WL14sm82mBQsWOLaZkZGh5557Tv/zP/+jkSNHOubDBAQEyNfX92TsJwAAGMZcDiwzZszQwYMHtWjRIjU0NCg2NlZlZWWOibh1dXVO81Pa2tqUm5ur2tpa+fn5KTk5WWvXrlVgYKCjZvny5ZKkf/u3f3N6r9WrV+v22293fa8AAIBbcfk5LGbFc1gAABh+BuQ5LAAAAEOBwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEyPwAIAAEzPe6gbAADAZV99JZWUSJ9/Lnl7S+PHS9deK40YMdSdYYAQWAAAw0dTk7RwofTcc1J7uxQcLHV1SYcOSWFhUkaGtGCBPcTArXBJCAAwPOzfLyUkSKWlUk6O/ezKgQPSwYPSP/4hXX+9tGiRdOONUmfnUHeLk4zAAgAwv85O+yUfm0167z0pN1cKDT3y+oQJ0ooV0l//KpWVSVlZQ9crBgSBBQBgfi++KH3wgfSXv0hjx/ZeN3Wq9OCD0sqV9stHcBsEFgCA+T3+uDRlihQXd/zaOXMkLy/pyScHvi93Z7NJb7whrVplD4FbttjnDA0BAgsAwNwMQ3rrLemmm/pWP2qUdNVV9jHon+5uqbBQOu886YorpF//Wpo3T7rmGunss6Xf/15qaxvUlggsAABz6+y0/6s+IKDvYwIDpdbWAWvJrXV0SNOn2+cBTZpkD35dXfb/DlVV9tDy4IP2UHj48KC1RWABAJibxSKddppUX9/3MZ9/bj/TAtfdfbf0yiv2eUPPPSddfrnk6WlfLrnEfnno9delf/5TuvXWQWuLwAIAML+UFGnNGvuciuOprbXPu0hJGfC23M6+ffZA8sgj9ruyepOQYJ8j9NJL9ru2BgGBBQBgfvPmSXv2SC+8cPzaP/zBfvnol78c+L7czRNPSH5+0h13HL/2xhuls86SiosHvi8RWAAAw8GUKdK0aVJamlRe3nONYUi/+539Q/fBB+2XkeCa8nL7mSk/v+PXennZQ+Frrw18X+LR/ACA4cDDQ1q3zj4ZNCnJ/ryV9HT7A+M6O6W//c1+6/M//iE98ICUmTnUHQ9Phw9LQUF9rw8KklpaBq6f7yGwAACGh9NOsz/J9k9/sl+G+PnPj7zm6Wl/NH9BgXTllUPX43AXECA1Nva9vqnJtbu3TgCBBQAwfHh7S7/6lf3S0K5dR76t+bzzpPDwoe5u+LvmGunRR+1nTfz9j13b1WW/iyg5eVBa69ccluLiYo0dO1ZWq1Xx8fHavn17r7WdnZ1asmSJoqKiZLVaFRMTo7KyMqeat956S9dff73CwsLk4eGh0tLS/rQFADhVeHhI48dLV19tP6NCWDk55syxPxBuxYrj165fb7/VfN68ge9L/QgsGzZsUFZWlvLy8rRjxw7FxMQoKSlJTb18Z0Nubq5WrlypoqIi1dTUKD09XdOmTVN1dbWjprW1VTExMSoepJnGAACgB+HhUkaGdN99x74j6/XX7U+//cUvpNjYQWnNwzAMw5UB8fHxmjRpkpYtWyZJstlsioiI0Pz585WdnX1UfVhYmBYuXKiMjAzHuunTp8vX11fr1q07uiEPD23atEk33HCDSzvS0tKigIAANTc3y/94p7EAAEDPurqk1FTpz3+WrrvOfgblssvsZ7WqqqTly+1h5oorpNLSE74bq6+f3y7NYeno6FBVVZVycnIc6zw9PZWYmKjKysoex7S3t8tqtTqt8/X11datW1156x63297e7vi9ZZBmKQMA4Na8vaW1a6Wf/Uz64x/td2R937hx9nku8+ZJI0YMXluuFB86dEjd3d0KCQlxWh8SEqLdu3f3OCYpKUkFBQX6yU9+oqioKFVUVKikpETd3d3971pSfn6+HnjggRPaBgAA6IGn55HJzVVV9of22Wz2B8Vdeqn99cFuaaDfoLCwUOPGjVN0dLQsFosyMzOVlpYmzxPc2ZycHDU3NzuW/fv3n6SOAQCAJPtloLg46ZZbpJkz7ZeGhiCsSC4GlqCgIHl5eanxB/doNzY2KjQ0tMcxwcHBKi0tVWtrq/bt26fdu3fLz89PkZGR/e9ako+Pj/z9/Z0WAADgnlwKLBaLRRMnTlRFRYVjnc1mU0VFhRISEo451mq1Kjw8XF1dXdq4caNS+FIqAADQRy4/OC4rK0upqamKi4vT5MmTtXTpUrW2tiotLU2SNHv2bIWHhys/P1+StG3bNtXX1ys2Nlb19fVavHixbDabFixY4Njm119/rU8++cTx+969e7Vz506NGjVKZ5555onuIwAAGOZcDiwzZszQwYMHtWjRIjU0NCg2NlZlZWWOibh1dXVO81Pa2tqUm5ur2tpa+fn5KTk5WWvXrlVgYKCj5v3339cVV1zh+D0rK0uSlJqaqmeeeaafuwYAANyFy89hMSuewwIAwPDT18/voZnqCwAA4AICCwAAMD23+bbm765s8cRbAACGj+8+t483Q8VtAsvhw4clSREREUPcCQAAcNXhw4cVEBDQ6+tuM+nWZrPpiy++0MiRI+Xh4XHSttvS0qKIiAjt37+fybwDiOM8eDjWg4PjPDg4zoNjII+zYRg6fPiwwsLCjvkUfLc5w+Lp6akzzjhjwLbP03QHB8d58HCsBwfHeXBwnAfHQB3nY51Z+Q6TbgEAgOkRWAAAgOkRWI7Dx8dHeXl58vHxGepW3BrHefBwrAcHx3lwcJwHhxmOs9tMugUAAO6LMywAAMD0CCwAAMD0CCwAAMD0CCwAAMD0CCySiouLNXbsWFmtVsXHx2v79u3HrP/LX/6i6OhoWa1WXXTRRdq8efMgdTq8uXKcV61apcsvv1ynn366Tj/9dCUmJh73vwvsXP3z/J3169fLw8NDN9xww8A26EZcPdZfffWVMjIyNGbMGPn4+Ojcc8/l748+cPU4L126VOedd558fX0VERGhe++9V21tbYPU7fD01ltv6frrr1dYWJg8PDxUWlp63DFvvPGGLrnkEvn4+Oicc87RM888M7BNGqe49evXGxaLxXj66aeNDz/80JgzZ44RGBhoNDY29lj/zjvvGF5eXsYjjzxi1NTUGLm5ucaIESOMDz74YJA7H15cPc633nqrUVxcbFRXVxu7du0ybr/9diMgIMD4/PPPB7nz4cXV4/ydvXv3GuHh4cbll19upKSkDE6zw5yrx7q9vd2Ii4szkpOTja1btxp79+413njjDWPnzp2D3Pnw4upxfvbZZw0fHx/j2WefNfbu3Wts2bLFGDNmjHHvvfcOcufDy+bNm42FCxcaJSUlhiRj06ZNx6yvra01TjvtNCMrK8uoqakxioqKDC8vL6OsrGzAejzlA8vkyZONjIwMx+/d3d1GWFiYkZ+f32P9zTffbFx77bVO6+Lj441f//rXA9rncOfqcf6hrq4uY+TIkcaaNWsGqkW30J/j3NXVZVx66aXGk08+aaSmphJY+sjVY718+XIjMjLS6OjoGKwW3YKrxzkjI8O48sorndZlZWUZU6ZMGdA+3UlfAsuCBQuMCy64wGndjBkzjKSkpAHr65S+JNTR0aGqqiolJiY61nl6eioxMVGVlZU9jqmsrHSql6SkpKRe69G/4/xD33zzjTo7OzVq1KiBanPY6+9xXrJkiUaPHq077rhjMNp0C/051i+++KISEhKUkZGhkJAQXXjhhXrooYfU3d09WG0PO/05zpdeeqmqqqocl41qa2u1efNmJScnD0rPp4qh+Cx0my8/7I9Dhw6pu7tbISEhTutDQkK0e/fuHsc0NDT0WN/Q0DBgfQ53/TnOP/Sf//mfCgsLO+p/EBzRn+O8detWPfXUU9q5c+cgdOg++nOsa2tr9be//U0zZ87U5s2b9cknn2jevHnq7OxUXl7eYLQ97PTnON966606dOiQLrvsMhmGoa6uLqWnp+u+++4bjJZPGb19Fra0tOjbb7+Vr6/vSX/PU/oMC4aHhx9+WOvXr9emTZtktVqHuh23cfjwYc2aNUurVq1SUFDQULfj9mw2m0aPHq0nnnhCEydO1IwZM7Rw4UKtWLFiqFtzK2+88YYeeughPf7449qxY4dKSkr08ssv68EHHxzq1nCCTukzLEFBQfLy8lJjY6PT+sbGRoWGhvY4JjQ01KV69O84f+fRRx/Vww8/rNdee00TJkwYyDaHPVeP86effqrPPvtM119/vWOdzWaTJHl7e+ujjz5SVFTUwDY9TPXnz/SYMWM0YsQIeXl5Odadf/75amhoUEdHhywWy4D2PBz15zjff//9mjVrlu68805J0kUXXaTW1lbdddddWrhwoTw9+Xf6ydDbZ6G/v/+AnF2RTvEzLBaLRRMnTlRFRYVjnc1mU0VFhRISEnock5CQ4FQvSeXl5b3Wo3/HWZIeeeQRPfjggyorK1NcXNxgtDqsuXqco6Oj9cEHH2jnzp2O5ec//7muuOIK7dy5UxEREYPZ/rDSnz/TU6ZM0SeffOIIhZL08ccfa8yYMYSVXvTnOH/zzTdHhZLvQqLBV+edNEPyWThg03mHifXr1xs+Pj7GM888Y9TU1Bh33XWXERgYaDQ0NBiGYRizZs0ysrOzHfXvvPOO4e3tbTz66KPGrl27jLy8PG5r7gNXj/PDDz9sWCwW44UXXjAOHDjgWA4fPjxUuzAsuHqcf4i7hPrO1WNdV1dnjBw50sjMzDQ++ugj46WXXjJGjx5t/O53vxuqXRgWXD3OeXl5xsiRI40///nPRm1trfHqq68aUVFRxs033zxUuzAsHD582Kiurjaqq6sNSUZBQYFRXV1t7Nu3zzAMw8jOzjZmzZrlqP/utubf/va3xq5du4zi4mJuax4MRUVFxplnnmlYLBZj8uTJxrvvvut47ac//amRmprqVP/8888b5557rmGxWIwLLrjAePnllwe54+HJleN81llnGZKOWvLy8ga/8WHG1T/P30dgcY2rx/rvf/+7ER8fb/j4+BiRkZHG73//e6Orq2uQux5+XDnOnZ2dxuLFi42oqCjDarUaERERxrx584wvv/xy8BsfRl5//fUe/8797timpqYaP/3pT48aExsba1gsFiMyMtJYvXr1gPboYRicIwMAAOZ2Ss9hAQAAwwOBBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmB6BBQAAmN7/B8bilQw0RDOHAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n",
        "x = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
        "y = [calculate_alignment(i, data) for i in x]\n",
        "\n",
        "plt.scatter(x, y, s=80, facecolors='none', edgecolors='r')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "fEYeBp_u5E11",
        "outputId": "2fec9c32-0990-4a5f-f299-9bcc52cbf039"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHHCAYAAABZbpmkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA58klEQVR4nO3deVhWdf7/8dcNAm4sIrKNuOROohaWkUsuJCqZJi3uS5haaCq5xGS5NaJWZtOY1lRqM5JlV8vkjppaSZqmaWruhsWipYLiV2Q5vz+6vH/dgia3wH1zej6u61zD+ZzPfc77c2Li1ecst8UwDEMAAAAm5eLoAgAAAMoSYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcwkXr16mno0KGOLsP0XnrpJd12221ydXVVq1atyu24S5YskcVi0cmTJ8vtmIAZEHYAJ3X1D9vOnTuL3d6xY0c1b978lo+zevVqTZs27Zb381exfv16TZo0SW3bttXixYs1a9asIn3y8vLk5+endu3aXXc/hmEoJCREd955Z1mWC0BSJUcXAKD0HDp0SC4uJftvmNWrV2vBggUEnpu0adMmubi46J133pG7u3uxfdzc3PTII4/ozTff1E8//aS6desW6bN161b9/PPPGj9+fFmXDPzlMbMDmIiHh4fc3NwcXUaJ5OTkOLqEEjl9+rSqVKly3aBz1YABA2QYht5///1ityclJcnFxUV9+/YtizIB/AFhBzCRa+/ZycvL0/Tp09WoUSNVrlxZNWvWVLt27ZScnCxJGjp0qBYsWCBJslgs1uWqnJwcPfPMMwoJCZGHh4eaNGmil19+WYZh2Bz3//7v//T000/Lz89Pnp6eevDBB/XLL7/IYrHYzBhNmzZNFotFBw4cUP/+/VWjRg3rpZ69e/dq6NChuu2221S5cmUFBgbq8ccf12+//WZzrKv7OHz4sAYOHChvb2/VqlVLzz//vAzD0KlTp9SrVy95eXkpMDBQr7zyyk2du/z8fM2cOVMNGjSQh4eH6tWrp7///e/Kzc219rFYLFq8eLFycnKs52rJkiXF7q9t27aqV6+ekpKSimzLy8vTRx99pE6dOik4OPimx16ca8/xVcXdv3X+/HmNGzfO+s+zYcOGmjNnjgoLC236LV++XOHh4fL09JSXl5fCwsL02muv/WktgLPiMhbg5LKysvTrr78Wac/Ly/vTz06bNk2JiYkaPny47r77bmVnZ2vnzp367rvvdP/992vkyJFKS0tTcnKy/vOf/9h81jAMPfjgg/riiy8UGxurVq1aad26dZo4caJ++eUXvfrqq9a+Q4cO1YcffqhBgwbpnnvu0ZYtWxQdHX3duh555BE1atRIs2bNsgan5ORkHT9+XMOGDVNgYKD279+vt956S/v379c333xjE8Ik6bHHHlOzZs00e/ZsrVq1Si+++KJ8fX315ptvqnPnzpozZ46WLVumCRMm6K677lKHDh1ueK6GDx+upUuX6uGHH9Yzzzyj7du3KzExUQcPHtQnn3wiSfrPf/6jt956Szt27NDbb78tSbr33nuL3Z/FYlH//v01a9Ys7d+/X7fffrt129q1a3X27FkNGDDArrHb49KlS7rvvvv0yy+/aOTIkapTp462bdumhIQEpaena/78+dZa+vXrpy5dumjOnDmSpIMHD+rrr7/W2LFjb7kOwCEMAE5p8eLFhqQbLrfffrvNZ+rWrWsMGTLEut6yZUsjOjr6hseJi4szivtXwaeffmpIMl588UWb9ocfftiwWCzG0aNHDcMwjF27dhmSjHHjxtn0Gzp0qCHJmDp1qrVt6tSphiSjX79+RY536dKlIm3vv/++IcnYunVrkX2MGDHC2pafn2/Url3bsFgsxuzZs63t586dM6pUqWJzToqzZ88eQ5IxfPhwm/YJEyYYkoxNmzZZ24YMGWJUq1bthvu7av/+/YYkIyEhwaa9b9++RuXKlY2srCzDMG5+7Fd/J06cOGFtu/YcX3Xt78LMmTONatWqGYcPH7bp9+yzzxqurq5GamqqYRiGMXbsWMPLy8vIz8+/qTECFQGXsQAnt2DBAiUnJxdZWrRo8aef9fHx0f79+3XkyJESH3f16tVydXXV008/bdP+zDPPyDAMrVmzRtLvsxSS9NRTT9n0GzNmzHX3PWrUqCJtVapUsf58+fJl/frrr7rnnnskSd99912R/sOHD7f+7OrqqtatW8swDMXGxlrbfXx81KRJEx0/fvy6tUi/j1WS4uPjbdqfeeYZSdKqVatu+PnrCQ0N1R133KHly5db23JycvS///1PDzzwgLy8vCSVfOz2WLFihdq3b68aNWro119/tS6RkZEqKCjQ1q1bJf1+znJycqyXOgEzIOwATu7uu+9WZGRkkaVGjRp/+tkZM2bo/Pnzaty4scLCwjRx4kTt3bv3po77008/KTg4WJ6enjbtzZo1s26/+r8uLi6qX7++Tb+GDRted9/X9pWks2fPauzYsQoICFCVKlVUq1Yta7+srKwi/evUqWOz7u3trcqVK8vPz69I+7lz565byx/HcG3NgYGB8vHxsY7VHgMGDNCJEye0bds2SdKnn36qS5cuWS9hSSUfuz2OHDmitWvXqlatWjZLZGSkpN9vvJZ+D62NGzdW9+7dVbt2bT3++OPWQAtUVNyzA5hYhw4ddOzYMX322Wdav3693n77bb366qtatGiRzcxIefvjTMZVjz76qLZt26aJEyeqVatWql69ugoLC9WtW7ciN9BKv8/m3EybpCI3VF9Padwbc61+/fpp0qRJSkpK0r333qukpCTVqFFDPXr0sPYp6dhvRkFBgc16YWGh7r//fk2aNKnY/o0bN5Yk+fv7a8+ePVq3bp3WrFmjNWvWaPHixRo8eLCWLl1qVy2AoxF2AJPz9fXVsGHDNGzYMF28eFEdOnTQtGnTrGHnen/g69atqw0bNujChQs2szs//vijdfvV/y0sLNSJEyfUqFEja7+jR4/edI3nzp3Txo0bNX36dL3wwgvWdnsuv9nj6hiOHDlinbmSpMzMTJ0/f77Y9+TcrODgYHXq1EkrVqzQ888/r+TkZA0dOtT66Pqtjr1GjRo6f/68TduVK1eUnp5u09agQQNdvHjROpNzI+7u7urZs6d69uypwsJCPfXUU3rzzTf1/PPP33DGDnBWXMYCTOzaR5erV6+uhg0b2jxOXa1aNUkq8gezR48eKigo0L/+9S+b9ldffVUWi0Xdu3eXJEVFRUmS3njjDZt+r7/++k3XeXVG5toZmKtPCJW1q7Ms1x5v3rx5knTDJ8tuxoABA3T69GmNHDlSeXl5NpewbnXsDRo0sN5vc9Vbb71VZGbn0UcfVUpKitatW1dkH+fPn1d+fr6kor8zLi4u1vvD/vh7A1QkzOwAJhYaGqqOHTsqPDxcvr6+2rlzpz766CONHj3a2ic8PFyS9PTTTysqKkqurq7q27evevbsqU6dOum5557TyZMn1bJlS61fv16fffaZxo0bpwYNGlg/HxMTo/nz5+u3336zPnp++PBhSTd3acjLy0sdOnTQ3LlzlZeXp7/97W9av369Tpw4UQZnpaiWLVtqyJAheuutt3T+/Hndd9992rFjh5YuXarevXurU6dOt7T/mJgYPfXUU/rss88UEhJi8xj8rY59+PDhGjVqlGJiYnT//ffr+++/17p164rcuzRx4kTrjdFDhw5VeHi4cnJytG/fPn300Uc6efKk/Pz8NHz4cJ09e1adO3dW7dq19dNPP+n1119Xq1atbGa9gArFoc+CAbiuq48Zf/vtt8Vuv++++/700fMXX3zRuPvuuw0fHx+jSpUqRtOmTY1//OMfxpUrV6x98vPzjTFjxhi1atUyLBaLzWPoFy5cMMaPH28EBwcbbm5uRqNGjYyXXnrJKCwstDluTk6OERcXZ/j6+hrVq1c3evfubRw6dMiQZPMo+NXHxs+cOVNkPD///LPx0EMPGT4+Poa3t7fxyCOPGGlpadd9fP3afVzvkfDizlNx8vLyjOnTpxv169c33NzcjJCQECMhIcG4fPnyTR3nzzzyyCOGJGPSpElFtt3s2It79LygoMCYPHmy4efnZ1StWtWIiooyjh49WuR3wTB+/+eZkJBgNGzY0HB3dzf8/PyMe++913j55ZetvxMfffSR0bVrV8Pf399wd3c36tSpY4wcOdJIT08v8ZgBZ2ExjJu8cw8ASmDPnj2644479N///tfmsg0AlDfu2QFwy/7v//6vSNv8+fPl4uLyp28uBoCyxj07AG7Z3LlztWvXLnXq1EmVKlWyPrI8YsQIhYSEOLo8AH9xXMYCcMuSk5M1ffp0HThwQBcvXlSdOnU0aNAgPffcc6pUif+mAuBYhB0AAGBq3LMDAABMjbADAABMjYvp+v07Y9LS0uTp6Vkm340DAABKn2EYunDhgoKDg+Xicv35G8KOpLS0NJ4YAQCggjp16pRq16593e2EHcn6JYenTp2Sl5eXg6sBAAA3Izs7WyEhITZfVlwcwo7+/3f3eHl5EXYAAKhg/uwWFG5QBgAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApkbYAQAApubQsLNw4UK1aNHC+jUNERERWrNmjXV7x44dZbFYbJZRo0bZ7CM1NVXR0dGqWrWq/P39NXHiROXn55f3UAAAgJNy6Hdj1a5dW7Nnz1ajRo1kGIaWLl2qXr16affu3br99tslSU888YRmzJhh/UzVqlWtPxcUFCg6OlqBgYHatm2b0tPTNXjwYLm5uWnWrFnlPh4AAOB8LIZhGI4u4o98fX310ksvKTY2Vh07dlSrVq00f/78YvuuWbNGDzzwgNLS0hQQECBJWrRokSZPnqwzZ87I3d39po6ZnZ0tb29vZWVl8UWgAABUEDf799tp7tkpKCjQ8uXLlZOTo4iICGv7smXL5Ofnp+bNmyshIUGXLl2ybktJSVFYWJg16EhSVFSUsrOztX///nKtHwAAOCeHXsaSpH379ikiIkKXL19W9erV9cknnyg0NFSS1L9/f9WtW1fBwcHau3evJk+erEOHDunjjz+WJGVkZNgEHUnW9YyMjOseMzc3V7m5udb17Ozs0h4WAABwEg4PO02aNNGePXuUlZWljz76SEOGDNGWLVsUGhqqESNGWPuFhYUpKChIXbp00bFjx9SgQQO7j5mYmKjp06eXRvlwEvWeXeWQ456cHe2Q4wIAbp7DL2O5u7urYcOGCg8PV2Jiolq2bKnXXnut2L5t2rSRJB09elSSFBgYqMzMTJs+V9cDAwOve8yEhARlZWVZl1OnTpXGUAAAgBNyeNi5VmFhoc0lpj/as2ePJCkoKEiSFBERoX379un06dPWPsnJyfLy8rJeCiuOh4eH9XH3qwsAADAnh17GSkhIUPfu3VWnTh1duHBBSUlJ2rx5s9atW6djx44pKSlJPXr0UM2aNbV3716NHz9eHTp0UIsWLSRJXbt2VWhoqAYNGqS5c+cqIyNDU6ZMUVxcnDw8PBw5NAAA4CQcGnZOnz6twYMHKz09Xd7e3mrRooXWrVun+++/X6dOndKGDRs0f/585eTkKCQkRDExMZoyZYr1866urlq5cqWefPJJRUREqFq1ahoyZIjNe3kAAMBfm9O9Z8cReM9OxccNygDw11Ph3rMDAABQFgg7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1BwadhYuXKgWLVrIy8tLXl5eioiI0Jo1a6zbL1++rLi4ONWsWVPVq1dXTEyMMjMzbfaRmpqq6OhoVa1aVf7+/po4caLy8/PLeygAAMBJOTTs1K5dW7Nnz9auXbu0c+dOde7cWb169dL+/fslSePHj9fnn3+uFStWaMuWLUpLS1OfPn2sny8oKFB0dLSuXLmibdu2aenSpVqyZIleeOEFRw0JAAA4GYthGIaji/gjX19fvfTSS3r44YdVq1YtJSUl6eGHH5Yk/fjjj2rWrJlSUlJ0zz33aM2aNXrggQeUlpamgIAASdKiRYs0efJknTlzRu7u7jd1zOzsbHl7eysrK0teXl5lNjaUnXrPrnLIcU/OjnbIcQEAN//322nu2SkoKNDy5cuVk5OjiIgI7dq1S3l5eYqMjLT2adq0qerUqaOUlBRJUkpKisLCwqxBR5KioqKUnZ1tnR0qTm5urrKzs20WAABgTg4PO/v27VP16tXl4eGhUaNG6ZNPPlFoaKgyMjLk7u4uHx8fm/4BAQHKyMiQJGVkZNgEnavbr267nsTERHl7e1uXkJCQ0h0UAABwGg4PO02aNNGePXu0fft2PfnkkxoyZIgOHDhQpsdMSEhQVlaWdTl16lSZHg8AADhOJUcX4O7uroYNG0qSwsPD9e233+q1117TY489pitXruj8+fM2szuZmZkKDAyUJAUGBmrHjh02+7v6tNbVPsXx8PCQh4dHKY8EAAA4I4fP7FyrsLBQubm5Cg8Pl5ubmzZu3GjddujQIaWmpioiIkKSFBERoX379un06dPWPsnJyfLy8lJoaGi51w4AAJyPQ2d2EhIS1L17d9WpU0cXLlxQUlKSNm/erHXr1snb21uxsbGKj4+Xr6+vvLy8NGbMGEVEROiee+6RJHXt2lWhoaEaNGiQ5s6dq4yMDE2ZMkVxcXHM3AAAAEkODjunT5/W4MGDlZ6eLm9vb7Vo0ULr1q3T/fffL0l69dVX5eLiopiYGOXm5ioqKkpvvPGG9fOurq5auXKlnnzySUVERKhatWoaMmSIZsyY4aghAQAAJ+N079lxBN6zU/Hxnh0A+OupcO/ZAQAAKAuEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGqEHQAAYGoODTuJiYm666675OnpKX9/f/Xu3VuHDh2y6dOxY0dZLBabZdSoUTZ9UlNTFR0drapVq8rf318TJ05Ufn5+eQ4FAAA4qUqOPPiWLVsUFxenu+66S/n5+fr73/+url276sCBA6pWrZq13xNPPKEZM2ZY16tWrWr9uaCgQNHR0QoMDNS2bduUnp6uwYMHy83NTbNmzSrX8QAAAOfj0LCzdu1am/UlS5bI399fu3btUocOHaztVatWVWBgYLH7WL9+vQ4cOKANGzYoICBArVq10syZMzV58mRNmzZN7u7uZToGAADg3Jzqnp2srCxJkq+vr037smXL5Ofnp+bNmyshIUGXLl2ybktJSVFYWJgCAgKsbVFRUcrOztb+/fuLPU5ubq6ys7NtFgAAYE4Ondn5o8LCQo0bN05t27ZV8+bNre39+/dX3bp1FRwcrL1792ry5Mk6dOiQPv74Y0lSRkaGTdCRZF3PyMgo9liJiYmaPn16GY0EAAA4E6cJO3Fxcfrhhx/01Vdf2bSPGDHC+nNYWJiCgoLUpUsXHTt2TA0aNLDrWAkJCYqPj7euZ2dnKyQkxL7CAQCAU3OKy1ijR4/WypUr9cUXX6h27do37NumTRtJ0tGjRyVJgYGByszMtOlzdf169/l4eHjIy8vLZgEAAObk0LBjGIZGjx6tTz75RJs2bVL9+vX/9DN79uyRJAUFBUmSIiIitG/fPp0+fdraJzk5WV5eXgoNDS2TugEAQMXh0MtYcXFxSkpK0meffSZPT0/rPTbe3t6qUqWKjh07pqSkJPXo0UM1a9bU3r17NX78eHXo0EEtWrSQJHXt2lWhoaEaNGiQ5s6dq4yMDE2ZMkVxcXHy8PBw5PAAAIATcOjMzsKFC5WVlaWOHTsqKCjIunzwwQeSJHd3d23YsEFdu3ZV06ZN9cwzzygmJkaff/65dR+urq5auXKlXF1dFRERoYEDB2rw4ME27+UBAAB/XQ6d2TEM44bbQ0JCtGXLlj/dT926dbV69erSKgsAAJiIU9ygDAAAUFYIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNQIOwAAwNTsCjvHjx8v7ToAAADKhF1hp2HDhurUqZP++9//6vLly6VdEwAAQKmxK+x89913atGiheLj4xUYGKiRI0dqx44dpV0bAADALbMr7LRq1Uqvvfaa0tLS9O677yo9PV3t2rVT8+bNNW/ePJ05c6a06wQAALDLLd2gXKlSJfXp00crVqzQnDlzdPToUU2YMEEhISEaPHiw0tPTS6tOAAAAu9xS2Nm5c6eeeuopBQUFad68eZowYYKOHTum5ORkpaWlqVevXqVVJwAAgF0q2fOhefPmafHixTp06JB69Oih9957Tz169JCLy+/ZqX79+lqyZInq1atXmrUCAACUmF1hZ+HChXr88cc1dOhQBQUFFdvH399f77zzzi0VBwAAcKvsCjtHjhz50z7u7u4aMmSIPbsHAAAoNXbds7N48WKtWLGiSPuKFSu0dOnSWy4KAACgtNgVdhITE+Xn51ek3d/fX7NmzbrlogAAAEqLXWEnNTVV9evXL9Jet25dpaam3nJRAAAApcWusOPv76+9e/cWaf/+++9Vs2bNm95PYmKi7rrrLnl6esrf31+9e/fWoUOHbPpcvnxZcXFxqlmzpqpXr66YmBhlZmba9ElNTVV0dLSqVq0qf39/TZw4Ufn5+fYMDQAAmIxdYadfv356+umn9cUXX6igoEAFBQXatGmTxo4dq759+970frZs2aK4uDh98803Sk5OVl5enrp27aqcnBxrn/Hjx+vzzz/XihUrtGXLFqWlpalPnz7W7QUFBYqOjtaVK1e0bds2LV26VEuWLNELL7xgz9AAAIDJWAzDMEr6oStXrmjQoEFasWKFKlX6/YGuwsJCDR48WIsWLZK7u7tdxZw5c0b+/v7asmWLOnTooKysLNWqVUtJSUl6+OGHJUk//vijmjVrppSUFN1zzz1as2aNHnjgAaWlpSkgIECStGjRIk2ePFlnzpy5qVqys7Pl7e2trKwseXl52VU7HKves6scctyTs6MdclwAwM3//bZrZsfd3V0ffPCBfvzxRy1btkwff/yxjh07pnfffdfuoCNJWVlZkiRfX19J0q5du5SXl6fIyEhrn6ZNm6pOnTpKSUmRJKWkpCgsLMwadCQpKipK2dnZ2r9/v921AAAAc7DrPTtXNW7cWI0bNy6VQgoLCzVu3Di1bdtWzZs3lyRlZGTI3d1dPj4+Nn0DAgKUkZFh7fPHoHN1+9VtxcnNzVVubq51PTs7u1TGAAAAnI9dYaegoEBLlizRxo0bdfr0aRUWFtps37RpU4n3GRcXpx9++EFfffWVPSWVSGJioqZPn17mxwEAAI5nV9gZO3aslixZoujoaDVv3lwWi+WWihg9erRWrlyprVu3qnbt2tb2wMBAXblyRefPn7eZ3cnMzFRgYKC1z44dO2z2d/Vprat9rpWQkKD4+HjrenZ2tkJCQm5pDAAAwDnZFXaWL1+uDz/8UD169LilgxuGoTFjxuiTTz7R5s2bi7y7Jzw8XG5ubtq4caNiYmIkSYcOHVJqaqoiIiIkSREREfrHP/6h06dPy9/fX5KUnJwsLy8vhYaGFntcDw8PeXh43FLtAACgYrAr7Li7u6thw4a3fPC4uDglJSXps88+k6enp/UeG29vb1WpUkXe3t6KjY1VfHy8fH195eXlpTFjxigiIkL33HOPJKlr164KDQ3VoEGDNHfuXGVkZGjKlCmKi4sj0AAAAPuexnrmmWf02muvyY6n1m0sXLhQWVlZ6tixo4KCgqzLBx98YO3z6quv6oEHHlBMTIw6dOigwMBAffzxx9btrq6uWrlypVxdXRUREaGBAwdq8ODBmjFjxi3VBgAAzMGu9+w89NBD+uKLL+Tr66vbb79dbm5uNtv/GEYqAt6zU/Hxnh0A+Ou52b/fdl3G8vHx0UMPPWR3cQAAAOXFrrCzePHi0q4DAACgTNh1z44k5efna8OGDXrzzTd14cIFSVJaWpouXrxYasUBAADcKrtmdn766Sd169ZNqampys3N1f333y9PT0/NmTNHubm5WrRoUWnXCQAAYBe7ZnbGjh2r1q1b69y5c6pSpYq1/aGHHtLGjRtLrTgAAIBbZdfMzpdffqlt27YV+dLPevXq6ZdffimVwgAAAEqDXTM7hYWFKigoKNL+888/y9PT85aLAgAAKC12hZ2uXbtq/vz51nWLxaKLFy9q6tSpt/wVEgAAAKXJrstYr7zyiqKiohQaGqrLly+rf//+OnLkiPz8/PT++++Xdo0AAAB2syvs1K5dW99//72WL1+uvXv36uLFi4qNjdWAAQNsblgGAABwNLvCjiRVqlRJAwcOLM1aAAAASp1dYee999674fbBgwfbVQwAAEBpsyvsjB071mY9Ly9Ply5dkru7u6pWrUrYAQAATsOup7HOnTtns1y8eFGHDh1Su3btuEEZAAA4Fbu/G+tajRo10uzZs4vM+gAAADhSqYUd6febltPS0kpzlwAAALfErnt2/ve//9msG4ah9PR0/etf/1Lbtm1LpTAAAIDSYFfY6d27t826xWJRrVq11LlzZ73yyiulURcAAECpsCvsFBYWlnYdAAAAZaJU79kBAABwNnbN7MTHx99033nz5tlzCAAAgFJhV9jZvXu3du/erby8PDVp0kSSdPjwYbm6uurOO++09rNYLKVTJQAAgJ3sCjs9e/aUp6enli5dqho1akj6/UWDw4YNU/v27fXMM8+UapEAAAD2suuenVdeeUWJiYnWoCNJNWrU0IsvvsjTWAAAwKnYFXays7N15syZIu1nzpzRhQsXbrkoAACA0mLXZayHHnpIw4YN0yuvvKK7775bkrR9+3ZNnDhRffr0KdUCATiXes+ucshxT86OdshxAVR8doWdRYsWacKECerfv7/y8vJ+31GlSoqNjdVLL71UqgUCAADcCrvCTtWqVfXGG2/opZde0rFjxyRJDRo0ULVq1Uq1OAAAgFt1Sy8VTE9PV3p6uho1aqRq1arJMIzSqgsAAKBU2BV2fvvtN3Xp0kWNGzdWjx49lJ6eLkmKjY3lsXMAAOBU7Ao748ePl5ubm1JTU1W1alVr+2OPPaa1a9eWWnEAAAC3yq57dtavX69169apdu3aNu2NGjXSTz/9VCqFAQAAlAa7ZnZycnJsZnSuOnv2rDw8PG65KAAAgNJi18xO+/bt9d5772nmzJmSfv8OrMLCQs2dO1edOnUq1QIBFM9R77sBgIrGrrAzd+5cdenSRTt37tSVK1c0adIk7d+/X2fPntXXX39d2jUCAADYza7LWM2bN9fhw4fVrl079erVSzk5OerTp492796tBg0alHaNAAAAdivxzE5eXp66deumRYsW6bnnniuLmgAAAEpNiWd23NzctHfv3rKoBQAAoNTZdRlr4MCBeuedd0q7FgAAgFJn1w3K+fn5evfdd7VhwwaFh4cX+U6sefPmlUpxAAAAt6pEMzvHjx9XYWGhfvjhB915553y9PTU4cOHtXv3buuyZ8+em97f1q1b1bNnTwUHB8tisejTTz+12T506FBZLBabpVu3bjZ9zp49qwEDBsjLy0s+Pj6KjY3VxYsXSzIsAABgYiWa2WnUqJHS09P1xRdfSPr96yH++c9/KiAgwK6D5+TkqGXLlnr88cfVp0+fYvt069ZNixcvtq5f+9LCAQMGKD09XcnJycrLy9OwYcM0YsQIJSUl2VUTAAAwlxKFnWu/1XzNmjXKycmx++Ddu3dX9+7db9jHw8NDgYGBxW47ePCg1q5dq2+//VatW7eWJL3++uvq0aOHXn75ZQUHB9tdGwAAMAe7blC+6trwUxY2b94sf39/NWnSRE8++aR+++0367aUlBT5+PhYg44kRUZGysXFRdu3b7/uPnNzc5WdnW2zAAAAcypR2Ll638y1bWWlW7dueu+997Rx40bNmTNHW7ZsUffu3VVQUCBJysjIkL+/v81nKlWqJF9fX2VkZFx3v4mJifL29rYuISEhZTYGAADgWCW+jDV06FDrfTOXL1/WqFGjijyN9fHHH5dKcX379rX+HBYWphYtWqhBgwbavHmzunTpYvd+ExISFB8fb13Pzs4m8AAAYFIlCjtDhgyxWR84cGCpFvNnbrvtNvn5+eno0aPq0qWLAgMDdfr0aZs++fn5Onv27HXv85F+vw+Ib2cHAOCvoURh549PRTnCzz//rN9++01BQUGSpIiICJ0/f167du1SeHi4JGnTpk0qLCxUmzZtHFkqAABwEna9VLC0XLx4UUePHrWunzhxQnv27JGvr698fX01ffp0xcTEKDAwUMeOHdOkSZPUsGFDRUVFSZKaNWumbt266YknntCiRYuUl5en0aNHq2/fvjyJBQAAJN3i01i3aufOnbrjjjt0xx13SJLi4+N1xx136IUXXpCrq6v27t2rBx98UI0bN1ZsbKzCw8P15Zdf2lyCWrZsmZo2baouXbqoR48eateund566y1HDQkAADgZh87sdOzY8YaPr69bt+5P9+Hr68sLBAEAwHU5dGYHAACgrBF2AACAqTn0MtZfQb1nVznkuCdnRzvkuAAAOBtmdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKkRdgAAgKlVcnQBMJd6z65ydAkAANhgZgcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJgaYQcAAJiaQ8PO1q1b1bNnTwUHB8tisejTTz+12W4Yhl544QUFBQWpSpUqioyM1JEjR2z6nD17VgMGDJCXl5d8fHwUGxurixcvluMoAACAM3No2MnJyVHLli21YMGCYrfPnTtX//znP7Vo0SJt375d1apVU1RUlC5fvmztM2DAAO3fv1/JyclauXKltm7dqhEjRpTXEAAAgJOr5MiDd+/eXd27dy92m2EYmj9/vqZMmaJevXpJkt577z0FBATo008/Vd++fXXw4EGtXbtW3377rVq3bi1Jev3119WjRw+9/PLLCg4OLrexAAAA5+S09+ycOHFCGRkZioyMtLZ5e3urTZs2SklJkSSlpKTIx8fHGnQkKTIyUi4uLtq+fft1952bm6vs7GybBQAAmJPThp2MjAxJUkBAgE17QECAdVtGRob8/f1ttleqVEm+vr7WPsVJTEyUt7e3dQkJCSnl6gEAgLNw2rBTlhISEpSVlWVdTp065eiSAABAGXHasBMYGChJyszMtGnPzMy0bgsMDNTp06dttufn5+vs2bPWPsXx8PCQl5eXzQIAAMzJacNO/fr1FRgYqI0bN1rbsrOztX37dkVEREiSIiIidP78ee3atcvaZ9OmTSosLFSbNm3KvWYAAOB8HPo01sWLF3X06FHr+okTJ7Rnzx75+vqqTp06GjdunF588UU1atRI9evX1/PPP6/g4GD17t1bktSsWTN169ZNTzzxhBYtWqS8vDyNHj1affv25UksAAAgycFhZ+fOnerUqZN1PT4+XpI0ZMgQLVmyRJMmTVJOTo5GjBih8+fPq127dlq7dq0qV65s/cyyZcs0evRodenSRS4uLoqJidE///nPch8LAABwTg4NOx07dpRhGNfdbrFYNGPGDM2YMeO6fXx9fZWUlFQW5QEAABNw2nt2AAAASgNhBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmJpTh51p06bJYrHYLE2bNrVuv3z5suLi4lSzZk1Vr15dMTExyszMdGDFAADA2Th12JGk22+/Xenp6dblq6++sm4bP368Pv/8c61YsUJbtmxRWlqa+vTp48BqAQCAs6nk6AL+TKVKlRQYGFikPSsrS++8846SkpLUuXNnSdLixYvVrFkzffPNN7rnnnvKu1QAAOCEnD7sHDlyRMHBwapcubIiIiKUmJioOnXqaNeuXcrLy1NkZKS1b9OmTVWnTh2lpKTcMOzk5uYqNzfXup6dnV2mYwCAkqr37CqHHPfk7GiHHBcoS059GatNmzZasmSJ1q5dq4ULF+rEiRNq3769Lly4oIyMDLm7u8vHx8fmMwEBAcrIyLjhfhMTE+Xt7W1dQkJCynAUAADAkZx6Zqd79+7Wn1u0aKE2bdqobt26+vDDD1WlShW795uQkKD4+HjrenZ2NoEHdnHUf30DAG6eU8/sXMvHx0eNGzfW0aNHFRgYqCtXruj8+fM2fTIzM4u9x+ePPDw85OXlZbMAAABzqlBh5+LFizp27JiCgoIUHh4uNzc3bdy40br90KFDSk1NVUREhAOrBAAAzsSpL2NNmDBBPXv2VN26dZWWlqapU6fK1dVV/fr1k7e3t2JjYxUfHy9fX195eXlpzJgxioiI4EksAABg5dRh5+eff1a/fv3022+/qVatWmrXrp2++eYb1apVS5L06quvysXFRTExMcrNzVVUVJTeeOMNB1cNAACciVOHneXLl99we+XKlbVgwQItWLCgnCoCAAAVTYW6ZwcAAKCkCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUCDsAAMDUnPoNygCA8lXv2VUOOe7J2dEOOS7+GpjZAQAApkbYAQAApkbYAQAApsY9OwBwA466hwVA6WFmBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBphBwAAmBpfBAoAcDhHfeHqydnRDjkuyhczOwAAwNQIOwAAwNQIOwAAwNS4ZwdAheCoezoAVHzM7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFMj7AAAAFPjPTsAgL8svpPrr8E0MzsLFixQvXr1VLlyZbVp00Y7duxwdEkAAMAJmCLsfPDBB4qPj9fUqVP13XffqWXLloqKitLp06cdXRoAAHAwU1zGmjdvnp544gkNGzZMkrRo0SKtWrVK7777rp599lkHVwcAgHP4q162q/AzO1euXNGuXbsUGRlpbXNxcVFkZKRSUlIcWBkAAHAGFX5m59dff1VBQYECAgJs2gMCAvTjjz8W+5nc3Fzl5uZa17OysiRJ2dnZpV5fYe6lUt/nzSiLsdwMR40XACqSOuNXOLqEclVWf5Ou7tcwjBv2q/Bhxx6JiYmaPn16kfaQkBAHVFM2vOc7ugIAAH5X1n+TLly4IG9v7+tur/Bhx8/PT66ursrMzLRpz8zMVGBgYLGfSUhIUHx8vHW9sLBQZ8+eVc2aNWWxWMq03puVnZ2tkJAQnTp1Sl5eXo4u5y+Fc+8YnHfH4dw7Buf91hmGoQsXLig4OPiG/Sp82HF3d1d4eLg2btyo3r17S/o9vGzcuFGjR48u9jMeHh7y8PCwafPx8SnjSu3j5eXF/wkchHPvGJx3x+HcOwbn/dbcaEbnqgofdiQpPj5eQ4YMUevWrXX33Xdr/vz5ysnJsT6dBQAA/rpMEXYee+wxnTlzRi+88IIyMjLUqlUrrV27tshNywAA4K/HFGFHkkaPHn3dy1YVkYeHh6ZOnVrkchvKHufeMTjvjsO5dwzOe/mxGH/2vBYAAEAFVuFfKggAAHAjhB0AAGBqhB0AAGBqhB0AAGBqhB0HWrBggerVq6fKlSurTZs22rFjxw37r1ixQk2bNlXlypUVFham1atXl1Ol5lOSc//vf/9b7du3V40aNVSjRg1FRkb+6T8rFK+kv/NXLV++XBaLxfriUJRMSc/7+fPnFRcXp6CgIHl4eKhx48b8+8ZOJT338+fPV5MmTVSlShWFhIRo/Pjxunz5cjlVa2IGHGL58uWGu7u78e677xr79+83nnjiCcPHx8fIzMwstv/XX39tuLq6GnPnzjUOHDhgTJkyxXBzczP27dtXzpVXfCU99/379zcWLFhg7N692zh48KAxdOhQw9vb2/j555/LufKKraTn/aoTJ04Yf/vb34z27dsbvXr1Kp9iTaSk5z03N9do3bq10aNHD+Orr74yTpw4YWzevNnYs2dPOVde8ZX03C9btszw8PAwli1bZpw4ccJYt26dERQUZIwfP76cKzcfwo6D3H333UZcXJx1vaCgwAgODjYSExOL7f/oo48a0dHRNm1t2rQxRo4cWaZ1mlFJz/218vPzDU9PT2Pp0qVlVaIp2XPe8/PzjXvvvdd4++23jSFDhhB27FDS875w4ULjtttuM65cuVJeJZpWSc99XFyc0blzZ5u2+Ph4o23btmVa518Bl7Ec4MqVK9q1a5ciIyOtbS4uLoqMjFRKSkqxn0lJSbHpL0lRUVHX7Y/i2XPur3Xp0iXl5eXJ19e3rMo0HXvP+4wZM+Tv76/Y2NjyKNN07Dnv//vf/xQREaG4uDgFBASoefPmmjVrlgoKCsqrbFOw59zfe++92rVrl/VS1/Hjx7V69Wr16NGjXGo2M9O8Qbki+fXXX1VQUFDk6ywCAgL0448/FvuZjIyMYvtnZGSUWZ1mZM+5v9bkyZMVHBxcJHzi+uw571999ZXeeecd7dmzpxwqNCd7zvvx48e1adMmDRgwQKtXr9bRo0f11FNPKS8vT1OnTi2Psk3BnnPfv39//frrr2rXrp0Mw1B+fr5GjRqlv//97+VRsqkxswOUwOzZs7V8+XJ98sknqly5sqPLMa0LFy5o0KBB+ve//y0/Pz9Hl/OXUlhYKH9/f7311lsKDw/XY489pueee06LFi1ydGmmt3nzZs2aNUtvvPGGvvvuO3388cdatWqVZs6c6ejSKjxmdhzAz89Prq6uyszMtGnPzMxUYGBgsZ8JDAwsUX8Uz55zf9XLL7+s2bNna8OGDWrRokVZlmk6JT3vx44d08mTJ9WzZ09rW2FhoSSpUqVKOnTokBo0aFC2RZuAPb/vQUFBcnNzk6urq7WtWbNmysjI0JUrV+Tu7l6mNZuFPef++eef16BBgzR8+HBJUlhYmHJycjRixAg999xzcnFhfsJenDkHcHd3V3h4uDZu3GhtKyws1MaNGxUREVHsZyIiImz6S1JycvJ1+6N49px7SZo7d65mzpyptWvXqnXr1uVRqqmU9Lw3bdpU+/bt0549e6zLgw8+qE6dOmnPnj0KCQkpz/IrLHt+39u2baujR49aw6UkHT58WEFBQQSdErDn3F+6dKlIoLkaOg2+xvLWOPoO6b+q5cuXGx4eHsaSJUuMAwcOGCNGjDB8fHyMjIwMwzAMY9CgQcazzz5r7f/1118blSpVMl5++WXj4MGDxtSpU3n03E4lPfezZ8823N3djY8++shIT0+3LhcuXHDUECqkkp73a/E0ln1Ket5TU1MNT09PY/To0cahQ4eMlStXGv7+/saLL77oqCFUWCU991OnTjU8PT2N999/3zh+/Lixfv16o0GDBsajjz7qqCGYBmHHgV5//XWjTp06hru7u3H33Xcb33zzjXXbfffdZwwZMsSm/4cffmg0btzYcHd3N26//XZj1apV5VyxeZTk3NetW9eQVGSZOnVq+RdewZX0d/6PCDv2K+l537Ztm9GmTRvDw8PDuO2224x//OMfRn5+fjlXbQ4lOfd5eXnGtGnTjAYNGhiVK1c2QkJCjKeeeso4d+5c+RduMhbDYG4MAACYF/fsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsAAAAUyPsADCtjh07aty4cY4uA4CDEXYAOKWePXuqW7duxW778ssvZbFYtHfv3nKuCkBFRNgB4JRiY2OVnJysn3/+uci2xYsXq3Xr1nz7PICbQtgB4JQeeOAB1apVS0uWLLFpv3jxolasWKHevXurX79++tvf/qaqVasqLCxM77///g33abFY9Omnn9q0+fj42Bzj1KlTevTRR+Xj4yNfX1/16tVLJ0+eLJ1BAXAIwg4Ap1SpUiUNHjxYS5Ys0R+/wm/FihUqKCjQwIEDFR4erlWrVumHH37QiBEjNGjQIO3YscPuY+bl5SkqKkqenp768ssv9fXXX6t69erq1q2brly5UhrDAuAAhB0ATuvxxx/XsWPHtGXLFmvb4sWLFRMTo7p162rChAlq1aqVbrvtNo0ZM0bdunXThx9+aPfxPvjgAxUWFurtt99WWFiYmjVrpsWLFys1NVWbN28uhREBcATCDgCn1bRpU91777169913JUlHjx7Vl19+qdjYWBUUFGjmzJkKCwuTr6+vqlevrnXr1ik1NdXu433//fc6evSoPD09Vb16dVWvXl2+vr66fPmyjh07VlrDAlDOKjm6AAC4kdjYWI0ZM0YLFizQ4sWL1aBBA913332aM2eOXnvtNc2fP19hYWGqVq2axo0bd8PLTRaLxeaSmPT7paurLl68qPDwcC1btqzIZ2vVqlV6gwJQrgg7AJzao48+qrFjxyopKUnvvfeennzySVksFn399dfq1auXBg4cKEkqLCzU4cOHFRoaet191apVS+np6db1I0eO6NKlS9b1O++8Ux988IH8/f3l5eVVdoMCUK64jAXAqVWvXl2PPfaYEhISlJ6erqFDh0qSGjVqpOTkZG3btk0HDx7UyJEjlZmZecN9de7cWf/617+0e/du7dy5U6NGjZKbm5t1+4ABA+Tn56devXrpyy+/1IkTJ7R582Y9/fTTxT4CD6BiIOwAcHqxsbE6d+6coqKiFBwcLEmaMmWK7rzzTkVFRaljx44KDAxU7969b7ifV155RSEhIWrfvr369++vCRMmqGrVqtbtVatW1datW1WnTh316dNHzZo1U2xsrC5fvsxMD1CBWYxrL2ADAACYCDM7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1Ag7AADA1P4fHuh3lPrr50kAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.hist(data['entropy'], bins=15, align='left')\n",
        "\n",
        "# Add axis labels and a title\n",
        "plt.xlabel('Value')\n",
        "plt.ylabel('Frequency')\n",
        "plt.title('Histogram of Values')\n",
        "# Display the plot\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9cvrSMQnfpFd",
        "outputId": "06375d84-a06e-4b7b-cc88-cc8af6c4051f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "92"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df=data\n",
        "df_high=df[df['entropy']>1.4]\n",
        "df_high.shape[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sw0vHm8j2wWy",
        "outputId": "5abeac8f-b8a6-4063-c138-bf006bfb52da"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 918 entries, 0 to 917\n",
            "Data columns (total 36 columns):\n",
            " #   Column          Non-Null Count  Dtype  \n",
            "---  ------          --------------  -----  \n",
            " 0   Unnamed: 0      918 non-null    int64  \n",
            " 1   text            918 non-null    object \n",
            " 2   humor           918 non-null    bool   \n",
            " 3   prompt1         918 non-null    object \n",
            " 4   prompt2         918 non-null    object \n",
            " 5   prompt3         918 non-null    object \n",
            " 6   prompt4         918 non-null    object \n",
            " 7   prompt5         918 non-null    object \n",
            " 8   prompt6         918 non-null    object \n",
            " 9   prompt7         918 non-null    object \n",
            " 10  prompt8         918 non-null    object \n",
            " 11  prompt9         918 non-null    object \n",
            " 12  label1          918 non-null    object \n",
            " 13  label2          918 non-null    object \n",
            " 14  label3          918 non-null    object \n",
            " 15  label4          918 non-null    object \n",
            " 16  label5          918 non-null    object \n",
            " 17  label6          918 non-null    object \n",
            " 18  label7          918 non-null    object \n",
            " 19  label8          918 non-null    object \n",
            " 20  label9          918 non-null    object \n",
            " 21  pred1           918 non-null    int64  \n",
            " 22  pred2           918 non-null    int64  \n",
            " 23  pred3           918 non-null    int64  \n",
            " 24  pred4           918 non-null    int64  \n",
            " 25  pred5           918 non-null    int64  \n",
            " 26  pred6           918 non-null    int64  \n",
            " 27  pred7           918 non-null    int64  \n",
            " 28  pred8           918 non-null    int64  \n",
            " 29  pred9           918 non-null    int64  \n",
            " 30  entropy         918 non-null    float64\n",
            " 31  MV              918 non-null    int64  \n",
            " 32  ann             918 non-null    object \n",
            " 33  funny           918 non-null    int64  \n",
            " 34  notfunny        918 non-null    int64  \n",
            " 35  one_hot_labels  918 non-null    object \n",
            "dtypes: bool(1), float64(1), int64(13), object(21)\n",
            "memory usage: 252.0+ KB\n"
          ]
        }
      ],
      "source": [
        "df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GDXsD2fEhJvf",
        "outputId": "73040e72-3cd4-4db9-bdff-516fd2f555fc"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Martha stewart tweets hideous food photo, twitter responds accordingly',\n",
              " 'Donald trump has found something mysterious for rudy giuliani to do',\n",
              " \"How donald trump and ted cruz's love affair is all relationships\",\n",
              " 'Alec baldwin and wife hilaria welcome another baby boy',\n",
              " 'Watch: former british open champ makes embarrassing putting fail',\n",
              " 'Interesting fact: by the year 2020 all actors on american tv shows will be australian.',\n",
              " 'On set with paul mitchell: from our network',\n",
              " \"My sons ebola joke what do africans have for breakfast? ebola cereal :) (be kind,he's only 14 lol)\",\n",
              " 'How do you know if someone is using recursion?',\n",
              " \"Valentine's dinner stress: 4 things not to worry about\"]"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_high['text'].tolist()[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "saaQnjXcZirg",
        "outputId": "65f072a8-ada5-4bab-8aad-093d6464e6f7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "665"
            ]
          },
          "execution_count": 191,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_low=data[data['entropy']<0.3]\n",
        "df_low.shape[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AXG26rU75seo",
        "outputId": "ee0a6ce8-87e5-4fb3-cf50-e11f36b21ed6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 232,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "sum(df_low['length'].tolist())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_HOfcw_O43lC",
        "outputId": "df41e4cd-17b3-4622-9583-07f47f9092f0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1154\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "989297"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_low=data[data['entropy']<1]\n",
        "print(df_low.shape[0])\n",
        "df=df_low\n",
        "df['length1']=df['prompt1'].apply(lambda x:len(x.split()))\n",
        "df['length2']=df['prompt2'].apply(lambda x:len(x.split()))\n",
        "df['length3']=df['prompt3'].apply(lambda x:len(x.split()))\n",
        "df['length4']=df['prompt4'].apply(lambda x:len(x.split()))\n",
        "df['length5']=df['prompt5'].apply(lambda x:len(x.split()))\n",
        "df['length6']=df['prompt6'].apply(lambda x:len(x.split()))\n",
        "df['length7']=df['prompt7'].apply(lambda x:len(x.split()))\n",
        "df['length8']=df['prompt8'].apply(lambda x:len(x.split()))\n",
        "df['length9']=df['prompt9'].apply(lambda x:len(x.split()))\n",
        "df['length10']=df['prompt10'].apply(lambda x:len(x.split()))\n",
        "df['length11']=df['prompt11'].apply(lambda x:len(x.split()))\n",
        "df['length12']=df['prompt12'].apply(lambda x:len(x.split()))\n",
        "df['length13']=df['prompt13'].apply(lambda x:len(x.split()))\n",
        "df['length']=df['length1']+df['length2']+df['length3']+df['length4']+df['length5']+df['length6']+df['length7']+df['length8']+df['length9']+df['length10']+df['length11']+df['length12']+df['length13']\n",
        "sum(df['length'].tolist())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lljHhLxuaChh",
        "outputId": "55701f5b-810c-4a5f-d4b8-761180d08273"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['Study finds strong link between zika and guillain-barre syndrome',\n",
              " \"Reflections from davos: leaders deliberate what's next for climate action after paris deal\",\n",
              " 'What do you call an explanation of an asian cooking show? a wok-through.',\n",
              " 'Why did little miss muffet have gps on her tuffet? to keep her from losing her whey.',\n",
              " 'Oscars 2016 red carpet: all the stunning looks from the academy awards',\n",
              " 'What do chicken families do on saturday afternoon? they go on peck-nics !',\n",
              " \"My friend is dealing with a really severe viagra addiction. he's having a hard time with it.\",\n",
              " 'Did you hear about the flying german pancakes in ww2? they were in the luftwaffle!',\n",
              " \"What's black and always in the back of a police car? the seat.\",\n",
              " \"Veterinarian accused of shooting neighbors' dog in the head\"]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_low['text'].tolist()[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nwYWgVJoaFBs",
        "outputId": "bf1e8d90-987e-414f-813f-0359ce2bc4e7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[False, False, True, True, False, True, True, True, True, False]"
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_low['humor'].tolist()[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dTMiGKjD7YGF",
        "outputId": "9ca0424e-cf73-4780-da90-754fbc09f2fc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.2157801421416577: 350\n",
            "0.3636983367872396: 161\n",
            "0.4315602842833155: 157\n",
            "-0.0: 141\n",
            "0.1479181946455818: 125\n",
            "0.2958363892911637: 49\n",
            "0.6100053707958025: 27\n",
            "0.5794784789288974: 24\n",
            "0.5116165314328215: 23\n",
            "0.6778673182918784: 22\n",
            "0.7579235654413844: 11\n",
            "0.4620871761502206: 10\n",
            "0.5421434232997266: 8\n",
            "0.3942252286541448: 6\n",
            "0.4437545839367456: 6\n",
            "0.6473404264249732: 5\n",
            "0.6405322626627077: 5\n",
            "0.8257855129374603: 4\n",
            "0.4926140680171257: 4\n",
            "0.6595347260784034: 3\n",
            "0.7083942101587836: 3\n",
            "0.7273966735744792: 2\n",
            "0.9547012441673464: 1\n",
            "0.7083942101587835: 1\n",
            "0.8868392966712706: 1\n",
            "0.8936474604335362: 1\n",
            "0.7884504573082896: 1\n",
            "0.7389211020256886: 1\n",
            "0.7952586210705551: 1\n",
            "0.9241743523004412: 1\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-6-707faf383544>:5: FutureWarning: iteritems is deprecated and will be removed in a future version. Use .items instead.\n",
            "  for index, count in counts.iteritems():\n"
          ]
        }
      ],
      "source": [
        "# count the unique values in the 'fruit' column\n",
        "counts = data['entropy'].value_counts()\n",
        "\n",
        "# print the unique values and their count\n",
        "for index, count in counts.iteritems():\n",
        "    print(f\"{index}: {count}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "id": "xYI-E6DfHEdz",
        "outputId": "e45ba0b8-a553-4283-8ea0-ea83879510da"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            1  Title: Pope Struggles Through Mass at Lourdes ...   \n",
              "1            1  Title: China Typhoon Death Toll Rises to 164 (...   \n",
              "2            1  Title: Wanted: Sleeping Space for Protesters. ...   \n",
              "3            1  Title: Clashes persist in South Ossetia Descri...   \n",
              "4            1  Title: U.S. Awaits Judgment on Venezuela Votin...   \n",
              "\n",
              "         label1        label2        label3 label4  label5  label6  label7  \\\n",
              "0         world         world  topic: world   TRUE  false.  false.  false.   \n",
              "1         world         world  topic: world   TRUE  false.  false.  false.   \n",
              "2  topic: world  topic: world  topic: world  true.  false.  false.  false.   \n",
              "3         world         world  topic: world  true.  false.  false.  false.   \n",
              "4         world         world  topic: world  true.  false.  false.  false.   \n",
              "\n",
              "                  label8  ... world7 sports7 business7 sci7 entropy_world  \\\n",
              "0                  world  ...      1       0         0    0           0.0   \n",
              "1                  world  ...      1       0         0    0           0.0   \n",
              "2  the topic is \"world\".  ...      1       0         0    0           0.0   \n",
              "3                  world  ...      1       0         0    0           0.0   \n",
              "4                  world  ...      1       0         0    0           0.0   \n",
              "\n",
              "   entropy_sports  entropy_business  entropy_sci  entropy  allocate  \n",
              "0             0.0               0.0          0.0      0.0         1  \n",
              "1             0.0               0.0          0.0      0.0         1  \n",
              "2             0.0               0.0          0.0      0.0         1  \n",
              "3             0.0               0.0          0.0      0.0         1  \n",
              "4             0.0               0.0          0.0      0.0         1  \n",
              "\n",
              "[5 rows x 55 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3545034f-b264-423a-8579-394c1ac7bd41\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>world7</th>\n",
              "      <th>sports7</th>\n",
              "      <th>business7</th>\n",
              "      <th>sci7</th>\n",
              "      <th>entropy_world</th>\n",
              "      <th>entropy_sports</th>\n",
              "      <th>entropy_business</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "      <th>allocate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Pope Struggles Through Mass at Lourdes ...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: China Typhoon Death Toll Rises to 164 (...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Wanted: Sleeping Space for Protesters. ...</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>the topic is \"world\".</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Clashes persist in South Ossetia Descri...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: U.S. Awaits Judgment on Venezuela Votin...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 55 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3545034f-b264-423a-8579-394c1ac7bd41')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3545034f-b264-423a-8579-394c1ac7bd41 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3545034f-b264-423a-8579-394c1ac7bd41');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 214
        }
      ],
      "source": [
        "import pandas as pd\n",
        "df=data\n",
        "df=df.sort_values(by='entropy', ignore_index=True)\n",
        "# Assuming your dataframe is called 'df'\n",
        "total_rows = df.shape[0]\n",
        "num_rows_20_percent = int(total_rows * 0.8) #can change this to 0.2, 0.4, 0.6, 0.8\n",
        "\n",
        "\n",
        "df['allocate'] = 0  # Initialize the new column with 0\n",
        "df.loc[:num_rows_20_percent-1, 'allocate'] = 1  # Assign 1 to the first 20% of rows\n",
        "\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "id": "WxS4YXfA730N",
        "outputId": "aa5e34b8-da5b-4a59-dce3-433af48ecbd7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            1  Title: Pope Struggles Through Mass at Lourdes ...   \n",
              "1            1  Title: China Typhoon Death Toll Rises to 164 (...   \n",
              "2            1  Title: Wanted: Sleeping Space for Protesters. ...   \n",
              "3            1  Title: Clashes persist in South Ossetia Descri...   \n",
              "4            1  Title: U.S. Awaits Judgment on Venezuela Votin...   \n",
              "\n",
              "         label1        label2        label3 label4  label5  label6  label7  \\\n",
              "0         world         world  topic: world   TRUE  false.  false.  false.   \n",
              "1         world         world  topic: world   TRUE  false.  false.  false.   \n",
              "2  topic: world  topic: world  topic: world  true.  false.  false.  false.   \n",
              "3         world         world  topic: world  true.  false.  false.  false.   \n",
              "4         world         world  topic: world  true.  false.  false.  false.   \n",
              "\n",
              "                  label8  ... entropy_sports entropy_business entropy_sci  \\\n",
              "0                  world  ...            0.0              0.0         0.0   \n",
              "1                  world  ...            0.0              0.0         0.0   \n",
              "2  the topic is \"world\".  ...            0.0              0.0         0.0   \n",
              "3                  world  ...            0.0              0.0         0.0   \n",
              "4                  world  ...            0.0              0.0         0.0   \n",
              "\n",
              "  entropy allocate  gold_label  world  sports  business  sci  \n",
              "0     0.0        1           1      1       0         0    0  \n",
              "1     0.0        1           1      1       0         0    0  \n",
              "2     0.0        1           1      1       0         0    0  \n",
              "3     0.0        1           1      1       0         0    0  \n",
              "4     0.0        1           1      1       0         0    0  \n",
              "\n",
              "[5 rows x 60 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cf3e4a40-d01a-485e-a153-b614fa819d82\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>entropy_sports</th>\n",
              "      <th>entropy_business</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "      <th>allocate</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Pope Struggles Through Mass at Lourdes ...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: China Typhoon Death Toll Rises to 164 (...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Wanted: Sleeping Space for Protesters. ...</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>the topic is \"world\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Clashes persist in South Ossetia Descri...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: U.S. Awaits Judgment on Venezuela Votin...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 60 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cf3e4a40-d01a-485e-a153-b614fa819d82')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cf3e4a40-d01a-485e-a153-b614fa819d82 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cf3e4a40-d01a-485e-a153-b614fa819d82');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 215
        }
      ],
      "source": [
        "#combined\n",
        "df['gold_label']=df['Class Index']\n",
        "world=[]\n",
        "sports=[]\n",
        "business=[]\n",
        "sci=[]\n",
        "chatgpt_count=0\n",
        "\n",
        "for index, row in df.iterrows():\n",
        "    if row['allocate']==1:\n",
        "      world.append(row['MV_world'])\n",
        "      sports.append(row['MV_sports'])\n",
        "      business.append(row['MV_business'])\n",
        "      sci.append(row['MV_sci'])\n",
        "    else:\n",
        "      if row['gold_label']==1:\n",
        "        world.append(1)\n",
        "        sports.append(0)\n",
        "        business.append(0)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==2:\n",
        "        world.append(0)\n",
        "        sports.append(1)\n",
        "        business.append(0)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==3:\n",
        "        world.append(0)\n",
        "        sports.append(0)\n",
        "        business.append(1)\n",
        "        sci.append(0)\n",
        "      elif row['gold_label']==4:\n",
        "        world.append(0)\n",
        "        sports.append(0)\n",
        "        business.append(0)\n",
        "        sci.append(1)\n",
        "df['world']=world\n",
        "df['sports']=sports\n",
        "df['business']=business\n",
        "df['sci']=sci\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dLcetMjZFjSH"
      },
      "source": [
        "## Load and Preprocess Training Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2dal0ggBcYdD"
      },
      "source": [
        "Dataset will be tokenized then split into training and validation sets. The validation set will be used to monitor training. For testing a separate test set will be loaded for analysis."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0ecREc7GnISW",
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "#df = data #jigsaw-toxic-comment-classification-challenge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UVI59S9VaAfB",
        "outputId": "ec3640a7-9224-4831-df3f-121d4a58fd59"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Label columns:  ['world', 'sports', 'business', 'sci']\n"
          ]
        }
      ],
      "source": [
        "#combined\n",
        "cols = df.columns\n",
        "label_cols = list(cols[56:60])\n",
        "num_labels = len(label_cols)\n",
        "print('Label columns: ', label_cols)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xzgA5qQgYIBZ",
        "outputId": "bff9f968-d52e-4364-c0c1-c7b4c4f42d2b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Count of 1 per label: \n",
            " world       268\n",
            "sports      266\n",
            "business    356\n",
            "sci         265\n",
            "dtype: int64 \n",
            "\n",
            "Count of 0 per label: \n",
            " world       886\n",
            "sports      888\n",
            "business    798\n",
            "sci         889\n",
            "dtype: int64\n"
          ]
        }
      ],
      "source": [
        "print('Count of 1 per label: \\n', df[label_cols].sum(), '\\n') # Label counts, may need to downsample or upsample\n",
        "print('Count of 0 per label: \\n', df[label_cols].eq(0).sum())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uFpSd4JzaAae"
      },
      "outputs": [],
      "source": [
        "#df = df.sample(frac=1).reset_index(drop=True) #shuffle rows"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.drop(659)"
      ],
      "metadata": {
        "id": "WFeiLG64uGKJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df=df.drop(1093)\n",
        "df=df.drop(1043)"
      ],
      "metadata": {
        "id": "JR6aIi5LqgZh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "id": "0DF3ddjej5vd",
        "outputId": "6177cf2a-2c99-40c2-8322-bb893797844b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                               text  \\\n",
              "0            1  Title: Pope Struggles Through Mass at Lourdes ...   \n",
              "1            1  Title: China Typhoon Death Toll Rises to 164 (...   \n",
              "2            1  Title: Wanted: Sleeping Space for Protesters. ...   \n",
              "3            1  Title: Clashes persist in South Ossetia Descri...   \n",
              "4            1  Title: U.S. Awaits Judgment on Venezuela Votin...   \n",
              "\n",
              "         label1        label2        label3 label4  label5  label6  label7  \\\n",
              "0         world         world  topic: world   TRUE  false.  false.  false.   \n",
              "1         world         world  topic: world   TRUE  false.  false.  false.   \n",
              "2  topic: world  topic: world  topic: world  true.  false.  false.  false.   \n",
              "3         world         world  topic: world  true.  false.  false.  false.   \n",
              "4         world         world  topic: world  true.  false.  false.  false.   \n",
              "\n",
              "                  label8  ... entropy_business entropy_sci entropy allocate  \\\n",
              "0                  world  ...              0.0         0.0     0.0        1   \n",
              "1                  world  ...              0.0         0.0     0.0        1   \n",
              "2  the topic is \"world\".  ...              0.0         0.0     0.0        1   \n",
              "3                  world  ...              0.0         0.0     0.0        1   \n",
              "4                  world  ...              0.0         0.0     0.0        1   \n",
              "\n",
              "  gold_label  world  sports  business  sci  one_hot_labels  \n",
              "0          1      1       0         0    0    [1, 0, 0, 0]  \n",
              "1          1      1       0         0    0    [1, 0, 0, 0]  \n",
              "2          1      1       0         0    0    [1, 0, 0, 0]  \n",
              "3          1      1       0         0    0    [1, 0, 0, 0]  \n",
              "4          1      1       0         0    0    [1, 0, 0, 0]  \n",
              "\n",
              "[5 rows x 61 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-0f7dd051-acd5-4657-8a99-e97eee32a66c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>text</th>\n",
              "      <th>label1</th>\n",
              "      <th>label2</th>\n",
              "      <th>label3</th>\n",
              "      <th>label4</th>\n",
              "      <th>label5</th>\n",
              "      <th>label6</th>\n",
              "      <th>label7</th>\n",
              "      <th>label8</th>\n",
              "      <th>...</th>\n",
              "      <th>entropy_business</th>\n",
              "      <th>entropy_sci</th>\n",
              "      <th>entropy</th>\n",
              "      <th>allocate</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Pope Struggles Through Mass at Lourdes ...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: China Typhoon Death Toll Rises to 164 (...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>TRUE</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Wanted: Sleeping Space for Protesters. ...</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>the topic is \"world\".</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: Clashes persist in South Ossetia Descri...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>Title: U.S. Awaits Judgment on Venezuela Votin...</td>\n",
              "      <td>world</td>\n",
              "      <td>world</td>\n",
              "      <td>topic: world</td>\n",
              "      <td>true.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>false.</td>\n",
              "      <td>world</td>\n",
              "      <td>...</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>[1, 0, 0, 0]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 61 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0f7dd051-acd5-4657-8a99-e97eee32a66c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-0f7dd051-acd5-4657-8a99-e97eee32a66c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-0f7dd051-acd5-4657-8a99-e97eee32a66c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 224
        }
      ],
      "source": [
        "df['one_hot_labels'] = list(df[label_cols].values)\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MlhHifh5bW7e"
      },
      "outputs": [],
      "source": [
        "labels = list(df.one_hot_labels.values)\n",
        "comments = list(df.text.values)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlMHfElhGJzc"
      },
      "source": [
        "Load the pretrained tokenizer that corresponds to your choice in model. e.g.,\n",
        "\n",
        "```\n",
        "BERT:\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "\n",
        "XLNet:\n",
        "tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=False)\n",
        "\n",
        "RoBERTa:\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False)\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhVr8SziL_PY"
      },
      "source": [
        "In order to avoid memory issues with Google Colab, I enforce a max_length of 100 tokens. Note that some sentences may not adequately represent each label because of this."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HNsEu-vUur-4",
        "outputId": "1322466e-7aa2-4250-c041-87884f871842"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tokenizer outputs:  dict_keys(['input_ids', 'attention_mask'])\n"
          ]
        }
      ],
      "source": [
        "from transformers import RobertaTokenizer\n",
        "max_length = 100\n",
        "tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=False) # tokenizer\n",
        "encodings = tokenizer.batch_encode_plus(comments,max_length=max_length,pad_to_max_length=True) # tokenizer's encoding method\n",
        "print('tokenizer outputs: ', encodings.keys())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nG9vOTp92yZE",
        "outputId": "4258abef-382f-4559-96f3-e04a9869a03d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "dict_keys(['input_ids', 'attention_mask'])"
            ]
          },
          "execution_count": 62,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "encodings.keys()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6CCLSjfur-9"
      },
      "outputs": [],
      "source": [
        "input_ids = encodings['input_ids'] # tokenized and encoded sentences\n",
        "#token_type_ids = encodings['token_type_ids'] # token type ids\n",
        "attention_masks = encodings['attention_mask'] # attention masks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vSOFbThlYcpb",
        "outputId": "bfdf101e-b3ad-43b5-ac0e-8207c2e4657b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "df label indices with only one instance:  []\n"
          ]
        }
      ],
      "source": [
        "# Identifying indices of 'one_hot_labels' entries that only occur once - this will allow us to stratify split our training data later\n",
        "label_counts = df.one_hot_labels.astype(str).value_counts()\n",
        "one_freq = label_counts[label_counts==1].keys()\n",
        "one_freq_idxs = sorted(list(df[df.one_hot_labels.astype(str).isin(one_freq)].index), reverse=True)\n",
        "print('df label indices with only one instance: ', one_freq_idxs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CQQ7CoOag_r7"
      },
      "outputs": [],
      "source": [
        "# Gathering single instance inputs to force into the training set after stratified split\n",
        "one_freq_input_ids = [input_ids.pop(i) for i in one_freq_idxs]\n",
        "#one_freq_token_types = [token_type_ids.pop(i) for i in one_freq_idxs]\n",
        "one_freq_attention_masks = [attention_masks.pop(i) for i in one_freq_idxs]\n",
        "one_freq_labels = [labels.pop(i) for i in one_freq_idxs]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r9PxAt48HRRj"
      },
      "source": [
        "Be sure to handle all classes during validation using \"stratify\" during train/validation split:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WPFaq4ufnIT2"
      },
      "outputs": [],
      "source": [
        "# Use train_test_split to split our data into train and validation sets\n",
        "\n",
        "train_inputs, validation_inputs, train_labels, validation_labels, train_masks, validation_masks = train_test_split(input_ids, labels, attention_masks,\n",
        "                                                            random_state=20, test_size=0.10, stratify = labels)\n",
        "\n",
        "# Add one frequency data to train data\n",
        "train_inputs.extend(one_freq_input_ids)\n",
        "train_labels.extend(one_freq_labels)\n",
        "train_masks.extend(one_freq_attention_masks)\n",
        "#train_token_types.extend(one_freq_token_types)\n",
        "\n",
        "# Convert all of our data into torch tensors, the required datatype for our model\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "#train_token_types = torch.tensor(train_token_types)\n",
        "\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)\n",
        "#validation_token_types = torch.tensor(validation_token_types)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZRnuLna-nIT4"
      },
      "outputs": [],
      "source": [
        "# Select a batch size for training. For fine-tuning with XLNet, the authors recommend a batch size of 32, 48, or 128. We will use 32 here to avoid memory issues.\n",
        "batch_size = 16\n",
        "\n",
        "# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop,\n",
        "# with an iterator the entire dataset does not need to be loaded into memory\n",
        "\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iiFRnP_ZTBFa"
      },
      "outputs": [],
      "source": [
        "torch.save(validation_dataloader,'validation_data_loader')\n",
        "torch.save(train_dataloader,'train_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncGteBuSFuZM"
      },
      "source": [
        "## Load Model & Set Params"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0dL-Bz_NrGj"
      },
      "source": [
        "Load the appropriate model below, each model already contains a single dense layer for classification on top.\n",
        "\n",
        "\n",
        "\n",
        "```\n",
        "BERT:\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=num_labels)\n",
        "\n",
        "XLNet:\n",
        "model = XLNetForSequenceClassification.from_pretrained(\"xlnet-base-cased\", num_labels=num_labels)\n",
        "\n",
        "RoBERTa:\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ujk4k16DnIT6",
        "outputId": "1b1de86c-3182-4017-9699-3e034092c1e6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.out_proj.weight', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=4, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 233
        }
      ],
      "source": [
        "# Load model, the pretrained model will include a single linear classification layer on top for classification.\n",
        "from transformers import RobertaForSequenceClassification\n",
        "model = RobertaForSequenceClassification.from_pretrained('roberta-base', num_labels=num_labels)\n",
        "model.cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jGE4gv9qfhRG"
      },
      "source": [
        "Setting custom optimization parameters for the AdamW optimizer https://huggingface.co/transformers/main_classes/optimizer_schedules.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GsV8zwWYnIT9"
      },
      "outputs": [],
      "source": [
        "# setting custom optimization parameters. You may implement a scheduler here as well.\n",
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "     'weight_decay_rate': 0.0}\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aOomZIEIoHOL",
        "outputId": "430860eb-85b8-4e66-d5d3-19a0343935ff"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "from transformers import AdamW\n",
        "optimizer = AdamW(optimizer_grouped_parameters,lr=2e-5 ,correct_bias=True)\n",
        "# optimizer = AdamW(model.parameters(),lr=2e-5)  # Default optimization"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JRQQZ8zIFzLW"
      },
      "source": [
        "## Train Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ho6xx8cQfhvK"
      },
      "outputs": [],
      "source": [
        "def is_max(array):\n",
        "  max_val=max(array)\n",
        "  res=[i==max_val for i in array]\n",
        "  return res"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V2x2N2D3iVAD",
        "outputId": "d03e3afc-1685-47f6-e291-328ef66f0d23"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/10 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.4049999876664235\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  10%|█         | 1/10 [00:17<02:39, 17.69s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  92.24137931034483\n",
            "Flat Validation Accuracy:  92.24137931034483\n",
            "Train loss: 0.12892476784495208\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  20%|██        | 2/10 [00:35<02:23, 17.95s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.06896551724138\n",
            "Flat Validation Accuracy:  87.06896551724138\n",
            "Train loss: 0.08471723012626171\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  30%|███       | 3/10 [00:54<02:08, 18.29s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.93103448275863\n",
            "Flat Validation Accuracy:  87.93103448275862\n",
            "Train loss: 0.05749459808262495\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  40%|████      | 4/10 [01:13<01:51, 18.56s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  85.34482758620689\n",
            "Flat Validation Accuracy:  85.34482758620689\n",
            "Train loss: 0.045284357924874015\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  50%|█████     | 5/10 [01:32<01:33, 18.62s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  86.20689655172413\n",
            "Flat Validation Accuracy:  86.20689655172413\n",
            "Train loss: 0.03986267308489634\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  60%|██████    | 6/10 [01:50<01:14, 18.60s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  88.79310344827587\n",
            "Flat Validation Accuracy:  88.79310344827587\n",
            "Train loss: 0.033958851173520085\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  70%|███████   | 7/10 [02:09<00:55, 18.59s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.06896551724138\n",
            "Flat Validation Accuracy:  87.06896551724138\n",
            "Train loss: 0.02452939413487911\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  80%|████████  | 8/10 [02:28<00:37, 18.63s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  89.65517241379311\n",
            "Flat Validation Accuracy:  89.65517241379311\n",
            "Train loss: 0.0302388100240093\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  90%|█████████ | 9/10 [02:46<00:18, 18.67s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.93103448275863\n",
            "Flat Validation Accuracy:  87.93103448275862\n",
            "Train loss: 0.022198500957053443\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 10/10 [03:05<00:00, 18.55s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.06896551724138\n",
            "Flat Validation Accuracy:  87.06896551724138\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "# Store our loss and accuracy for plotting - 0\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 10\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5kXzCwJz0Whl",
        "outputId": "b1f675e1-6ebd-49ae-cff0-7c99e014a9a7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:   0%|          | 0/5 [00:00<?, ?it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train loss: 0.015534048477331032\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  20%|██        | 1/5 [00:18<01:15, 18.83s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.93103448275863\n",
            "Flat Validation Accuracy:  87.93103448275862\n",
            "Train loss: 0.020301148610619397\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  40%|████      | 2/5 [00:38<00:57, 19.16s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  86.20689655172413\n",
            "Flat Validation Accuracy:  86.20689655172413\n",
            "Train loss: 0.01689189364417241\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  60%|██████    | 3/5 [00:56<00:37, 18.96s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  86.20689655172413\n",
            "Flat Validation Accuracy:  86.20689655172413\n",
            "Train loss: 0.01650217930571391\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rEpoch:  80%|████████  | 4/5 [01:15<00:18, 18.75s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.93103448275863\n",
            "Flat Validation Accuracy:  87.93103448275862\n",
            "Train loss: 0.016973170107946946\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 5/5 [01:33<00:00, 18.77s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "F1 Validation Accuracy:  87.93103448275863\n",
            "Flat Validation Accuracy:  87.93103448275862\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "# Store our loss and accuracy for plotting - 0.2\n",
        "train_loss_set = []\n",
        "\n",
        "# Number of training epochs (authors recommend between 2 and 4)\n",
        "epochs = 5\n",
        "\n",
        "# trange is a tqdm wrapper around the normal python range\n",
        "for _ in trange(epochs, desc=\"Epoch\"):\n",
        "\n",
        "  # Training\n",
        "\n",
        "  # Set our model to training mode (as opposed to evaluation mode)\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0 #running loss\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  # Train the data for one epoch\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    # Add batch to GPU\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    # Clear out the gradients (by default they accumulate)\n",
        "    optimizer.zero_grad()\n",
        "\n",
        "    # # Forward pass for multiclass classification\n",
        "    # outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
        "    # loss = outputs[0]\n",
        "    # logits = outputs[1]\n",
        "\n",
        "    # Forward pass for multilabel classification\n",
        "    outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    logits = outputs[0]\n",
        "    loss_func = BCEWithLogitsLoss()\n",
        "    loss = loss_func(logits.view(-1,num_labels),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    # loss_func = BCELoss()\n",
        "    # loss = loss_func(torch.sigmoid(logits.view(-1,num_labels)),b_labels.type_as(logits).view(-1,num_labels)) #convert labels to float for calculation\n",
        "    train_loss_set.append(loss.item())\n",
        "\n",
        "    # Backward pass\n",
        "    loss.backward()\n",
        "    # Update parameters and take a step using the computed gradient\n",
        "    optimizer.step()\n",
        "    # scheduler.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n",
        "\n",
        "###############################################################################\n",
        "\n",
        "  # Validation\n",
        "\n",
        "  # Put model in evaluation mode to evaluate loss on the validation set\n",
        "  model.eval()\n",
        "\n",
        "  # Variables to gather full output\n",
        "  logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "  # Predict\n",
        "  for i, batch in enumerate(validation_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    # Unpack the inputs from our dataloader\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "      b_logit_pred = outs[0]\n",
        "      pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "      b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "      pred_label = pred_label.to('cpu').numpy()\n",
        "      b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "    tokenized_texts.append(b_input_ids)\n",
        "    logit_preds.append(b_logit_pred)\n",
        "    true_labels.append(b_labels)\n",
        "    pred_labels.append(pred_label)\n",
        "\n",
        "  # Flatten outputs\n",
        "  pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "  true_labels = [item for sublist in true_labels for item in sublist]\n",
        "\n",
        "  # Calculate Accuracy\n",
        "  #threshold = 0.30\n",
        "  #pred_bools = [pl>threshold for pl in pred_labels]\n",
        "  #print(pred_labels)\n",
        "  #max_val = max(pred_labels)\n",
        "  pred_bools = [is_max(arr) for arr in pred_labels]\n",
        "  true_bools = [tl==1 for tl in true_labels]\n",
        "  val_f1_accuracy = f1_score(true_bools,pred_bools,average='micro')*100\n",
        "  val_flat_accuracy = accuracy_score(true_bools, pred_bools)*100\n",
        "\n",
        "  print('F1 Validation Accuracy: ', val_f1_accuracy)\n",
        "  print('Flat Validation Accuracy: ', val_flat_accuracy)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_7dd2GE3F4yK"
      },
      "source": [
        "## Load and Preprocess Test Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        },
        "id": "G5Q7hC4GFOLJ",
        "outputId": "5ea03f97-6071-4ace-927a-47d81e50764b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     Class Index                                         Title  \\\n",
              "495            2                               Pressure points   \n",
              "496            3  Unions protest as overtime rules take effect   \n",
              "497            1              Serb denies siege terror charges   \n",
              "498            2                 11th-hour highlights too late   \n",
              "499            4                             No IE? No Can See   \n",
              "\n",
              "                                           Description  \\\n",
              "495  ATHENS -- The booing went on for nearly 10 min...   \n",
              "496  WASHINGTON -- Hundreds of workers rallied on t...   \n",
              "497  A Bosnian Serb general accused of organising t...   \n",
              "498  NBC's prime-time Olympic coverage is taped and...   \n",
              "499  One thing that #39;s always irritated those wh...   \n",
              "\n",
              "                                                  text  gold_label  world  \\\n",
              "495  Title: Pressure points Description: ATHENS -- ...           2      0   \n",
              "496  Title: Unions protest as overtime rules take e...           3      0   \n",
              "497  Title: Serb denies siege terror charges Descri...           1      1   \n",
              "498  Title: 11th-hour highlights too late Descripti...           2      0   \n",
              "499  Title: No IE? No Can See Description: One thin...           4      0   \n",
              "\n",
              "     sports  business  sci  \n",
              "495       1         0    0  \n",
              "496       0         1    0  \n",
              "497       0         0    0  \n",
              "498       1         0    0  \n",
              "499       0         0    1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-482611ef-c3dc-4dd1-a409-4c664193f420\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>text</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>495</th>\n",
              "      <td>2</td>\n",
              "      <td>Pressure points</td>\n",
              "      <td>ATHENS -- The booing went on for nearly 10 min...</td>\n",
              "      <td>Title: Pressure points Description: ATHENS -- ...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>496</th>\n",
              "      <td>3</td>\n",
              "      <td>Unions protest as overtime rules take effect</td>\n",
              "      <td>WASHINGTON -- Hundreds of workers rallied on t...</td>\n",
              "      <td>Title: Unions protest as overtime rules take e...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>497</th>\n",
              "      <td>1</td>\n",
              "      <td>Serb denies siege terror charges</td>\n",
              "      <td>A Bosnian Serb general accused of organising t...</td>\n",
              "      <td>Title: Serb denies siege terror charges Descri...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>498</th>\n",
              "      <td>2</td>\n",
              "      <td>11th-hour highlights too late</td>\n",
              "      <td>NBC's prime-time Olympic coverage is taped and...</td>\n",
              "      <td>Title: 11th-hour highlights too late Descripti...</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>499</th>\n",
              "      <td>4</td>\n",
              "      <td>No IE? No Can See</td>\n",
              "      <td>One thing that #39;s always irritated those wh...</td>\n",
              "      <td>Title: No IE? No Can See Description: One thin...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-482611ef-c3dc-4dd1-a409-4c664193f420')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-482611ef-c3dc-4dd1-a409-4c664193f420 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-482611ef-c3dc-4dd1-a409-4c664193f420');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "test_df = pd.read_csv('agnews_test.csv')\n",
        "#test_labels_df = pd.read_csv('test_labels.csv')\n",
        "#test_df = test_df.merge(test_labels_df, on='id', how='left')\n",
        "# test_df['text']=test_df['sentence']\n",
        "test_df['world']=test_df['gold_label'].apply(lambda x:1 if x==1 else 0)\n",
        "test_df['sports']=test_df['gold_label'].apply(lambda x:1 if x==2 else 0)\n",
        "test_df['business']=test_df['gold_label'].apply(lambda x:1 if x==3 else 0)\n",
        "test_df['sci']=test_df['gold_label'].apply(lambda x:1 if x==4 else 0)\n",
        "#print('Null values: ', test_df.isnull().values.any()) #should not be any null sentences or labels\n",
        "#print('Same columns between train and test: ', label_cols == test_label_cols) #columns should be the same\n",
        "test_df.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j_PZ1mVOhkK8",
        "outputId": "55960b34-41e2-4a36-adf9-7b6a1c89b078"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['world', 'sports', 'business', 'sci']"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "test_label_cols = list(test_df.columns[5:])\n",
        "test_label_cols"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441
        },
        "id": "77rjCrMGpYxz",
        "outputId": "f748524c-a267-485e-8494-03fab60ce4bf"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Class Index                                              Title  \\\n",
              "0            3                  Fears for T N pension after talks   \n",
              "1            4  The Race is On: Second Private Team Sets Launc...   \n",
              "2            4      Ky. Company Wins Grant to Study Peptides (AP)   \n",
              "3            4      Prediction Unit Helps Forecast Wildfires (AP)   \n",
              "4            4        Calif. Aims to Limit Farm-Related Smog (AP)   \n",
              "\n",
              "                                         Description  \\\n",
              "0  Unions representing workers at Turner   Newall...   \n",
              "1  SPACE.com - TORONTO, Canada -- A second\\team o...   \n",
              "2  AP - A company founded by a chemistry research...   \n",
              "3  AP - It's barely dawn when Mike Fitzpatrick st...   \n",
              "4  AP - Southern California's smog-fighting agenc...   \n",
              "\n",
              "                                                text  gold_label  world  \\\n",
              "0  Title: Fears for T N pension after talks Descr...           3      0   \n",
              "1  Title: The Race is On: Second Private Team Set...           4      0   \n",
              "2  Title: Ky. Company Wins Grant to Study Peptide...           4      0   \n",
              "3  Title: Prediction Unit Helps Forecast Wildfire...           4      0   \n",
              "4  Title: Calif. Aims to Limit Farm-Related Smog ...           4      0   \n",
              "\n",
              "   sports  business  sci one_hot_labels  \n",
              "0       0         1    0   [0, 0, 1, 0]  \n",
              "1       0         0    1   [0, 0, 0, 1]  \n",
              "2       0         0    1   [0, 0, 0, 1]  \n",
              "3       0         0    1   [0, 0, 0, 1]  \n",
              "4       0         0    1   [0, 0, 0, 1]  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8bcfec6e-e29d-4b7e-84b6-82950aa94fa5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class Index</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>text</th>\n",
              "      <th>gold_label</th>\n",
              "      <th>world</th>\n",
              "      <th>sports</th>\n",
              "      <th>business</th>\n",
              "      <th>sci</th>\n",
              "      <th>one_hot_labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Fears for T N pension after talks</td>\n",
              "      <td>Unions representing workers at Turner   Newall...</td>\n",
              "      <td>Title: Fears for T N pension after talks Descr...</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>[0, 0, 1, 0]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>4</td>\n",
              "      <td>The Race is On: Second Private Team Sets Launc...</td>\n",
              "      <td>SPACE.com - TORONTO, Canada -- A second\\team o...</td>\n",
              "      <td>Title: The Race is On: Second Private Team Set...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>Ky. Company Wins Grant to Study Peptides (AP)</td>\n",
              "      <td>AP - A company founded by a chemistry research...</td>\n",
              "      <td>Title: Ky. Company Wins Grant to Study Peptide...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>Prediction Unit Helps Forecast Wildfires (AP)</td>\n",
              "      <td>AP - It's barely dawn when Mike Fitzpatrick st...</td>\n",
              "      <td>Title: Prediction Unit Helps Forecast Wildfire...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>Calif. Aims to Limit Farm-Related Smog (AP)</td>\n",
              "      <td>AP - Southern California's smog-fighting agenc...</td>\n",
              "      <td>Title: Calif. Aims to Limit Farm-Related Smog ...</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>[0, 0, 0, 1]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8bcfec6e-e29d-4b7e-84b6-82950aa94fa5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8bcfec6e-e29d-4b7e-84b6-82950aa94fa5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8bcfec6e-e29d-4b7e-84b6-82950aa94fa5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "test_df = test_df[~test_df[test_label_cols].eq(-1).any(axis=1)] #remove irrelevant rows/comments with -1 values\n",
        "test_df['one_hot_labels'] = list(test_df[test_label_cols].values)\n",
        "test_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1a41OmU2i7qp"
      },
      "outputs": [],
      "source": [
        "# Gathering input data\n",
        "test_labels = list(test_df.one_hot_labels.values)\n",
        "test_comments = list(test_df.text.values)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "amySMO8EQzf2",
        "outputId": "cf9e4009-0e3b-404f-dc4a-c3387f335128"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "# Encoding input data\n",
        "test_encodings = tokenizer.batch_encode_plus(test_comments,max_length=max_length,pad_to_max_length=True)\n",
        "test_input_ids = test_encodings['input_ids']\n",
        "#test_token_type_ids = test_encodings['token_type_ids']\n",
        "test_attention_masks = test_encodings['attention_mask']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqOfi9fkRaRN"
      },
      "outputs": [],
      "source": [
        "# Make tensors out of data\n",
        "test_inputs = torch.tensor(test_input_ids)\n",
        "test_labels = torch.tensor(test_labels)\n",
        "test_masks = torch.tensor(test_attention_masks)\n",
        "#test_token_types = torch.tensor(test_token_type_ids)\n",
        "# Create test dataloader\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "#test_data = TensorDataset(test_inputs, test_masks)\n",
        "test_sampler = SequentialSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)\n",
        "# Save test dataloader\n",
        "torch.save(test_dataloader,'test_data_loader')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PFTWxCA_GBau"
      },
      "source": [
        "## Prediction and Metics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NPvrL6OFSQvf"
      },
      "outputs": [],
      "source": [
        "# Test\n",
        "\n",
        "# Put model in evaluation mode to evaluate loss on the validation set\n",
        "model.eval()\n",
        "\n",
        "#track variables\n",
        "logit_preds,true_labels,pred_labels,tokenized_texts = [],[],[],[]\n",
        "\n",
        "# Predict\n",
        "for i, batch in enumerate(test_dataloader):\n",
        "  batch = tuple(t.to(device) for t in batch)\n",
        "  # Unpack the inputs from our dataloader\n",
        "  b_input_ids, b_input_mask, b_labels = batch\n",
        "  #b_input_ids, b_input_mask = batch\n",
        "  with torch.no_grad():\n",
        "    # Forward pass\n",
        "    outs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
        "    b_logit_pred = outs[0]\n",
        "    pred_label = torch.sigmoid(b_logit_pred)\n",
        "\n",
        "    b_logit_pred = b_logit_pred.detach().cpu().numpy()\n",
        "    pred_label = pred_label.to('cpu').numpy()\n",
        "    b_labels = b_labels.to('cpu').numpy()\n",
        "\n",
        "  tokenized_texts.append(b_input_ids)\n",
        "  logit_preds.append(b_logit_pred)\n",
        "  true_labels.append(b_labels)\n",
        "  pred_labels.append(pred_label)\n",
        "\n",
        "# Flatten outputs\n",
        "tokenized_texts = [item for sublist in tokenized_texts for item in sublist]\n",
        "pred_labels = [item for sublist in pred_labels for item in sublist]\n",
        "true_labels = [item for sublist in true_labels for item in sublist]\n",
        "# Converting flattened binary values to boolean values\n",
        "true_bools = [tl==1 for tl in true_labels]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_INi5MSJtEkQ",
        "outputId": "70a67240-c004-43e8-f563-3a769c7df70e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[array([0.0045191 , 0.12284625, 0.78677535], dtype=float32),\n",
              " array([0.9635983 , 0.04758941, 0.00145089], dtype=float32),\n",
              " array([0.00207093, 0.01049966, 0.99107623], dtype=float32),\n",
              " array([0.01098563, 0.99250317, 0.00175095], dtype=float32),\n",
              " array([0.9887567 , 0.01733083, 0.00118418], dtype=float32),\n",
              " array([0.00506789, 0.00214649, 0.997235  ], dtype=float32),\n",
              " array([0.9978714 , 0.00287712, 0.00215554], dtype=float32),\n",
              " array([0.00341052, 0.996572  , 0.00284842], dtype=float32),\n",
              " array([0.9862281 , 0.00255858, 0.01361227], dtype=float32),\n",
              " array([0.10106141, 0.09945446, 0.10647077], dtype=float32)]"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "pred_labels[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dCiP1ZmTiewr",
        "outputId": "625ff3f1-db93-4b5a-b8ea-e3330f1e3e55"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[False, False, True]"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "is_max(pred_labels[0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GBxAD5B2ikLb",
        "outputId": "943787d7-47ee-4539-d427-b7833bce4250"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "10000"
            ]
          },
          "execution_count": 69,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(true_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bQeGWqeMzAoZ"
      },
      "source": [
        "We need to threshold our sigmoid function outputs which range from [0, 1]. Below I use 0.50 as a threshold."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cPRT6OFfDdiu",
        "outputId": "320c19a6-6e3f-4354-e52c-8c77f73f9ac3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Test F1 Accuracy:  0.882\n",
            "Test Flat Accuracy:  0.882 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.81      0.93      0.86       122\n",
            "      sports       0.97      0.92      0.94       145\n",
            "    business       0.83      0.82      0.82       106\n",
            "         sci       0.92      0.85      0.88       127\n",
            "\n",
            "   micro avg       0.88      0.88      0.88       500\n",
            "   macro avg       0.88      0.88      0.88       500\n",
            "weighted avg       0.89      0.88      0.88       500\n",
            " samples avg       0.88      0.88      0.88       500\n",
            "\n"
          ]
        }
      ],
      "source": [
        "#combined - 0\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.2\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u1CuYXk0npJc",
        "outputId": "7536d98b-e117-4502-e054-a36d274222d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8838277474132114\n",
            "Micro Test F1 Accuracy:  0.89\n",
            "Weighted Test F1 Accuracy:  0.8899179032409382\n",
            "Test Flat Accuracy:  0.89 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.87      0.89      0.88       122\n",
            "      sports       0.97      0.95      0.96       145\n",
            "    business       0.82      0.79      0.80       106\n",
            "         sci       0.89      0.90      0.89       127\n",
            "\n",
            "   micro avg       0.89      0.89      0.89       500\n",
            "   macro avg       0.88      0.88      0.88       500\n",
            "weighted avg       0.89      0.89      0.89       500\n",
            " samples avg       0.89      0.89      0.89       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.4\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lioyGO1fpsX1",
        "outputId": "3bcaf37e-2126-49b2-9ebf-9aab189298d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8815924915510439\n",
            "Micro Test F1 Accuracy:  0.886\n",
            "Weighted Test F1 Accuracy:  0.886853086237362\n",
            "Test Flat Accuracy:  0.886 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.92      0.85      0.89       122\n",
            "      sports       0.95      0.97      0.96       145\n",
            "    business       0.75      0.90      0.82       106\n",
            "         sci       0.92      0.82      0.87       127\n",
            "\n",
            "   micro avg       0.89      0.89      0.89       500\n",
            "   macro avg       0.89      0.88      0.88       500\n",
            "weighted avg       0.89      0.89      0.89       500\n",
            " samples avg       0.89      0.89      0.89       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.6\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r5dNKqr3qpej",
        "outputId": "988e6e5b-e17e-4102-8aa0-7d43313748fa"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8742179386308181\n",
            "Micro Test F1 Accuracy:  0.88\n",
            "Weighted Test F1 Accuracy:  0.8805429776426997\n",
            "Test Flat Accuracy:  0.88 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.85      0.87      0.86       122\n",
            "      sports       0.95      0.97      0.96       145\n",
            "    business       0.77      0.83      0.80       106\n",
            "         sci       0.93      0.83      0.88       127\n",
            "\n",
            "   micro avg       0.88      0.88      0.88       500\n",
            "   macro avg       0.88      0.87      0.87       500\n",
            "weighted avg       0.88      0.88      0.88       500\n",
            " samples avg       0.88      0.88      0.88       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 0.8\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xra6Ukijwr7z",
        "outputId": "ff658510-193c-46bb-d8a9-65e31c06237b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8398007293268176\n",
            "Micro Test F1 Accuracy:  0.848\n",
            "Weighted Test F1 Accuracy:  0.8472770593659139\n",
            "Test Flat Accuracy:  0.848 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.83      0.84      0.84       122\n",
            "      sports       0.94      0.99      0.96       145\n",
            "    business       0.69      0.86      0.77       106\n",
            "         sci       0.94      0.69      0.79       127\n",
            "\n",
            "   micro avg       0.85      0.85      0.85       500\n",
            "   macro avg       0.85      0.84      0.84       500\n",
            "weighted avg       0.86      0.85      0.85       500\n",
            " samples avg       0.85      0.85      0.85       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#combined - 1\n",
        "pred_bools = [is_max(pl) for pl in pred_labels] #boolean output after thresholding\n",
        "\n",
        "# Print and save classification report\n",
        "print('Macro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='macro'))\n",
        "print('Micro Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='micro'))\n",
        "print('Weighted Test F1 Accuracy: ', f1_score(true_bools, pred_bools,average='weighted'))\n",
        "print('Test Flat Accuracy: ', accuracy_score(true_bools, pred_bools),'\\n')\n",
        "clf_report = classification_report(true_bools,pred_bools,target_names=test_label_cols)\n",
        "pickle.dump(clf_report, open('classification_report.txt','wb')) #save report\n",
        "print(clf_report)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cEr5HmwbsvqK",
        "outputId": "79a299ed-f788-43c1-e3e7-89b0ba7e76d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Macro Test F1 Accuracy:  0.8265653472085007\n",
            "Micro Test F1 Accuracy:  0.836\n",
            "Weighted Test F1 Accuracy:  0.8343256624733804\n",
            "Test Flat Accuracy:  0.836 \n",
            "\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "       world       0.86      0.84      0.85       122\n",
            "      sports       0.94      0.99      0.96       145\n",
            "    business       0.65      0.90      0.75       106\n",
            "         sci       0.94      0.61      0.74       127\n",
            "\n",
            "   micro avg       0.84      0.84      0.84       500\n",
            "   macro avg       0.85      0.83      0.83       500\n",
            "weighted avg       0.86      0.84      0.83       500\n",
            " samples avg       0.84      0.84      0.84       500\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Significance Testing"
      ],
      "metadata": {
        "id": "Ukk1t-ifhZVj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "size = 500\n",
        "proportion_of_ones = 0.847\n",
        "\n",
        "# Calculate the number of 1s based on the desired proportion\n",
        "num_ones = int(size * proportion_of_ones)\n",
        "\n",
        "# Create the list with num_ones 1s and (size - num_ones) 0s\n",
        "my_list = random.choices([0, 1], k=num_ones) + [0] * (size - num_ones)\n",
        "\n",
        "# Shuffle the list to randomize the order\n",
        "random.shuffle(my_list)\n",
        "\n",
        "import statistics\n",
        "\n",
        "\n",
        "std_dev = statistics.stdev(my_list)\n",
        "\n",
        "print(std_dev)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Og6WQKh2Uj-8",
        "outputId": "9b8b9111-0d71-46ca-af9c-26d8b3aa9bc4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.49873185875164194\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "\n",
        "size = 500\n",
        "proportion_of_ones = 0.818\n",
        "\n",
        "# Calculate the number of 1s based on the desired proportion\n",
        "num_ones = int(size * proportion_of_ones)\n",
        "\n",
        "# Create the list with num_ones 1s and (size - num_ones) 0s\n",
        "my_list = random.choices([0, 1], k=num_ones) + [0] * (size - num_ones)\n",
        "\n",
        "# Shuffle the list to randomize the order\n",
        "random.shuffle(my_list)\n",
        "\n",
        "import statistics\n",
        "\n",
        "\n",
        "std_dev = statistics.stdev(my_list)\n",
        "\n",
        "print(std_dev)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GO1ioys4V2Cx",
        "outputId": "a0bbf988-8257-4851-f0ff-780922722510"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.48823841871263474\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#0.2\n",
        "import scipy.stats as stats\n",
        "\n",
        "t_statistic, p_value = stats.ttest_ind_from_stats(\n",
        "    0.8838277474132114, 0.49662726305543786, 500, 0.8793896106883861, 0.49638371607776816, 500, alternative='greater'\n",
        ")\n",
        "\n",
        "if p_value < 0.1:\n",
        "    print(\"The mean of Group A is significantly higher than Group B.\")\n",
        "else:\n",
        "    print(\"There is no significant difference between the means of Group A and Group B.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "12rEw7AvW8J7",
        "outputId": "9f86fb6f-d203-4f67-973a-102cc897829a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There is no significant difference between the means of Group A and Group B.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#0.4\n",
        "import scipy.stats as stats\n",
        "\n",
        "t_statistic, p_value = stats.ttest_ind_from_stats(\n",
        "    0.8815924915510439, 0.4949889171043249, 500, 0.8583741668153195, 0.48868561187588594, 500, alternative='greater'\n",
        ")\n",
        "\n",
        "if p_value < 0.1:\n",
        "    print(\"The mean of Group A is significantly higher than Group B.\")\n",
        "else:\n",
        "    print(\"There is no significant difference between the means of Group A and Group B.\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lDGjX0VEVmgB",
        "outputId": "0a192bc3-27fb-4717-e9a1-9213fd4cc1c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There is no significant difference between the means of Group A and Group B.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#0.6\n",
        "import scipy.stats as stats\n",
        "\n",
        "t_statistic, p_value = stats.ttest_ind_from_stats(\n",
        "    0.8742179386308181, 0.49157633648211585, 500, 0.7980504439471048, 0.49038858257900714, 500, alternative='greater'\n",
        ")\n",
        "\n",
        "if p_value < 0.05:\n",
        "    print(\"The mean of Group A is significantly higher than Group B.\")\n",
        "else:\n",
        "    print(\"There is no significant difference between the means of Group A and Group B.\")\n",
        "p_value"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1SGBW_iwZAmR",
        "outputId": "5334acaa-0a92-4107-8405-7e979dbd0b81"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The mean of Group A is significantly higher than Group B.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.007171316416932223"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#0.8\n",
        "import scipy.stats as stats\n",
        "\n",
        "t_statistic, p_value = stats.ttest_ind_from_stats(\n",
        "    0.847, 0.49873185875164194, 500, 0.818, 0.48823841871263474, 500, alternative='greater'\n",
        ")\n",
        "\n",
        "if p_value < 0.05:\n",
        "    print(\"The mean of Group A is significantly higher than Group B.\")\n",
        "else:\n",
        "    print(\"There is no significant difference between the means of Group A and Group B.\")\n",
        "p_value"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VD5ZkigxpAcn",
        "outputId": "61fbe75a-5819-499e-d887-a35f9356dc04"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There is no significant difference between the means of Group A and Group B.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.1765271293296073"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}