term_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fundamental-geneva",
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataset import get_dataset_by_name\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "import utils\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from train_bert import compute_negative_entropy, LMForSequenceClassification\n",
    "from collections import defaultdict\n",
    "from typing import Dict\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "sns.set_theme(\"notebook\")\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "\n",
    "from string import punctuation\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "swedish-harvest",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tokens_entropy(model, tokenizer, dataset, device=\"cpu\", join=True, batch_size=32):\n",
    "    loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)\n",
    "    entropy_scores = defaultdict(list)\n",
    "    word_occ = defaultdict(int)\n",
    "    fps = defaultdict(int)\n",
    "    fns = defaultdict(int)\n",
    "    \n",
    "    entropy_fps = defaultdict(list)\n",
    "    entropy_fns = defaultdict(list)\n",
    "    \n",
    "    num_positives = defaultdict(int)\n",
    "    num_negatives = defaultdict(int)\n",
    "        \n",
    "    with torch.no_grad():\n",
    "        for idx, batch in tqdm(enumerate(loader), total=len(loader)):\n",
    "            encoding = tokenizer(\n",
    "                batch[\"text\"],\n",
    "                add_special_tokens=True,\n",
    "                padding=True,\n",
    "                truncation=True,\n",
    "                max_length=120,\n",
    "                return_tensors=\"pt\",\n",
    "            ).to(device)\n",
    "            \n",
    "            output = model(**encoding, output_attentions=True)\n",
    "            y_preds = output[\"logits\"].argmax(-1)\n",
    "            y_trues = batch[\"label\"]\n",
    "            \n",
    "            neg_entropy, entropies = compute_negative_entropy(\n",
    "                output[\"attentions\"], encoding[\"attention_mask\"], return_values=True\n",
    "            )\n",
    "                        \n",
    "            # process each batch\n",
    "            for i_batch in range(y_preds.shape[0]):\n",
    "                y_pred = y_preds[i_batch]\n",
    "                y_true = y_trues[i_batch]\n",
    "                curr_e = -entropies[i_batch]\n",
    "                curr_e = torch.flipud(curr_e)\n",
    "\n",
    "                input_ids = encoding[\"input_ids\"][i_batch]\n",
    "                input_ids = input_ids[input_ids != 0]\n",
    "                tokens = tokenizer.convert_ids_to_tokens(input_ids)\n",
    "\n",
    "                # if remove_special:\n",
    "                #     tokens = tokens[1:-1]\n",
    "\n",
    "                if join:\n",
    "                    # join subwords for better visualization\n",
    "                    new_tokens, pop_idxs, spans = utils.join_subwords(tokens)\n",
    "                    #  print(\"Len new tokens\", len(new_tokens))\n",
    "                    tokens = new_tokens\n",
    "                    \n",
    "                # average subwords\n",
    "                if join and len(spans) > 0:\n",
    "                    curr_e = utils.average_2d_over_spans(curr_e, spans)\n",
    "                    \n",
    "                curr_e = curr_e.mean(0).unsqueeze(0)    \n",
    "                assert curr_e.shape[1] == len(tokens)\n",
    "\n",
    "                for i, t in enumerate(tokens):\n",
    "                    entr = curr_e[0, i].cpu().item()\n",
    "                    entropy_scores[t].append(entr)\n",
    "                    \n",
    "                    word_occ[t] += 1\n",
    "                    if y_true == 1:\n",
    "                        num_positives[t] += 1\n",
    "                        num_negatives[t] += 0\n",
    "                    else:\n",
    "                        num_negatives[t] += 1\n",
    "                        num_positives[t] += 0\n",
    "                    \n",
    "                    # false positives\n",
    "                    if y_true == 0 and y_pred == 1:\n",
    "                        fps[t] += 1\n",
    "                        fns[t] += 0\n",
    "                        entropy_fps[t].append(entr)\n",
    "                        \n",
    "                    # false negatives\n",
    "                    elif y_true == 1 and y_pred == 0:\n",
    "                        fns[t] += 1\n",
    "                        fps[t] += 0\n",
    "                        entropy_fns[t].append(entr)\n",
    "                        \n",
    "                    else:\n",
    "                        fns[t] += 0\n",
    "                        fps[t] += 0\n",
    "\n",
    "        # return the average\n",
    "        entropy_scores = {k: np.mean(v) for k, v in entropy_scores.items()}\n",
    "        entropy_fps = {k: np.mean(v) for k, v in entropy_fps.items()}\n",
    "        entropy_fns = {k: np.mean(v) for k, v in entropy_fns.items()}\n",
    "        return entropy_scores, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives\n",
    "\n",
    "\n",
    "def filter_stats(stats):\n",
    "    len_m = stats[\"token\"].apply(len) > 3\n",
    "    count_min = stats[\"count\"] > 10\n",
    "    count_max = stats[\"count\"] < 3600\n",
    "    punct = stats[\"token\"].isin(list(punctuation))\n",
    "    \n",
    "    return stats.loc[\n",
    "        len_m &\n",
    "        count_min &\n",
    "        count_max &\n",
    "        ~punct\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "detected-africa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "    \n",
    "def twitter_tokenizer(doc):    \n",
    "    tokens = TweetTokenizer().tokenize(doc)\n",
    "    \n",
    "    tokens_new = list()\n",
    "    for t in tokens:\n",
    "        if t.startswith(\"@\") and len(t) > 1:\n",
    "            tokens_new.append(\"USER\")\n",
    "        \n",
    "        elif len(t) < 3:\n",
    "            continue\n",
    "        \n",
    "        else:\n",
    "            tokens_new.append(t)\n",
    "            \n",
    "    return tokens_new\n",
    "\n",
    "\n",
    "def preprocess_collection(documents, min_df=0.05, max_df=0.95):\n",
    "    cv = CountVectorizer(min_df=min_df, max_df=max_df, tokenizer=twitter_tokenizer)\n",
    "    new_docs = cv.fit_transform(documents)\n",
    "    new_docs = cv.inverse_transform(new_docs)\n",
    "    new_corpus = [\" \".join(doc) for doc in new_docs]\n",
    "    return cv, new_corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "interesting-elements",
   "metadata": {},
   "source": [
    "# Misogyny (EN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "confident-observation",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    \"BERT-0/\"\n",
    ").to(device)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "train, dev, test = get_dataset_by_name(\"miso\")\n",
    "cv, docs = preprocess_collection(train.get_texts(), 0.01, 0.95)\n",
    "train.texts = docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "pressed-lesbian",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 113/113 [00:05<00:00, 20.20it/s]\n"
     ]
    }
   ],
   "source": [
    "entropy_dict, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives = get_tokens_entropy(model, tokenizer, train, device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "exterior-lending",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entropy</th>\n",
       "      <th>entropy_fps</th>\n",
       "      <th>entropy_fns</th>\n",
       "      <th>count</th>\n",
       "      <th>fps</th>\n",
       "      <th>fns</th>\n",
       "      <th>num_pos</th>\n",
       "      <th>num_neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>[CLS]</th>\n",
       "      <td>2.237726</td>\n",
       "      <td>2.271439</td>\n",
       "      <td>2.230080</td>\n",
       "      <td>3600.0</td>\n",
       "      <td>614.0</td>\n",
       "      <td>256.0</td>\n",
       "      <td>1606.0</td>\n",
       "      <td>1994.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user</th>\n",
       "      <td>2.250743</td>\n",
       "      <td>2.279840</td>\n",
       "      <td>2.262625</td>\n",
       "      <td>1235.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>549.0</td>\n",
       "      <td>686.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>every</th>\n",
       "      <td>2.508682</td>\n",
       "      <td>2.643061</td>\n",
       "      <td>2.153221</td>\n",
       "      <td>46.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <td>2.533932</td>\n",
       "      <td>2.596356</td>\n",
       "      <td>2.377192</td>\n",
       "      <td>82.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>46.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>there</th>\n",
       "      <td>2.430238</td>\n",
       "      <td>2.100990</td>\n",
       "      <td>2.621234</td>\n",
       "      <td>56.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fuckin</th>\n",
       "      <td>2.520665</td>\n",
       "      <td>2.444635</td>\n",
       "      <td>2.699345</td>\n",
       "      <td>39.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>take</th>\n",
       "      <td>2.416907</td>\n",
       "      <td>2.314340</td>\n",
       "      <td>2.284142</td>\n",
       "      <td>65.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>did</th>\n",
       "      <td>2.447385</td>\n",
       "      <td>2.561686</td>\n",
       "      <td>2.269706</td>\n",
       "      <td>53.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keep</th>\n",
       "      <td>2.389684</td>\n",
       "      <td>2.303536</td>\n",
       "      <td>2.520922</td>\n",
       "      <td>40.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>big</th>\n",
       "      <td>2.381730</td>\n",
       "      <td>2.490798</td>\n",
       "      <td>1.977379</td>\n",
       "      <td>39.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>177 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         entropy  entropy_fps  entropy_fns   count    fps    fns  num_pos  \\\n",
       "[CLS]   2.237726     2.271439     2.230080  3600.0  614.0  256.0   1606.0   \n",
       "user    2.250743     2.279840     2.262625  1235.0  204.0  115.0    549.0   \n",
       "every   2.508682     2.643061     2.153221    46.0    6.0    3.0     28.0   \n",
       "time    2.533932     2.596356     2.377192    82.0   15.0    4.0     36.0   \n",
       "there   2.430238     2.100990     2.621234    56.0    6.0    4.0     24.0   \n",
       "...          ...          ...          ...     ...    ...    ...      ...   \n",
       "fuckin  2.520665     2.444635     2.699345    39.0    5.0    1.0     22.0   \n",
       "take    2.416907     2.314340     2.284142    65.0    7.0    3.0     32.0   \n",
       "did     2.447385     2.561686     2.269706    53.0    9.0    6.0     25.0   \n",
       "keep    2.389684     2.303536     2.520922    40.0    5.0    4.0     21.0   \n",
       "big     2.381730     2.490798     1.977379    39.0    2.0    2.0     30.0   \n",
       "\n",
       "        num_neg  \n",
       "[CLS]    1994.0  \n",
       "user      686.0  \n",
       "every      18.0  \n",
       "time       46.0  \n",
       "there      32.0  \n",
       "...         ...  \n",
       "fuckin     17.0  \n",
       "take       33.0  \n",
       "did        28.0  \n",
       "keep       19.0  \n",
       "big         9.0  \n",
       "\n",
       "[177 rows x 8 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy_df = pd.DataFrame(\n",
    "    [\n",
    "        entropy_dict, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives\n",
    "    ], index=[\"entropy\", \"entropy_fps\", \"entropy_fns\", \"count\", \"fps\", \"fns\", \"num_pos\", \"num_neg\"]\n",
    ").T\n",
    "entropy_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "contemporary-poison",
   "metadata": {},
   "outputs": [],
   "source": [
    "entropy_df.sort_values(\"entropy\", ascending=True).to_csv(\"latex/term_extraction/miso_eng.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alone-hepatitis",
   "metadata": {},
   "source": [
    "# Misogyny (IT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "violent-enough",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    \"BERT-0/\"\n",
    ").to(device)\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"dbmdz/bert-base-italian-uncased\")\n",
    "train, dev, test = get_dataset_by_name(\"miso-ita-raw\")\n",
    "cv, docs = preprocess_collection(train.get_texts(), 0.01, 0.95)\n",
    "train.texts = docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "equal-italian",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 141/141 [00:06<00:00, 22.15it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entropy</th>\n",
       "      <th>entropy_fps</th>\n",
       "      <th>entropy_fns</th>\n",
       "      <th>count</th>\n",
       "      <th>fps</th>\n",
       "      <th>fns</th>\n",
       "      <th>num_pos</th>\n",
       "      <th>num_neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>[CLS]</th>\n",
       "      <td>2.067716</td>\n",
       "      <td>2.160825</td>\n",
       "      <td>2.029560</td>\n",
       "      <td>4500.0</td>\n",
       "      <td>344.0</td>\n",
       "      <td>274.0</td>\n",
       "      <td>2103.0</td>\n",
       "      <td>2397.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ora</th>\n",
       "      <td>2.378925</td>\n",
       "      <td>2.354428</td>\n",
       "      <td>2.482992</td>\n",
       "      <td>126.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>88.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alle</th>\n",
       "      <td>2.477005</td>\n",
       "      <td>2.451033</td>\n",
       "      <td>2.619496</td>\n",
       "      <td>56.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>che</th>\n",
       "      <td>2.260726</td>\n",
       "      <td>2.299474</td>\n",
       "      <td>2.266860</td>\n",
       "      <td>1862.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>777.0</td>\n",
       "      <td>1085.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>siete</th>\n",
       "      <td>2.428688</td>\n",
       "      <td>2.710807</td>\n",
       "      <td>2.516716</td>\n",
       "      <td>60.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ciao</th>\n",
       "      <td>2.224712</td>\n",
       "      <td>2.386989</td>\n",
       "      <td>2.069853</td>\n",
       "      <td>57.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>11.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tette</th>\n",
       "      <td>2.273221</td>\n",
       "      <td>1.980642</td>\n",
       "      <td>2.687062</td>\n",
       "      <td>45.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>quindi</th>\n",
       "      <td>2.522248</td>\n",
       "      <td>2.651324</td>\n",
       "      <td>2.321795</td>\n",
       "      <td>48.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>32.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>voi</th>\n",
       "      <td>2.484247</td>\n",
       "      <td>2.474970</td>\n",
       "      <td>2.463013</td>\n",
       "      <td>77.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>56.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>donne</th>\n",
       "      <td>2.456921</td>\n",
       "      <td>2.295880</td>\n",
       "      <td>2.517311</td>\n",
       "      <td>46.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>22.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>162 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         entropy  entropy_fps  entropy_fns   count    fps    fns  num_pos  \\\n",
       "[CLS]   2.067716     2.160825     2.029560  4500.0  344.0  274.0   2103.0   \n",
       "ora     2.378925     2.354428     2.482992   126.0   10.0   10.0     38.0   \n",
       "alle    2.477005     2.451033     2.619496    56.0    7.0    3.0     15.0   \n",
       "che     2.260726     2.299474     2.266860  1862.0  157.0  135.0    777.0   \n",
       "siete   2.428688     2.710807     2.516716    60.0    2.0    8.0     12.0   \n",
       "...          ...          ...          ...     ...    ...    ...      ...   \n",
       "ciao    2.224712     2.386989     2.069853    57.0    2.0    1.0     46.0   \n",
       "tette   2.273221     1.980642     2.687062    45.0    7.0    5.0     28.0   \n",
       "quindi  2.522248     2.651324     2.321795    48.0    8.0    5.0     16.0   \n",
       "voi     2.484247     2.474970     2.463013    77.0    9.0    9.0     21.0   \n",
       "donne   2.456921     2.295880     2.517311    46.0    2.0   14.0     24.0   \n",
       "\n",
       "        num_neg  \n",
       "[CLS]    2397.0  \n",
       "ora        88.0  \n",
       "alle       41.0  \n",
       "che      1085.0  \n",
       "siete      48.0  \n",
       "...         ...  \n",
       "ciao       11.0  \n",
       "tette      17.0  \n",
       "quindi     32.0  \n",
       "voi        56.0  \n",
       "donne      22.0  \n",
       "\n",
       "[162 rows x 8 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy_dict, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives = get_tokens_entropy(model, tokenizer, train, device)\n",
    "entropy_df = pd.DataFrame(\n",
    "    [\n",
    "        entropy_dict, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives\n",
    "    ], index=[\"entropy\", \"entropy_fps\", \"entropy_fns\", \"count\", \"fps\", \"fns\", \"num_pos\", \"num_neg\"]\n",
    ").T\n",
    "entropy_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "dirty-reservoir",
   "metadata": {},
   "outputs": [],
   "source": [
    "entropy_df.sort_values(\"entropy\", ascending=True).to_csv(\"latex/term_extraction/miso_ita.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dirty-summer",
   "metadata": {},
   "source": [
    "# MlMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "moderate-designation",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "%capture\n",
    "\n",
    "device = \"cuda\"\n",
    "\n",
    "model = LMForSequenceClassification.load_from_checkpoint(\n",
    "    \"BERT-0/\"\n",
    ").to(device)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "train, dev, test = get_dataset_by_name(\"mlma\")\n",
    "cv, docs = preprocess_collection(test.get_texts(), 0.01, 0.95)\n",
    "test.texts = docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "enabling-macintosh",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/18 [00:00<?, ?it/s]/home/dauin_user/gattanasio/venvs/unbias_venv/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:2227: UserWarning: `max_length` is ignored when `padding`=`True`.\n",
      "  warnings.warn(\"`max_length` is ignored when `padding`=`True`.\")\n",
      "100%|██████████| 18/18 [00:00<00:00, 22.49it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>entropy</th>\n",
       "      <th>entropy_fps</th>\n",
       "      <th>entropy_fns</th>\n",
       "      <th>count</th>\n",
       "      <th>fps</th>\n",
       "      <th>fns</th>\n",
       "      <th>num_pos</th>\n",
       "      <th>num_neg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>[CLS]</th>\n",
       "      <td>1.817048</td>\n",
       "      <td>1.798705</td>\n",
       "      <td>NaN</td>\n",
       "      <td>565.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>499.0</td>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>stupid</th>\n",
       "      <td>2.117414</td>\n",
       "      <td>1.918335</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cunt</th>\n",
       "      <td>1.771495</td>\n",
       "      <td>1.970147</td>\n",
       "      <td>NaN</td>\n",
       "      <td>51.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user</th>\n",
       "      <td>1.854205</td>\n",
       "      <td>1.821117</td>\n",
       "      <td>NaN</td>\n",
       "      <td>431.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>382.0</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[SEP]</th>\n",
       "      <td>1.788042</td>\n",
       "      <td>1.768903</td>\n",
       "      <td>NaN</td>\n",
       "      <td>565.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>499.0</td>\n",
       "      <td>66.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         entropy  entropy_fps  entropy_fns  count   fps  fns  num_pos  num_neg\n",
       "[CLS]   1.817048     1.798705          NaN  565.0  66.0  0.0    499.0     66.0\n",
       "stupid  2.117414     1.918335          NaN   11.0   1.0  0.0     10.0      1.0\n",
       "cunt    1.771495     1.970147          NaN   51.0   2.0  0.0     49.0      2.0\n",
       "user    1.854205     1.821117          NaN  431.0  49.0  0.0    382.0     49.0\n",
       "[SEP]   1.788042     1.768903          NaN  565.0  66.0  0.0    499.0     66.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entropy_dict, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives = get_tokens_entropy(model, tokenizer, test, device)\n",
    "entropy_df = pd.DataFrame(\n",
    "    [\n",
    "        entropy_dict, entropy_fps, entropy_fns, word_occ, fps, fns, num_positives, num_negatives\n",
    "    ], index=[\"entropy\", \"entropy_fps\", \"entropy_fns\", \"count\", \"fps\", \"fns\", \"num_pos\", \"num_neg\"]\n",
    ").T\n",
    "entropy_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "musical-chance",
   "metadata": {},
   "outputs": [],
   "source": [
    "entropy_df.sort_values(\"entropy\", ascending=True).to_csv(\"latex/term_extraction/mlma.csv\")"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "be9233e59ab1f58386d6c7ed9ac3c1cc25c801ceabc31d17529d54705ed180c1"
  },
  "kernelspec": {
   "display_name": "Python 3.8.5 64-bit ('unbias': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}