ulmfit_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifying OUV using ULMFiT model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/opt/miniconda3/bin/python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm import tqdm\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from scipy.special import softmax\n",
    "\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "import torchtext\n",
    "from torchtext.data import get_tokenizer\n",
    "tokenizer = get_tokenizer('spacy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fastai\n",
    "from fastai.text.all import * \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from functools import partial\n",
    "import io\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.7.0\n",
      "GPU-enabled installation? False\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary(object):\n",
    "    \"\"\"Class to process text and extract vocabulary for mapping\"\"\"\n",
    "\n",
    "    def __init__(self, token_to_idx=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            token_to_idx (dict): a pre-existing map of tokens to indices\n",
    "            add_unk (bool): a flag that indicates whether to add the UNK token\n",
    "            unk_token (str): the UNK token to add into the Vocabulary\n",
    "        \"\"\"\n",
    "\n",
    "        if token_to_idx is None:\n",
    "            token_to_idx = {}\n",
    "        self._token_to_idx = token_to_idx\n",
    "\n",
    "        self._idx_to_token = {idx: token \n",
    "                              for token, idx in self._token_to_idx.items()}\n",
    "                \n",
    "    def to_serializable(self):\n",
    "        \"\"\" returns a dictionary that can be serialized \"\"\"\n",
    "        return {'token_to_idx': self._token_to_idx}\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        \"\"\" instantiates the Vocabulary from a serialized dictionary \"\"\"\n",
    "        return cls(**contents)\n",
    "\n",
    "    def add_token(self, token):\n",
    "        \"\"\"Update mapping dicts based on the token.\n",
    "\n",
    "        Args:\n",
    "            token (str): the item to add into the Vocabulary\n",
    "        Returns:\n",
    "            index (int): the integer corresponding to the token\n",
    "        \"\"\"\n",
    "        if token in self._token_to_idx:\n",
    "            index = self._token_to_idx[token]\n",
    "        else:\n",
    "            index = len(self._token_to_idx)\n",
    "            self._token_to_idx[token] = index\n",
    "            self._idx_to_token[index] = token\n",
    "        return index\n",
    "    \n",
    "    def add_many(self, tokens):\n",
    "        \"\"\"Add a list of tokens into the Vocabulary\n",
    "        \n",
    "        Args:\n",
    "            tokens (list): a list of string tokens\n",
    "        Returns:\n",
    "            indices (list): a list of indices corresponding to the tokens\n",
    "        \"\"\"\n",
    "        return [self.add_token(token) for token in tokens]\n",
    "\n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "          or the UNK index if token isn't present.\n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        Notes:\n",
    "            `unk_index` needs to be >=0 (having been added into the Vocabulary) \n",
    "              for the UNK functionality \n",
    "        \"\"\"\n",
    "        return self._token_to_idx[token]\n",
    "\n",
    "    def lookup_index(self, index):\n",
    "        \"\"\"Return the token associated with the index\n",
    "        \n",
    "        Args: \n",
    "            index (int): the index to look up\n",
    "        Returns:\n",
    "            token (str): the token corresponding to the index\n",
    "        Raises:\n",
    "            KeyError: if the index is not in the Vocabulary\n",
    "        \"\"\"\n",
    "        if index not in self._idx_to_token:\n",
    "            raise KeyError(\"the index (%d) is not in the Vocabulary\" % index)\n",
    "        return self._idx_to_token[index]\n",
    "\n",
    "    def __str__(self):\n",
    "        return \"<Vocabulary(size=%d)>\" % len(self)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._token_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SequenceVocabulary(Vocabulary):\n",
    "    def __init__(self, token_to_idx=None, unk_token='xxunk',\n",
    "                 mask_token='xxpad', begin_seq_token=\"xxbos\",\n",
    "                 end_seq_token=\"xxeos\"):\n",
    "\n",
    "        super(SequenceVocabulary, self).__init__(token_to_idx)\n",
    "\n",
    "        self._mask_token = mask_token\n",
    "        self._unk_token = unk_token\n",
    "        self._begin_seq_token = begin_seq_token\n",
    "        self._end_seq_token = end_seq_token\n",
    "\n",
    "        self.mask_index = 1\n",
    "        self.unk_index = 0\n",
    "        self.begin_seq_index = 2\n",
    "        self.end_seq_index = 3\n",
    "\n",
    "    def to_serializable(self):\n",
    "        contents = super(SequenceVocabulary, self).to_serializable()\n",
    "        contents.update({'unk_token': self._unk_token,\n",
    "                         'mask_token': self._mask_token,\n",
    "                         'begin_seq_token': self._begin_seq_token,\n",
    "                         'end_seq_token': self._end_seq_token})\n",
    "        return contents\n",
    "    \n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        \"\"\" instantiates the Vocabulary from a serialized dictionary \"\"\"\n",
    "        return cls(**contents)\n",
    "\n",
    "    \n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "          or the UNK index if token isn't present.\n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        Notes:\n",
    "            `unk_index` needs to be >=0 (having been added into the Vocabulary) \n",
    "              for the UNK functionality \n",
    "        \"\"\"\n",
    "        if self.unk_index >= 0:\n",
    "            return self._token_to_idx.get(token, self.unk_index)\n",
    "        else:\n",
    "            return self._token_to_idx[token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OuvVectorizer(object):\n",
    "    \"\"\" The Vectorizer which coordinates the Vocabularies and puts them to use\"\"\"\n",
    "    def __init__(self, ouv_vocab):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            review_vocab (Vocabulary): maps words to integers\n",
    "        \"\"\"\n",
    "        self.ouv_vocab = ouv_vocab\n",
    "        \n",
    "    def vectorize(self, data, vector_length = -1):\n",
    "        \"\"\"Create a collapsed one-hit vector for the ouv data\n",
    "        \n",
    "        Args:\n",
    "            data (str): the ouv description data\n",
    "            vector_length (int): an argument for forcing the length of index vector\n",
    "        Returns:\n",
    "            the vectorized data (np.ndarray)\n",
    "        \"\"\"\n",
    "        indices = []\n",
    "        indices.extend(self.ouv_vocab.lookup_token(token) for token in data.split(' '))\n",
    "        #indices.append(self.ouv_vocab.end_seq_index)\n",
    "        \n",
    "        if vector_length < 0:\n",
    "            vector_length = len(indices)\n",
    "            \n",
    "        out_vector = np.zeros(vector_length, dtype=np.int64)\n",
    "        out_vector[:len(indices)] = indices\n",
    "        out_vector[len(indices):] = self.ouv_vocab.mask_index\n",
    "        \n",
    "        return out_vector, len(indices)\n",
    "\n",
    "    @classmethod\n",
    "    def from_dataframe(cls, vocab, cutoff=5):\n",
    "        \"\"\"Instantiate the vectorizer from the dataset dataframe\n",
    "        \n",
    "        Args:\n",
    "            ouv_df (pandas.DataFrame): the ouv dataset\n",
    "            cutoff (int): the parameter for frequency-based filtering\n",
    "        Returns:\n",
    "            an instance of the OuvVectorizer\n",
    "        \"\"\"\n",
    "        \n",
    "        # Add top words if count > provided count\n",
    "        #word_counts = Counter()\n",
    "        #for data in ouv_df.data:\n",
    "        #    for word in data.split(' '):\n",
    "        #        if word not in string.punctuation:\n",
    "        #            word_counts[word] += 1\n",
    "        \n",
    "        ouv_vocab = SequenceVocabulary()\n",
    "        for token in vocab:\n",
    "            ouv_vocab.add_token(token)\n",
    "        #for word, count in word_counts.items():\n",
    "        #    if count > cutoff:\n",
    "        #        ouv_vocab.add_token(word)\n",
    "\n",
    "        return cls(ouv_vocab)\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        \"\"\"Instantiate a OuvVectorizer from a serializable dictionary\n",
    "        \n",
    "        Args:\n",
    "            contents (dict): the serializable dictionary\n",
    "        Returns:\n",
    "            an instance of the OuvVectorizer class\n",
    "        \"\"\"\n",
    "        ouv_vocab = SequenceVocabulary.from_serializable(contents['ouv_vocab'])\n",
    "        \n",
    "        return cls(ouv_vocab=ouv_vocab)\n",
    "\n",
    "    def to_serializable(self):\n",
    "        \"\"\"Create the serializable dictionary for caching\n",
    "        \n",
    "        Returns:\n",
    "            contents (dict): the serializable dictionary\n",
    "        \"\"\"\n",
    "        return {'ouv_vocab': self.ouv_vocab.to_serializable()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OuvDataset(Dataset):\n",
    "    def __init__(self, ouv_df, vectorizer, vocab):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            ouv_df (pandas.DataFrame): the dataset\n",
    "            vectorizer (ReviewVectorizer): vectorizer instantiated from dataset\n",
    "        \"\"\"\n",
    "        self.ouv_df = ouv_df\n",
    "        self._vectorizer = vectorizer\n",
    "        self.vocab = vocab\n",
    "        \n",
    "        # +0 if not using begin_seq and end seq, +1 if only using begin_seq, +2 if using both begin and end seq tokens\n",
    "        measure_len = lambda context: len(context.split(\" \"))\n",
    "        self._max_seq_length = max(map(measure_len, ouv_df.data)) + 0\n",
    "\n",
    "        self.train_df = self.ouv_df[self.ouv_df.split=='train']\n",
    "        self.train_size = len(self.train_df)\n",
    "\n",
    "        self.val_df = self.ouv_df[self.ouv_df.split=='dev']\n",
    "        self.validation_size = len(self.val_df)\n",
    "\n",
    "        self.test_df = self.ouv_df[self.ouv_df.split=='test']\n",
    "        self.test_size = len(self.test_df)\n",
    "\n",
    "        self._lookup_dict = {'train': (self.train_df, self.train_size),\n",
    "                             'val': (self.val_df, self.validation_size),\n",
    "                             'test': (self.test_df, self.test_size)}\n",
    "\n",
    "        self.set_split('train')\n",
    "\n",
    "    @classmethod\n",
    "    def load_dataset_and_make_vectorizer(cls, ouv_csv, cutoff, vocab):\n",
    "        \"\"\"Load dataset and make a new vectorizer from scratch\n",
    "        \n",
    "        Args:\n",
    "            ouv_csv (str): location of the dataset\n",
    "            cutoff (int): the boundary to set the words into unknown\n",
    "        Returns:\n",
    "            an instance of OuvDataset\n",
    "        \"\"\"\n",
    "        ouv_df = pd.read_csv(ouv_csv)\n",
    "        train_ouv_df = ouv_df[ouv_df.split=='train']\n",
    "        return cls(ouv_df, OuvVectorizer.from_dataframe(cutoff=cutoff, vocab=vocab), vocab=vocab)\n",
    "    \n",
    "    @classmethod\n",
    "    def load_dataset_and_load_vectorizer(cls, ouv_csv, vectorizer_filepath, vocab):\n",
    "        \"\"\"Load dataset and the corresponding vectorizer. \n",
    "        Used in the case in the vectorizer has been cached for re-use\n",
    "        \n",
    "        Args:\n",
    "            ouv_csv (str): location of the dataset\n",
    "            vectorizer_filepath (str): location of the saved vectorizer\n",
    "        Returns:\n",
    "            an instance of OuvDataset\n",
    "        \"\"\"\n",
    "        ouv_df = pd.read_csv(ouv_csv)\n",
    "        vectorizer = cls.load_vectorizer_only(vectorizer_filepath)\n",
    "        return cls(ouv_df, vectorizer, vocab=vocab)\n",
    "\n",
    "    @staticmethod\n",
    "    def load_vectorizer_only(vectorizer_filepath):\n",
    "        \"\"\"a static method for loading the vectorizer from file\n",
    "        \n",
    "        Args:\n",
    "            vectorizer_filepath (str): the location of the serialized vectorizer\n",
    "        Returns:\n",
    "            an instance of ReviewVectorizer\n",
    "        \"\"\"\n",
    "        with open(vectorizer_filepath) as fp:\n",
    "            return OuvVectorizer.from_serializable(json.load(fp))\n",
    "\n",
    "    def save_vectorizer(self, vectorizer_filepath):\n",
    "        \"\"\"saves the vectorizer to disk using json\n",
    "        \n",
    "        Args:\n",
    "            vectorizer_filepath (str): the location to save the vectorizer\n",
    "        \"\"\"\n",
    "        with open(vectorizer_filepath, \"w\") as fp:\n",
    "            json.dump(self._vectorizer.to_serializable(), fp)\n",
    "\n",
    "    def get_vectorizer(self):\n",
    "        \"\"\" returns the vectorizer \"\"\"\n",
    "        return self._vectorizer\n",
    "\n",
    "    def set_split(self, split=\"train\"):\n",
    "        \"\"\" selects the splits in the dataset using a column in the dataframe \n",
    "        \n",
    "        Args:\n",
    "            split (str): one of \"train\", \"val\", or \"test\"\n",
    "        \"\"\"\n",
    "        self._target_split = split\n",
    "        self._target_df, self._target_size = self._lookup_dict[split]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self._target_size\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"the primary entry point method for PyTorch datasets\n",
    "        \n",
    "        Args:\n",
    "            index (int): the index to the data point \n",
    "        Returns:\n",
    "            a dictionary holding the data point's features (x_data) and component for labels (y_target and y_fuzzy)\n",
    "        \"\"\"\n",
    "        row = self._target_df.iloc[index]\n",
    "\n",
    "        ouv_vector, vec_length = \\\n",
    "            self._vectorizer.vectorize(row.data, self._max_seq_length)\n",
    "\n",
    "        true_label = \\\n",
    "            np.fromstring(row.true[1:-1],dtype=float, sep=' ')\n",
    "        \n",
    "        if len(true_label)==10:\n",
    "            true_label = np.append(true_label,0.0)\n",
    "        \n",
    "        fuzzy_label = \\\n",
    "            np.fromstring(row.fuzzy[1:-1],dtype=float, sep=' ')\n",
    "\n",
    "        return {'x_data': ouv_vector,\n",
    "                'y_target': true_label,\n",
    "                'y_fuzzy': fuzzy_label,\n",
    "                'x_length': vec_length,\n",
    "                'id': row.id\n",
    "               }\n",
    "\n",
    "    def get_num_batches(self, batch_size):\n",
    "        \"\"\"Given a batch size, return the number of batches in the dataset\n",
    "        \n",
    "        Args:\n",
    "            batch_size (int)\n",
    "        Returns:\n",
    "            number of batches in the dataset\n",
    "        \"\"\"\n",
    "        return len(self) // batch_size  \n",
    "    \n",
    "def generate_batches(dataset, batch_size, shuffle=True,\n",
    "                     drop_last=True, device=\"cpu\"):\n",
    "    \"\"\"\n",
    "    A generator function which wraps the PyTorch DataLoader. It will \n",
    "      ensure each tensor is on the write device location.\n",
    "    \"\"\"\n",
    "    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,\n",
    "                            shuffle=shuffle, drop_last=drop_last)\n",
    "\n",
    "    for data_dict in dataloader:\n",
    "        out_data_dict = {}\n",
    "        for name, tensor in data_dict.items():\n",
    "            out_data_dict[name] = data_dict[name].to(device)\n",
    "        yield out_data_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if cuda:\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)\n",
    "\n",
    "def load_glove_from_file(glove_filepath):\n",
    "    \"\"\"\n",
    "    Load the GloVe embeddings \n",
    "    \n",
    "    Args:\n",
    "        glove_filepath (str): path to the glove embeddings file \n",
    "    Returns:\n",
    "        word_to_index (dict), embeddings (numpy.ndarary)\n",
    "    \"\"\"\n",
    "\n",
    "    word_to_index = {}\n",
    "    embeddings = []\n",
    "    with open(glove_filepath, \"r\") as fp:\n",
    "        for index, line in enumerate(fp):\n",
    "            line = line.split(\" \") # each line: word num1 num2 ...\n",
    "            word_to_index[line[0]] = index # word = line[0] \n",
    "            embedding_i = np.array([float(val) for val in line[1:]])\n",
    "            embeddings.append(embedding_i)\n",
    "    return word_to_index, np.stack(embeddings)\n",
    "\n",
    "def make_embedding_matrix(glove_filepath, words):\n",
    "    \"\"\"\n",
    "    Create embedding matrix for a specific set of words.\n",
    "    \n",
    "    Args:\n",
    "        glove_filepath (str): file path to the glove embeddigns\n",
    "        words (list): list of words in the dataset\n",
    "    \"\"\"\n",
    "    word_to_idx, glove_embeddings = load_glove_from_file(glove_filepath)\n",
    "    embedding_size = glove_embeddings.shape[1]\n",
    "    \n",
    "    final_embeddings = np.zeros((len(words), embedding_size))\n",
    "\n",
    "    for i, word in enumerate(words):\n",
    "        if word in word_to_idx:\n",
    "            final_embeddings[i, :] = glove_embeddings[word_to_idx[word]]\n",
    "        else:\n",
    "            embedding_i = torch.ones(1, embedding_size)\n",
    "            torch.nn.init.xavier_uniform_(embedding_i)\n",
    "            final_embeddings[i, :] = embedding_i\n",
    "\n",
    "    return final_embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/ulmfit/vectorizer.json\n",
      "\tmodel_storage/ulmfit/model.pth\n",
      "Using CUDA: False\n"
     ]
    }
   ],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    frequency_cutoff=1,\n",
    "    model_state_file='model.pth',\n",
    "    ouv_csv='Data/ouv_with_splits_full.csv',\n",
    "    big_ouv_csv='Data/all_with_splits_full.csv',\n",
    "    prior_csv = 'Data/Coappearance_matrix.csv',\n",
    "    sm_csv = 'Data/Social_media.csv',\n",
    "    save_dir='model_storage/ulmfit/',\n",
    "    vectorizer_file='vectorizer.json',\n",
    "    # Model hyper parameters\n",
    "    glove_filepath='Data/glove/glove.6B.300d.txt', \n",
    "    use_glove=False,\n",
    "    freeze = True,\n",
    "    embedding_size=400, \n",
    "    hidden_dim=64, \n",
    "    bi = False,\n",
    "    # Training hyper parameters\n",
    "    batch_size=64,\n",
    "    early_stopping_criteria=3,\n",
    "    learning_rate=2e-2,\n",
    "    l2=1e-5,\n",
    "    dropout_p=0,\n",
    "    k = 3,\n",
    "    fuzzy = True,\n",
    "    fuzzy_how = 'prior',\n",
    "    fuzzy_lambda = 0.1,\n",
    "    num_epochs=10,\n",
    "    seed=1337,\n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")\n",
    "\n",
    "classes = ['Criteria i', 'Criteria ii', 'Criteria iii', 'Criteria iv', 'Criteria v', 'Criteria vi', \n",
    "              'Criteria vii', 'Criteria viii', 'Criteria ix', 'Criteria x', 'Others']\n",
    "\n",
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.vectorizer_file = os.path.join(args.save_dir,\n",
    "                                        args.vectorizer_file)\n",
    "\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.vectorizer_file))\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "    \n",
    "# Check CUDA\n",
    "if not torch.cuda.is_available():\n",
    "    args.cuda = False\n",
    "\n",
    "print(\"Using CUDA: {}\".format(args.cuda))\n",
    "\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "\n",
    "# Set seed for reproducibility\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_k_acc_val': 0,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_1_acc': [],\n",
    "            'train_k_acc': [],\n",
    "            'train_k_jac': [],\n",
    "            'val_loss': [],\n",
    "            'val_1_acc': [],\n",
    "            'val_k_acc': [],\n",
    "            'val_k_jac': [],\n",
    "            'test_loss': -1,\n",
    "            'test_1_acc': -1,\n",
    "            'test_k_acc':-1,\n",
    "            'test_k_jac':-1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        acc_tm1, acc_t = train_state['val_k_acc'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        if acc_t <= train_state['early_stopping_best_k_acc_val']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if acc_t > train_state['early_stopping_best_k_acc_val']:\n",
    "                train_state['early_stopping_best_k_acc_val'] = acc_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_sensitive(T):\n",
    "    T = np.exp(T) - np.exp(0) + 1e-9\n",
    "    if len(T.shape)==1:\n",
    "        return T/T.sum()\n",
    "    return  T/(T.sum(axis=1).unsqueeze(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert a df to tensor to be used in pytorch\n",
    "def df_to_tensor(df):\n",
    "    device = args.device\n",
    "    return torch.from_numpy(df.values).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prior():\n",
    "    prior = pd.read_csv(args.prior_csv,sep=';',names=classes[:-1], skiprows=1)\n",
    "    prior['Others'] = 1\n",
    "    prior = prior.T\n",
    "    prior['Others'] = 1\n",
    "    prior = df_to_tensor(prior)\n",
    "    return(prior)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_fuzzy_label(y_target, y_fuzzy, fuzzy=False, how='uni', lbd=0):\n",
    "    '''\n",
    "    Using two sets of prediction labels and fuzziness parameters to compute the fuzzy label in the form as \n",
    "    a distribution over classes\n",
    "    \n",
    "    Args:\n",
    "    y_target (torch.Tensor) of shape (n_batch, n_classes): the true label of the ouv description\n",
    "    y_fuzzy (torch.Tensor) of shape (n_batch, n_classes): the fuzzy label of the ouv description\n",
    "    fuzzy (bool): whether or not to turn on the fuzziness option\n",
    "    how (string): the way fuzziness weights are used, one of the options in {'uni', 'prior'}\n",
    "    lbd (float): the scaler applied to the fuzziness of the label\n",
    "    \n",
    "    Returns:\n",
    "    A pytorch Tensor of shape (n_batch, n_classes): The processed label in the form of distribution that add to 1\n",
    "    '''\n",
    "    assert y_target.shape == y_fuzzy.shape, 'target labels must have the same size'\n",
    "    assert how in {'uni', 'prior', 'origin'}, '''how must be one of the two options in {'uni', 'prior'}'''\n",
    "    \n",
    "    if not fuzzy:\n",
    "        return softmax_sensitive(y_target)\n",
    "    \n",
    "    if how == 'uni':\n",
    "        y_label = y_target + lbd * y_fuzzy\n",
    "        return softmax_sensitive(y_label)\n",
    "    \n",
    "    ### TO DO ###\n",
    "    elif how == 'prior':\n",
    "        prior = get_prior()\n",
    "        y_inter = torch.matmul(y_target.float(),prior)\n",
    "        y_inter = y_inter/(y_inter.max(dim=1, keepdim=True)[0])\n",
    "        y_label = y_target + lbd * y_fuzzy * y_inter\n",
    "        return softmax_sensitive(y_label)\n",
    "    \n",
    "    else:\n",
    "        y_label = y_target + lbd\n",
    "        return softmax_sensitive(y_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_encoder(model, file, device=None):\n",
    "        \"Load the encoder `file` from the model directory, optionally ensuring it's on `device`\"\n",
    "        encoder = model[0]\n",
    "        if device is None: device = args.device\n",
    "        if hasattr(encoder, 'module'): encoder = encoder.module\n",
    "        distrib_barrier()\n",
    "        wgts = torch.load(args.save_dir + file+'.pth', map_location=device)\n",
    "        encoder.load_state_dict(clean_raw_keys(wgts))\n",
    "        for param in encoder.parameters():\n",
    "            param.requires_grad = False\n",
    "        #model.freeze()\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "ouv_df = pd.read_csv(args.big_ouv_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/lib/python3.7/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "data_lm = TextDataLoaders.from_df(ouv_df, text_col = 'data', label_col = 'true', path = \"\", is_lm=True)\n",
    "vocab = data_lm.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not using pre-trained embeddings\n"
     ]
    }
   ],
   "source": [
    "if args.reload_from_files:\n",
    "    # training from a checkpoint\n",
    "    dataset = OuvDataset.load_dataset_and_load_vectorizer(args.ouv_csv, args.vectorizer_file, vocab=vocab)\n",
    "\n",
    "else:\n",
    "    # create dataset and vectorizer\n",
    "    dataset = OuvDataset.load_dataset_and_make_vectorizer(args.ouv_csv, cutoff=args.frequency_cutoff, vocab=vocab)\n",
    "    dataset.save_vectorizer(args.vectorizer_file)    \n",
    "\n",
    "vectorizer = dataset.get_vectorizer()\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "\n",
    "# Use GloVe or randomly initialized embeddings\n",
    "if args.use_glove:\n",
    "    words = vectorizer.ouv_vocab._token_to_idx.keys()\n",
    "    embeddings = make_embedding_matrix(glove_filepath=args.glove_filepath, \n",
    "                                       words=words)\n",
    "    print(\"Using pre-trained embeddings\")\n",
    "else:\n",
    "    print(\"Not using pre-trained embeddings\")\n",
    "    embeddings = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train a Domain-specific Language Model using whole data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Data and Prepare Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "ouv_df = pd.read_csv(args.big_ouv_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/lib/python3.7/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "data_lm = TextDataLoaders.from_df(ouv_df, text_col = 'data', label_col = 'true', path = \"\", is_lm=True)\n",
    "vocab = data_lm.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10536"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/lib/python3.7/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "# Language model data\n",
    "data_lm = TextDataLoaders.from_df(ouv_df, text_col = 'data', label_col = 'true', path = \"\", is_lm=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = language_model_learner(data_lm, AWD_LSTM, metrics=[accuracy, Perplexity()], path='', wd=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = learn.load('lm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialRNN(\n",
       "  (0): AWD_LSTM(\n",
       "    (encoder): Embedding(10536, 400, padding_idx=1)\n",
       "    (encoder_dp): EmbeddingDropout(\n",
       "      (emb): Embedding(10536, 400, padding_idx=1)\n",
       "    )\n",
       "    (rnns): ModuleList(\n",
       "      (0): WeightDropout(\n",
       "        (module): LSTM(400, 1152, batch_first=True)\n",
       "      )\n",
       "      (1): WeightDropout(\n",
       "        (module): LSTM(1152, 1152, batch_first=True)\n",
       "      )\n",
       "      (2): WeightDropout(\n",
       "        (module): LSTM(1152, 400, batch_first=True)\n",
       "      )\n",
       "    )\n",
       "    (input_dp): RNNDropout()\n",
       "    (hidden_dps): ModuleList(\n",
       "      (0): RNNDropout()\n",
       "      (1): RNNDropout()\n",
       "      (2): RNNDropout()\n",
       "    )\n",
       "  )\n",
       "  (1): LinearDecoder(\n",
       "    (decoder): Linear(in_features=400, out_features=10536, bias=True)\n",
       "    (output_dp): RNNDropout()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fitting Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.465606</td>\n",
       "      <td>3.843865</td>\n",
       "      <td>0.341763</td>\n",
       "      <td>46.705639</td>\n",
       "      <td>13:27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1,1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Path('models/1epoch.pth')"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.save('1epoch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = learn.load('1epoch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>perplexity</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.051659</td>\n",
       "      <td>3.704584</td>\n",
       "      <td>0.355655</td>\n",
       "      <td>40.633137</td>\n",
       "      <td>20:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.915759</td>\n",
       "      <td>3.627175</td>\n",
       "      <td>0.362101</td>\n",
       "      <td>37.606422</td>\n",
       "      <td>19:40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.793126</td>\n",
       "      <td>3.554804</td>\n",
       "      <td>0.367985</td>\n",
       "      <td>34.980976</td>\n",
       "      <td>19:53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.680459</td>\n",
       "      <td>3.509872</td>\n",
       "      <td>0.372307</td>\n",
       "      <td>33.443985</td>\n",
       "      <td>19:34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.585886</td>\n",
       "      <td>3.476479</td>\n",
       "      <td>0.375953</td>\n",
       "      <td>32.345627</td>\n",
       "      <td>19:20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.493525</td>\n",
       "      <td>3.454393</td>\n",
       "      <td>0.377256</td>\n",
       "      <td>31.639082</td>\n",
       "      <td>20:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>3.430653</td>\n",
       "      <td>3.438308</td>\n",
       "      <td>0.379044</td>\n",
       "      <td>31.134241</td>\n",
       "      <td>19:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>3.364055</td>\n",
       "      <td>3.429951</td>\n",
       "      <td>0.380467</td>\n",
       "      <td>30.875135</td>\n",
       "      <td>19:27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>3.321345</td>\n",
       "      <td>3.427366</td>\n",
       "      <td>0.380578</td>\n",
       "      <td>30.795437</td>\n",
       "      <td>20:24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>3.303921</td>\n",
       "      <td>3.427146</td>\n",
       "      <td>0.380355</td>\n",
       "      <td>30.788652</td>\n",
       "      <td>20:49</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, 1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving Trained Encoder and Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save_encoder('finetuned')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Path('models/lm.pth')"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.save('lm')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Some Inference on the Effect of Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "TEXT = \"This site is unique because\"\n",
    "N_WORDS = 50\n",
    "N_SENTENCES = 2\n",
    "preds = [learn.predict(TEXT, N_WORDS, temperature=0.75) \n",
    "         for _ in range(N_SENTENCES)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This site is unique because it is the only example of a complex of karst complexes that is clearly recognised as being of outstanding universal value . the island of zanzibar has been inscribed as a world heritage site in < num > the inscriptions , which bear witness to the civilisation of\n",
      "This site is unique because of the large number of species of birds that make it into permanent , species rich and impressive populations the churches of the romanesque cathedral , the cathedral of st demetrius and st john s cemetery have outstanding architectural and artistic significance for the settlers who were northern merchants\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(preds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "TEXT = \"This architecture has a special layout\"\n",
    "N_WORDS = 50\n",
    "N_SENTENCES = 2\n",
    "preds = [learn.predict(TEXT, N_WORDS, temperature=0.75) \n",
    "         for _ in range(N_SENTENCES)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This architecture has a special layout , , especially in the form of the body of the building . the planet s primary feature is the addition of the ideal island , which lies at an elevation of < num > m above the sea floor , and is home to some < num >\n",
      "This architecture has a special layout with a high degree of integrity and integrity it was composed of fortified fortified fortifications built in the < num > th century , the first of which was built in < num > on the site of the old city , and which still exists today the\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(preds))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading Model for Classifier Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = get_text_classifier(AWD_LSTM, len(vocab), len(classes), seq_len=72, config=None, drop_mult=0.5, lin_ftrs=None,\n",
    "                        ps=None, pad_idx=1, max_len=72*20, y_range=None)\n",
    "classifier = load_encoder(classifier, 'finetuned')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_to(model,n):\n",
    "    param_lists = [param for param in model[0].parameters()]\n",
    "    frozen_idx = n if n >= 0 else len(param_lists) + n*4\n",
    "    if frozen_idx >= len(param_lists):\n",
    "        warn(f\"Freezing {frozen_idx} groups; model has {len(param_lists)}; whole model is frozen.\")\n",
    "    for i in range(len(param_lists)):\n",
    "        if i < frozen_idx:\n",
    "            param_lists[i].requires_grad = False\n",
    "        else:\n",
    "            param_lists[i].requires_grad = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Trained Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 1 LS Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:\n",
    "    hyperdict_fuzzy = pickle.load(fp)\n",
    "    train_state = hyperdict_fuzzy[('prior', 0.1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialRNN(\n",
       "  (0): SentenceEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(10536, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(10536, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): LinBnDrop(\n",
       "        (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): Dropout(p=0.2, inplace=False)\n",
       "        (2): Linear(in_features=1200, out_features=50, bias=False)\n",
       "        (3): ReLU(inplace=True)\n",
       "      )\n",
       "      (1): LinBnDrop(\n",
       "        (0): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): Dropout(p=0.1, inplace=False)\n",
       "        (2): Linear(in_features=50, out_features=11, bias=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.load_state_dict(torch.load(args.save_dir+'1337/model.pth',map_location=torch.device('cpu')))\n",
    "classifier.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 2 Baseline w/o LS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:\n",
    "    hyperdict_fuzzy = pickle.load(fp)\n",
    "    train_state = hyperdict_fuzzy[('prior', 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SequentialRNN(\n",
       "  (0): SentenceEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(10536, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(10536, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1152, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1152, 1152, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1152, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): LinBnDrop(\n",
       "        (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): Dropout(p=0.2, inplace=False)\n",
       "        (2): Linear(in_features=1200, out_features=50, bias=False)\n",
       "        (3): ReLU(inplace=True)\n",
       "      )\n",
       "      (1): LinBnDrop(\n",
       "        (0): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (1): Dropout(p=0.1, inplace=False)\n",
       "        (2): Linear(in_features=50, out_features=11, bias=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.load_state_dict(torch.load(args.save_dir+'baseline/model.pth',map_location=torch.device('cpu')))\n",
    "classifier.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24550730"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters())\n",
    "count_parameters(classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the loss & accuracy on the test set using the best available model\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "loss_func = cross_entropy\n",
    "#classifier.load_state_dict(torch.load(train_state['model_filename']))\n",
    "#classifier = classifier.to(args.device)\n",
    "\n",
    "dataset.set_split('test')\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "running_loss = 0.\n",
    "running_1_acc = 0.\n",
    "running_k_acc = 0.\n",
    "running_k_jac = 0.\n",
    "classifier.eval()\n",
    "\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    with torch.no_grad():\n",
    "        y_pred = classifier(X)[0]\n",
    "\n",
    "    # compute the loss\n",
    "    loss = loss_func(y_pred, Y)\n",
    "    loss_t = loss.item()\n",
    "    running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "    # compute the accuracy\n",
    "    acc_1_t = compute_1_accuracy(y_pred, y_target)\n",
    "    acc_k_t = compute_k_accuracy(y_pred, y_target, args.k)\n",
    "    jac_k_t = compute_jaccard_index(y_pred, y_target, args.k)\n",
    "\n",
    "    running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)\n",
    "    running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)\n",
    "    running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)\n",
    "\n",
    "train_state['test_loss'] = running_loss\n",
    "train_state['test_1_acc'] = running_1_acc\n",
    "train_state['test_k_acc'] = running_k_acc\n",
    "train_state['test_k_jac'] = running_k_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 3,\n",
       " 'early_stopping_best_k_acc_val': 94.3359375,\n",
       " 'learning_rate': 0.02,\n",
       " 'epoch_index': 7,\n",
       " 'train_loss': [1.317077281648767,\n",
       "  1.2605138109592569,\n",
       "  1.0280631605861255,\n",
       "  0.8563412818072178,\n",
       "  0.8161277029345488,\n",
       "  0.799169392660451,\n",
       "  0.7933862136873495,\n",
       "  0.7829414117187636],\n",
       " 'train_1_acc': [52.745535714285715,\n",
       "  55.15625,\n",
       "  66.87500000000001,\n",
       "  75.93750000000001,\n",
       "  77.61160714285718,\n",
       "  78.77232142857142,\n",
       "  79.01785714285711,\n",
       "  80.35714285714286],\n",
       " 'train_k_acc': [87.70089285714286,\n",
       "  88.48214285714286,\n",
       "  93.59375000000003,\n",
       "  96.8526785714286,\n",
       "  97.47767857142858,\n",
       "  98.23660714285717,\n",
       "  98.01339285714289,\n",
       "  97.92410714285717],\n",
       " 'train_k_jac': [0.19667853808828764,\n",
       "  0.2042666726878711,\n",
       "  0.21074838978903632,\n",
       "  0.21338205337524413,\n",
       "  0.21399385694946568,\n",
       "  0.2147677530135427,\n",
       "  0.2148703260081155,\n",
       "  0.21484508088656837],\n",
       " 'val_loss': [1.1491646554898864,\n",
       "  1.1290186723900748,\n",
       "  1.0530891137511653,\n",
       "  1.0253822197406395,\n",
       "  1.0060079722589403,\n",
       "  1.0004815254454278,\n",
       "  1.0093779732617738,\n",
       "  0.9832991686452636],\n",
       " 'val_1_acc': [63.28125,\n",
       "  64.453125,\n",
       "  69.33593750000001,\n",
       "  69.140625,\n",
       "  70.1171875,\n",
       "  70.8984375,\n",
       "  70.3125,\n",
       "  72.265625],\n",
       " 'val_k_acc': [91.2109375,\n",
       "  91.60156250000001,\n",
       "  92.38281250000001,\n",
       "  92.7734375,\n",
       "  94.3359375,\n",
       "  93.359375,\n",
       "  93.359375,\n",
       "  93.5546875],\n",
       " 'val_k_jac': [0.20777065493166447,\n",
       "  0.21173270605504513,\n",
       "  0.21013997867703438,\n",
       "  0.21314639411866665,\n",
       "  0.21215820498764515,\n",
       "  0.2117513082921505,\n",
       "  0.21409040689468384,\n",
       "  0.21462984010577202],\n",
       " 'test_loss': 1.0505487003554266,\n",
       " 'test_1_acc': 67.1875,\n",
       " 'test_k_acc': 93.16406250000001,\n",
       " 'test_k_jac': 0.2173874657601118,\n",
       " 'model_filename': 'model_storage/rnn/ulmfit/multigroup/savemodel/model.pth'}"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#LS Model\n",
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 3,\n",
       " 'early_stopping_best_k_acc_val': 93.9453125,\n",
       " 'learning_rate': 0.02,\n",
       " 'epoch_index': 7,\n",
       " 'train_loss': [1.2564197204045884,\n",
       "  1.190402439515455,\n",
       "  0.9138767724905221,\n",
       "  0.7031986134363143,\n",
       "  0.6537914813493433,\n",
       "  0.6345267694804869,\n",
       "  0.6277358806501516,\n",
       "  0.6202104856876341],\n",
       " 'train_1_acc': [52.65625,\n",
       "  55.3125,\n",
       "  66.69642857142857,\n",
       "  76.31696428571428,\n",
       "  78.10267857142856,\n",
       "  79.21875000000003,\n",
       "  79.8660714285714,\n",
       "  79.93303571428574],\n",
       " 'train_k_acc': [87.58928571428574,\n",
       "  88.39285714285714,\n",
       "  93.88392857142857,\n",
       "  96.6964285714286,\n",
       "  97.00892857142857,\n",
       "  97.90178571428571,\n",
       "  97.52232142857139,\n",
       "  97.76785714285714],\n",
       " 'train_k_jac': [0.1981719507702759,\n",
       "  0.20549957730940405,\n",
       "  0.21371926580156597,\n",
       "  0.21728670937674383,\n",
       "  0.21784828922578273,\n",
       "  0.21953054538794925,\n",
       "  0.2196302837559155,\n",
       "  0.21900271049567638],\n",
       " 'val_loss': [1.073857385544383,\n",
       "  1.1019409043499477,\n",
       "  0.9323346340491093,\n",
       "  0.9113813801002052,\n",
       "  0.8923210292044449,\n",
       "  0.8835381399065128,\n",
       "  0.8921374937794404,\n",
       "  0.8638266738246135],\n",
       " 'val_1_acc': [61.328125,\n",
       "  58.59375,\n",
       "  68.1640625,\n",
       "  68.94531250000001,\n",
       "  69.3359375,\n",
       "  70.3125,\n",
       "  70.3125,\n",
       "  71.2890625],\n",
       " 'val_k_acc': [91.2109375,\n",
       "  89.84375,\n",
       "  93.1640625,\n",
       "  93.16406249999999,\n",
       "  93.9453125,\n",
       "  93.5546875,\n",
       "  93.9453125,\n",
       "  93.9453125],\n",
       " 'val_k_jac': [0.20890764892101288,\n",
       "  0.21362149715423584,\n",
       "  0.21269531548023224,\n",
       "  0.21675967425107953,\n",
       "  0.21723400987684727,\n",
       "  0.21632255241274836,\n",
       "  0.21755952388048172,\n",
       "  0.21769903413951397],\n",
       " 'test_loss': 1.0794167117083788,\n",
       " 'test_1_acc': 66.40625,\n",
       " 'test_k_acc': 92.3828125,\n",
       " 'test_k_jac': 0.2222702857106924,\n",
       " 'model_filename': 'model_storage/rnn/ulmfit/multigroup/savemodel/baseline/model.pth'}"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Baseline\n",
    "train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"([.,!?])\", r\" \\1 \", text)\n",
    "    text = re.sub(r\"[^a-zA-Z.,!?]+\", r\" \", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_rating(text, classifier, vectorizer, classes, k=1):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        text (str): the text of the description\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    ouv = preprocess_text(text)\n",
    "    \n",
    "    classifier.eval()\n",
    "    vectorized_ouv = torch.tensor(vectorizer.vectorize(ouv)[0])\n",
    "    X = vectorized_ouv.view(1,-1)\n",
    "    result = classifier(vectorized_ouv.unsqueeze(0))[0]\n",
    "    result = F.softmax(result, dim=1)\n",
    "    \n",
    "    if k==1:\n",
    "        pred_id = result.argmax().item()\n",
    "        return (classes[pred_id], result[0][pred_id])\n",
    "    else:\n",
    "        pred_indices = [i.item() for i in result.topk(k)[1][0]]\n",
    "        output = []\n",
    "        for pred_id in pred_indices:\n",
    "            output.append((classes[pred_id],result[0][pred_id].item()))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a very old building dating back to 13th century -> Criteria iii with a probability of 0.67\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'this is a very old building dating back to 13th century'\n",
    "\n",
    "prediction = predict_rating(test_ouv,classifier,vectorizer,classes)\n",
    "print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "this is a very old building dating back to 13th century -> Criteria iii with a probability of 0.67\n",
      "this is a very old building dating back to 13th century -> Criteria iv with a probability of 0.22\n",
      "this is a very old building dating back to 13th century -> Criteria ii with a probability of 0.05\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'this is a very old building dating back to 13th century'\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "The particular layout of the complex is unique to this site -> Criteria iv with a probability of 0.44\n",
      "The particular layout of the complex is unique to this site -> Criteria iii with a probability of 0.23\n",
      "The particular layout of the complex is unique to this site -> Criteria ii with a probability of 0.11\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'The particular layout of the complex is unique to this site'\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty -> Criteria i with a probability of 0.69\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty -> Criteria iv with a probability of 0.12\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty -> Criteria ii with a probability of 0.05\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
    "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
    "age are represented by monuments of incomparable beauty'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world -> Criteria i with a probability of 0.33\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world -> Criteria vii with a probability of 0.28\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world -> Criteria iv with a probability of 0.11\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "from torcellos cathedral to the church of santa maria della salute -> Criteria vi with a probability of 0.38\n",
      "from torcellos cathedral to the church of santa maria della salute -> Criteria i with a probability of 0.20\n",
      "from torcellos cathedral to the church of santa maria della salute -> Criteria ii with a probability of 0.15\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''from torcellos cathedral to the church of santa maria della salute'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the years of the republics extraordinary golden age are represented by monuments of incomparable beauty -> Criteria i with a probability of 0.25\n",
      "the years of the republics extraordinary golden age are represented by monuments of incomparable beauty -> Criteria iv with a probability of 0.25\n",
      "the years of the republics extraordinary golden age are represented by monuments of incomparable beauty -> Criteria iii with a probability of 0.22\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the years of the republics extraordinary golden age are represented by monuments of incomparable beauty'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed: 0.05233001708984375\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "class Timer(object):\n",
    "    def __init__(self, name=None):\n",
    "        self.name = name\n",
    "\n",
    "    def __enter__(self):\n",
    "        self.tstart = time.time()\n",
    "\n",
    "    def __exit__(self, type, value, traceback):\n",
    "        if self.name:\n",
    "            print('[%s]' % self.name,)\n",
    "        print('Elapsed: %s' % (time.time() - self.tstart))\n",
    "        \n",
    "set_seed_everywhere(args.seed, args.cuda)        \n",
    "test_ouv = 'The particular layout of the complex is unique to this site'\n",
    "k=3\n",
    "with Timer():\n",
    "    predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpretability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngrams_iterator(token_list, ngrams):\n",
    "    def _get_ngrams(n):\n",
    "        return zip(*[token_list[i:] for i in range(n)])\n",
    "    for x in token_list:\n",
    "        yield x\n",
    "    for n in range(2, ngrams+1):\n",
    "        for x in _get_ngrams(n):\n",
    "            yield ' '.join(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "ouv_df = pd.read_csv(args.ouv_csv)\n",
    "word_counts = Counter()\n",
    "for data in ouv_df.data:\n",
    "    token_list = data.split(' ')\n",
    "    for word in ngrams_iterator(token_list, 5):\n",
    "        temp = 0\n",
    "        for element in word:\n",
    "            if element in string.punctuation:\n",
    "                temp = 1\n",
    "                break\n",
    "        if temp==0:\n",
    "            word_counts[word] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2353"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_new = [word for word, count in word_counts.items() if count>15 and count<600]\n",
    "len(vocab_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_inf = DataLoader(vocab_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1160,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#vocab = list(vectorizer.ouv_vocab._token_to_idx.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        vocab (list of str): the whole vocabulary\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    vectorized_token = []    \n",
    "    for token in vocab:\n",
    "        vectorized_token.append(torch.tensor(vectorizer.vectorize(token, vector_length=dataset._max_seq_length)[0]))\n",
    "    \n",
    "    result=torch.zeros((int(len(vocab)/args.batch_size)+1)*args.batch_size,len(classes))\n",
    "    #print(result.shape)\n",
    "    for i in range(int(len(vocab)/args.batch_size)+1):\n",
    "        X = torch.stack(vectorized_token[i*args.batch_size:(i+1)*args.batch_size])\n",
    "        X = torch.cat([X,torch.zeros(args.batch_size-X.shape[0], dataset._max_seq_length).long()])\n",
    "        #print(X.shape)\n",
    "        classifier.eval()\n",
    "        res = classifier(X)[0]\n",
    "        #print(res.shape)\n",
    "        result[i*args.batch_size:(i+1)*args.batch_size]=res\n",
    "        #print(result.shape)\n",
    "    result = result[:len(vocab)]\n",
    "    \n",
    "    vocab_id = result.topk(k, dim=0)[1]\n",
    "    vocab_weight = result.topk(k, dim=0)[0]\n",
    "    return vocab_id, vocab_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=10):\n",
    "    \n",
    "    vocab_id = infer_tokens_importance(vocab, classifier, vectorizer, classes, k)[0]\n",
    "    df = pd.DataFrame(columns = classes)\n",
    "    for i in range(len(classes)):\n",
    "        \n",
    "        indices = vocab_id[:,i].tolist()\n",
    "        words = pd.Series([vocab[j] for j in indices])\n",
    "        df[classes[i]] = words\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Criteria i</th>\n",
       "      <th>Criteria ii</th>\n",
       "      <th>Criteria iii</th>\n",
       "      <th>Criteria iv</th>\n",
       "      <th>Criteria v</th>\n",
       "      <th>Criteria vi</th>\n",
       "      <th>Criteria vii</th>\n",
       "      <th>Criteria viii</th>\n",
       "      <th>Criteria ix</th>\n",
       "      <th>Criteria x</th>\n",
       "      <th>Others</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>masterpiece of</td>\n",
       "      <td>interchange of</td>\n",
       "      <td>ritual</td>\n",
       "      <td>architectural and</td>\n",
       "      <td>traditional human settlement</td>\n",
       "      <td>orthodox</td>\n",
       "      <td>superlative</td>\n",
       "      <td>continental</td>\n",
       "      <td>tropical</td>\n",
       "      <td>critically</td>\n",
       "      <td>san</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>creative</td>\n",
       "      <td>exchange of</td>\n",
       "      <td>greek</td>\n",
       "      <td>architectural ensemble</td>\n",
       "      <td>traditional human</td>\n",
       "      <td>tangibly</td>\n",
       "      <td>spectacular</td>\n",
       "      <td>tectonic</td>\n",
       "      <td>biological</td>\n",
       "      <td>viable</td>\n",
       "      <td>directly and</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>creative genius</td>\n",
       "      <td>interchange of human values</td>\n",
       "      <td>buddhist</td>\n",
       "      <td>design and</td>\n",
       "      <td>farming</td>\n",
       "      <td>buddhist</td>\n",
       "      <td>stunning</td>\n",
       "      <td>ice</td>\n",
       "      <td>ecological</td>\n",
       "      <td>globally</td>\n",
       "      <td>el</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>art of</td>\n",
       "      <td>interchange of human</td>\n",
       "      <td>bears exceptional testimony to the</td>\n",
       "      <td>construction and</td>\n",
       "      <td>agricultural</td>\n",
       "      <td>directly and</td>\n",
       "      <td>scenic</td>\n",
       "      <td>oceanic</td>\n",
       "      <td>viable</td>\n",
       "      <td>bird</td>\n",
       "      <td>directly and tangibly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>art in</td>\n",
       "      <td>influence on the</td>\n",
       "      <td>funerary</td>\n",
       "      <td>architectural and artistic</td>\n",
       "      <td>traditional</td>\n",
       "      <td>arab</td>\n",
       "      <td>underwater</td>\n",
       "      <td>geological</td>\n",
       "      <td>sub</td>\n",
       "      <td>endangered</td>\n",
       "      <td>interchange of human values</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>artistic and</td>\n",
       "      <td>influence on</td>\n",
       "      <td>roman</td>\n",
       "      <td>architectural</td>\n",
       "      <td>sustainable</td>\n",
       "      <td>christian</td>\n",
       "      <td>dramatic</td>\n",
       "      <td>glacial</td>\n",
       "      <td>altitudinal</td>\n",
       "      <td>species are</td>\n",
       "      <td>tropical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>masterpieces of</td>\n",
       "      <td>influenced the</td>\n",
       "      <td>ancestral</td>\n",
       "      <td>ecclesiastical</td>\n",
       "      <td>rural</td>\n",
       "      <td>tangibly associated with</td>\n",
       "      <td>pristine</td>\n",
       "      <td>earth</td>\n",
       "      <td>biological processes</td>\n",
       "      <td>species include</td>\n",
       "      <td>directly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>artistic achievement</td>\n",
       "      <td>influenced by</td>\n",
       "      <td>byzantine</td>\n",
       "      <td>urban planning</td>\n",
       "      <td>vernacular</td>\n",
       "      <td>tangibly associated</td>\n",
       "      <td>ocean</td>\n",
       "      <td>worlds</td>\n",
       "      <td>biological diversity</td>\n",
       "      <td>critically endangered</td>\n",
       "      <td>earth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>art and</td>\n",
       "      <td>influence of</td>\n",
       "      <td>christian</td>\n",
       "      <td>urban and</td>\n",
       "      <td>urban and</td>\n",
       "      <td>ancestral</td>\n",
       "      <td>snow</td>\n",
       "      <td>pleistocene</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>species in</td>\n",
       "      <td>viable</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>art</td>\n",
       "      <td>influenced</td>\n",
       "      <td>neolithic</td>\n",
       "      <td>architecture and</td>\n",
       "      <td>wine</td>\n",
       "      <td>directly and tangibly associated with</td>\n",
       "      <td>majestic</td>\n",
       "      <td>longest</td>\n",
       "      <td>montane</td>\n",
       "      <td>wildlife</td>\n",
       "      <td>oceanic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>artistic</td>\n",
       "      <td>influence on the development</td>\n",
       "      <td>testimony to the</td>\n",
       "      <td>planning and</td>\n",
       "      <td>old</td>\n",
       "      <td>islamic</td>\n",
       "      <td>exceptional natural beauty</td>\n",
       "      <td>giant</td>\n",
       "      <td>ecological processes</td>\n",
       "      <td>endemic</td>\n",
       "      <td>directly and tangibly associated with</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>a masterpiece of</td>\n",
       "      <td>developments in</td>\n",
       "      <td>religious and</td>\n",
       "      <td>technological</td>\n",
       "      <td>land and</td>\n",
       "      <td>greek</td>\n",
       "      <td>giant</td>\n",
       "      <td>pacific</td>\n",
       "      <td>temperate</td>\n",
       "      <td>species of</td>\n",
       "      <td>interchange of human</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>human creative genius</td>\n",
       "      <td>influence on the development of</td>\n",
       "      <td>testimony to a</td>\n",
       "      <td>buildings and</td>\n",
       "      <td>residential</td>\n",
       "      <td>directly and tangibly</td>\n",
       "      <td>striking</td>\n",
       "      <td>ongoing</td>\n",
       "      <td>lowland</td>\n",
       "      <td>biodiversity</td>\n",
       "      <td>directly and tangibly associated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>architect</td>\n",
       "      <td>exerted</td>\n",
       "      <td>cultural and</td>\n",
       "      <td>ceremonial</td>\n",
       "      <td>cultural landscape of</td>\n",
       "      <td>german</td>\n",
       "      <td>huge</td>\n",
       "      <td>ocean</td>\n",
       "      <td>vascular</td>\n",
       "      <td>species have</td>\n",
       "      <td>world s</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>masterpiece</td>\n",
       "      <td>profound influence on</td>\n",
       "      <td>arab</td>\n",
       "      <td>colonial</td>\n",
       "      <td>human settlement</td>\n",
       "      <td>holy</td>\n",
       "      <td>impressive</td>\n",
       "      <td>full</td>\n",
       "      <td>ecological and</td>\n",
       "      <td>vascular</td>\n",
       "      <td>creative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>gothic</td>\n",
       "      <td>fusion of</td>\n",
       "      <td>prehistoric</td>\n",
       "      <td>technological ensemble</td>\n",
       "      <td>cultural tradition of</td>\n",
       "      <td>spiritual</td>\n",
       "      <td>barrier</td>\n",
       "      <td>fossil</td>\n",
       "      <td>relict</td>\n",
       "      <td>mammal</td>\n",
       "      <td>creative genius</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>art and architecture</td>\n",
       "      <td>interchange</td>\n",
       "      <td>ottoman</td>\n",
       "      <td>construction of</td>\n",
       "      <td>land use</td>\n",
       "      <td>directly and tangibly associated</td>\n",
       "      <td>viable</td>\n",
       "      <td>altitudinal</td>\n",
       "      <td>evergreen</td>\n",
       "      <td>globally threatened</td>\n",
       "      <td>worlds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>work of</td>\n",
       "      <td>inspired</td>\n",
       "      <td>trading</td>\n",
       "      <td>art and</td>\n",
       "      <td>living</td>\n",
       "      <td>religious and</td>\n",
       "      <td>rugged</td>\n",
       "      <td>lava</td>\n",
       "      <td>terrestrial</td>\n",
       "      <td>nesting</td>\n",
       "      <td>world heritage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>works of</td>\n",
       "      <td>exchanges</td>\n",
       "      <td>hellenistic</td>\n",
       "      <td>urban and architectural</td>\n",
       "      <td>urban and architectural</td>\n",
       "      <td>directly associated with</td>\n",
       "      <td>exceptional natural</td>\n",
       "      <td>spectacular</td>\n",
       "      <td>critically</td>\n",
       "      <td>bird species</td>\n",
       "      <td>not</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>monumental</td>\n",
       "      <td>exchange</td>\n",
       "      <td>portuguese</td>\n",
       "      <td>monastic</td>\n",
       "      <td>economic and</td>\n",
       "      <td>pilgrimage</td>\n",
       "      <td>magnificent</td>\n",
       "      <td>snow</td>\n",
       "      <td>restricted</td>\n",
       "      <td>habitat for</td>\n",
       "      <td>last</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               Criteria i                      Criteria ii  \\\n",
       "0          masterpiece of                   interchange of   \n",
       "1                creative                      exchange of   \n",
       "2         creative genius      interchange of human values   \n",
       "3                  art of             interchange of human   \n",
       "4                  art in                 influence on the   \n",
       "5            artistic and                     influence on   \n",
       "6         masterpieces of                   influenced the   \n",
       "7    artistic achievement                    influenced by   \n",
       "8                 art and                     influence of   \n",
       "9                     art                       influenced   \n",
       "10               artistic     influence on the development   \n",
       "11       a masterpiece of                  developments in   \n",
       "12  human creative genius  influence on the development of   \n",
       "13              architect                          exerted   \n",
       "14            masterpiece            profound influence on   \n",
       "15                 gothic                        fusion of   \n",
       "16   art and architecture                      interchange   \n",
       "17                work of                         inspired   \n",
       "18               works of                        exchanges   \n",
       "19             monumental                         exchange   \n",
       "\n",
       "                          Criteria iii                 Criteria iv  \\\n",
       "0                               ritual           architectural and   \n",
       "1                                greek      architectural ensemble   \n",
       "2                             buddhist                  design and   \n",
       "3   bears exceptional testimony to the            construction and   \n",
       "4                             funerary  architectural and artistic   \n",
       "5                                roman               architectural   \n",
       "6                            ancestral              ecclesiastical   \n",
       "7                            byzantine              urban planning   \n",
       "8                            christian                   urban and   \n",
       "9                            neolithic            architecture and   \n",
       "10                    testimony to the                planning and   \n",
       "11                       religious and               technological   \n",
       "12                      testimony to a               buildings and   \n",
       "13                        cultural and                  ceremonial   \n",
       "14                                arab                    colonial   \n",
       "15                         prehistoric      technological ensemble   \n",
       "16                             ottoman             construction of   \n",
       "17                             trading                     art and   \n",
       "18                         hellenistic     urban and architectural   \n",
       "19                          portuguese                    monastic   \n",
       "\n",
       "                      Criteria v                            Criteria vi  \\\n",
       "0   traditional human settlement                               orthodox   \n",
       "1              traditional human                               tangibly   \n",
       "2                        farming                               buddhist   \n",
       "3                   agricultural                           directly and   \n",
       "4                    traditional                                   arab   \n",
       "5                    sustainable                              christian   \n",
       "6                          rural               tangibly associated with   \n",
       "7                     vernacular                    tangibly associated   \n",
       "8                      urban and                              ancestral   \n",
       "9                           wine  directly and tangibly associated with   \n",
       "10                           old                                islamic   \n",
       "11                      land and                                  greek   \n",
       "12                   residential                  directly and tangibly   \n",
       "13         cultural landscape of                                 german   \n",
       "14              human settlement                                   holy   \n",
       "15         cultural tradition of                              spiritual   \n",
       "16                      land use       directly and tangibly associated   \n",
       "17                        living                          religious and   \n",
       "18       urban and architectural               directly associated with   \n",
       "19                  economic and                             pilgrimage   \n",
       "\n",
       "                  Criteria vii Criteria viii           Criteria ix  \\\n",
       "0                  superlative   continental              tropical   \n",
       "1                  spectacular      tectonic            biological   \n",
       "2                     stunning           ice            ecological   \n",
       "3                       scenic       oceanic                viable   \n",
       "4                   underwater    geological                   sub   \n",
       "5                     dramatic       glacial           altitudinal   \n",
       "6                     pristine         earth  biological processes   \n",
       "7                        ocean        worlds  biological diversity   \n",
       "8                         snow   pleistocene               ongoing   \n",
       "9                     majestic       longest               montane   \n",
       "10  exceptional natural beauty         giant  ecological processes   \n",
       "11                       giant       pacific             temperate   \n",
       "12                    striking       ongoing               lowland   \n",
       "13                        huge         ocean              vascular   \n",
       "14                  impressive          full        ecological and   \n",
       "15                     barrier        fossil                relict   \n",
       "16                      viable   altitudinal             evergreen   \n",
       "17                      rugged          lava           terrestrial   \n",
       "18         exceptional natural   spectacular            critically   \n",
       "19                 magnificent          snow            restricted   \n",
       "\n",
       "               Criteria x                                 Others  \n",
       "0              critically                                    san  \n",
       "1                  viable                           directly and  \n",
       "2                globally                                     el  \n",
       "3                    bird                  directly and tangibly  \n",
       "4              endangered            interchange of human values  \n",
       "5             species are                               tropical  \n",
       "6         species include                               directly  \n",
       "7   critically endangered                                  earth  \n",
       "8              species in                                 viable  \n",
       "9                wildlife                                oceanic  \n",
       "10                endemic  directly and tangibly associated with  \n",
       "11             species of                   interchange of human  \n",
       "12           biodiversity       directly and tangibly associated  \n",
       "13           species have                                world s  \n",
       "14               vascular                               creative  \n",
       "15                 mammal                        creative genius  \n",
       "16    globally threatened                                 worlds  \n",
       "17                nesting                         world heritage  \n",
       "18           bird species                                    not  \n",
       "19            habitat for                                   last  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_top_k_DataFrame(vocab_new, classifier, vectorizer, classes, k=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_top_k_DataFrame(vocab_new, classifier, vectorizer, classes, k=50).to_csv(args.save_dir+'top_words.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('test')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_test = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    with torch.no_grad():\n",
    "        y_pred = classifier(X)[0]\n",
    "    \n",
    "    conf_mat_test = np.add(conf_mat_test,confusion_matrix(y_target.argmax(axis=1), y_pred.argmax(axis=1)\n",
    "                                                          ,labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[19.,  1.,  4.,  9.,  0.,  3.,  0.,  0.,  0.,  0.],\n",
       "       [ 1., 51.,  2., 15.,  1.,  2.,  2.,  0.,  0.,  0.],\n",
       "       [ 1.,  1., 43., 14.,  3.,  0.,  1.,  0.,  0.,  0.],\n",
       "       [ 4.,  8.,  9., 53.,  5.,  1.,  0.,  1.,  0.,  0.],\n",
       "       [ 2.,  0.,  4.,  8., 15.,  0.,  1.,  0.,  2.,  0.],\n",
       "       [ 0.,  5., 11.,  3.,  1., 22.,  1.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  1., 33.,  3.,  0.,  4.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  4., 24.,  1.,  1.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  1.,  1.,  4., 31.,  9.],\n",
       "       [ 0.,  0.,  0.,  1.,  0.,  0.,  0.,  0., 10., 53.]])"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('val')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_val = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    with torch.no_grad():\n",
    "        y_pred = classifier(X)[0]\n",
    "    \n",
    "    conf_mat_val = np.add(conf_mat_val,confusion_matrix(y_target.argmax(axis=1), y_pred.argmax(axis=1)\n",
    "                                                       ,labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[17.,  2.,  2., 11.,  0.,  3.,  0.,  0.,  0.,  0.],\n",
       "       [ 2., 48.,  5.,  7.,  1.,  2.,  1.,  0.,  0.,  0.],\n",
       "       [ 2.,  4., 52.,  9.,  7.,  5.,  0.,  0.,  0.,  0.],\n",
       "       [ 2.,  3., 14., 53.,  4.,  3.,  1.,  2.,  0.,  0.],\n",
       "       [ 0.,  1.,  2.,  6., 14.,  0.,  1.,  0.,  1.,  0.],\n",
       "       [ 1.,  4.,  8.,  1.,  0., 26.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0., 33.,  0.,  2.,  4.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  0.,  3., 29.,  4.,  3.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  0.,  4.,  2., 25.,  8.],\n",
       "       [ 0.,  0.,  0.,  0.,  1.,  0.,  1.,  0.,  4., 60.]])"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('train')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_train = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    with torch.no_grad():\n",
    "        y_pred = classifier(X)[0]\n",
    "    \n",
    "    conf_mat_train = np.add(conf_mat_train,confusion_matrix(y_target.argmax(axis=1), y_pred.argmax(axis=1)\n",
    "                                                            ,labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[263.,   4.,   6.,  54.,   0.,   3.,   0.,   0.,   0.,   0.],\n",
       "       [  9., 502.,  13.,  90.,   6.,   8.,   0.,   0.,   0.,   0.],\n",
       "       [  6.,  17., 528.,  62.,  11.,  19.,   0.,   0.,   1.,   1.],\n",
       "       [ 14.,  45.,  52., 647.,   9.,   3.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   4.,  14.,  21., 168.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,  15.,  32.,   3.,   1., 272.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   1.,   1.,   1.,   0., 349.,   5.,   8.,  17.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,  21., 234.,   3.,   2.],\n",
       "       [  0.,   0.,   0.,   1.,   0.,   0.,   9.,   4., 295.,  60.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   3.,   1.,  26., 536.]])"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'confusion_matrix.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'baseline_confusion_matrix.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/row.sum()\n",
    "        recall = val/col.sum()\n",
    "        F1 = 2*(precision*recall)/(precision+recall)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test'] = per_class_metrics(conf_mat_test, classes[:-1])\n",
    "metrics_dict['val'] = per_class_metrics(conf_mat_val, classes[:-1])\n",
    "metrics_dict['train'] = per_class_metrics(conf_mat_train, classes[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'baseline_per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try on totally Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_ouv_csv='Data/sd_full.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_jac_k_accuracy(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "        \n",
    "    n_correct = torch.tensor([torch.tensor([y_pred_indices[j][i] in y_target_indices[j] for i in range(k)]).sum()>0 \n",
    "                              for j in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_jac_1_accuracy(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    y_pred_indices = y_pred.topk(1, dim=1)[1]\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "        \n",
    "    n_correct = torch.tensor([torch.tensor([y_pred_indices[j] in y_target_indices[j] for i in range(k)]).sum()>0 \n",
    "                              for j in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed: 153.16160893440247\n"
     ]
    }
   ],
   "source": [
    "with Timer():\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    loss_func = cross_entropy\n",
    "    train_state = make_train_state(args)\n",
    "    dataset = OuvDataset.load_dataset_and_load_vectorizer(new_ouv_csv, args.vectorizer_file, vocab=vocab)\n",
    "\n",
    "    dataset.set_split('val')\n",
    "    verbose=False\n",
    "    try:\n",
    "        # Iterate over training dataset\n",
    "\n",
    "        # setup: batch generator, set loss and acc to 0, set train mode on\n",
    "        dataset.set_split('val')\n",
    "        batch_generator = generate_batches(dataset, \n",
    "                                           batch_size=args.batch_size, \n",
    "                                           device=args.device)\n",
    "        running_loss = 0.0\n",
    "        running_1_acc = 0.0\n",
    "        running_k_acc = 0.0\n",
    "        running_k_jac = 0.0\n",
    "        classifier.eval()\n",
    "\n",
    "        for batch_index, batch_dict in enumerate(batch_generator):\n",
    "\n",
    "            # step 2. get the data compute fuzzy labels\n",
    "            X = batch_dict['x_data']\n",
    "\n",
    "            y_target = batch_dict['y_target']\n",
    "            y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "            Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                                    how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "            # step 3. compute the output\n",
    "            with torch.no_grad():\n",
    "                y_pred = classifier(X)[0]\n",
    "\n",
    "            # step 4. compute the loss\n",
    "            loss = loss_func(y_pred, Y)\n",
    "            loss_t = loss.item()\n",
    "            running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "            # -----------------------------------------\n",
    "            # compute the accuracies\n",
    "            acc_1_t = compute_jac_1_accuracy(y_pred, y_target)\n",
    "            acc_k_t = compute_jac_k_accuracy(y_pred, y_target, args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_pred, y_target, len(classes))\n",
    "\n",
    "            running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)\n",
    "            running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)\n",
    "            running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)\n",
    "\n",
    "            # update bar\n",
    "            if verbose:\n",
    "                val_bar.set_postfix(loss=running_loss, \n",
    "                                acc_1=running_1_acc,\n",
    "                                acc_k=running_k_acc,\n",
    "                                jac_k=running_k_jac,\n",
    "                                epoch=epoch_index)\n",
    "                val_bar.update()\n",
    "\n",
    "        train_state['val_loss'].append(running_loss)\n",
    "        train_state['val_1_acc'].append(running_1_acc)\n",
    "        train_state['val_k_acc'].append(running_k_acc)\n",
    "        train_state['val_k_jac'].append(running_k_jac)\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 0,\n",
       " 'early_stopping_best_k_acc_val': 0,\n",
       " 'learning_rate': 0.02,\n",
       " 'epoch_index': 0,\n",
       " 'train_loss': [],\n",
       " 'train_1_acc': [],\n",
       " 'train_k_acc': [],\n",
       " 'train_k_jac': [],\n",
       " 'val_loss': [2.25682575315162],\n",
       " 'val_1_acc': [70.65104166666666],\n",
       " 'val_k_acc': [96.22395833333331],\n",
       " 'val_k_jac': [0.3615849067767461],\n",
       " 'test_loss': -1,\n",
       " 'test_1_acc': -1,\n",
       " 'test_k_acc': -1,\n",
       " 'test_k_jac': -1,\n",
       " 'model_filename': 'model_storage/ulmfit/model.pth'}"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#LS Model\n",
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 0,\n",
       " 'early_stopping_best_k_acc_val': 0,\n",
       " 'learning_rate': 0.02,\n",
       " 'epoch_index': 0,\n",
       " 'train_loss': [],\n",
       " 'train_1_acc': [],\n",
       " 'train_k_acc': [],\n",
       " 'train_k_jac': [],\n",
       " 'val_loss': [2.3552627651099343],\n",
       " 'val_1_acc': [70.20833333333333],\n",
       " 'val_k_acc': [96.1458333333333],\n",
       " 'val_k_jac': [0.36160394102334986],\n",
       " 'test_loss': -1,\n",
       " 'test_1_acc': -1,\n",
       " 'test_k_acc': -1,\n",
       " 'test_k_jac': -1,\n",
       " 'model_filename': 'model_storage/ulmfit/model.pth'}"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Baseline\n",
    "train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting scores for Social Media Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_df = pd.read_csv(args.sm_csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def record_ouv(line, classifier=classifier, vectorizer=vectorizer, classes=classes, k=11):\n",
    "    predictions = predict_rating(line['processed'], classifier, vectorizer, classes, k)\n",
    "    for key, value in predictions:\n",
    "        line[key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_dict = {}\n",
    "for i in range(len(sm_df)):\n",
    "    new_dict[i]={}\n",
    "    line = sm_df.iloc[i].copy()\n",
    "    predictions = predict_rating(line['processed'], classifier, vectorizer, classes, k=11)\n",
    "    for key, value in predictions:\n",
    "        new_dict[i][key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "des_dict = {}\n",
    "for i in range(len(des_df)):\n",
    "    des_dict[i]={}\n",
    "    line = des_df.iloc[i].copy()\n",
    "    predictions = predict_rating(line['data'], classifier, vectorizer, classes, k=11)\n",
    "    for key, value in predictions:\n",
    "        des_dict[i][key] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Criteria i</th>\n",
       "      <th>Criteria ii</th>\n",
       "      <th>Criteria iii</th>\n",
       "      <th>Criteria iv</th>\n",
       "      <th>Criteria v</th>\n",
       "      <th>Criteria vi</th>\n",
       "      <th>Criteria vii</th>\n",
       "      <th>Criteria viii</th>\n",
       "      <th>Criteria ix</th>\n",
       "      <th>Criteria x</th>\n",
       "      <th>Others</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.071549</td>\n",
       "      <td>0.066268</td>\n",
       "      <td>0.249283</td>\n",
       "      <td>0.548340</td>\n",
       "      <td>0.036276</td>\n",
       "      <td>0.020212</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.003252</td>\n",
       "      <td>0.001464</td>\n",
       "      <td>0.001303</td>\n",
       "      <td>0.000158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.024733</td>\n",
       "      <td>0.053539</td>\n",
       "      <td>0.346672</td>\n",
       "      <td>0.127028</td>\n",
       "      <td>0.290864</td>\n",
       "      <td>0.091671</td>\n",
       "      <td>0.035114</td>\n",
       "      <td>0.003927</td>\n",
       "      <td>0.012178</td>\n",
       "      <td>0.014135</td>\n",
       "      <td>0.000138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.355913</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>0.111262</td>\n",
       "      <td>0.253084</td>\n",
       "      <td>0.005347</td>\n",
       "      <td>0.156860</td>\n",
       "      <td>0.001084</td>\n",
       "      <td>0.001213</td>\n",
       "      <td>0.000570</td>\n",
       "      <td>0.000786</td>\n",
       "      <td>0.000157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.024738</td>\n",
       "      <td>0.057970</td>\n",
       "      <td>0.406587</td>\n",
       "      <td>0.216162</td>\n",
       "      <td>0.114056</td>\n",
       "      <td>0.164718</td>\n",
       "      <td>0.004133</td>\n",
       "      <td>0.004567</td>\n",
       "      <td>0.004407</td>\n",
       "      <td>0.002570</td>\n",
       "      <td>0.000091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.292953</td>\n",
       "      <td>0.252021</td>\n",
       "      <td>0.077721</td>\n",
       "      <td>0.250269</td>\n",
       "      <td>0.026984</td>\n",
       "      <td>0.065994</td>\n",
       "      <td>0.012496</td>\n",
       "      <td>0.008707</td>\n",
       "      <td>0.006200</td>\n",
       "      <td>0.006277</td>\n",
       "      <td>0.000378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1127</th>\n",
       "      <td>0.086426</td>\n",
       "      <td>0.118610</td>\n",
       "      <td>0.122076</td>\n",
       "      <td>0.571597</td>\n",
       "      <td>0.039693</td>\n",
       "      <td>0.055296</td>\n",
       "      <td>0.000902</td>\n",
       "      <td>0.001556</td>\n",
       "      <td>0.001775</td>\n",
       "      <td>0.001837</td>\n",
       "      <td>0.000231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>0.837418</td>\n",
       "      <td>0.077769</td>\n",
       "      <td>0.021447</td>\n",
       "      <td>0.056075</td>\n",
       "      <td>0.000523</td>\n",
       "      <td>0.006144</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>0.000165</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.000343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1129</th>\n",
       "      <td>0.084014</td>\n",
       "      <td>0.045307</td>\n",
       "      <td>0.220497</td>\n",
       "      <td>0.112530</td>\n",
       "      <td>0.224691</td>\n",
       "      <td>0.270244</td>\n",
       "      <td>0.027755</td>\n",
       "      <td>0.003435</td>\n",
       "      <td>0.004544</td>\n",
       "      <td>0.006103</td>\n",
       "      <td>0.000880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1130</th>\n",
       "      <td>0.311289</td>\n",
       "      <td>0.252788</td>\n",
       "      <td>0.145648</td>\n",
       "      <td>0.140554</td>\n",
       "      <td>0.021969</td>\n",
       "      <td>0.115754</td>\n",
       "      <td>0.004693</td>\n",
       "      <td>0.003648</td>\n",
       "      <td>0.001502</td>\n",
       "      <td>0.001392</td>\n",
       "      <td>0.000765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1131</th>\n",
       "      <td>0.058580</td>\n",
       "      <td>0.193856</td>\n",
       "      <td>0.150152</td>\n",
       "      <td>0.149800</td>\n",
       "      <td>0.018335</td>\n",
       "      <td>0.345900</td>\n",
       "      <td>0.015731</td>\n",
       "      <td>0.021735</td>\n",
       "      <td>0.013992</td>\n",
       "      <td>0.031084</td>\n",
       "      <td>0.000834</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1132 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Criteria i  Criteria ii  Criteria iii  Criteria iv  Criteria v  \\\n",
       "0       0.071549     0.066268      0.249283     0.548340    0.036276   \n",
       "1       0.024733     0.053539      0.346672     0.127028    0.290864   \n",
       "2       0.355913     0.113725      0.111262     0.253084    0.005347   \n",
       "3       0.024738     0.057970      0.406587     0.216162    0.114056   \n",
       "4       0.292953     0.252021      0.077721     0.250269    0.026984   \n",
       "...          ...          ...           ...          ...         ...   \n",
       "1127    0.086426     0.118610      0.122076     0.571597    0.039693   \n",
       "1128    0.837418     0.077769      0.021447     0.056075    0.000523   \n",
       "1129    0.084014     0.045307      0.220497     0.112530    0.224691   \n",
       "1130    0.311289     0.252788      0.145648     0.140554    0.021969   \n",
       "1131    0.058580     0.193856      0.150152     0.149800    0.018335   \n",
       "\n",
       "      Criteria vi  Criteria vii  Criteria viii  Criteria ix  Criteria x  \\\n",
       "0        0.020212      0.001894       0.003252     0.001464    0.001303   \n",
       "1        0.091671      0.035114       0.003927     0.012178    0.014135   \n",
       "2        0.156860      0.001084       0.001213     0.000570    0.000786   \n",
       "3        0.164718      0.004133       0.004567     0.004407    0.002570   \n",
       "4        0.065994      0.012496       0.008707     0.006200    0.006277   \n",
       "...           ...           ...            ...          ...         ...   \n",
       "1127     0.055296      0.000902       0.001556     0.001775    0.001837   \n",
       "1128     0.006144      0.000077       0.000165     0.000018    0.000021   \n",
       "1129     0.270244      0.027755       0.003435     0.004544    0.006103   \n",
       "1130     0.115754      0.004693       0.003648     0.001502    0.001392   \n",
       "1131     0.345900      0.015731       0.021735     0.013992    0.031084   \n",
       "\n",
       "        Others  \n",
       "0     0.000158  \n",
       "1     0.000138  \n",
       "2     0.000157  \n",
       "3     0.000091  \n",
       "4     0.000378  \n",
       "...        ...  \n",
       "1127  0.000231  \n",
       "1128  0.000343  \n",
       "1129  0.000880  \n",
       "1130  0.000765  \n",
       "1131  0.000834  \n",
       "\n",
       "[1132 rows x 11 columns]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_df = pd.DataFrame(new_dict).reindex(classes).T\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = score_df[classes].to_numpy().argsort()[:, :-4:-1]\n",
    "c = np.array(classes)[a]\n",
    "d = score_df[classes].to_numpy()[np.arange(a.shape[0])[:, None], a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.DataFrame(c).rename(columns=lambda x : f'max_{x+1}_col')\n",
    "df2 = pd.DataFrame(d).rename(columns=lambda x : f'max_{x+1}_val')\n",
    "c = score_df.columns.tolist() + [y for x in zip(df2.columns, df1.columns) for y in x]\n",
    "\n",
    "score_df = pd.concat([score_df, df1, df2], axis=1).reindex(c, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Criteria i</th>\n",
       "      <th>Criteria ii</th>\n",
       "      <th>Criteria iii</th>\n",
       "      <th>Criteria iv</th>\n",
       "      <th>Criteria v</th>\n",
       "      <th>Criteria vi</th>\n",
       "      <th>Criteria vii</th>\n",
       "      <th>Criteria viii</th>\n",
       "      <th>Criteria ix</th>\n",
       "      <th>Criteria x</th>\n",
       "      <th>Others</th>\n",
       "      <th>max_1_val</th>\n",
       "      <th>max_1_col</th>\n",
       "      <th>max_2_val</th>\n",
       "      <th>max_2_col</th>\n",
       "      <th>max_3_val</th>\n",
       "      <th>max_3_col</th>\n",
       "      <th>max_1</th>\n",
       "      <th>max_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.071549</td>\n",
       "      <td>0.066268</td>\n",
       "      <td>0.249283</td>\n",
       "      <td>0.548340</td>\n",
       "      <td>0.036276</td>\n",
       "      <td>0.020212</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.003252</td>\n",
       "      <td>0.001464</td>\n",
       "      <td>0.001303</td>\n",
       "      <td>0.000158</td>\n",
       "      <td>0.548340</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.249283</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.071549</td>\n",
       "      <td>Criteria i</td>\n",
       "      <td>0.548340</td>\n",
       "      <td>0.869173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.024733</td>\n",
       "      <td>0.053539</td>\n",
       "      <td>0.346672</td>\n",
       "      <td>0.127028</td>\n",
       "      <td>0.290864</td>\n",
       "      <td>0.091671</td>\n",
       "      <td>0.035114</td>\n",
       "      <td>0.003927</td>\n",
       "      <td>0.012178</td>\n",
       "      <td>0.014135</td>\n",
       "      <td>0.000138</td>\n",
       "      <td>0.346672</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.290864</td>\n",
       "      <td>Criteria v</td>\n",
       "      <td>0.127028</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.346672</td>\n",
       "      <td>0.764564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.355913</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>0.111262</td>\n",
       "      <td>0.253084</td>\n",
       "      <td>0.005347</td>\n",
       "      <td>0.156860</td>\n",
       "      <td>0.001084</td>\n",
       "      <td>0.001213</td>\n",
       "      <td>0.000570</td>\n",
       "      <td>0.000786</td>\n",
       "      <td>0.000157</td>\n",
       "      <td>0.355913</td>\n",
       "      <td>Criteria i</td>\n",
       "      <td>0.253084</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.156860</td>\n",
       "      <td>Criteria vi</td>\n",
       "      <td>0.355913</td>\n",
       "      <td>0.765857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.024738</td>\n",
       "      <td>0.057970</td>\n",
       "      <td>0.406587</td>\n",
       "      <td>0.216162</td>\n",
       "      <td>0.114056</td>\n",
       "      <td>0.164718</td>\n",
       "      <td>0.004133</td>\n",
       "      <td>0.004567</td>\n",
       "      <td>0.004407</td>\n",
       "      <td>0.002570</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.406587</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.216162</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.164718</td>\n",
       "      <td>Criteria vi</td>\n",
       "      <td>0.406587</td>\n",
       "      <td>0.787467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.292953</td>\n",
       "      <td>0.252021</td>\n",
       "      <td>0.077721</td>\n",
       "      <td>0.250269</td>\n",
       "      <td>0.026984</td>\n",
       "      <td>0.065994</td>\n",
       "      <td>0.012496</td>\n",
       "      <td>0.008707</td>\n",
       "      <td>0.006200</td>\n",
       "      <td>0.006277</td>\n",
       "      <td>0.000378</td>\n",
       "      <td>0.292953</td>\n",
       "      <td>Criteria i</td>\n",
       "      <td>0.252021</td>\n",
       "      <td>Criteria ii</td>\n",
       "      <td>0.250269</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.292953</td>\n",
       "      <td>0.795244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1127</th>\n",
       "      <td>0.086426</td>\n",
       "      <td>0.118610</td>\n",
       "      <td>0.122076</td>\n",
       "      <td>0.571597</td>\n",
       "      <td>0.039693</td>\n",
       "      <td>0.055296</td>\n",
       "      <td>0.000902</td>\n",
       "      <td>0.001556</td>\n",
       "      <td>0.001775</td>\n",
       "      <td>0.001837</td>\n",
       "      <td>0.000231</td>\n",
       "      <td>0.571597</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.122076</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.118610</td>\n",
       "      <td>Criteria ii</td>\n",
       "      <td>0.571597</td>\n",
       "      <td>0.812284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>0.837418</td>\n",
       "      <td>0.077769</td>\n",
       "      <td>0.021447</td>\n",
       "      <td>0.056075</td>\n",
       "      <td>0.000523</td>\n",
       "      <td>0.006144</td>\n",
       "      <td>0.000077</td>\n",
       "      <td>0.000165</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.000343</td>\n",
       "      <td>0.837418</td>\n",
       "      <td>Criteria i</td>\n",
       "      <td>0.077769</td>\n",
       "      <td>Criteria ii</td>\n",
       "      <td>0.056075</td>\n",
       "      <td>Criteria iv</td>\n",
       "      <td>0.837418</td>\n",
       "      <td>0.971262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1129</th>\n",
       "      <td>0.084014</td>\n",
       "      <td>0.045307</td>\n",
       "      <td>0.220497</td>\n",
       "      <td>0.112530</td>\n",
       "      <td>0.224691</td>\n",
       "      <td>0.270244</td>\n",
       "      <td>0.027755</td>\n",
       "      <td>0.003435</td>\n",
       "      <td>0.004544</td>\n",
       "      <td>0.006103</td>\n",
       "      <td>0.000880</td>\n",
       "      <td>0.270244</td>\n",
       "      <td>Criteria vi</td>\n",
       "      <td>0.224691</td>\n",
       "      <td>Criteria v</td>\n",
       "      <td>0.220497</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.270244</td>\n",
       "      <td>0.715433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1130</th>\n",
       "      <td>0.311289</td>\n",
       "      <td>0.252788</td>\n",
       "      <td>0.145648</td>\n",
       "      <td>0.140554</td>\n",
       "      <td>0.021969</td>\n",
       "      <td>0.115754</td>\n",
       "      <td>0.004693</td>\n",
       "      <td>0.003648</td>\n",
       "      <td>0.001502</td>\n",
       "      <td>0.001392</td>\n",
       "      <td>0.000765</td>\n",
       "      <td>0.311289</td>\n",
       "      <td>Criteria i</td>\n",
       "      <td>0.252788</td>\n",
       "      <td>Criteria ii</td>\n",
       "      <td>0.145648</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.311289</td>\n",
       "      <td>0.709724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1131</th>\n",
       "      <td>0.058580</td>\n",
       "      <td>0.193856</td>\n",
       "      <td>0.150152</td>\n",
       "      <td>0.149800</td>\n",
       "      <td>0.018335</td>\n",
       "      <td>0.345900</td>\n",
       "      <td>0.015731</td>\n",
       "      <td>0.021735</td>\n",
       "      <td>0.013992</td>\n",
       "      <td>0.031084</td>\n",
       "      <td>0.000834</td>\n",
       "      <td>0.345900</td>\n",
       "      <td>Criteria vi</td>\n",
       "      <td>0.193856</td>\n",
       "      <td>Criteria ii</td>\n",
       "      <td>0.150152</td>\n",
       "      <td>Criteria iii</td>\n",
       "      <td>0.345900</td>\n",
       "      <td>0.689909</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1132 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Criteria i  Criteria ii  Criteria iii  Criteria iv  Criteria v  \\\n",
       "0       0.071549     0.066268      0.249283     0.548340    0.036276   \n",
       "1       0.024733     0.053539      0.346672     0.127028    0.290864   \n",
       "2       0.355913     0.113725      0.111262     0.253084    0.005347   \n",
       "3       0.024738     0.057970      0.406587     0.216162    0.114056   \n",
       "4       0.292953     0.252021      0.077721     0.250269    0.026984   \n",
       "...          ...          ...           ...          ...         ...   \n",
       "1127    0.086426     0.118610      0.122076     0.571597    0.039693   \n",
       "1128    0.837418     0.077769      0.021447     0.056075    0.000523   \n",
       "1129    0.084014     0.045307      0.220497     0.112530    0.224691   \n",
       "1130    0.311289     0.252788      0.145648     0.140554    0.021969   \n",
       "1131    0.058580     0.193856      0.150152     0.149800    0.018335   \n",
       "\n",
       "      Criteria vi  Criteria vii  Criteria viii  Criteria ix  Criteria x  \\\n",
       "0        0.020212      0.001894       0.003252     0.001464    0.001303   \n",
       "1        0.091671      0.035114       0.003927     0.012178    0.014135   \n",
       "2        0.156860      0.001084       0.001213     0.000570    0.000786   \n",
       "3        0.164718      0.004133       0.004567     0.004407    0.002570   \n",
       "4        0.065994      0.012496       0.008707     0.006200    0.006277   \n",
       "...           ...           ...            ...          ...         ...   \n",
       "1127     0.055296      0.000902       0.001556     0.001775    0.001837   \n",
       "1128     0.006144      0.000077       0.000165     0.000018    0.000021   \n",
       "1129     0.270244      0.027755       0.003435     0.004544    0.006103   \n",
       "1130     0.115754      0.004693       0.003648     0.001502    0.001392   \n",
       "1131     0.345900      0.015731       0.021735     0.013992    0.031084   \n",
       "\n",
       "        Others  max_1_val     max_1_col  max_2_val     max_2_col  max_3_val  \\\n",
       "0     0.000158   0.548340   Criteria iv   0.249283  Criteria iii   0.071549   \n",
       "1     0.000138   0.346672  Criteria iii   0.290864    Criteria v   0.127028   \n",
       "2     0.000157   0.355913    Criteria i   0.253084   Criteria iv   0.156860   \n",
       "3     0.000091   0.406587  Criteria iii   0.216162   Criteria iv   0.164718   \n",
       "4     0.000378   0.292953    Criteria i   0.252021   Criteria ii   0.250269   \n",
       "...        ...        ...           ...        ...           ...        ...   \n",
       "1127  0.000231   0.571597   Criteria iv   0.122076  Criteria iii   0.118610   \n",
       "1128  0.000343   0.837418    Criteria i   0.077769   Criteria ii   0.056075   \n",
       "1129  0.000880   0.270244   Criteria vi   0.224691    Criteria v   0.220497   \n",
       "1130  0.000765   0.311289    Criteria i   0.252788   Criteria ii   0.145648   \n",
       "1131  0.000834   0.345900   Criteria vi   0.193856   Criteria ii   0.150152   \n",
       "\n",
       "         max_3_col     max_1     max_3  \n",
       "0       Criteria i  0.548340  0.869173  \n",
       "1      Criteria iv  0.346672  0.764564  \n",
       "2      Criteria vi  0.355913  0.765857  \n",
       "3      Criteria vi  0.406587  0.787467  \n",
       "4      Criteria iv  0.292953  0.795244  \n",
       "...            ...       ...       ...  \n",
       "1127   Criteria ii  0.571597  0.812284  \n",
       "1128   Criteria iv  0.837418  0.971262  \n",
       "1129  Criteria iii  0.270244  0.715433  \n",
       "1130  Criteria iii  0.311289  0.709724  \n",
       "1131  Criteria iii  0.345900  0.689909  \n",
       "\n",
       "[1132 rows x 19 columns]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score_df['max_1'] = score_df['max_1_val']\n",
    "score_df['max_3'] = score_df['max_1_val']+score_df['max_2_val']+score_df['max_3_val']\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "513"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(score_df[(score_df.max_3>0.8) & (score_df.max_1>1./3)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD7CAYAAACPDORaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAX60lEQVR4nO3df2xVd+H/8deFQrGGbcbdu8qPQESESRUITGFjRcwoSHtBClEKAxdcqGEjARGE0lHdZMPZiCGAglmy6Ei2rlpAMsqPMJqQGpFrBC9DxhwF+dFyC+NCf67tfX//4NP7pbS09zd3b56PZEnvOffc9+v+4HXvzr3nfRzGGCMAgJV63e8AAID4oeQBwGKUPABYjJIHAItR8gBgsZT7HaBdU1OTvF6vnE6nevfufb/jAMBnQltbm3w+nzIyMtSvX79O65Om5L1erxYsWHC/YwDAZ9LOnTs1fvz4TsuTpuSdTqek20HT09MTPr7X61VGRkbCxw0V+aJDvuiQLzrxzFddXa0FCxYEO/RuSVPy7bto0tPTNWjQoISPX1NTc1/GDRX5okO+6JAvOonId6/d3HzxCgAWo+QBwGKUPABYjJIHAItR8gBgMUoeACxGyQOAxUL6nfyWLVu0b98+SdLkyZO1evVqrV27Vh6PR5/73OckSS+++KKmTp2q06dPa926daqvr9f48eP1i1/8QikpSfNz/Li41fCpmlva4jqGc8BQ1fobOyxL7dNb/dP6xnVcAJ9tPbZvZWWljh49qrKyMjkcDj3//PM6ePCgvF6v3nrrLblcrg7XX7VqlX75y19qzJgxKigoUElJiebPnx+3O5AMmlvatK30RFzH8N/w6+FHHu6wbOnc0eof11EBfNb1uLvG6XRqzZo16tu3r/r06aNhw4bp8uXLunz5sgoKCuR2u7V582YFAgFdunRJTU1NGjNmjCQpNzdX5eXl8b4PAIB76PGT/PDhw4N/V1VVad++fdq5c6eOHTumoqIi9e/fX/n5+SotLdXw4cM7zJ/gdDpVU1MTViCv1xv2NrHi8Xgi2s45YKj8N/wxTtPZ3WM0NDTI89EHcR83VJE+folCvuiQLzrxyufz+bpdH/LO8rNnzyo/P1+rV6/Wl7/8ZW3dujW4buHChdq1a5eGDRsmh8MRXG6M6XA5FBkZGfdlDgqPx6Nx48ZFtG2tv7HTrpRY62p3TVpaWsSZYy2axy8RyBcd8kUnnvkuXrzY7fqQfl3j8Xj03HPPaeXKlZo9e7bOnDmj/fv3B9cbY5SSkqL09PQO7yq1tbWd9tkDABKnx5K/cuWKXnjhBRUXFys7O1vS7VJ/9dVX5ff71dLSonfeeUdTp07VwIEDlZqaGvzfkt27dyszMzO+9wAAcE897q5544031NzcrI0bNwaXzZs3T0uWLFFeXp5aW1uVlZWlnJwcSVJxcbEKCwtVV1enUaNGadGiRfFLDwDoVo8lX1hYqMLCwi7XdXUmp5EjR6q0tDT6ZACAqNl9lJLlHFKnA6QShQOxgM8GSv4zrKUtoB1l/74vY3MgFvDZwNw1AGAxSh4ALEbJA4DFKHkAsBglDwAWo+QBwGKUPABYjJIHAItR8gBgMUoeACxGyQOAxSh5ALAYJQ8AFqPkAcBilDwAWIySBwCLUfIAYDHODIWI3H3qQeeAoQk5FSGnHQTCQ8kjInefetB/w6+HH3k47uNy2kEgPOyuAQCLUfIAYDFKHgAsRskDgMUoeQCwGCUPABaj5AHAYpQ8AFiMkgcAi1HyAGAxSh4ALBZSyW/ZskXZ2dnKzs7W66+/LkmqrKyU2+1WVlaWNm3aFLzu6dOnlZubq2nTpmndunVqbW2NT3IAQI96LPnKykodPXpUZWVl2rVrl06dOqW9e/eqoKBA27Zt03vvvSev16uKigpJ0qpVq7R+/Xrt379fxhiVlJTE/U4AALrWY8k7nU6tWbNGffv2VZ8+fTRs2DBVVVVpyJAhGjx4sFJSUuR2u1VeXq5Lly6pqalJY8aMkSTl5uaqvLw83vcBAHAPPU41PHz48ODfVVVV2rdvn5599lk5nc7gcpfLpZqaGl29erXDcqfTqZqamrACeb3esLeJFY/HE9F2zgFD5b/hj3Gazu4ewwQCCRm3K12NnYgsDQ0N8nz0QUTbRvr8Jgr5ovOg5vP5fN2uD3k++bNnzyo/P1+rV69W7969VVVVFVxnjJHD4VAgEJDD4ei0PBwZGRkaNGhQWNvEgsfj0bhx4yLattbfGPe51Luar93Rq1dC5nDvyt1jJ2o++bS0tIiep2ie30QgX3Qe5HwXL17sdn1IX7x6PB4999xzWrlypWbPnq309PQO7x4+n08ul6vT8traWrlcrgijAwCi1WPJX7lyRS+88IKKi4uVnZ0tSRo9erTOnTun8+fPq62tTXv37lVmZqYGDhyo1NTU4P+W7N69W5mZmfG9BwCAe+pxd80bb7yh5uZmbdy4Mbhs3rx52rhxo5YtW6bm5mZNnjxZ06dPlyQVFxersLBQdXV1GjVqlBYtWhS/9ACAbvVY8oWFhSosLOxy3Z49ezotGzlypEpLS6NPBgCIGke8AoDFKHkAsBglDwAWo+QBwGKUPABYjJIHAItR8gBgMUoeACxGyQOAxSh5ALAYJQ8AFqPkAcBilDwAWIySBwCLUfIAYDFKHgAsRskDgMUoeQCwGCUPABaj5AHAYpQ8AFiMkgcAi1HyAGAxSh4ALEbJA4DFKHkAsBglDwAWo+QBwGKUPABYjJIHAItR8gBgsZBKvq6uTjk5Obp48aIkae3atcrKytKsWbM0a9YsHTx4UJJ0+vRp5ebmatq0aVq3bp1aW1vjlxwA0KMeS/7EiRPKy8tTVVVVcJnX69Vbb72l3bt3a/fu3Zo6daokadWqVVq/fr32798vY4xKSkriFhwA0LMeS76kpERFRUVyuVySpMbGRl2+fFkFBQVyu93avHmzAoGALl26pKamJo0ZM0aSlJubq/Ly8riGBwB0L6WnK2zYsKHD5draWk2YMEFFRUXq37+/8vPzVVpaquHDh8vpdAav53Q6VVNTE/vEAICQ9Vjydxs8eLC2bt0avLxw4ULt2rVLw4YNk8PhCC43xnS4HCqv13vf3hw8Hk9E2zkHDJX/hj/GaTq7ewwTCCRk3K50NXYisjQ0NMjz0QcRbRvp85so5IvOg5rP5/N1uz7skj9z5oyqqqo0bdo0SbfLPCUlRenp6R0Gq62tDe7iCUdGRoYGDRoU9nbR8ng8GjduXETb1vob9fAjD8c4UUf+G/5OYzh69Yr7uPdy99hd5YuHtLS0iJ6naJ7fRCBfdB7kfO0/iLmXsH9CaYzRq6++Kr/fr5aWFr3zzjuaOnWqBg4cqNTU1OC71e7du5WZmRlZagBATIT9SX7kyJFasmSJ8vLy1NraqqysLOXk5EiSiouLVVhYqLq6Oo0aNUqLFi2KeWAAQOhCLvnDhw8H/16wYIEWLFjQ6TojR45UaWlpbJIBAKLGEa8AYDFKHgAsRskDgMUoeQCwGCUPABaj5AHAYpQ8AFgs7IOhgPvJodvTSITLOWBoRNu1S+3TW/3T+ka8PXC/UPL4TGlpC2hH2b/D3i7auXWWzh2t/hFvDdw/7K4BAItR8gBgMUoeACxGyQOAxSh5ALAYJQ8AFqPkAcBilDwAWIySBwCLUfIAYDFKHgAsRskDgMUoeQCwGCUPABaj5AHAYpQ8AFiMkgcAi1HyAGAxSh4ALEbJA4DFKHkAsBglDwAWo+QBwGIhlXxdXZ1ycnJ08eJFSVJlZaXcbreysrK0adOm4PVOnz6t3NxcTZs2TevWrVNra2t8UgMAQtJjyZ84cUJ5eXmqqqqSJDU1NamgoEDbtm3Te++9J6/Xq4qKCknSqlWrtH79eu3fv1/GGJWUlMQ1PACgez2WfElJiYqKiuRyuSRJJ0+e1JAhQzR48GClpKTI7XarvLxcly5dUlNTk8aMGSNJys3NVXl5eVzDAwC6l9LTFTZs2NDh8tWrV+V0OoOXXS6XampqOi13Op2qqamJYVQAQLh6LPm7BQIBORyO4GVjjBwOxz2Xh8vr9d63NwePxxPRds4BQ+W/4Y9xms7uHsMEAgkZtytdjZ2ILNHc52jyNTQ0yPPRBxFvH4pIX3+JQr7oxCufz+frdn3YJZ+ent7hRn0+n1wuV6fltbW1wV084cjIyNCgQYPC3i5aHo9H48aNi2jbWn+jHn7k4Rgn6sh/w99pDEevXnEf917uHrurfIkYN1TR5ktLS4v49RGKaF5/iUC+6MQzX/sPYu4l7J9Qjh49WufOndP58+fV1tamvXv3KjMzUwMHDlRqamrw3Wr37t3KzMyMLDUAICbC/iSfmpqqjRs3atmyZWpubtbkyZM1ffp0SVJxcbEKCwtVV1enUaNGadGiRTEPDAAIXcglf/jw4eDfEydO1J49ezpdZ+TIkSotLY1NMgBA1DjiFQAsRskDgMUoeQCwGCUPABaj5AHAYmH/hBJ4EDl0+6C3eHEOGHrP20/t01v90/rGbWzYjZIHQtDSFtCOsn/H7fa7OyJ36dzR6h+3kWE7dtcAgMUoeQCwGCUPABaj5AHAYpQ8AFiMkgcAi1HyAGAxSh4ALEbJA4DFKHkAsBglDwAWo+QBwGKUPABYjJIHAItR8gBgMUoeACxGyQOAxSh5ALAYJQ8AFuMcrwC6dKvhUzW3tEnq/kTj8cDJy2OHkgfQpeaWNm0rPSGp+xONxwMnL48ddtcAgMUoeQCwGCUPABaj5AHAYlF98bpw4UJdv35dKSm3b+bll19WfX29XnvtNTU3N+u73/2uVqxYEZOgAIDwRVzyxhhVVVXp/fffD5Z8U1OTpk+frj/96U/60pe+pPz8fFVUVGjy5MkxCwwACF3EJf/xxx9LkhYvXqwbN27o+9//vr761a9qyJAhGjx4sCTJ7XarvLyckgeA+yTiffI3b97UxIkTtXXrVr355pt6++23dfnyZTmdzuB1XC6XampqYhIUABC+iD/Jjx07VmPHjg1enjt3rjZv3qxx48YFlxlj5HA4wrpdr9d7394YPB5PRNs5BwyV/4Y/xmk6u3sMEwgkZNyudDV2IrJEc5+jyZeIx/pet9/Q0CDPRx/Edeyu3P26TuRrLZL7HOm/30SJVz6fz9ft+ohL/vjx42ppadHEiRMl3S70gQMHdhjQ5/PJ5XKFdbsZGRkaNGhQpLEi5vF4OrxBhaPW3xj3owG7OuLQ0atXQo9C7G7sRB0RGel9jjZfvB/r7vKlpaVF/NqMxp2v60Qf8RrufY7m328ixDPfxYsXu10f8e6aW7du6fXXX1dzc7Pq6upUVlamn/zkJzp37pzOnz+vtrY27d27V5mZmZEOAQCIUsSf5KdMmaITJ07oe9/7ngKBgObPn6+xY8dq48aNWrZsmZqbmzV58mRNnz49lnkBAGGI6nfyy5cv1/Llyzssmzhxovbs2RPNzUbkzhnzIhHNLHuBNhPxuEBPHFJCZ4Bsx+vaDtbMQnnnjHmRiGaf45LZX494XKAnLW0B7Sj7d8LH5XVtB6Y1AACLUfIAYDFKHgAsRskDgMUoeQCwGCUPABaj5AHAYtb8Th6APcI9ACyagxnvlNqnt/qn9Y36dpIJJQ8g6YR7AFisJlBbOne0+kd9K8mF3TUAYDFKHgAsRskDgMUoeQCwGCUPABaj5AHAYpQ8AFiMkgcAi1HyAGAxSh4ALEbJA4DFKHkAsBglDwAWo+QBwGKUPABYjJIHAItR8gBgMUoeACzG6f8A4P+Ee27ZUIVyDtp4nV+WkgeA/xPuuWVDFco5aON1fll21wCAxSh5ALBYXEr+r3/9q2bMmKGsrCzt3LkzHkMAAEIQ833yNTU12rRpk/7yl7+ob9++mjdvnr71rW/pK1/5SqyHAgD0IOYlX1lZqQkTJuiRRx6RJE2bNk3l5eV68cUXu92ura1NklRdXR3RuJ/calKd3xfRtpLUcOuWejs+jWjbK1cuRzV2KLrKl4hx7+XusaN5/KIZN1TR5ov3Y91dvvv1PN85bqKe367GDkWs8sXrsQ4l35XLl9R0q1/Yt93eme0dejeHMcaEfavd2L59uxoaGrRixQpJ0rvvvquTJ0/qlVde6Xa748ePa8GCBbGMAgAPjJ07d2r8+PGdlsf8k3wgEJDD4QheNsZ0uHwvGRkZ2rlzp5xOp3r37h3rWABgpba2Nvl8PmVkZHS5PuYln56eruPHjwcv+3w+uVyuHrfr169fl+9CAIDuDRky5J7rYv7rmieffFJ/+9vfdP36dTU2NurAgQPKzMyM9TAAgBDE/JP8Y489phUrVmjRokVqaWnR3Llz9Y1vfCPWwwAAQhDzL14BAMmDI14BwGKUPABYjJIHAItR8gBgsQeu5HuaPO3QoUOaNWuWZs6cqaVLl8rv9ydVvoMHD8rtdis7O1tr1qzRp58m7lDzUPK1O3LkiL7zne8kMNltPeXbsmWLpkyZolmzZmnWrFkJn0Cvp3wff/yxFi5cqJkzZ+pHP/pRUr3+Tp8+HXzcZs2apaefflo5OTlJk0+STp06pTlz5mjmzJnKz8/XzZs3kypfRUWF3G633G63Vq5cqfr6+viHMg+Q6upqM2XKFPPJJ5+Y+vp643a7zdmzZ4Prb926ZZ566ilTXV1tjDHmt7/9rXnllVeSJl99fb2ZNGmS8fl8xhhjli9fbt5+++2kydfO5/OZ6dOnmylTpiQsW6j58vPzzT//+c+E5go1XyAQMFlZWaaiosIYY8yvf/1r8/rrrydNvjs1NDSY7Oxs849//COp8uXl5ZkjR44YY4x57bXXzG9+85ukyef3+82ECROCy3bs2JGQfnmgPsnfOXlaWlpacPK0di0tLSoqKtJjjz0mSRoxYoSuXLmSNPnS0tJ0+PBhPfroo2psbNS1a9f00EMPJU2+doWFhT1OSHe/8nm9Xm3fvl1ut1svv/yympubkybfqVOnlJaWFjx48Mc//nFC53MK9fmVbs9R9cQTTyT0KPVQ8gUCgeCn48bGRvXrF/6EX/HKV1VVpQEDBgRn5J0yZYoOHToU91wPVMlfvXpVTqczeNnlcqmmpiZ4+Qtf+IKmTp0qSWpqatKOHTv0zDPPJE0+SerTp48qKir07W9/W5988okmTZqUVPn++Mc/6mtf+5pGjx6dsFztespXX1+vxx9/XKtWrVJZWZlu3rypbdu2JU2+Cxcu6NFHH1VBQYFmz56toqIipaWlJU2+drdu3VJJSUnC38hDybdmzRoVFhZq0qRJqqys1Lx585Im39ChQ1VdXa3//Oc/kqR9+/aptrY27rkeqJIPdfK0W7duacmSJRo5cqRmz56ddPkmT56sv//975oyZYp+/vOfJ02+Dz/8UAcOHNDSpUsTlulOPeX7/Oc/rz/84Q8aNmyYUlJStHjxYlVUVCRNvtbWVh07dkx5eXkqKyvT4MGDtXHjxqTJ127Pnj165pln9MUvfjFh2aSe8zU1NWndunV68803dfToUc2fP18/+9nPkibfQw89pF/96ld66aWXNGfOHLlcLvXp0yfuuR6okk9PT5fP9//niu5q8rSrV69q/vz5GjFihDZs2JBU+W7cuKGjR48GL7vdbp05cyZp8pWXl8vn82nOnDlasmRJ8LFMlnyXL19WaWlp8LIxRikpiTuXfU/5nE6nhgwZoq9//euSpJycHJ08eTJp8rU7dOiQZsyYkbBc7XrK9+GHHyo1NTU4jcoPfvADHTt2LGnytbW1KT09Xe+++67+/Oc/6/HHH9fgwYPjHyzue/2TSPsXI9euXTMNDQ1m5syZ5sSJE8H1ra2tZvbs2Wbr1q1Jme/69etmwoQJ5tKlS8YYYzZv3mxeeumlpMl3p//973/37YvXe+W7du2a+eY3v2kuXLhgAoGAWbt2rfn973+fNPkaGxvNU089ZU6fPm2MMWb79u3mpz/9adLkM+b2l8NPPPGEaWpqSliuUPPduHHDTJw40fz3v/81xhizZ88e8+yzzyZNvra2NvP000+b6upqEwgEzIoVK8zvfve7uOd6oEremNtPfHZ2tsnKyjI7duwwxhjz/PPPm5MnT5oDBw6YESNGmJkzZwb/KygoSJp8xhhz8OBBk5OTY9xut1mxYoW5efNmUuVrdz9KPpR85eXlwfVr1qwxzc3NSZXvX//6l5kzZ46ZMWOGWbx4samtrU2qfLW1tebJJ59MaKZw8h05csS43W6Tk5NjfvjDH5oLFy4kVb7333/f5OTkmKysLFNUVGQ+/fTTuGdigjIAsNgDtU8eAB40lDwAWIySBwCLUfIAYDFKHgAsRskDgMUoeQCwGCUPABb7f2rn2tK6hOLIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.set(style=\"white\")\n",
    "plt.figure()\n",
    "score_df['max_1'].hist(alpha=.7, bins=10)\n",
    "#des_score_df['max_1'].hist(alpha=.7, bins=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_df = sm_df.merge(score_df, how = 'left', left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_df.to_csv(args.save_dir+'social_media_pred.csv', encoding='utf-8-sig')\n",
    "score_df.to_csv(args.save_dir+'social_media_score.csv', encoding='utf-8-sig')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## END"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}