notebooks/modeling/4.1_train_text_classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Modeling\n",
    "\n",
    "## New York Time Articles - Text Classifier\n",
    "\n",
    "### Goal of this notebook\n",
    "\n",
    "> Train and select best Text Classifier for predicting article subject, `new_desk`, based on its text."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Load"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import libraries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\muril\\miniconda3\\envs\\ds-env\\lib\\site-packages\\tqdm\\std.py:668: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    }
   ],
   "source": [
    "# visualization and processing packages\n",
    "import joblib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "from src.paths import LOCAL_INTERIM_DATA_PATH, LOCAL_MODELS_PATH\n",
    "\n",
    "# custom packages for simplifing readibility of current notebook\n",
    "from src.processing.IOController import load_normalized_nyt_df\n",
    "from src.visualization.Plotter import plot_confusion_matrix, compare_models\n",
    "from src.processing.DataManipulator import get_most_correlated_tokens, predict_text, train_text_classifier, plot_prediction_mistakes\n",
    "\n",
    "# sklearn packages for machine learning / text classifier\n",
    "from sklearn import metrics\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "\n",
    "tqdm.pandas()\n",
    "pd.options.display.max_colwidth=300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load consolidated DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>abstract</th>\n",
       "      <td>A Times video investigation identified Chinese...</td>\n",
       "      <td>Anger at lockdowns and mask mandates provides ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>web_url</th>\n",
       "      <td>https://www.nytimes.com/2020/07/19/world/asia/...</td>\n",
       "      <td>https://www.nytimes.com/2020/07/19/business/me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>snippet</th>\n",
       "      <td>A Times video investigation identified Chinese...</td>\n",
       "      <td>Anger at lockdowns and mask mandates provides ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lead_paragraph</th>\n",
       "      <td>As the coronavirus pandemic continues to drive...</td>\n",
       "      <td>COLUMBUS, Ohio — If you’ve been following Gov....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>source</th>\n",
       "      <td>The New York Times</td>\n",
       "      <td>The New York Times</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>multimedia</th>\n",
       "      <td>[]</td>\n",
       "      <td>[{'rank': 0, 'subtype': 'xlarge', 'caption': N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>headline</th>\n",
       "      <td>{'main': 'China Is Using Uighur Labor to Produ...</td>\n",
       "      <td>{'main': 'An Ex-Times Reporter. An Ohio Weddin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keywords</th>\n",
       "      <td>[{'name': 'subject', 'value': 'Uighurs (Chines...</td>\n",
       "      <td>[{'name': 'subject', 'value': 'Coronavirus (20...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pub_date</th>\n",
       "      <td>2020-07-19 20:00:09+00:00</td>\n",
       "      <td>2020-07-20 00:30:07+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>document_type</th>\n",
       "      <td>article</td>\n",
       "      <td>article</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>news_desk</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>Business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>section_name</th>\n",
       "      <td>World</td>\n",
       "      <td>Business Day</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>subsection_name</th>\n",
       "      <td>Asia Pacific</td>\n",
       "      <td>Media</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>byline</th>\n",
       "      <td>{'original': 'By Muyi Xiao, Haley Willis, Chri...</td>\n",
       "      <td>{'original': 'By Ben Smith', 'person': [{'firs...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>type_of_material</th>\n",
       "      <td>News</td>\n",
       "      <td>News</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>_id</th>\n",
       "      <td>nyt://article/01f6e842-bd72-5372-8dee-9a6805c0...</td>\n",
       "      <td>nyt://article/0a910bd9-7727-5f9d-9a14-d75644a9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>word_count</th>\n",
       "      <td>400</td>\n",
       "      <td>2199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uri</th>\n",
       "      <td>nyt://article/01f6e842-bd72-5372-8dee-9a6805c0...</td>\n",
       "      <td>nyt://article/0a910bd9-7727-5f9d-9a14-d75644a9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>print_section</th>\n",
       "      <td>NaN</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>print_page</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slideshow_credits</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                   0  \\\n",
       "abstract           A Times video investigation identified Chinese...   \n",
       "web_url            https://www.nytimes.com/2020/07/19/world/asia/...   \n",
       "snippet            A Times video investigation identified Chinese...   \n",
       "lead_paragraph     As the coronavirus pandemic continues to drive...   \n",
       "source                                            The New York Times   \n",
       "multimedia                                                        []   \n",
       "headline           {'main': 'China Is Using Uighur Labor to Produ...   \n",
       "keywords           [{'name': 'subject', 'value': 'Uighurs (Chines...   \n",
       "pub_date                                   2020-07-19 20:00:09+00:00   \n",
       "document_type                                                article   \n",
       "news_desk                                                    Foreign   \n",
       "section_name                                                   World   \n",
       "subsection_name                                         Asia Pacific   \n",
       "byline             {'original': 'By Muyi Xiao, Haley Willis, Chri...   \n",
       "type_of_material                                                News   \n",
       "_id                nyt://article/01f6e842-bd72-5372-8dee-9a6805c0...   \n",
       "word_count                                                       400   \n",
       "uri                nyt://article/01f6e842-bd72-5372-8dee-9a6805c0...   \n",
       "print_section                                                    NaN   \n",
       "print_page                                                       NaN   \n",
       "slideshow_credits                                                NaN   \n",
       "\n",
       "                                                                   1  \n",
       "abstract           Anger at lockdowns and mask mandates provides ...  \n",
       "web_url            https://www.nytimes.com/2020/07/19/business/me...  \n",
       "snippet            Anger at lockdowns and mask mandates provides ...  \n",
       "lead_paragraph     COLUMBUS, Ohio — If you’ve been following Gov....  \n",
       "source                                            The New York Times  \n",
       "multimedia         [{'rank': 0, 'subtype': 'xlarge', 'caption': N...  \n",
       "headline           {'main': 'An Ex-Times Reporter. An Ohio Weddin...  \n",
       "keywords           [{'name': 'subject', 'value': 'Coronavirus (20...  \n",
       "pub_date                                   2020-07-20 00:30:07+00:00  \n",
       "document_type                                                article  \n",
       "news_desk                                                   Business  \n",
       "section_name                                            Business Day  \n",
       "subsection_name                                                Media  \n",
       "byline             {'original': 'By Ben Smith', 'person': [{'firs...  \n",
       "type_of_material                                                News  \n",
       "_id                nyt://article/0a910bd9-7727-5f9d-9a14-d75644a9...  \n",
       "word_count                                                      2199  \n",
       "uri                nyt://article/0a910bd9-7727-5f9d-9a14-d75644a9...  \n",
       "print_section                                                      B  \n",
       "print_page                                                         1  \n",
       "slideshow_credits                                                NaN  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = load_normalized_nyt_df(LOCAL_INTERIM_DATA_PATH / 'nyt-articles-consolidated.csv')\n",
    "df = df.reset_index().drop('index', axis=1)\n",
    "df.head(2).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classes Balancing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove data with missing `abstract`\n",
    "\n",
    "> Just noticed there were some missing spots on our target variable, better to get rid of them prior to diving deeper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.abstract.notna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check classes distribution.\n",
    "\n",
    "> It's important to check the classes distribution before training our model. Usually classes which a predominant tend to bias the classifier. Also, classes which have too few samples tend to be ignored by the model and treated as outliers. The idea here is to avoid overly amplifing or reducing the signal caught from our features and thus minimize biased predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvEAAAGDCAYAAACiOk+2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7hdVX3v//fHBAMUCHKphihEMJWjXCJEKggYFesl1kvFI5Qjl1pzsB6ttbRS9UGwtY3Vo6hYORERoSr+vNaCileughAgJICACFG5KDcNVyPE7++PNbcsNnsn2cnaa60Z3q/n2c+ea8wxx/zOEZ7wydhjrZ2qQpIkSVJ7PG7QBUiSJEmaGEO8JEmS1DKGeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXpA1YkuVJDujheN9McliPxtovybVdr3td61VJ5vVqPEkaJlMHXYAkPdYlWQ48EXgIWAVcDZwKLKqq3/exjgLuBwpYCSxpavjCSJ+qeukExppdVdeP16eqzgOevl5FP3y/U4CbqurdXeM/sxdjS9IwciVekobDn1fV5sAOwELgHcCnBlDH7lW1GZ1wfQpwQpL39PomSVxEkqT1YIiXpCFSVSuq6uvA64DDkuwCkGRakg8m+XmSXyU5MckmzbltkpyR5DdJ7kpyXpJH/f2eZOckNyY5aC3quKOqTgPeBPxTkq2bMc5O8tfN8dOSnJNkRZI7knyhaT+3GeaKJPcmeV2SeUluSvKOJL8EPj3SNurWz05ydZJfJ/l0ko2bMQ9Pcv6o56mmhgXAIcA/Nvf77+b8H7bnNPN3fJJbmq/jk0xrzo3U9vdJbktya5Ij1viHJUkDZIiXpCFUVRcDNwH7NU3vB/4EmAM8DZgJHNOc+/um77Z0tuW8k86WmD9IsgfwbeAtVXX6BEr5LzpbL/ca49w/N2M+AXgy8LGm9v2b87tX1WZd23GeBGxF56cNC8a53yHAi4Gd6Dzvu8fp9wdVtQj4LPDvzf3+fIxu7wKeQ2f+dm+ep3vsJwHT6czrG4CPJ3nCmu4tSYNiiJek4XULsFWSAG8E/q6q7qqqe4B/BUZW1B8EZgA7VNWDVXVeVXWH+P2ArwOHVdUZEymgqh4E7qATvkd7kE4g366qfltV54/Rp9vvgfdU1cqqemCcPidU1S+q6i7gfcDBE6l3NQ4B3ltVt1XV7cBxwOu7zj/YnH+wqr4B3EuP9utL0mQwxEvS8JoJ3EVnhX1T4NJmy8xvgG817QAfAK4Hvp3khiRHjxrnSOCHVfWDiRaQZKPmPneNcfofgQAXN58E81drGO72qvrtGvr8ouv4Z8B2a13s6m3XjDfe2HdW1UNdr+8HNuvRvSWp5wzxkjSEkjybTog/n85K+APAM6tqy+ZrevMGVKrqnqr6+6raEfhz4O1JXtg13JHA9kk+vA6lvJLOp+ZcPPpEVf2yqt5YVdsB/xv4jyRPW81YtZpzI57Sdbw9nZ9GANxH5x8yACR50gTHvoXOTw3GGluSWscQL0lDJMkWSV4OnA78Z1Utaz5m8pPAh5P8cdNvZpIXN8cvb97gGeBuOh9Tuapr2HuAlwD7J1m4lnVsleQQ4OPA+6vqzjH6vDbJk5uXv6YTpEfu+ytgxwk9fMebkzw5yVZ09vaP7Ke/AnhmkjnNm12PHXXdmu73eeDdSbZNsg2d9xP85zrUJ0lDwRAvScPhv5PcQ2c7ybuADwHdn5DyDjpbZi5KcjfwXR7esz27eX0vcCHwH1V1dvfgVfUb4EXAS5P882rquCLJvc29/prOPvxjxun7bOBHTf+vA39bVTc2544FPtNs//mfa3r4Lp+j82bZG5qvf2nqvw54b/OcP6HzE4punwKe0dzva2OM+y/AYmApsAy4bGRsSWqjPPK9T5IkSZKGnSvxkiRJUssY4iVJkqSWMcRLkiRJLWOIlyRJklrGEC9JkiS1zNRBFzCsttlmm5o1a9agy5AkSdIG7tJLL72jqrZdc8+HGeLHMWvWLBYvXjzoMiRJkrSBS/KziV7jdhpJkiSpZQzxkiRJUssY4iVJkqSWMcRLkiRJLWOIlyRJklrGT6cZx7KbVzDr6DMHXYYkSZIm2fKF8wddwoS5Ei9JkiS1jCFekiRJahlDvCRJktQyfQ3xSVYlWdL1NasHY743yQHrX50kSZLUDv1+Y+sDVTVnohclmVpVD411rqqOWf+yJEmSpPYY+HaaJHOSXJRkaZKvJnlC0352kn9Ncg7wt0n2THJOkkuTnJVkRtPvlCQHNscvS3JNkvOTfDTJGU37sUlObsa8IclbB/bAkiRJ0nrqd4jfpGsrzVebtlOBd1TVbsAy4D1d/besqucBHwU+BhxYVXsCJwPv6x44ycbA/wNeWlX7AtuOuvfOwIuBvYD3JNlodHFJFiRZnGTxqvtXrPfDSpIkSZNhoNtpkkynE9TPaZo+A3yxq/8Xmu9PB3YBvpMEYApw66ixdwZuqKobm9efBxZ0nT+zqlYCK5PcBjwRuKl7gKpaBCwCmDZjdq3TE0qSJEmTbNh/2dN9zfcAV1XV3qvpmzWMtbLreBXD/+ySJEnSmAa6J76qVgC/TrJf0/R64Jwxul4LbJtkb4AkGyV55qg+1wA7dn3izet6X7EkSZI0eMOwGn0YcGKSTYEbgCNGd6iq3zVvXv1oswVnKnA8cFVXnweS/A3wrSR3ABf3pXpJkiSpz/oa4qtqszHalgDPGaN93hj99h+j3+FdL39QVTuns3H+48Dips+xo67ZZeLVS5IkScNh4B8x2WNvTLKEzgr9dDqfViNJkiRtUIZhO03PVNWHgQ8Pug5JkiRpMm1QIb6Xdp05ncUL5w+6DEmSJOlRNrTtNJIkSdIGzxAvSZIktYwhXpIkSWoZQ7wkSZLUMoZ4SZIkqWUM8ZIkSVLLGOIlSZKkljHES5IkSS1jiJckSZJaxhAvSZIktYwhXpIkSWoZQ7wkSZLUMoZ4SZIkqWUM8ZIkSVLLTB10AcNq2c0rmHX0mYMuQ5IkrYPlC+cPugRpUrkSL0mSJLWMIV6SJElqGUO8JEmS1DI9D/FJViVZkuSKJJcl2WcdxzkyyaG9rk+SJElqu8l4Y+sDVTUHIMmLgX8DnjfRQarqxF4XJkmSJG0IJns7zRbArwGSzEtyxsiJJCckObw5Xpjk6iRLk3ywaTs2yVHN8dlJ3p/k4iTXJdmvaZ+S5ANJLmmu/d9N+4wk5zY/EbgyyX5N31Oa18uS/N0kP7skSZI0KSZjJX6TJEuAjYEZwAtW1znJVsCrgZ2rqpJsOU7XqVW1V5KXAe8BDgDeAKyoqmcnmQZckOTbwF8AZ1XV+5JMATYF5gAzq2qX5r6Puk+SBcACgClbbDvhB5ckSZL6YTJW4h+oqjlVtTPwEuDUJFlN/7uB3wInJfkL4P5x+n2l+X4pMKs5/jPg0OYfDT8CtgZmA5cARyQ5Fti1qu4BbgB2TPKxJC9p7vsIVbWoquZW1dwpm05f+yeWJEmS+mhSt9NU1YXANsC2wEOj7rdx0+chYC/gy8CrgG+NM9zK5vsqHv4JQoC3NP9omFNVT62qb1fVucD+wM3AaUkOrapfA7sDZwNvBk7qzVNKkiRJ/TWpv7E1yc7AFOBO4GfAM5ptLxsDLwTOT7IZsGlVfSPJRcD1E7jFWcCbkny/qh5M8id0gvs2wM1V9ckkfwTskeQbwO+q6stJfgqc0qvnlCRJkvppMvfEQ2el/LCqWgX8Isn/BywFfgJc3vTZHPivJBs3/SfyhtOT6GytuazZsnM7ndX8ecA/JHkQuBc4FJgJfDrJyE8D/mndHk+SJEkarFTVoGsYStNmzK4Zhx0/6DIkSdI6WL5w/qBLkNZakkurau5ErvE3tkqSJEktY4iXJEmSWmZS39jaZrvOnM5ifxQnSZKkIeRKvCRJktQyhnhJkiSpZQzxkiRJUssY4iVJkqSWMcRLkiRJLWOIlyRJklrGEC9JkiS1jCFekiRJahlDvCRJktQyhnhJkiSpZQzxkiRJUssY4iVJkqSWMcRLkiRJLTN10AUMq2U3r2DW0WcOugxJ2uAsXzh/0CVIUuu5Ei9JkiS1jCFekiRJahlDvCRJktQyQxHik7wryVVJliZZkuRPezDmvCT79KI+SZIkaZgM/I2tSfYGXg7sUVUrk2wDPH49x5wKzAPuBX643kVKkiRJQ2TgIR6YAdxRVSsBquoOgCTLgS8Az2/6/WVVXZ9kB+BkYFvgduCIqvp5klOAu4BnNd+fC6xK8r+AtwBPAt4DrAJWVNX+/Xk8SZIkqbeGYTvNt4GnJLkuyX8keV7Xuburai/gBOD4pu0E4NSq2g34LPDRrv5/AhxQVa8BTgQ+XFVzquo84BjgxVW1O/CKSX4mSZIkadIMPMRX1b3AnsACOivrX0hyeHP6813f926O9wY+1xyfBuzbNdwXq2rVOLe6ADglyRuBKWN1SLIgyeIki1fdv2JdHkeSJEmadMOwnYYmeJ8NnJ1kGXDYyKnubuNd3nV832rucWTzhtn5wJIkc6rqzlF9FgGLAKbNmD3e/SRJkqSBGvhKfJKnJ5nd1TQH+Flz/Lqu7xc2xz8EDmqODwHOH2foe4DNu+6zU1X9qKqOAe4AntKD8iVJkqS+G4aV+M2AjyXZEngIuJ7O1pqXA9OS/IjOPzYObvq/FTg5yT/QvLF1nHH/G/hSklfSeWPr3zX/WAjwPeCKSXoeSZIkaVINPMRX1aXAoz7PPQnAx6vquFH9lwMvGGOcw0e9vg7YravpvPWvVpIkSRq8gW+nkSRJkjQxA1+JH09VzRp0DZIkSdIwGtoQP2i7zpzO4oXzB12GJEmS9Chup5EkSZJaxhAvSZIktYwhXpIkSWoZQ7wkSZLUMoZ4SZIkqWUM8ZIkSVLLGOIlSZKkljHES5IkSS1jiJckSZJaxhAvSZIktYwhXpIkSWoZQ7wkSZLUMoZ4SZIkqWUM8ZIkSVLLTB10AcNq2c0rmHX0mYMuQ1IPLF84f9AlSJLUU67ES5IkSS1jiJckSZJaxhAvSZIktcxAQnySVUmWJLkyyReTbLqavocnOaE5PjLJoV3t23X1OynJMya/ekmSJGmwBrUS/0BVzamqXYDfAUeuzUVVdWJVndq8PBzYruvcX1fV1T2vVJIkSRoyw7Cd5jzgaUm2SvK1JEuTXJRkt9Edkxyb5KgkBwJzgc82K/qbJDk7ydym30uSXJbkiiTfa9qe1/RdkuTyJJv39SklSZKkHhloiE8yFXgpsAw4Dri8qnYD3gmcOt51VfUlYDFwSLOi/0DXmNsCnwReU1W7A69tTh0FvLmq5gD7AQ+MGpYkC5IsTrJ41f0revKMkiRJUq8NKsRvkmQJnSD+c+BTwL7AaQBV9X1g6yTT12Hs5wDnVtWNzVh3Ne0XAB9K8lZgy6p6aPSFVbWoquZW1dwpm67LrSVJkqTJN6hf9vRAsyL+B0kyRr9ah7Ez1nVVtTDJmcDLgIuSHFBV16zD+JIkSdJADcOe+BHnAocAJJkH3FFVd6+m/z3AWPvaLwSel+SpzVhbNd93qqplVfV+Oj8B2LmHtUuSJEl9M6iV+LEcC3w6yVLgfuCwNfQ/BTgxyQPA3iONVXV7kgXAV5I8DrgNeBHwtiTPB1YBVwPf7PkTSJIkSX2QqnXZsbLhmzZjds047PhBlyGpB5YvnD/oEiRJGleSS6tq7kSuGabtNJIkSZLWgiFekiRJaplh2hM/VHadOZ3F/ghekiRJQ8iVeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLXM1EEXMKyW3byCWUefOegypEdZvnD+oEuQJEkD5kq8JEmS1DKGeEmSJKllDPGSJElSy/QtxCd5V5KrkixNsiTJn47Tb26Sj/arLkmSJKlt+vLG1iR7Ay8H9qiqlUm2AR4/Vt+qWgws7kddkiRJUhv1ayV+BnBHVa0EqKo7quqWJM9O8sMkVyS5OMnmSeYlOQMgyR8lOTnJJUkuT/LKpv3wJF9J8q0kP0ny7yM3SvKSJJc1Y35vdeNIkiRJbdSvj5j8NnBMkuuA7wJfAC5svr+uqi5JsgXwwKjr3gV8v6r+KsmWwMVJvtucmwM8C1gJXJvkY8BvgU8C+1fVjUm2Wt04VXVf982SLAAWAEzZYtueToAkSZLUK30J8VV1b5I9gf2A59MJ7+8Dbq2qS5o+dwMk6b70z4BXJDmqeb0xsH1z/L2qWtFcczWwA/AE4NyqurEZ8641jPPjUXUuAhYBTJsxu9b/ySVJkqTe69sve6qqVcDZwNlJlgFvBtYUlAO8pqqufURj502xK7uaVtF5lowz5pjjSJIkSW3Ulz3xSZ6eZHZX0xw6q+DbJXl202fzJKP/UXEW8JY0y/NJnrWGW10IPC/JU5v+I9tpJjqOJEmSNLT6tRK/GfCxZj/6Q8D1dPaef7pp34TOfvgDRl33z8DxwNImgC+n8yk3Y6qq25t97V9J8jjgNuBFEx1HkiRJGmapcuv3WKbNmF0zDjt+0GVIj7J84fxBlyBJknooyaVVNXci1/gbWyVJkqSW6dsbW9tm15nTWeyKpyRJkoaQK/GSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqmamDLmBYLbt5BbOOPnPQZUyK5QvnD7oESZIkrQdX4iVJkqSWMcRLkiRJLWOIlyRJklqmdSE+yawkV45qOzbJUaPanpjkjCRXJLk6yTf6W6kkSZI0OTbkN7a+F/hOVX0EIMluA65HkiRJ6onWrcRPwAzgppEXVbV0gLVIkiRJPbMhh/iPA59K8oMk70qy3aALkiRJknqhjSG+1qa9qs4CdgQ+CewMXJ5k29UNnGRBksVJFq+6f0VPipUkSZJ6rY0h/k7gCaPatgLuGN2xqu6qqs9V1euBS4D9VzdwVS2qqrlVNXfKptN7VrAkSZLUS60L8VV1L3BrkhcCJNkKeAlwfne/JC9IsmlzvDmwE/DzPpcrSZIk9VxbP53mUODjSf5v8/q4qvppkiMBqupEYE/ghCQP0fnHyklVdQlAkiVVNWcQhUuSJEnrq5UhvqquBp4/RvuJXccfAD4wzvUGeEmSJLVW67bTSJIkSY91hnhJkiSpZVq5naYfdp05ncUL5w+6DEmSJOlRXImXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSy0wddAHDatnNK5h19Jmr7bN84fw+VSNJkiQ9zJV4SZIkqWUM8ZIkSVLLGOIlSZKklhmqEJ/kw0ne1vX6rCQndb3+v0nevpZjnZLkwDHa5yU5ozcVS5IkSf03VCEe+CGwD0CSxwHbAM/sOr8PcMGaBkkyZVKqkyRJkobAsIX4C2hCPJ3wfiVwT5InJJkG/A9gyySXJ1mW5OSmnSTLkxyT5Hzgtd2DJnlJkmuac3/Rv8eRJEmSem+oQnxV3QI8lGR7OmH+QuBHwN7AXOA64CTgdVW1K52PyHxT1xC/rap9q+r0kYYkGwOfBP4c2A940nj3T7IgyeIki1fdv6K3DydJkiT1yFCF+MbIavxIiL+w6/XNwI1VdV3T9zPA/l3XfmGM8XZurvlJVRXwn+PduKoWVdXcqpo7ZdPp6/8kkiRJ0iQYxhA/si9+VzrbaS6isxK/D3DZGq69b5z26ll1kiRJ0oANY4i/AHg5cFdVraqqu4At6QT5TwOzkjyt6ft64Jw1jHcN8NQkOzWvD56EmiVJkqS+GcYQv4zOp9JcNKptRVXdBBwBfDHJMuD3wImrG6yqfgssAM5s3tj6s0mpWpIkSeqTqYMuYLSqWgVsMart8K7j7wHPGuO6Wau55lt09sZLkiRJrTeMK/GSJEmSVsMQL0mSJLXM0G2nGRa7zpzO4oXzB12GJEmS9CiuxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllpg66gGG17OYVzDr6zEe0LV84f0DVSJIkSQ9zJV6SJElqGUO8JEmS1DKGeEmSJKllWhnik7w6SSXZeZzzWyb5m37XJUmSJPVDK0M8cDBwPnDQ6BNJpgBbAoZ4SZIkbZBaF+KTbAY8F3gDTYhPMi/JD5J8DlgGLAR2SrIkyQeSzEhybvP6yiT7De4JJEmSpPXTxo+YfBXwraq6LsldSfZo2vcCdqmqG5PMao7nACT5e+Csqnpfs1K/6VgDJ1kALACYssW2k/wYkiRJ0rpp3Uo8na00pzfHpzevAS6uqhvHueYS4IgkxwK7VtU9Y3WqqkVVNbeq5k7ZdHova5YkSZJ6plUr8Um2Bl4A7JKkgClAAd8A7hvvuqo6N8n+wHzgtCQfqKpT+1GzJEmS1GttW4k/EDi1qnaoqllV9RTgRmDfUf3uATYfeZFkB+C2qvok8ClgDyRJkqSWatVKPJ2tMwtHtX0ZeBPw05GGqrozyQVJrgS+CVwJ/EOSB4F7gUP7VK8kSZLUc60K8VU1b4y2jwIfHaP9L0c1fWaSypIkSZL6qm3baSRJkqTHvFatxPfTrjOns3jh/EGXIUmSJD2KK/GSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSyxjiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUP8OJbdvGLQJUiSJEljMsRLkiRJLWOIlyRJklrGEC9JkiS1zGpDfJKtkyxpvn6Z5Oau149f0+BJ5iU5oxeFJjk8yQm9GEuSJElqs6mrO1lVdwJzAJIcC9xbVR/sQ12SJEmSxjHh7TRJ9kxyTpJLk5yVZEbT/rQk301yRZLLkuzUXLJZki8luSbJZ5Ok6b88yXFN32VJdm7at0rytSRLk1yUZLcxatghyfeaPt9Lsn3TvlNzzSVJ3pvk3qb9tCSv7Lr+s0leMeHZkiRJkobAREN8gI8BB1bVnsDJwPuac58FPl5VuwP7ALc27c8C3gY8A9gReG7XeHdU1R7AJ4CjmrbjgMurajfgncCpY9RxAnBq0+ezwEeb9o8AH6mqZwO3dPU/CTgCIMn0pr5vTPDZJUmSpKEw0RA/DdgF+E6SJcC7gScn2RyYWVVfBaiq31bV/c01F1fVTVX1e2AJMKtrvK803y/tat8XOK0Z5/vA1k3w7rY38Lnm+LTmmpH2LzbHI+epqnOApyX5Y+Bg4MtV9dDoh0uyIMniJItX3e/nxEuSJGk4rXZP/BgCXFVVez+iMdliNdes7DpeNeqeK8dozxhj1BrqWtN56IT9Q4CDgL8ac5CqRcAigGkzZq/NmJIkSVLfTXQlfiWwbZK9AZJslOSZVXU3cFOSVzXt05Jsuo41nUsnbJNkHp0tN3eP6vNDOmGcpu/5zfFFwGua44NGXXMKnW09VNVV61ibJEmSNHATDfG/Bw4E3p/kCjrbY/Zpzr0eeGuSpXRC9pPWsaZjgbnNOAuBw8bo81bgiKbP64G/bdrfBrw9ycXADOAPe2Kq6lfAj4FPr2NdkiRJ0lBI1Yaza6RZ/X+gqirJQcDBVfXKrnPLgD2qao0b3qfNmF0rb/3J5BYsSZKkx7wkl1bV3IlcM9E98cNuT+CE5mMsf0Oz9z3JAXQ+SedDaxPgJUmSpGG2QYX4qjoP2H2M9u8C2/e/IkmSJKn3JvzLnh4rdp05+lMtJUmSpOFgiJckSZJaxhAvSZIktYwhXpIkSWoZQ7wkSZLUMoZ4SZIkqWUM8ZIkSVLLGOIlSZKkljHES5IkSS1jiJckSZJaxhAvSZIktYwhXpIkSWoZQ7wkSZLUMoZ4SZIkqWUM8ZIkSVLLGOLHsezmFYMuQZIkSRqTIV6SJElqGUO8JEmS1DKGeEmSJKllBhLik1SS07peT01ye5Iz1nG8I5Mc2rsKJUmSpOE1dUD3vQ/YJckmVfUA8CLg5nUdrKpO7FllkiRJ0pAb5HaabwLzm+ODgc+PnEiyVZKvJVma5KIkuyV5XJLlSbbs6nd9kicmOTbJUU3bTkm+leTSJOcl2blpf22SK5NckeTcPj6nJEmS1FODDPGnAwcl2RjYDfhR17njgMurajfgncCpVfV74L+AVwMk+VNgeVX9atS4i4C3VNWewFHAfzTtxwAvrqrdgVeMVVCSBUkWJ1m86n4/YlKSJEnDaWAhvqqWArPorMJ/Y9TpfYHTmn7fB7ZOMh34AvC6ps9Bzes/SLIZsA/wxSRLgP8HzGhOXwCckuSNwJRxalpUVXOrau6UTaev3wNKkiRJk2RQe+JHfB34IDAP2LqrPWP0LeBC4GlJtgVeBfzLqD6PA35TVXMedXHVkc3q/XxgSZI5VXXn+j+CJEmS1F+D/ojJk4H3VtWyUe3nAocAJJkH3FFVd1dVAV8FPgT8eHQIr6q7gRuTvLa5Nkl2b453qqofVdUxwB3AUybxuSRJkqRJM9CV+Kq6CfjIGKeOBT6dZClwP3BY17kvAJcAh48z7CHAJ5K8G9iIzt77K4APJJlNZ5X/e02bJEmS1DrpLG5rtGkzZtfKW38y6DIkSZK0gUtyaVXNncg1g95OI0mSJGmCDPGSJElSyxjix7HrTD9iUpIkScPJEC9JkiS1jCFekiRJahlDvCRJktQyhnhJkiSpZQzxkiRJUssY4iVJkqSWMcRLkiRJLWOIlyRJklrGEC9JkiS1jCFekiRJahlDvCRJktQyhnhJkiSpZQzxkiRJUssY4iVJkqSWMcRLkiRJLWOIlyRJklrGEC9JkiS1jCFekiRJapmBhvgk9xJC46wAAAmcSURBVI56fXiSE9ZxrHlJzug63qfr3ClJDly/aiVJkqThsKGuxM8D9llTJ0mSJKmNhjbEJ9k2yZeTXNJ8Pbdp3yvJD5Nc3nx/+qjrZgFHAn+XZEmS/ZpT+zf9b3BVXpIkSW02dcD33yTJkq7XWwFfb44/Any4qs5Psj1wFvA/gGuA/avqoSQHAP8KvGZkgKpanuRE4N6q+iBAkjcAM4B9gZ2be3xpdDFJFgALALbffvuePqgkSZLUK4MO8Q9U1ZyRF0kOB+Y2Lw8AnpFk5PQWSTYHpgOfSTIbKGCjtbzX16rq98DVSZ44VoeqWgQsApg7d25N8FkkSZKkvhh0iF+dxwF7V9UD3Y1JPgb8oKpe3WydOXstx1vZPUwvCpQkSZIGYWj3xAPfBv7PyIskIyv204Gbm+PDx7n2HmDzSatMkiRJGqBhDvFvBeYmWZrkajpvVgX4d+DfklwATBnn2v8GXj3qja2SJEnSBiFVbv0ey9y5c2vx4sWDLkOSJEkbuCSXVtXcNfd82DCvxEuSJEkagyFekiRJahlDvCRJktQyhnhJkiSpZQzxkiRJUssY4iVJkqSWMcRLkiRJLWOIlyRJklrGEC9JkiS1jL+xdRxJ7gGuHXQdj1HbAHcMuojHKOd+cJz7wXDeB8e5HxznfnDGm/sdqmrbiQw0tTf1bJCuneivv1VvJFns3A+Gcz84zv1gOO+D49wPjnM/OL2ce7fTSJIkSS1jiJckSZJaxhA/vkWDLuAxzLkfHOd+cJz7wXDeB8e5HxznfnB6Nve+sVWSJElqGVfiJUmSpJYxxI+S5CVJrk1yfZKjB13PhiDJyUluS3JlV9tWSb6T5CfN9yd0nfunZv6vTfLirvY9kyxrzn00Sfr9LG2T5ClJfpDkx0muSvK3TbvzP8mSbJzk4iRXNHN/XNPu3PdBkilJLk9yRvPaee+DJMubOVuSZHHT5tz3QZItk3wpyTXN3/l7O/eTL8nTm//eR77uTvK2vsx9VfnVfAFTgJ8COwKPB64AnjHoutr+BewP7AFc2dX278DRzfHRwPub42c08z4NeGrz5zGlOXcxsDcQ4JvASwf9bMP+BcwA9miONweua+bY+Z/8uQ+wWXO8EfAj4DnOfd/m/+3A54AzmtfOe3/mfTmwzag2574/c/8Z4K+b48cDWzr3ff8zmAL8EtihH3PvSvwj7QVcX1U3VNXvgNOBVw64ptarqnOBu0Y1v5LOXzg031/V1X56Va2sqhuB64G9kswAtqiqC6vzX/qpXddoHFV1a1Vd1hzfA/wYmInzP+mq497m5UbNV+HcT7okTwbmAyd1NTvvg+PcT7IkW9BZMPsUQFX9rqp+g3Pfby8EflpVP6MPc2+If6SZwC+6Xt/UtKn3nlhVt0InaAJ/3LSP92cwszke3a61lGQW8Cw6K8LOfx80WzqWALcB36kq574/jgf+Efh9V5vz3h8FfDvJpUkWNG3O/eTbEbgd+HSzjeykJH+Ec99vBwGfb44nfe4N8Y801t4jP76nv8b7M/DPZj0k2Qz4MvC2qrp7dV3HaHP+11FVraqqOcCT6ay07LKa7s59DyR5OXBbVV26tpeM0ea8r7vnVtUewEuBNyfZfzV9nfvemUpn2+onqupZwH10tnCMx7nvsSSPB14BfHFNXcdoW6e5N8Q/0k3AU7pePxm4ZUC1bOh+1fzoiOb7bU37eH8GNzXHo9u1Bkk2ohPgP1tVX2manf8+an6sfTbwEpz7yfZc4BVJltPZEvmCJP+J894XVXVL8/024Kt0tqk695PvJuCm5qd9AF+iE+qd+/55KXBZVf2qeT3pc2+If6RLgNlJntr8i+og4OsDrmlD9XXgsOb4MOC/utoPSjItyVOB2cDFzY+i7knynObd2od2XaNxNHP1KeDHVfWhrlPO/yRLsm2SLZvjTYADgGtw7idVVf1TVT25qmbR+Tv8+1X1v3DeJ12SP0qy+cgx8GfAlTj3k66qfgn8IsnTm6YXAlfj3PfTwTy8lQb6MfeDePfuMH8BL6PzCR4/Bd416Ho2hK/mP+pbgQfp/EvzDcDWwPeAnzTft+rq/65m/q+l653ZwFw6/0P4KXACzS8r82u1c78vnR/HLQWWNF8vc/77Mve7AZc3c38lcEzT7tz3789gHg9/Oo3zPvnzvSOdT924Arhq5P+hzn3f5n8OsLj5O+drwBOc+77N/abAncD0rrZJn3t/Y6skSZLUMm6nkSRJklrGEC9JkiS1jCFekiRJahlDvCRJktQyhnhJkiSpZQzxkrSBSfLqJJVk59X02TLJ33S93i7Jl9Yw7tlJ5k6wlqOSXJPkyiRXJDl0ItevxfiPeA5JeqwwxEvShudg4Hw6v+zoUZJMAbYE/hB+q+qWqjqwl0UkORJ4EbBXVe0C7M/Yv1p8fTziOSTpscIQL0kbkCSbAc+l80vVDupqn5fkB0k+BywDFgI7JVmS5ANJZiW5suk7JckHkyxLsjTJW8a4z58luTDJZUm+2Nx3tHcCf1NVdwNU1Yqq+kxz/QuTXN7c4+Qk05r25Um2aY7nJjm7OT626Xd2khuSvLW5xyOeowdTKEmtMHXQBUiSeupVwLeq6rokdyXZo6oua87tBexSVTcmmdUczwFoXo9YADwVeFZVPZRkq+4bNCH73cABVXVfkncAbwfe29Vnc2Dzqvrp6AKTbAycArywqfNU4E3A8Wt4tp2B5wObA9cm+QRwdPdzSNJjhSvxkrRhORg4vTk+vXk94uKqunEtxjgAOLGqHgKoqrtGnX8O8AzggiRLgMOAHUb1CTDerwR/OnBjVV3XvP4Mna02a3JmVa2sqjuA24AnrsU1krRBciVekjYQSbYGXgDskqSAKUAl+cemy31rOxTjB/CR89+pqoPH61BVdye5L8mOVXXDGNeP5yEeXmDaeNS5lV3Hq/D/YZIew1yJl6QNx4HAqVW1Q1XNqqqnADcC+47R9x4621LG8m3gyCRTAUZvpwEuAp6b5GnN+U2T/MkY4/wb8PEkWzT9tkiyALgGmDVyPfB64JzmeDmwZ3P8mtU+7ZqfQ5I2WIZ4SdpwHAx8dVTbl4G/HN2xqu6ksx3myjHeEHoS8HNgaZIrRl9fVbcDhwOfT7KUTqgf6+MsPwH8ALikedPsOcD9VfVb4Ajgi0mWAb8HTmyuOQ74SJLz6Ky2r9YankOSNlipWt1PTCVJkiQNG1fiJUmSpJYxxEuSJEktY4iXJEmSWsYQL0mSJLWMIV6SJElqGUO8JEmS1DKGeEmSJKllDPGSJElSy/z/ZJ64+YcRQlgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "_ = df.news_desk.value_counts(ascending=True).plot(kind='barh', title='Desk Distribution', figsize=(12,6)).set(xlabel='Article Count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove minor classes.\n",
    "\n",
    "> For simplicity purposes I've just removed everything with sample size lower than 100, since these examples would hardly be predicted anyways. In a real world solution though this has to be checked with the business department and alternatives (such as gathering more data, yahoo!) must be considered. Besides, the Foreign class might be over predicted, and that's something we are going to see further on our confusion matrix!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuYAAAGDCAYAAABnSNUnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7RdVX328e/TRAMIBBCqASwBjTAUMEKkUgVRUMFYbat9lVIBq6a0Du3NKhYHoq1trG2lCi0vIiLW2+uttWAVq4aLohAgJICAXKJyUQhouBoh/t4/9opuDuckOSfn7D2TfD9j7LHXmmuuueaaJ+PkOXPPvXeqCkmSJEnD9WvD7oAkSZIkg7kkSZLUBIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRtRJIsT3LYJLb3P0mOmaS2DkpyXd/+ZPf16iSHTFZ7ktSa6cPugCRtapIsB54APAysBq4BzgZOr6pfDLAfBTwAFLAKWNL14dNr6lTVEeNoa05V3TBWnaq6ENhzgzr9q+udBdxSVe/oa//pk9G2JLXKGXNJmhq/XVXbALsBC4G3AR8eQj+eUVVb0wvMZwGnJHnnZF8kiRM9krSBDOaSNIWqamVVfRF4FXBMkr0BksxI8k9JfpDkx0lOS7Jld2zHJOck+WmSu5NcmORRv6+T7JXk5iSvXo9+rKiqjwF/Arw9yeO7NhYleX23/ZQk5ydZmWRFkk935Rd0zVyZ5L4kr0pySJJbkrwtyY+Aj6wpG3HpZyW5JslPknwkyRZdm8cmuWjE/VTXhwXAUcBbu+v9d3f8l0tjuvE7Oclt3ePkJDO6Y2v69ldJ7khye5LXrvOHJUlDZjCXpAGoqkuAW4CDuqL3Ak8F5gJPAXYBTuyO/VVXdyd6S2L+ht5ylF9Ksh9wHvCmqvrUOLryX/SWMR4wyrG/7drcHtgV+GDX94O748+oqq37lsI8EdiB3qsCC8a43lHAi4En07vfd4xR75eq6nTg48A/dtf77VGqnQA8m974PaO7n/62nwjMpDeurwNOTbL9uq4tScNkMJekwbkN2CFJgDcAf1FVd1fVvcDfA2tmvh8CZgG7VdVDVXVhVfUH84OALwLHVNU54+lAVT0ErKAXqEd6iF7I3rmqflZVF41Sp98vgHdW1aqqenCMOqdU1Q+r6m7gPcCR4+nvWhwFvLuq7qiqO4F3Aa/pO/5Qd/yhqvoScB+TtP5dkqaKwVySBmcX4G56M+FbAZd1y1V+Cny5Kwd4H3ADcF6Sm5IcP6Kd44BvVdU3xtuBJI/prnP3KIffCgS4pPsElD9aR3N3VtXP1lHnh33b3wd2Xu/Ort3OXXtjtX1XVT3ct/8AsPUkXVuSpoTBXJIGIMmz6AXzi+jNWD8IPL2qtuseM7s3aVJV91bVX1XVHsBvA3+Z5NC+5o4DfiPJ+yfQlZfT+7SYS0YeqKofVdUbqmpn4I+Bf0vylLW0VWs5tsaT+rZ/g96rBgD30/vjBIAkTxxn27fRm90frW1J2igZzCVpCiXZNslLgU8B/1FVy7qPTPwQ8P4kv97V2yXJi7vtl3ZvggxwD72PXFzd1+y9wOHAwUkWrmc/dkhyFHAq8N6qumuUOr+fZNdu9yf0wvGa6/4Y2GNcN9/zxiS7JtmB3lr5NevTrwSenmRu94bQk0act67rfRJ4R5KdkuxIb33+f0ygf5LUDIO5JE2N/05yL72lHCcA/wL0fzLI2+gtV/l2knuA/+VXa6DndPv3ARcD/1ZVi/obr6qfAi8Ejkjyt2vpx5VJ7uuu9Xp669pPHKPus4DvdPW/CPxZVd3cHTsJ+Gi39Ob/rOvm+3yC3htKb+oef9f1/3rg3d19fo/eKwn9Pgw8rbvef47S7t8Bi4GlwDLg8jVtS9LGKo98P5EkSZKkYXDGXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqwPRhd2CQdtxxx5o9e/awuyFJkqRN2GWXXbaiqnZad81H2qyC+ezZs1m8ePGwuyFJkqRNWJLvT+Q8l7JIkiRJDTCYS5IkSQ0wmEuSJEkNMJhLkiRJDTCYS5IkSQ0wmEuSJEkNMJhLkiRJDTCYS5IkSQ0wmEuSJEkNMJhLkiRJDTCYS5IkSQ2YPuwODNKyW1cy+/hzh90NSZIkTaHlC+cPuwsT4oy5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUgA0O5klWJ1nS95g9CW2+O8lhG9qOJEmStLGYPgltPFhVc8d7UpLpVfXwaMeq6sQN75YkSZK08ZiSpSxJ5ib5dpKlSb6QZPuufFGSv09yPvBnSfZPcn6Sy5J8Jcmsrt5ZSV7Zbb8kybVJLkrygSTndOUnJTmza/OmJG+einuRJEmSBmEygvmWfctYvtCVnQ28rar2BZYB7+yrv11VPQ/4APBB4JVVtT9wJvCe/oaTbAH8X+CIqnousNOIa+8FvBg4AHhnksdMwv1IkiRJAzfpS1mSzKQXvs/vij4KfKav/qe75z2BvYGvJgGYBtw+ou29gJuq6uZu/5PAgr7j51bVKmBVkjuAJwC39DeQZMGac6ZtOzLXS5IkSW2YjGA+Xvd3zwGurqoD11I362hrVd/2aka5n6o6HTgdYMasOTWOfkqSJEkDM+lrzKtqJfCTJAd1Ra8Bzh+l6nXATkkOBEjymCRPH1HnWmCPvk96edVk91eSJElqwVTNmB8DnJZkK+Am4LUjK1TVz7s3eH6gW/4yHTgZuLqvzoNJ/hT4cpIVwCVT1F9JkiRpqDY4mFfV1qOULQGePUr5IaPUO3iUesf27X6jqvZKbyH6qcDirs5JI87Ze/y9lyRJktqwMXzz5xuSLKE3kz6T3qe0SJIkSZuUYbz5c1yq6v3A+4fdD0mSJGkqbQwz5pIkSdImz2AuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDWg+Y9LnEz77DKTxQvnD7sbkiRJ0qM4Yy5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDVg+rA7MEjLbl3J7OPPHXY3JEnSBCxfOH/YXZCmlDPmkiRJUgMM5pIkSVIDDOaSJElSAwzmkiRJUgMM5pIkSVIDDOaSJElSAwzmkiRJUgMM5pIkSVIDDOaSJElSA9YZzJOsTrIkyZVJLk/yWxO5UJLjkhw9kXMlSZKkTd309ajzYFXNBUjyYuAfgOeN90JVddp4z5EkSZI2F+NdyrIt8BOAJIckOWfNgSSnJDm2216Y5JokS5P8U1d2UpK3dNuLkrw3ySVJrk9yUFc+Lcn7klzanfvHXfmsJBd0M/dXJTmoq3tWt78syV9s8GhIkiRJQ7I+M+ZbJlkCbAHMAl6wtspJdgB+F9irqirJdmNdu6oOSPIS4J3AYcDrgJVV9awkM4BvJjkP+D3gK1X1niTTgK2AucAuVbV3d91Rr5NkAbAAYNq2O63H7UqSJEmDtz4z5g9W1dyq2gs4HDg7SdZS/x7gZ8AZSX4PeGCMep/vni8DZnfbLwKO7v4Q+A7weGAOcCnw2iQnAftU1b3ATcAeST6Y5PDuuo9SVadX1byqmjdtq5nrcbuSJEnS4I1rKUtVXQzsCOwEPDzi/C26Og8DBwCfA34H+PIYza3qnlfzq5n7AG/q/hCYW1W7V9V5VXUBcDBwK/CxJEdX1U+AZwCLgDcCZ4znXiRJkqSWrM9Sll9KshcwDbgL+D7wtG7JyRbAocBFSbYGtqqqLyX5NnDDOC7xFeBPkny9qh5K8lR6YXxH4Naq+lCSxwH7JfkS8POq+lySG4GzxnMvkiRJUkvGs8YcejPax1TVauCHSf4fsBT4HnBFV2cb4L+SbNHVH8+bMs+gt6zl8m65zJ30Zt0PAf46yUPAfcDRwC7AR5KsmbV/+ziuI0mSJDUlVTXsPgzMjFlzatYxJw+7G5IkaQKWL5w/7C5I6yXJZVU1b7zn+c2fkiRJUgMM5pIkSVIDDOaSJElSAwzmkiRJUgMM5pIkSVIDDOaSJElSAwzmkiRJUgMM5pIkSVID1uebPzcZ++wyk8V+OYEkSZIa5Iy5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktSA6cPuwCAtu3Uls48/d9jdkKRNzvKF84fdBUna6DljLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNWBKgnmSE5JcnWRpkiVJfnMS2jwkyW9NRv8kSZKk1kyf7AaTHAi8FNivqlYl2RF47Aa2OR04BLgP+NYGd1KSJElqzKQHc2AWsKKqVgFU1QqAJMuBTwPP7+r9QVXdkGQ34ExgJ+BO4LVV9YMkZwF3A8/snp8DrE7yh8CbgCcC7wRWAyur6uApuBdJkiRpIKZiKct5wJOSXJ/k35I8r+/YPVV1AHAKcHJXdgpwdlXtC3wc+EBf/acCh1XVK4DTgPdX1dyquhA4EXhxVT0DeNkU3IckSZI0MJMezKvqPmB/YAG9GfBPJzm2O/zJvucDu+0DgU902x8DntvX3GeqavUYl/omcFaSNwDTxupPkgVJFidZvPqBleO9HUmSJGkgpmIpC12YXgQsSrIMOGbNof5qY53et33/Wq5xXPem0vnAkiRzq+quUeqdDpwOMGPWnLGuKUmSJA3VpM+YJ9kzyZy+ornA97vtV/U9X9xtfwt4dbd9FHDRGE3fC2zTd50nV9V3qupEYAXwpEnoviRJkjQUUzFjvjXwwSTbAQ8DN9Bb1vJSYEaS79D7g+DIrv6bgTOT/DXdmz/HaPe/gc8meTm9N3/+RfcHQICvAVdOwb1IkiRJAzHpwbyqLgMe9XnjSQBOrap3jai/HHjBKO0cO2L/emDfvqILN7y3kiRJUhv85k9JkiSpAVPy5s/RVNXsQV1LkiRJ2tg4Yy5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDVgYB+X2IJ9dpnJ4oXzh90NSZIk6VGcMZckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGjB92B0YpGW3rmT28ecOuxuSNtDyhfOH3QVJkiadM+aSJElSAwzmkiRJUgMM5pIkSVIDDOaSJElSAwzmkiRJUgMM5pIkSVIDDOaSJElSAwzmkiRJUgMM5pIkSVIDJi2YJ1mdZEmSq5J8JslWa6l7bJJTuu3jkhzdV75zX70zkjxtsvooSZIktWoyZ8wfrKq5VbU38HPguPU5qapOq6qzu91jgZ37jr2+qq6ZxD5KkiRJTZqqpSwXAk9JskOS/0yyNMm3k+w7smKSk5K8JckrgXnAx7uZ9y2TLEoyr6t3eJLLk1yZ5Gtd2fO6ukuSXJFkmym6H0mSJGlKTXowTzIdOAJYBrwLuKKq9gX+Bjh7rPOq6rPAYuCobub9wb42dwI+BLyiqp4B/H536C3AG6tqLnAQ8CCSJEnSRmgyg/mWSZbQC9c/AD4MPBf4GEBVfR14fJKZE2j72cAFVXVz19bdXfk3gX9J8mZgu6p6eOSJSRYkWZxk8eoHVk7g0pIkSdLUmz6JbT3YzVz/UpKMUq8m0HZGO6+qFiY5F3gJ8O0kh1XVtSPqnA6cDjBj1pyJXFuSJEmaclP9cYkXAEcBJDkEWFFV96yl/r3AaOvELwael2T3rq0duucnV9WyqnovvZn6vSax75IkSdLATOaM+WhOAj6SZCnwAHDMOuqfBZyW5EHgwDWFVXVnkgXA55P8GnAH8ELgz5M8H1gNXAP8z6TfgSRJkjQAqdp8VnfMmDWnZh1z8rC7IWkDLV84f9hdkCRpTEkuq6p54z3Pb/6UJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhpgMJckSZIaYDCXJEmSGmAwlyRJkhow1d/82ZR9dpnJYr+YRJIkSQ1yxlySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWrA9GF3YJCW3bqS2cefO+xuSI+wfOH8YXdBkiQ1wBlzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpARMO5klOSHJ1kqVJliT5zTHqzUvygYl3UZIkSdr0TZ/ISUkOBF4K7FdVq5LsCDx2tLpVtRhYPPEuSpIkSZu+ic6YzwJWVNUqgKpaUVW3JXlWkm8luTLJJUm2SXJIknMAkjwuyZlJLk1yRZKXd+XHJvl8ki8n+V6Sf1xzoSSHJ7m8a/Nra2tHkiRJ2lhNaMYcOA84Mcn1wP8CnwYu7p5fVVWXJtkWeHDEeScAX6+qP0qyHXBJkv/tjs0FngmsAq5L8kHgZ8CHgIOr6uYkO6ytnaq6f2RHkywAFgBM23anCd6uJEmSNLUmFMyr6r4k+wMHAc+nF8jfA9xeVZd2de4BSNJ/6ouAlyV5S7e/BfAb3fbXqmpld841wG7A9sAFVXVz1+bd62jnu6P09XTgdIAZs+bURO5XkiRJmmoTnTGnqlYDi4BFSZYBbwTWFXwDvKKqrntEYe+No6v6ilZ3fcsYbY7ajiRJkrSxmtAa8yR7JpnTVzSX3mz1zkme1dXZJsnI4P8V4E3pptGTPHMdl7oYeF6S3bv6a5ayjLcdSZIkqWkTnTHfGvhgt777YeAGeuu4P9KVb0lvfflhI877W+BkYGkXqpfT+3SXUVXVnd0a8c8n+TXgDuCF421HkiRJal2qNp9l1zNmzalZx5w87G5Ij7B84fxhd0GSJE2iJJdV1bzxnuc3f0qSJEkNMJhLkiRJDTCYS5IkSQ0wmEuSJEkNMJhLkiRJDTCYS5IkSQ0wmEuSJEkNMJhLkiRJDZjoN39ulPbZZSaL/TIXSZIkNcgZc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAQZzSZIkqQEGc0mSJKkBBnNJkiSpAdOH3YFBWnbrSmYff+6wuzHpli+cP+wuSJIkaQM5Yy5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDXAYC5JkiQ1wGAuSZIkNcBgLkmSJDVg6ME8yewkV40oOynJW0aUPSHJOUmuTHJNki8NtqeSJEnS1Jk+7A6Mw7uBr1bVvwIk2XfI/ZEkSZImzdBnzMdhFnDLmp2qWjrEvkiSJEmTamMK5qcCH07yjSQnJNl52B2SJEmSJksLwbzWp7yqvgLsAXwI2Au4IslO62o8yYIki5MsXv3Ayg3urCRJkjQVWgjmdwHbjyjbAVgxsmJV3V1Vn6iq1wCXAgevq/GqOr2q5lXVvGlbzZyUDkuSJEmTbejBvKruA25PcihAkh2Aw4GL+usleUGSrbrtbYAnAz8YcHclSZKkKdHKp7IcDZya5J+7/XdV1Y1JjgOoqtOA/YFTkjxM7w+KM6rqUoAkS6pq7jA6LkmSJE2GJoJ5VV0DPH+U8tP6tt8HvG+M8w3lkiRJ2qgNfSmLJEmSJIO5JEmS1ASDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1IAmvmBoUPbZZSaLF84fdjckSZKkR3HGXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWqAwVySJElqgMFckiRJaoDBXJIkSWpAqmrYfRiYJPcC1w27H5uhHYEVw+7EZsqxHx7Hfngc++Fx7IfHsR+e0cZ+t6raabwNTZ+c/mw0rquqecPuxOYmyWLHfTgc++Fx7IfHsR8ex354HPvhmcyxdymLJEmS1ACDuSRJktSAzS2Ynz7sDmymHPfhceyHx7EfHsd+eBz74XHsh2fSxn6zevOnJEmS1KrNbcZckiRJatJmEcyTHJ7kuiQ3JDl+2P3ZFCQ5M8kdSa7qK9shyVeTfK973r7v2Nu78b8uyYv7yvdPsqw79oEkGfS9bEySPCnJN5J8N8nVSf6sK3fsp1iSLZJckuTKbuzf1ZU79gOSZFqSK5Kc0+079gOQZHk3ZkuSLO7KHPsBSLJdks8mubb7vX+gYz/1kuzZ/Xtf87gnyZ8PZOyrapN+ANOAG4E9gMcCVwJPG3a/NvYHcDCwH3BVX9k/Asd328cD7+22n9aN+wxg9+7nMa07dglwIBDgf4Ajhn1vLT+AWcB+3fY2wPXd+Dr2Uz/2Abbuth8DfAd4tmM/0J/BXwKfAM7p9h37wYz7cmDHEWWO/WDG/qPA67vtxwLbOfYD/xlMA34E7DaIsd8cZswPAG6oqpuq6ufAp4CXD7lPG72qugC4e0Txy+n9EqF7/p2+8k9V1aqquhm4ATggySxg26q6uHr/es/uO0ejqKrbq+rybvte4LvALjj2U6567ut2H9M9Csd+IJLsCswHzugrduyHx7GfYkm2pTcJ9mGAqvp5Vf0Ux37QDgVurKrvM4Cx3xyC+S7AD/v2b+nKNPmeUFW3Qy9AAr/elY/1M9il2x5ZrvWQZDbwTHozt479AHRLKZYAdwBfrSrHfnBOBt4K/KKvzLEfjALOS3JZkgVdmWM/9fYA7gQ+0i3hOiPJ43DsB+3VwCe77Skf+80hmI+2lsePohmssX4G/mwmKMnWwOeAP6+qe9ZWdZQyx36Cqmp1Vc0FdqU3G7L3Wqo79pMkyUuBO6rqsvU9ZZQyx37inlNV+wFHAG9McvBa6jr2k2c6vSWj/15VzwTup7d8YiyO/SRL8ljgZcBn1lV1lLIJjf3mEMxvAZ7Ut78rcNuQ+rKp+3H3sg3d8x1d+Vg/g1u67ZHlWoskj6EXyj9eVZ/vih37AepeTl4EHI5jPwjPAV6WZDm95YgvSPIfOPYDUVW3dc93AF+gt0TUsZ96twC3dK/MAXyWXlB37AfnCODyqvpxtz/lY785BPNLgTlJdu/+8nk18MUh92lT9UXgmG77GOC/+spfnWRGkt2BOcAl3ctA9yZ5dvcu5aP7ztEounH6MPDdqvqXvkOO/RRLslOS7brtLYHDgGtx7KdcVb29qnatqtn0fod/var+EMd+yiV5XJJt1mwDLwKuwrGfclX1I+CHSfbsig4FrsGxH6Qj+dUyFhjE2A/jHa6DfgAvoffpFTcCJwy7P5vCo/uHejvwEL2/CF8HPB74GvC97nmHvvondON/HX3vSAbm0fslfyNwCt2XXvkYc9yfS+9lsKXAku7xEsd+IGO/L3BFN/ZXASd25Y79YH8Oh/CrT2Vx7Kd+vPeg92kTVwJXr/k/1LEf2PjPBRZ3v3f+E9jesR/Y2G8F3AXM7Cub8rH3mz8lSZKkBmwOS1kkSZKk5hnMJUmSpAYYzCVJkqQGGMwlSZKkBhjMJUmSpAYYzCVpI5Dkd5NUkr3WUme7JH/at79zks+uo91FSeaNsy9vSXJtkquSXJnk6PGcvx7tP+I+JGlzYTCXpI3DkcBF9L5g51GSTAO2A34ZaKvqtqp65WR2IslxwAuBA6pqb+BgRv/a6Q3xiPuQpM2FwVySGpdka3pfS/86+oJ5kkOSfCPJJ4BlwELgyUmWJHlfktlJrurqTkvyT0mWJVma5E2jXOdFSS5OcnmSz3TXHelvgD+tqnsAqmplVX20O//QJFd01zgzyYyufHmSHbvteUkWddsndfUWJbkpyZu7azziPiZhCCVpozB92B2QJK3T7wBfrqrrk9ydZL+qurw7dgCwd1XdnGR2tz0XoNtfYwGwO/DMqno4yQ79F+iC8zuAw6rq/iRvA/4SeHdfnW2AbarqxpEdTLIFcBZwaNfPs4E/AU5ex73tBTwf2Aa4Lsm/A8f334ckbS6cMZek9h0JfKrb/lS3v8YlVXXzerRxGHBaVT0MUFV3jzj+bOBpwDeTLAGOAXYbUSfAWF8XvSdwc1Vd3+1/lN4yl3U5t6pWVdUK4A7gCetxjiRtkpwxl6SGJXk88AJg7yQFTAMqyVu7Kvevb1OMHarXHP9qVR05VoWquifJ/Un2qKqbRjl/LA/zq4mgLUYcW9W3vRr/X5K0GXPGXJLa9krg7KrarapmV80vxPEAAAD4SURBVNWTgJuB545S9156S0JGcx5wXJLpACOXsgDfBp6T5Cnd8a2SPHWUdv4BODXJtl29bZMsAK4FZq85H3gNcH63vRzYv9t+xVrvdt33IUmbLIO5JLXtSOALI8o+B/zByIpVdRe9pShXjfKmyTOAHwBLk1w58vyquhM4FvhkkqX0gvpoH83478A3gEu7N5aeDzxQVT8DXgt8Jsky4BfAad057wL+NcmF9GbF12od9yFJm6xUre2VTUmSJEmD4Iy5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktQAg7kkSZLUAIO5JEmS1ACDuSRJktSA/w8M+o2Nywb5ugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "allowed_labels = df.news_desk.value_counts().where(lambda x: x > 100).dropna().index\n",
    "df = df[df.news_desk.isin(allowed_labels)]\n",
    "_ = df.news_desk.value_counts(ascending=True).plot(kind='barh', title='Desk Distribution', figsize=(12,6)).set(xlabel='Article Count')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract features from text.\n",
    "\n",
    "> Before training our model we need to convert our text into something the Classifier can actually process. Since we're talking about machines here, nothing better than converting text into numbers! That's exactly what we do here with `TfidfVectorizer`. It follows the bag of words concept, which converts the whole corpus into a words dictionary and each document (or article in this case) gets transformed into a array of numbers signaling which words appeared on the text. This modeling is oversimplified though since it doesn't take into consideration the order of words, only it they happend or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20508, 18881)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['news_desk_id'] = df.news_desk.factorize()[0]\n",
    "id_to_news_desk = df.set_index('news_desk_id')['news_desk'].to_dict()\n",
    "news_desk_to_id = {v: k for k, v in id_to_news_desk.items()}\n",
    "tfidf = TfidfVectorizer(sublinear_tf=True, min_df=3, norm='l2', encoding='latin-1', ngram_range=(1, 3), stop_words='english')\n",
    "features = tfidf.fit_transform(df.abstract).toarray()\n",
    "labels = df.news_desk_id\n",
    "features.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check Vectorizer result.\n",
    "\n",
    "> The result features from Vectorizer is a sparse matrix, have a look below. Nice! - There are indeed many zeros there... "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check correlation between words and classes.\n",
    "\n",
    "> Oh how I love statistics! Chi Squared methodology here enabled us to understadt which features (or tokens) have the higher correlation with each class. It's a good indication of how our model might perform when certain keywords appear on the text to predict."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3ca\" ><caption>Most Correlated Tokens to Category</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >1-grams</th>        <th class=\"col_heading level0 col1\" >2-grams</th>        <th class=\"col_heading level0 col2\" >3-grams</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3calevel0_row0\" class=\"row_heading level0 row0\" >Business</th>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow0_col0\" class=\"data row0 col0\" >['business', 'companies', 'company']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow0_col1\" class=\"data row0 col1\" >['real estate', 'commercial real', 'stock market']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow0_col2\" class=\"data row0 col2\" >['recent commercial real', 'real estate transactions', 'commercial real estate']</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3calevel0_row1\" class=\"row_heading level0 row1\" >Foreign</th>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow1_col0\" class=\"data row1 col0\" >['country', 'prime', 'minister']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow1_col1\" class=\"data row1 col1\" >['european union', 'benjamin netanyahu', 'prime minister']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow1_col2\" class=\"data row1 col2\" >['president vladimir putin', 'prime minister benjamin', 'minister benjamin netanyahu']</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3calevel0_row2\" class=\"row_heading level0 row2\" >Politics</th>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow2_col0\" class=\"data row2 col0\" >['sanders', 'democratic', 'biden']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow2_col1\" class=\"data row2 col1\" >['tip sheet', 'mr biden', 'joe biden']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow2_col2\" class=\"data row2 col2\" >['democratic presidential candidates', 'joseph biden jr', 'morning tip sheet']</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3calevel0_row3\" class=\"row_heading level0 row3\" >Science</th>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow3_col0\" class=\"data row3 col0\" >['patients', 'study', 'scientists']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow3_col1\" class=\"data row3 col1\" >['study suggests', 'scientists say', 'new study']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow3_col2\" class=\"data row3 col2\" >['new study finds', 'food drug administration', 'new study suggests']</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3calevel0_row4\" class=\"row_heading level0 row4\" >Sports</th>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow4_col0\" class=\"data row4 col0\" >['players', 'league', 'team']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow4_col1\" class=\"data row4 col1\" >['grand slam', 'premier league', 'world cup']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow4_col2\" class=\"data row4 col2\" >['new england patriots', 'newsletter marc stein', 'week newsletter marc']</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3calevel0_row5\" class=\"row_heading level0 row5\" >U.S.</th>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow5_col0\" class=\"data row5 col0\" >['county', 'detailed', 'maps']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow5_col1\" class=\"data row5 col1\" >['shows extent', 'asked candidates', 'results maps']</td>\n",
       "                        <td id=\"T_dab64ad0_cc59_11ea_8dda_988389d3b3carow5_col2\" class=\"data row5 col2\" >['county map shows', 'extent coronavirus outbreak', 'shows extent coronavirus']</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x2872ccfe388>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_df = get_most_correlated_tokens(features, labels, tfidf, news_desk_to_id, 3)\n",
    "corr_df.style.set_caption(\"Most Correlated Tokens to Category\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Selection\n",
    "\n",
    "Train Naive Bayes classifier.\n",
    "\n",
    "> Finally we've achieve the moment to actually train a Machine Leraning model!! But let's take it easy hehe... We're starting with a probability classifier which uses the Bayes Theorem. Here it basically compounds each feature probabilty to infer the result considering their combination in a given test value. Let's see it in action!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_vect, clf = train_text_classifier(df, 'abstract', 'news_desk', MultinomialNB)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check results.\n",
    "\n",
    "> After training it we can play around with our first Text Classifier. For that I've taken some article news from NYT web site to see if it actually performs well. And a priori it sure does!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': 'Russian historian jailed in sex abuse case', 'prediction': 'Foreign'}\n",
      "{'text': 'It’s time for the weirdest season of baseball in recent memory.', 'prediction': 'Sports'}\n"
     ]
    }
   ],
   "source": [
    "test_texts = [\n",
    "    \"Russian historian jailed in sex abuse case\",\n",
    "    \"It’s time for the weirdest season of baseball in recent memory.\"\n",
    "]\n",
    "\n",
    "for txt in test_texts:\n",
    "    print(predict_text(txt, clf, count_vect))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare more models.\n",
    "\n",
    "> Should we be happy with our first trained model? Yes! And how about being satisfied with it? No! Before moving on we should definetly check if our model performs well in comparison to other algorithms. For that we use cross validation. That's what we have below. Apparantly LinearSVC is a good choice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 4/4 [05:18<00:00, 79.62s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEXCAYAAABoPamvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZgV5Zn+8e/dO82OICoIuOCCRklsjSZm1KAZY1zib5y4zSRm0TEZg47ZzBgzOGOiMYmjZDfGaDSjiUsCrmgWjAsuoCiLGhFBEZVmp4Gmt+f3R1XD4dBNHzh9+nTD/bmuvrpOrU/VqVNPvW9VvaWIwMzMLB8lxQ7AzMx6PicTMzPLm5OJmZnlzcnEzMzy5mRiZmZ5czIxM7O8OZnYDkHSKEkhqSyHcc+T9ERXxNXVJP1c0hXFjsN2Pk4m1uUkLZDUIGlwVv+ZaUIYVZzINsZRIWmCpNckrU3jvbnYceUiIi6MiP8pdhy283EysWJ5Azi79YOk9wG9ihfOZu4GTgXOAfoDhwIzgHHFDKojkkqLHYPtvJxMrFhuAz6d8fkzwG8yR5DUX9JvJNVKWijpW5JK0mGlkn4gaamk+cAn2pj2V5LekfS2pKtyOdhKOh44ATgtIp6LiKaIWBURP4mIX6Xj7CFpsqTlkuZJOj9j+gmS7pJ0u6Q1kmZJ2k/SNyUtkfSWpI9ljD9V0tWSnpW0StIkSYMyht8l6d102N8kHZQx7BZJP5P0oKS1wHFpv6vS4YMl3S9pZRrr4xnb78B02SslzZF0atZ8fyLpgXQdnpG0T0fbznZuTiZWLE8D/dKDWilwJnB71jg/IikZ7A0cQ5J8PpsOOx84GXg/UAOckTXtrUATsG86zseAL+QQ1/HAsxHx1lbGuQNYBOyRLve7kjJLLaeQJMuBwAvAFJLf2jDgv4FfZM3v08Dn0vk1ARMzhj0EjAZ2BZ4Hfps17TnAd4C+QPZ1oK+kcQ4BhgL/CYSkcuA+4JF0vl8Gfitp/4xpzwauTNdhXroMs3Y5mVgxtZZOTgBeAd5uHZCRYL4ZEWsiYgHwQ+Bf01E+BVwfEW9FxHLg6oxphwIfBy6JiLURsQT4X+CsHGLaBXinvYGS9gSOBr4REfURMRO4KSMugMcjYkpENAF3kRzMr4mIRuBOYJSkAZnbISJmR8Ra4ArgU62lqIi4OV3/DcAE4FBJ/TOmnRQRT0ZES0TUZ4XbCOwOjIyIxoh4PJLG+I4E+qQxNUTEX4D7yah2BO6NiGfTdfgtMLbDLWc7NScTK6bbSM6szyOrigsYDFQACzP6LSQ5u4fkLP6trGGtRgLlwDtpNc5KktLArjnEtIzkANyePYDlEbGmnbgA3svoXg8sjYjmjM+QHMxbZa9HOTA4rcq7RtLrklYDC9JxBrczbbbvk5QqHpE0X9JlGevwVkS0bGUd3s3oXpcVr9kWnEysaCJiIcmF+JOAe7MGLyU5sx6Z0W8Em0ov7wB7Zg1r9RawARgcEQPSv34RcRAd+xNwhKTh7QxfDAyS1LeduLZH9no0kqz/OcBpJFVv/YFR6TjKGL/dZr/TEs1XImJvkqq3S9PquMXAnq3XTzppHWwn52RixfZ54KNpFc9G6Zn874HvSOoraSRwKZuuq/weGC9puKSBwGUZ075Dcj3gh5L6SSqRtI+kYzoKJiL+BDwK/EHSYZLK0uVfKOlz6bWUp4CrJVVJOiRdh+xrGdviXySNkVRNck3l7nT9+5IkxWVANfDdbZmppJMl7StJwGqgOf17BlgLfF1SuaRjSZLNnXmsg+3knEysqCLi9YiY3s7gL5Mc9OaTXFz+P+DmdNgvSS5sv0hyYTq7ZPNpkmqyucAKktt9t1Z9lekM4EHgd8AqYDbJRf4/pcPPJiklLAb+APxXRDya47zbchtwC0nVUhUwPu3/G5Lqp7fT9Xh6G+c7Oo25DpgG/DQipkZEA8mtzx8nKQH9FPh0RLySxzrYTk5+OZZZ8UiaCtweETcVOxazfLhkYmZmeXMyMTOzvLmay8zM8uaSiZmZ5a3D5rq7m8GDB8eoUaOKHYaZWY8yY8aMpRExpFDz73HJZNSoUUyf3t6dpGZm1hZJCzsea/u5msvMzPLmZGJmZnlzMjEzs7z1uGsm3dWCBQt46KGHWLJkCf3792fcuHEcfPDBJM0imZnt2JxM8tTc3Mz111/PpEmTNut/7733cuSRRzJhwgSqq6uLFJ2ZWddwNVeebrrpJiZNmkSolLpdDmD9gL1oquhDIJ5++mm+/e1vFztEM7OCczLJw5o1a7j77ruT7qFj6b3sVXqtfIOyhjqUvmbimWef5Z577ilmmGZmBedqriwTJ05k3rx5OY27YsUKNmzYQEP1EPq9OwOAuiEHs3bIQailib7vzaTXite54YYbmDJlClVVVdsUy7777sv48eM7HtHMrMh2uGSyLcmgLYsWLWL9+vUdjwg0NjYCUNKYjL9q+IdYNeIjG4fXD9yHXV67j95LX2bevHlUVlZucyz5rAs4IZlZ19jhksnUqVOpXboMSrtg1aIFAaWNdQRi9R6Hbz5cYs0eh9N76cs0NjXRGNt2Z1ddfQO1K1Ztf3zNTSxatMjJxMwKbodLJgCUltFSvUvhlxNBSd0SRNBSUkqUblnyaC7vnXSopGtiylCyblmXLs+6r3Xr1nHfffdtvH29X79+jBs3jtNPP53BgwcXOzzbAexwyWT48OG8t6GM+jEnd8nySpcvpPK1RylpaaJq5RvUD9x7s+G9lv8dgJa+u1F/4EldElOrqrn3M3z4bl26TOt+Vq5cySWXXML8+fM39qurq+O2227j/vvv57rrrmOfffYpYoS2I/DdXHlqHjSSpsH7AjDo9SlUrFmcDIigavnrDHjzcQAadz2gWCHaTu7aa69l/vz5tFS00FKd/DXt0kRT/yZWrFjB5ZdfTlNTU7HDtB5uhyuZFEPDqKMpWb+KsrW17DbrNhqrBqKWJsoa1gDQNGgvmgftVeQobWe0ePFinnjiCYKgpKEEGpL+JeuS88goCxYvXsy0adP4yEc+spU5mW2dk0lnKC2j/sCTqFg0g7Lav1NevwKAlvJqmoaOoXGPQ8DNqlgn2Na7FWtrawkCIRpGNFB/SD1RHZS/VU7VjKokwQDXX389d911V87z9V2Cls3JpLOUltMw8kga9qxB9auAEqKqP5S4JtES+d62Dtt26zpAfX09QjTu1sja49dCek6zYcwGmgc00/ehvgTB8uXLWbdu3TbF4dvWLVNBk4mkE4EbgFLgpoi4Jmt4f+B2YEQayw8i4teFjKngSsqILr5ry3qGqVOnUrustmtP4VpAiA0HbNiYSFo17d5Ec/9mSleV0hRN1DXU5TzbuoY6alfVbn9cTfi29R1MwXZrSaXAT4ATgEXAc5ImR8TcjNH+HZgbEadIGgK8Kum3EdFQqLjMiqoMGNCFy1sJNNL2L10QpUmzP/QFKrouLFZ24bKsSxTyHOkIYF5EzAeQdCdwGpCZTALoq6Sd9j7AciDv20pK1i2nau79+c5mu6l+NQBR1a9oMUCyHcC3BncXw4cPp1a1tBzb0mXL1CyhV0TF6xU0jmzcbFjJihLKlpcRJUHLcS1Q3mVhUTK1hOHDhnfdAq3gCplMhgFvZXxeBHwwa5wfA5OBxSTnRmdGxBa/NEkXABcAjBgxYqsL3Xfffbc/4k7y2mvJXVyj9yn2gXy3brE9rHhi7yBeDSreqKB5ejMb3reBqAjK3i2j+snk1QgxMro0kdiOqZDJpK3blyLr8z8CM4GPAvsAj0p6PCJWbzZRxI3AjQA1NTXZ89hMd6iDbY1h4sSJRY5kx7Z8+XJefvllJHHggQcycODAYofU/fSG+EDADOj1Yi+qXqqCMlBj8vOM/kEcstWflFlOCplMFgF7ZnweTlICyfRZ4JqICGCepDeAA4BnCxiX9XCrV6/mhhtu4C9/+QvNzc0AlJWVMW7cOMaPH0/fvn2LHGH3EnsH0TsoebUEvSdohKiMpP/+LpVY5yhkMnkOGC1pL+Bt4CzgnKxx3gTGAY9LGgrsD8zHrB3r1q3j4osv5vXXX6eE4P0DkkY2X1rZmylTprBgwQImTpxIr169ih1q9zIUWoa2JFckm0kutvvRJ+tEBUsmEdEk6SJgCsmtwTdHxBxJF6bDfw78D3CLpFkku/Y3ImJpoWIqtvfee4/Jkyczd+5cJDF27FhOPvlkBg0aVOzQuq2I4IUXXuCZZ56hoaGBJUuW8Prrr7NndT3XHvIGw6qTG//eWlfB12buzauvvsqkSZM466yzihx5O1YmF5+LpvXu3z7FCwFI7uYaVuQYrFMV9I73iHgQeDCr388zuhcDHytkDN3Fww8/zLXXXrtZG0jTp0/n9ttv58orr+Soo44qYnTd05IlS/jWt77FK6+8kjUk+OiuKzcmEoA9qxv499GLuXzWXjzwwAPdMpl0h5shXnvtNQBGDxtd3ECGdY/tYZ3HT8B3gVmzZnHNNdfQ0tLCul32Z+2QgyBa6PPeTFi5gCuuuIJf/epXjBw5stihdrrtfeq7paWFV199lQ0bNjCoopETd1tBv/ImHq/tz5zVvbltwVAOG1jH2IFrN05TMyg57X7zzTe3uBGjOzxtXezlZ8bgm0OsszmZdIE777yTlpYWVu9ew8q9xm3sv37Qfuzy2n2w9GXuueceLr300iJGuaViNP/RqrGxkYaGBkZU1/PTw+bRrzy50H72iFp+9Noe3L1oCLcsGMr1AzddYnt7XfLUXURsPAPPjGNnaP6jubmZp556iilTprBs2TIGDRrECSecwNFHH01ZmX/uVjjeu7J0xgG09UA2fvx4IoIXX3wRgNXDsh6zkVg97IP0XvoyDzzwAAsWLNhscLEPXlOnTmX50loqS7v+1tGmZgHiX0Yu2ZhIIGkv83N7v8vkxbvw/Iq+rGgoY2BFE80BtywYCkCpgpb6NZvNb239Gl5dsWS749nQrG7f/MfatWv55je/ycyZMzfr//jjj3PQQQdx7bXXFiky2xk4mRRA5p1EyV3PECqhpfWtixmay5MroS0tXfdU9LaoLA1G9m3ueMRO9sbqMja0wF596rcY1qeshV0rG1m0vpI/LBrEoIpmHnp3IC+v7k0Jwcg+TVSUdm48C9d08gwL4Nprr2XmzJm09Gqh/n31NA9ppnRpKVWzqpgzZw5XXXVVsUO0HZiTSZZCnHmeeeaZvPPOO1StnE/9wM3faNdrRVIKOuCAA7pdPfbw4cOpb3qHb9Xk3gBgZ/nhzN68sLSCWSt7s3/fzavJajeUsXh9BRDcsmD3jf37lLUw/pC1jBnU+S96ump6H6qGd9/mP95++23++te/EqXBmpPX0NIvOTlp2q2JhlEN9L+nP9OmTWP//ff3bdNWEE4mXeCUU07hxhtvZND8R6jd/3Qa++wGEVSufosBCx8D4NRTTy1ylN3LR3Zv4IWlFdy2YFdqBq1hVO8NANQ3i+teHU4LYlTfJvbonZSaDhjQxId2a6BqB9+j26uGra1NWvBtHNm4MZG0ij5Bw94NVP69kvnz51NRUZH3SVOxq2Ct+9nBf3rdwxlnnMETTzzB3Llz2f2lW2noNRhFC+X1ywH44Ac/yAknnFDkKLuXml0bGTOwkbkryjnv2f05YtAa+pc38fSyfqxqLKO6rIUvHryWYb27Z/VgV2utJm3p1fb2aO1fVlbmkokVhJNJF6iqquK6667jl7/8JQ899BCsS57L7NevH6eddhrnnXee77TJUiK4dGwdt75SzVPvVvD0sk0tMI/s28QFY9btlImkvdLAtGnT+MY3vkH5W+WsP2I9ZD4XGVCxMLnT7bLLLuO4447rgkhtZ+MjWBeprq7m4osv5vzzz2fBggWUlJSw1157UVlZWezQuq2qUvi3g9bxqX3XM2d5OY0tMKJPM3v3a/ZbkLMcfvjhDB06lPfee4/qJ6tZf8R6ojKgAXrN6EXpylIGDRrE0UcfXexQbQflZNLFqqurGTNmTLHDyNmbdaVcNb24bW+8ty45zR5a3cKTRYrhzbpS9ivSsnNRVlbGZZddxte+/jX4O1S8XpG8RXF1KWoSpaWlScml3K06WmE4mVi7itncRUtLCy0tLZSWltKQXnCuGlW8JkD2o/s3/3HYYYfxo4k/4qabbmLGjBmULU9+3mPHjuXzn/88hx56aJEjtB2ZWp+D6Clqampi+vTpxQ7DCmT27NncfvvtPP3007S0tNCnd2+qevVi6NCh/OxnPyt2eD1GbW0ty5YtY+DAgQwdOrTY4Vg3IGlGRNQUav4umVi38dhjjzFhwgSam5spIehb1syatWupW7uW1atXs2TJEnbddddih9kjDBkyhCFDhhQ7DNuJFLEtbLNN1qxZw3e/8x2am5s5fdhS/nD0XB74hzncdPjfOaDvOhoaGrjuuuuKHaaZtcPJxLqFhx9+mPX19bx/wBou2e9tBlYkT7Hv13c9Vx/yBuVqYdq0abz77rtFjtTM2uJqLiuoXBvOXLhwIQDjhq7c4rbfXSqb+MDAOp5Z3o+vf/3rDBgwYJvj8BPbZoXlkol1C0ozSH1L27vk+uaSzcYzs+7FJRMrqFxLA4888ghXXXUVDywexOnDllFesukuw9frqnhpVR8qKiqYOHEi/fv3L1S4ZradXDKxbuGYY45hyJAhvLG2F5fO3JvnlvfhnfXl3L94EF+ZuTcAJ554ohOJWTflkol1C5WVlVx99dV85Stf4cWV8JWZmz91//73v5+LLrqoSNGZWUecTKzb2G+//bj11luZNGkSU6dOpa6ujj322INPfOITHH/88W4M06wb8xPwZmY7gUI/Ae9rJmZmljcnEzMzy5uTiZmZ5c3JxMzM8uZkYmZmeXMyMTOzvDmZmJlZ3pxMzMwsb04mZmaWt4ImE0knSnpV0jxJl7Ux/GuSZqZ/syU1SxpUyJjMzKzzFSyZSCoFfgJ8HBgDnC1pTOY4EfH9iBgbEWOBbwKPRcTyQsVkZmaFUciSyRHAvIiYHxENwJ3AaVsZ/2zgjgLGY2ZmBVLIZDIMeCvj86K03xYkVQMnAve0M/wCSdMlTa+tre30QM3MLD+FTCZtvV+1vSaKTwGebK+KKyJujIiaiKgZMmRIpwVoZmado5DJZBGwZ8bn4cDidsY9C1dxmZn1WIVMJs8BoyXtJamCJGFMzh5JUn/gGGBSAWMxM7MCKtir6yKiSdJFwBSgFLg5IuZIujAd/vN01NOBRyJibaFiMTOzwvKbFs3MdgJ+06KZmXV7TiZmZpY3JxMzM8ubk4mZmeXNycTMzPLmZGJmZnlzMjEzs7w5mZiZWd6cTMzMLG9OJmZmljcnEzMzy5uTiZmZ5c3JxMzM8uZkYmZmeXMyMTOzvDmZmJlZ3pxMzMwsb04mZmaWNycTMzPLm5OJmZnlzcnEzMzy5mRiZmZ5yymZSLpH0ickOfmYmdkWck0OPwPOAV6TdI2kAwoYk5mZ9TA5JZOI+FNEnAt8AFgAPCrpKUmflVReyADNzKz7y7naStIuwHnAF4AXgBtIksujBYnMzMx6jLJcRpJ0L3AAcBtwSkS8kw76naTphQrOzMx6hpySCfDjiPhLWwMioqYT4zEzsx4o12quAyUNaP0gaaCkLxUoJjMz62FyTSbnR8TK1g8RsQI4vzAhmZlZT5NrMimRpNYPkkqBisKEZGZmPU2uyWQK8HtJ4yR9FLgDeLijiSSdKOlVSfMkXdbOOMdKmilpjqTHcg/dzMy6i1wvwH8D+Dfgi4CAR4CbtjZBWnr5CXACsAh4TtLkiJibMc4A4KfAiRHxpqRdt30VzMys2HJKJhHRQvIU/M+2Yd5HAPMiYj6ApDuB04C5GeOcA9wbEW+my1myDfM3M7NuIte2uUZLulvSXEnzW/86mGwY8FbG50Vpv0z7AQMlTZU0Q9Kn21n+BZKmS5peW1ubS8hmZtaFcr1m8muSUkkTcBzwG5IHGLdGbfSLrM9lwGHAJ4B/BK6QtN8WE0XcGBE1EVEzZMiQHEM2M7Oukmsy6RURfwYUEQsjYgLw0Q6mWQTsmfF5OLC4jXEejoi1EbEU+BtwaI4xmZlZN5FrMqlPm59/TdJFkk4HOrpY/hwwWtJekiqAs4DJWeNMAj4iqUxSNfBB4OVtiN/MzLqBXO/mugSoBsYD/0NS1fWZrU0QEU2SLiK5rbgUuDki5ki6MB3+84h4WdLDwEtAC3BTRMzevlUxM7NiUUT2ZYysEZJbfK+JiK91TUhbV1NTE9Onu21JM7NtIWlGIdtS7LCaKyKagcMyn4A3MzPLlGs11wvAJEl3AWtbe0bEvQWJyszMepRck8kgYBmb38EVgJOJmZnl/AT8ZwsdiJmZ9Vy5vmnx12z5wCER8blOj8jMzHqcXKu57s/orgJOZ8sHEM3MbCeVazXXPZmfJd0B/KkgEZmZWY+T6xPw2UYDIzozEDMz67lyvWayhs2vmbxL8o4TMzOznKu5+hY6EDMz67lyfZ/J6ZL6Z3weIOmThQvLzMx6klyvmfxXRKxq/RARK4H/KkxIZmbW0+SaTNoaL9fbis3MbAeXazKZLuk6SftI2lvS/wIzChmYmZn1HLkmky8DDcDvgN8D64F/L1RQZmbWs+R6N9da4LICx2JmZj1UrndzPSppQMbngZKmFC4sMzPrSXKt5hqc3sEFQESsoON3wJuZ2U4i12TSImlj8ymSRtFGK8JmZrZzyvX23suBJyQ9ln7+B+CCwoRkZmY9Ta4X4B+WVEOSQGYCk0ju6DIzM8u5occvABcDw0mSyZHANDZ/ja+Zme2kcr1mcjFwOLAwIo4D3g/UFiwqMzPrUXJNJvURUQ8gqTIiXgH2L1xYZmbWk+R6AX5R+pzJH4FHJa3Ar+01M7NUrhfgT087J0j6K9AfeLhgUZmZWY+yzS3/RsRjHY9lZmY7k+19B7yZmdlGTiZmZpY3JxMzM8tbQZOJpBMlvSppnqQtmrCXdKykVZJmpn/fLmQ8ZmZWGAV79a6kUuAnwAnAIuA5SZMjYm7WqI9HxMmFisPMzAqvkCWTI4B5ETE/IhqAO4HTCrg8MzMrkkImk2HAWxmfF6X9sh0l6UVJD0k6qIDxmJlZgRSsmgtQG/2y34HyPDAyIuoknUTyhP3oLWYkXUDa5P2IESOyB5uZWZEVsmSyCNgz4/NwsppgiYjVEVGXdj8IlEsanD2jiLgxImoiombIkCEFDNnMzLZHIZPJc8BoSXtJqgDOAiZnjiBpN0lKu49I41lWwJjMzKwAClbNFRFNki4CpgClwM0RMUfShenwnwNnAF+U1ETysq2zIsKvAzYz62HU047dNTU1MX369GKHYWbWo0iaERE1hZq/n4A3M7O8OZmYmVnenEzMzCxvTiZmZpY3JxMzM8ubk4mZmeXNycTMzPLmZGJmZnlzMjEzs7w5mZiZWd6cTMzMLG9OJmZmljcnEzMzy5uTiZmZ5c3JxMzM8uZkYmZmeXMyMTOzvDmZmJlZ3pxMzMwsb04mZmaWNycTMzPLm5OJmZnlzcnEzMzy5mRiZmZ5czIxM7O8OZmYmVnenEzMzCxvTiZmZpY3JxMzM8ubk4mZmeXNycTMzPJW0GQi6URJr0qaJ+myrYx3uKRmSWcUMh4zMyuMgiUTSaXAT4CPA2OAsyWNaWe87wFTChWLmZkVViFLJkcA8yJifkQ0AHcCp7Ux3peBe4AlBYzFzMwKqJDJZBjwVsbnRWm/jSQNA04Hfl7AOMzMrMAKmUzURr/I+nw98I2IaN7qjKQLJE2XNL22trbTAjQzs85RVsB5LwL2zPg8HFicNU4NcKckgMHASZKaIuKPmSNFxI3AjQA1NTXZCcnMzIqskMnkOWC0pL2At4GzgHMyR4iIvVq7Jd0C3J+dSMzMrPsrWDKJiCZJF5HcpVUK3BwRcyRdmA73dRIzsx1EIUsmRMSDwINZ/dpMIhFxXiFjMTOzwvET8GZmljcnEzMzy5uTiZmZ5c3JxMzM8uZkYmZmeXMyMTOzvDmZmJlZ3pxMzMwsb04mZmaWNycTMzPLm5OJmZnlzcnEzMzy5mRiZmZ5czIxM7O8OZmYmVneCvo+EzOzHU19fT0vvfQS9fX1jBo1ihEjRhQ7pG7BycTMLAfNzc3ceuut3H333dTV1W3sf/DBB/PVr36Vvffeu4jRFZ+TiZlZByKC73//+zz44INbDJs9axbnn38+N954I/vssw9NTU08+OCDTJ48mTfeeIPKykqOPPJIzjzzTPbff/8iRN81FBHFjmGb1NTUxPTp04sdhpntRGbPns2XvvQliACJypYW9mtsZE1JCQvKywEYPHgwd9xxB5dffjnPPvvsFvMoKSnhiiuuYNy4cV0dPgCSZkRETaHm75KJmVkH7r///qRD4uANG/jC6jX0Sk/E/15ezo/792Pp0qVcddVVPPvss/RtaeGMNXWMbWhgTYl4pFc1f6vuxXe/8x0OOuggdttttyKuTWE4mZhZjzRx4kTmzZu33dMvWrSI9evX5zTuunXrIIIS4NNr6jYmEoD9Ghs5fv16Hujdm8ceewyA81etZv/GRgCqmoNz6+qoKynh+So499xzqaio2Gz+vXr1Yvjw4du9LgD77rsv48ePz2se+XAyMbMeaerUqSxdurTrFiixR2MT/Vtathh0QEMjD/ROundpbma/NJFk+nB9Pc9XVdLY2Ehj1vC1a9fmvS6LFi1yMjEz21YDBgzIuWTRlg0bNtDSRmJoS+t4y0tLaATKs4bXlpZu7Fa7c9lUmikp2fwRv5KSEiorK3OKpT0DBgzIa/p8OZmYWY908803d9myNmzYwEknncS6xkb+XN2LE9dtSmLrJR6p7gUk1VVL16/ntfLyLUonT1VVAXDOOedw4YUXdlnsXcVPwJuZdaCysnJjFdIf+vThp/378WRVFQ9V9+J/Bg3k3bIyhg0bxic/+UkAftm/H89VVtIILC8p4c4+vZlRVUVZWRmnnnpqEdekcFwyMTPLwWmnnUZzczM/mjiRFysreTGjWmrUqFFcc8017LLLLrz22mtMnz6dm/r322z6kpISLrvsMvbYY4+uDr1L+DkTM7NtsGzZMh566KGNDyR+6EMf4sgjj6SsLCtG4WQAAA3JSURBVDk3b2xs5L777mPy5MksWLCAiooKjjrqKM4880zGjBlTtLgL/ZyJk4mZWYFEBFL7l+S7UqGTia+ZmJkVSHdJJF3BycTMzPLmZGJmZnlzMjEzs7z1uAvwkmqBhcWOIweDgS5s62GH5+3ZebwtO1dP2Z4jI2JIoWbe45JJTyFpeiHvnNjZeHt2Hm/LzuXtmXA1l5mZ5c3JxMzM8uZkUjg3FjuAHYy3Z+fxtuxc3p74momZmXUCl0zMzCxvTiZmZpa3nTKZSKpro9+Fkj7dBcv+nKRZkl6SNFvSaZLOk3RH1niDJdVKqpRULukaSa+l0zwr6eMFjjMk3ZbxuSyN5/4cpq1L/4+SdE5G/xpJEwsT8cZlnCrpsg7GOU/Sj9PuCZLWSdo1Y3hdRnezpJmSXpT0vKQPZc1ri31pO2Le6nbZnu0oaUHGfvaYpJH5xrmtMrbdbEn3SRqQ9s/rt5b5/XVCjK3baWb696GOp9qu5YyVdFJWv49Lmi7pZUmvSPpB2n+CpK924rKfyuj+vqQ56f/OPeZFxE73B9QVYZkCRgCvA/3Tfn2AvYB+JA89VWeMfyHwq7T7GuBWoDL9PBT4VKG3EfAC0Cv9/HFgJnB/rtsXODaX8YvwXZwH/DjtngC8CXyvrf0jq/sfgce6el/anu0ILAAGp91XAr/shDgElGzLPpTRfStweWd/f50wr43baRunK8snZuDg9FhwQOv8gC+l3ROArxZoX1rdehzp7HXeKUsmbck8G5A0VdL30hLA3yV9JO1fmmb059Izvn9L+/eR9Of0zHWWpNPS/qPSs46fAs+TJI41JAdqIqIuIt6IiNXA34BTMkI6C7hDUjVwPvDliNiQTvdeRPy+CzbLQ8An0u6zgY2lp+yzp/Tsc1TW9NcAH0nP+P5D0rGtJZt0+pvTbT1f0viMeV2azm+2pEvSfqPSs7eb0v6/lXS8pCfTEtsR6XiZpY5TJD0j6QVJf5I0tJ31vBk4U9KgDrZHP2BFB+O0noU+ne4jf5A0MO1/eNpvWrofzU77Z26XYzLOkl+Q1LeD7dhH0q8zSiH/1EZI04Bh6fhDJN2T7sPPSfpwRv9H0334F5IWKikdZ+/De0r6WsZv4Mp0+t6SHlBSgpst6cy0/zWS5gLjgDPSfr9MSwQvZKzrS5JmKCnBPKyktPhMxm/wu+n/x4APZ2zrkelv76X0/4i0/y2Sfibpr+n+dUy6v70s6ZYOvr+tzfM6SX8FvidpnzTWGZIel3RAOt4/p9vgRUl/k1QB/DfJPjYz3TZfB74TEa8ARERTRPy0jVjOT7f1i+n3Vt3WMtJ+B6Xbq3V7jk77t9YUTAZ6A89IOlObH/PaW5fN1nlr263TM19P+KONs0kyzgaAqcAP0+6TgD+l3RcA30q7K4HpJAmiDOiX9h8MzCM5ixsFtABHpsNKgSkkZ8K/Bk7JWP4/A39Iu/cAFqfjHwK8UIxtlC77bqCKpFRyLOkZMllnT8BsYFTm9iXrjLqN6Z9Kt+NgYBlQDhwGzEp3+j7AHOD96bZsAt5HUj07gyQJCDgN+GM63/PYVOoYyKY7Fr+Q8Z1mjjMB+CrwbeDK7P0DaE7X/RVgFXBYDvvSS8Axafd/A9dnbKMPpd3XALPb2C73AR9Ou/uk+9bWtuP3Wuffus7p/wVsKplcD1yQdv8fcHTaPQJ4Oe3+MfDNtPtEINLvZRSb78MfI7kVVun3cD/wD8A/kVH6AfqT7EOvkuzHdwH/lA67Gvh62j2ftLQHTEq3cX+Sk6vVwJ7AuUA9MASoAJ7M+P7uAz6Tdn+OTfvBLcCdbNo/VrP5vjM2YzvNSr/jZ3KY5/1Aafr5z8DotPuDwF/S7lnAsLR7QPY+l35+Hji0nd/eBDYdi3bJ6H8VyUlle8v4EXBu2l3BplqF9kramctpb102W+et/fm1ve27N/0/g+QHBckP6RBJZ6Sf+wOjgUXAdyX9A8kPbxhJVRTAwoh4GiAimiWdCBxOcqb2v5IOi4gJJF/YTyX1Az4F3J2OX8BV3LqIeElJaeNs4MECLOKBSEpbGyQtIdlmR5Mk1bUAku4FPgJMBt6IiFlp/znAnyMiJM1i03eUaTjwO0m7k/y43thKLBOBmZJ+mNV/fUSMTZd5FPAbSQdH+kvLJqk/yY/7sbTXrcBdSq4X9I2I1vrr/wNObmMWTwLXSfotcG9ELOpgHziepBQLQERklpz+qqQ0tgT4Vsb4YzLm2U9J6edo4PR0Hg9LypzPxn2Y5DfwMZIqUEgS3mjgceAHkr5Hkugel9SLJGGtI0kqD6TTVAL/Kuk8ku+oIe3/BEkiXSWpheREYiRJQm2IiFoASb8D9kunOQr4f2n3bcC1GXHfl7F/vJe174wiSSAAx0VEZttaW5vnXenvsg/wIZLvtnVY63t8nwRukfR7Nh1HttfBkq4CBpBs6ylbWcY04HJJw0n2nddyWUAH6wLpOnc0H1dztW9D+r8ZNiZdkZwZjE3/9oqIR0jOnIaQnLWOBd4jOZsHWJs500g8GxFXkxwE/intvx54mOQHfRabqpTmASPSH3wxTAZ+kBFPqyY233+q2HYbMrpbt/PWjpyZ47dkfG6BNk+MfkRyNvg+4N+2FmNErCQ5wH9pK+NMIzlb357G8nI6K4iIa0hKUb2Ap1urGzqYb3sPix1HcjCeQ1JCguQ7OypjHx4WEWs6iC9zHxZwdcb0+0bEryLi72wqVV4t6dvAepKD4LkkJwovpfP4KMn1gqOAWjb/XloyuptIvtcWcj9WZW6LzP0je9/ZlhPpzHm2bosSYGXGdhgbEQcCRMSFJMl7T5ITlF3amOccku3VkVuAi9J9+ErSbdXWMiLi/4BTSbb7FEkfzXH92l2XrHXucCaWuynAFyWVA0jaT1JvkhLKkoholNT6A96CpD0kfSCj11g2bwH5DuBSkh9ea2lmHfArYGJa94qk3SX9S+euWrtuBv679awuwwLgA2k8HyCp7su2BtjWJPg34JOSqtNtezrJWe/26A+8nXZ/JofxryNJOm0eaNIDeylJlVybImIVsELpdTbgX0mqcVYAayQdmfY/q63pJe0TEbMi4nsk1agHsPXt+AhwUcb0A7PiWQ9cAnxayTWh7PHHpp1PkJSIkfQxkirCtkwBPpeezSJpmKRdJe0BrIuI20lOPlr38/4RcXc6733S304lsCrdVqVsOgv+MEn1brbngSpJu6TT/3PGsKfYtC3PTdcjXx3OM5LrnG9I+mcAJQ5Nu/eJiGci4tskN9bsyZbf4feB/5S0XzpNiaRL24ilL/BOut7ntvZsaxmS9gbmR8REkpPAQ3JZ2a2ty7bYWZNJtaRFGX9tfYltuQmYCzyv5OLpL0gOPL8FaiRNJ/nCX2ln+nKSqoBXJM0EzgQuzhj+CMn1kt9lVaN8i+QMbm663D+mnwsuIhZFxA1tDLoHGJSuxxeBv7cxzktAU3qR8D9yXN7zJGdjzwLPADdFxAtbnah9E0iK7o+TQxPhaVXHH9i8iN8rvaA5E/gdSV16ZpG/rX3pM8D3Jb1EcsLQWir4PHCjpGkkZ/ir2gjjEqUXVknOMB9i69vxKmBgxjTHtbFe75CcqPw7MJ5kX31JyYXxC9PRrgQ+Jul5kjv33iE5AGbP6xGSEty0tProbpID3vuAZ9PtdHkal4D70+3w43Q9ziKpojlJ0pMkB71B6TgjgOfa2CbvAStJqnH+RJJcWo0HPptO/69s/nvaXrnO81zg8+l2n0NybQaS735W+lv9G/Ai8FeS6sWZks6MiJdIkvwdkl4muZ62exvLuILkd/Aomx9X2lrGmcDs9Ds4APjNNqxze+uSMzenYtZFJPWJiNY7ay4Ddo+Izjj45U1SJdAcEU1Krg39rPVakVkufAHerOt8QtI3SX53C0nu8OkuRgC/l1RCckH8/CLHYz2MSyZmZpa3nfWaiZmZdSInEzMzy5uTiZmZ5c3JxMzM8uZkYtYOJY0RDs53HLOdgZOJmZnlzcnEdijKoal6SYMk/TF9CvxpSYek0+4i6RElTaP/goz2qiT9izY17/0LSaU5xvKykibX56Tz7pUOa69p8ZyaTpf0MSVN2T8v6S6lzZuYFYuTie2I9gVuIGmb6ADgHJJWcb8K/CdJ0yEvRMQh6efWZif+C3giIt5P0sxH63ssDiRpquLD6VPhzWS0k9SB0cBPIuIgkiZBWt83cm9EHB4RhwIvkzS10mogSWOI/0HSHPr/AgcB71PyrpTBJE3sHB8RHyBpwyvXJoHMCsJPwNuOqKOm6keyqbXmv6Qlkv4k7+X4f2n/B7SpGfZxJC28Pqekie5eJM265xpLa1Pnma8zaK9pcei46fThwBjgyTSeCpJ2q8yKxsnEdkQdNVXf1MY0kfU/k4BbI+KbecbSTJKIIGnM8pMR8aKS93oc28Y07TWd3gw8GhFnb0c8ZgXhai7bGf2NtJpK0rHA0tj06uTW/h9nUzPsfwbOkLRrOmyQpDZfM7AN2mxaPEdPAx+WtG8aT7XSpszNisUlE9sZTQB+nTYxvo5N7zq5kqRJ8OeBx0her0xEzJX0LeCRtCHERpLm3Bdmz3gbtDYtvpDkhVI5v/clImrT0swdaWu/kFxDaes1AGZdwg09mplZ3lzNZWZmeXM1l1melLzj+89tDBoXEe2+4tdsR+JqLjMzy5uruczMLG9OJmZmljcnEzMzy5uTiZmZ5e3/A7GyHorZhqjJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "classifiers = [\n",
    "    LinearSVC(),\n",
    "    MultinomialNB(),\n",
    "    LogisticRegression(random_state=0, max_iter=150),\n",
    "    RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0),\n",
    "]\n",
    "\n",
    "cv_df = compare_models(classifiers, features, labels, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate model predictions.\n",
    "\n",
    "> After choosing the best model, it's a good idea to check how well it performs in a greater detail. The confusion matrix is awesome for that! Here we compare predictions to actual values to check visually the best (and worst) performing classes. It looks like our Foreign class has in fact biased our model, since it has many word predictions specially from Business and Science... Maybe that due to familiar words in both contexts. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj0AAAHwCAYAAABND972AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3wU1frH8c+ThC5FmigqFkDFAioqAiKiIoiI12vvFRv6E69iFwuo14JdFK4NsKGoiAoqKFVFULGAIGCjN+k9yfP7YyZxgZAsmMks7Pf9es0rO2fKeXazmz15zjkz5u6IiIiIbO8y4g5AREREpCSo0SMiIiJpQY0eERERSQtq9IiIiEhaUKNHRERE0oIaPSIiIpIW1OiREmFm5cxskJktNbO3/sF5zjWzT4oztjiY2WAzu7CE67zNzP5XknXK38zMzaxuROfe4HNhZs3MbKqZrTCzU+J4v4mkIjV6ZANmdo6ZjQ//WM4J/1g2L4ZTnwbsBFRz99O39iTu/qq7ty6GeDZgZi3DL6V3NipvGJYPT/I8d5tZv6L2c/e27v7KVoZbWP17hPFmFVDn/e5+WXHXmWRcu5rZADNbGDZ8fzSzi8ysrJktMbNWBRzzmJm9nbCe1HszLF8RLuvNbF3C+nNbGX+RDRYz29nMXghjW25mk83sHjOrsDV1bokCPhf3Ak+7+w7u/l5U7zeRbY0aPZLPzG4AHgfuJ2ig7A48C3QohtPXAX5x9+xiOFdUFgBNzaxaQtmFwC/FVYEF0upzZ2aZQF9gBsH7oBpwATDP3dcAb4brGx9zNvBKuJ70ezP8gt/B3XcAXgUeylt39ysjeo5VgS+BcsCR7l4ROB6oAuwdRZ1FqANM/KcnKajxLLJNc3ctWgAqAyuA0wvZpwzBF8/scHkcKBNuawnMBP4DzAfmABeH2+4B1gHrwzouBe4G+iWcew/Agaxw/SLgV2A58BtwbkL56ITjmgLjgKXhz6YJ24YD9wFjwvN8AlTfzHPLi/854JqwLDMsuwsYnrDvEwRf4MuAb4CjwvI2Gz3P7xPi6B7GsRqoG5ZdFm7vCbydcP7/AsMA24rf4wav40bb8l/zhP0uBP4EFgK3J+ybAdwCTAcWAf2Bqgnb3wLmhq/7SGD/hG0vh8/pI2AlcFz4ejTaTMxNw99P+YSyE8P3URZJvDcLeT1eBrolrJ8ETACWAF8AB4XlZ4bvt0rhetvw+dUIn5+Hz2UFcGYB9XQDfgQyConFgbrh43bAd+F7aAZwd8J+ZYF+4eu+hOB9vVOyn4vwd5YbvtdWEHxu899v4T6XAD8Di4GPgTobxXkNMBX4Lc6/S1q0FPeSVv9xSqGOJPhj+24h+9wONAEaAQ2Bw4E7ErbXIviCqk3QsHnGzHZ0964E/6G/6cF/2y8UFkjYHfAk0NaD/5ibEnxRbbxfVeDDcN9qQA/gw40yNecAFwM1gdLAjYXVDfTh76zDCQT/Lc/eaJ9xBK9BVeA14C0zK+vuQzZ6ng0Tjjkf6AhUBP7Y6Hz/AQ4Ku3uOInjtLnT3krhHTHNgH+BY4C4z2y8svw44BTga2IXgy/GZhOMGA/UIXtdvCTIqic4haOhVBEYDXxG8H84ys90Td3T3LwgayacmFJ8PvOZBZjCZ92aRzOwQ4EXgCoL3y/PA+2ZWxt3fJMjUPBm+f14gaCQscPcW4Skahr/XNws4/XHAO+6em2Q4KwneZ1UIGkBXmdkp4bYLCT5Hu4VxXgmsTvZz4e57EzRk24fxrt3odTgFuI3g9a4BjAJe3+g0pwBHAA2SfD4i2wQ1eiRPNWChF979dC5wr7vPd/cFBBmc8xO2rw+3r3f3jwj+y9xnK+PJBQ4ws3LuPsfdC0rVtwOmuntfd89299eByUD7hH1ecvdf3H01QbaiUWGVhl/AVc1sH4IvpT4F7NPP3ReFdT5K8J90Uc/zZXefGB6zfqPzrQLOI2i09QOudfeZRZyvuNzj7qvd/Xvge4LGLAQNg9vdfWb4pXk3cFped4e7v+juyxO2NTSzygnnHejuY9w914MurNMJvlzvBH4zswlmdljC/vmNTTOrRNBtlTcGJZn3ZjIuB55397HunuPBGJe1BA15CLIbrQiyIoPc/YMtOHc1goZbUtx9uLv/GL4+PxA0Oo4ON68Pz1c3jPMbd18Wbkvmc1GUK4AH3P3n8DW9H2hkZnUS9nnA3f8KPzci2w01eiTPIqB6EX34u7BhluKPsCz/HBt9Ma0CdtjSQNx9JUF3w5XAHDP70Mz2TSKevJhqJ6zP3Yp4+gKdgGMoILtgZv8xs5/DAblLCP4rr17EOWcUttHdvybotjCCxlmBzGxiwqDco4qoMxmbe33qAO+Gg4yXEHSF5AA7mVmmmT1oZtPNbBnwe3hM4muwwfN198Xufou7708wJmcC8J6ZWbhLH+AYM6tNMOh9mrt/F25L5r2ZjDrAf/KeU/i8diN8D7v7EoJuuwOAR7fw3IuAnZPd2cyOMLPPzWyBmS0leK/nvX59Cbqc3jCz2Wb2kJmV2oLPRVHqAE8kvAZ/EbzvEj83hb5fRbZVavRIni+BNQRp7c2ZTfAHM8/ubNr1k6yVQPmE9VqJG939Y3c/nuCLZDLQO4l48mKatZUx5ekLXA18FGZh8oUNjZuBM4Ad3b0KwbiWvC/vzXVJFdpVZWbXEGSMZgNdNrefu+/vfw/KHZXMk9lKMwi6UaokLGXdfRZB11UHgi6dygTjg+Dv1wAKeb7uvhB4hKCxUTUs+5MgE3QuQfYwMcOWzHsz2efUfaPnVD7MEGJmjQjGurxO0I20JYYC/9qCQeqvAe8Du7l7ZYKxZAYQZkrvcfcGBF1YJxFmwZL8XBRlBnDFRq9DuTDLmackulZFSpwaPQKAuy8lGLD7jAXX9ShvZqXMrK2ZPRTu9jpwh5nVMLPq4f5FTs/ejAlACzPbPewWuTVvg5ntZGYnh2MY1hJ0k+UUcI6PgPrhVOYsMzuTYAzClnRLbMLdfyPoari9gM0VgWyCmV5ZZnYXUClh+zxgjy2ZoWVm9QkGwp5H8IXfJfwC/ifKhNPB85Yt/aw/B3TP6/IIf+d5M6UqEvxeFhE0XO8v6mRm9l8zOyD8PVUEriLI5ixK2O0VggxbMxLGCCX53kxGb+DKMMtiZlbBzNqZWUUzyxs8fBvBGLDaZnZ1wrHzgL0KOXcPgvfBKwmvWW0z62FmBxWwf0XgL3dfY2aHEzQkCY87xswODGewLSPo7srZgs9FUZ4DbjWz/cP6KpvZVl9GQmRbokaP5HP3HsANBIOTFxD8R9gJeC/cpRswHviBYKbKt2HZ1tT1KcFU5R8IZkAlNlQyCAb3ziZIvR9NkHnZ+ByLCP4L/g/BF3AX4KQwk/CPuPtody8oi/UxwSDeXwi60tawYVdA3oUXF5nZt0XVE3bZ9AP+6+7fu/tUgi/evmZW5h88hRUEs3fylk2ug1OEJwgyEZ+Y2XKCgchHhNv6EDz3WcCkcFtRyhN0FS4h6MarA5y80T5vAzsCw9x9g/ExSbw3i+Tu4wnG9TxNMDB7GsGsJ4AHgJnu3jMcp3Qe0M3M6oXb7yZo0CwxszMKOPdfBFmZ9cDY8DUbRpAFnFZAOFcD94b73cWGXZq1wtdiGUG34giC90hSn4skXod3CWYIvhF2T/5EMFtNZLtnJTNBRERERCReyvSIiIhIWlCjR0RERNKCGj0iIiKSFtToERERkbSgRo+IiIikhZS9g+76hb9qWlkS9q5fHDdATw9zVy6OO4RtQunMUnGHsE3ISfo2W7I+55/eQSR9ZK+bZUXvVXyi+K4tVX2vEn0OW0KZHhEREUkLKZvpERERkYjlbs1FvbddyvSIiIhIWlCmR0REJF2l2dg0ZXpEREQkLSjTIyIikq5y0yvTo0aPiIhImnJ1b4mIiIhsf5TpERERSVdp1r2lTI+IiIikBWV6RERE0lWajelRo0dERCRd6YrMIiIiItsfZXpERETSVZp1bynTIyIiImlBmR4REZF0lWZT1tXoERERSVO6IrOIiIjIdkiZHhERkXSVZt1byvSIiIhIWlCmR0REJF1pTI+IiIjI9keZHhERkXSVZrehUKNHREQkXal7S0RERGT7o0yPiIhIutKUdREREZHtjzI9IiIi6SrNxvSo0SMiIpKu1L0lIiIisv1J+0bPHff3oEW7szjlvCvzyyZP/ZVzO3bmX+dfxTVdurJi5UoAlixdxsWdbuaw4/5F90ef3eA8F3XqwklnXca/L7yGf194DYsWLynR51GSHn7qXr6dMpxPx7yzybaOnS7kz79+ZMeqVTYo36V2LX7+cywdO11YUmGmnF7PP8LMGRP47tuh+WUND2rAqJHvM+7rj/nyiw9p3LhRjBGmjok/j2Ls14P54qsPGTl6IAAHHrQfnw1/J7/s0MYNY44yXmXKlGHUqIGMHTuYb775lDvu6AzAQQc1YMSId/nqq48YPXoQjdP8ddrYrrvuwtBP3uLHH4bz/YTPuLbTpXGHFCv3nGJfUlnaN3pOOfF4nuvRbYOyrg8+zvVXXcy7fXtybIumvPTqAABKly7NtZefz43XXFbguR7s2oUBrzzDgFeeodqOVQrcZ3vw1msDueD0qzYp37n2ThzV8khmzpi9yba77u/C8GGjSyK8lNWn71uc1P68Dcruf+B2unV/jMMOP4F77n2UB+6/PaboUs+Jbc+haZN2tGjeAYBu3W7lgfufoGmTdnS77zG6dbsl5gjjtXbtWtq0OZsjjmjLEUe0pXXrozn88IPp3v1Wund/giZNTuS++3rQvfutcYeaUrKzs7mpyz0ceFBLmjVvz1VXXcR++9WLOywpIWnf6Gnc6EAqV6q4Qdnvf86kcaMDATjysEP4dETwZV2+XFkOaXgAZUqXLvE4U8nXX37DksVLNynv2r0L93ftgbtvUN76xFb8+ftMfpk8raRCTEmjR49l8UYZQHenUsUdAKhcqSJz5syLI7Rtgl6rTa1cuQqAUqWyyMoqhbsHr1Ol8HWqXJE5c+bHGWLKmTt3Pt9N+AmAFStWMnnyVGrvUivmqGLkucW/pLDIBzKbWSawU2Jd7v5n1PX+E3X32oPPR39Fq6OO5JPPRzF33sKkjrvz/sfIyMjg+JbNuOKiszGziCNNHce3acncOfP5eeIvG5SXK1+Oq/7vEs499XKu6HRRPMGlsBtvvJsPBr3Kgw/eSUZGBke37BB3SCnB3Rk4qA/uzosvvM5LL77OzV3u5b33X6H7A7eRkZHBscecFneYscvIyOCLLz5g77334Pnn+zBu3ARuuuleBg3qwwMP3E5GRgbHHHNq3GGmrDp1dqVRwwMY+/V3cYcSnxgGMpvZi8BJwHx3PyAsexPYJ9ylCrDE3RuZ2R7Az8CUcNtX7n5leMyhwMtAOeAj4P984/+6NxJppsfMrgXmAZ8CH4bLB1HWWRzuu60zrw8YxBmXXMvKVaspVarotuF/u3bh3b496fPsw3zz/U+8P2RYCUSaGsqWK0un/1zOo/c/s8m2G265mhd69mXVytUxRJb6Ona8gJtuuoe96x7OTTfdzfPPPxJ3SCnhuGNPo3nT9px6ysV07Hg+zZodzmWXn8ctXbqxb/1m3NKlG8/2fDDuMGOXm5tLkyYnUrduExo3bkSDBvXp2PE8unS5j3r1jqRLl3vp2fOhuMNMSRUqlKf/m7254cauLF++Iu5w0s3LQJvEAnc/090buXsjYACQOGh0et62vAZPqCfQEagXLhucsyBRd2/9H7CPu+/v7geGy0Gb29nMOprZeDMb/78+r0cc2ubtVWc3ej9+P/1ffIoTjzua3WrvXOQxO9WoDgQfpHbHH8NPk34p4ojtR509dmO33WszZNTbjJkwhJ132YmPhvenRs1qHHzogdx6d2fGTBjCJVeeR6fOl3PhZWfHHXLKOP+803j3vY8AeHvABxymgcwAzA27ZBYsWMSgQR9zaOOGnHPuqQwcOASAd975MO0HMidaunQZI0d+SevWLTn33H/z3nuDARgw4EMNZC5AVlYWb73Zm9dffzf/tUpbMXRvuftI4K+CtlnQRXIGUGgjwMx2Biq5+5dhdqcPcEpRdUfd6JkBbDr4YzPcvZe7N3b3xpddEN8XY97Mq9zcXJ5/5Q3OOOXEQvfPzs5h8ZLgaa7PzmbEF2Opu1edyONMFVN+nsoh+7SkWaM2NGvUhjmz53FiyzNYMH8Rp7W7KL/8xef68fRjvXnlf/E1aFPNnDnzaNHiSACOOaYZ06b9FnNE8Stfvhw77FAh/3GrY49i0qQpzJ0zn6OOOgKAli2bMn367zFGGb/q1atSuXIlAMqWLUOrVs2ZMmUac+bM56ijmgDQsmUzpk37PcYoU1PvXo/y8+RpPP5Er7hD2S4lJjDCpeMWHH4UMM/dpyaU7Wlm35nZCDM7KiyrDcxM2GdmWFaoqMf0/AoMN7MPgbV5he7eI+J6k3ZT1wcZ990PLFmyjGNPOY+rLz2fVatX88Y7QS/ccUc35V/tWufv3/rfF7Ji5SrWZ2fz2agv6PVYd3autRNX3HAH67Ozyc3JpclhB3PayUVm2bZZT/X+L0c2O4wdq1Vh7E9D6fHgM7zZ7924w0p5ffs8TYsWR1K9elV+nT6Oe+97lCuv6kKPR+8hKyuLNWvWctXVN8cdZuxq1qzO6288D0BWVib9+7/P0E9H0mnFrTz0yF1kZWaxZu1aru10W8yRxqtWrZr07t2DzMwMMjIyGDDgAwYP/oylS5fx8MN3k5WVydq1a+nUKb1nuW2sWdPDOP+80/jhx0mMH/cJAHfe+SCDh3wWc2QxyS3+Kebu3gvY2hbl2WyY5ZkD7O7ui8IxPO+Z2f5AQYNmCx3PA2BFjPn5R8ysa0Hl7n5PUceuX/hrdIFtR/aur4GvyZq7cnHcIWwTSmeWijuEbUJOis9SSSXrc7LjDmGbkb1uVonOgFnz9VvF/l1b9vDTi3wO4QDlD/IGModlWcAs4FB3n7mZ44YDN4b7fe7u+4blZwMt3f2KwuqNNNOTTONGREREBDgOmJzY4DGzGsBf7p5jZnsRDFj+1d3/MrPlZtYEGAtcADxVVAWRNnrMbBCbppuWAuOB5919TZT1i4iISCHimbL+OtASqG5mM4Gu7v4CcBabDmBuAdxrZtlADnClu+cNgr6Kv6esDw6XQpXEmJ4a/P0kziSYwl4f6A2cH3H9IiIikkLcvcCZSu5+UQFlAwimsBe0/3jggIK2bU7UjZ6D3b1FwvogMxvp7i3MbGLEdYuIiEhh0mxsWtRT1muY2e55K+Hj6uHquojrFhEREckXdabnP8BoM5tOML1sT+BqM6sAvBJx3SIiIlKYGMb0xCnq2VsfmVk9YF+CRs/khMHLj0dZt4iIiBRBjZ5/zsxauftnZrbxne72MjPc/Z0CDxQRERGJSFSZnqOBz4D2BWxzNryRmIiIiMTAvfivyJzKImn0uHvX8OfFUZxfREREZEtFOnvLzHYysxfMbHC43sDMLo2yThEREUlSbm7xLyks6inrLwMfA7uE678A10dcp4iIiCTDc4t/SWFRN3qqu3t/IBfA3fMuIy0iIiJSoqK+Ts9KM6tGeP+t8MZgSyOuU0RERJKR4t1RxS3qRs8NwPvA3mY2huA+XKdFXKeIiIjIJiJr9JhZJsHU9aOBfQguTjjF3ddHVaeIiIhsgRQfg1PcImv0uHuOmXVw98cA3VxUREQk1ah7q1iNMbOngTeBlXmF7v5txPWKiIiIbCDqRk/T8Oe9CWUOtIq4XhERESmKureKj7sfE+X5RURERJIVaaPHzCoDXYEWYdEI4F5317R1ERGRuKXZmJ6oL074IrAcOCNclgEvRVyniIiIyCaiHtOzt7v/O2H9HjObEHGdIiIikgxleorVajNrnrdiZs2A1RHXKSIiIslIs3tvRZ3puRLoE47tAVgMXBhxnSIiIiKbiKTRY2a7u/uf7v490NDMKgG4+7Io6hMREZGtoO6tYvFe3gMzG+Duy9TgERERkThF1b1lCY/3iqgOERER+SdSfAxOcYuq0eObeSwiIiKpIs26t6Jq9DQ0s2UEGZ9y4WPCdXf3ShHVKyIiIlKgSBo97p4ZxXlFRESkGKVZ91bU1+kRERERSQlRX6dnqx3U4Ky4Q9gmTL7tiLhD2GbUuGto3CFsE3LS7D+/reWu4YqyHdCYHhEREUkLadboUfeWiIiIpAVlekRERNJVmnXTKtMjIiIiaUGZHhERkXSlMT0iIiIi2x9lekRERNJVmmV61OgRERFJV2l2XS51b4mIiEhaUKZHREQkXaVZ95YyPSIiIpIWlOkRERFJV2l2cUI1ekRERNKVurdEREREtj/K9IiIiKQrZXpEREREtj/K9IiIiKQrXZxQRERE0oHnerEvRTGzF81svpn9lFB2t5nNMrMJ4XJiwrZbzWyamU0xsxMSyg81sx/DbU+amRVVtxo9IiIiUpJeBtoUUP6YuzcKl48AzKwBcBawf3jMs2aWGe7fE+gI1AuXgs65ATV6RERE0lVubvEvRXD3kcBfSUbYAXjD3de6+2/ANOBwM9sZqOTuX7q7A32AU4o6mRo9IiIikgo6mdkPYffXjmFZbWBGwj4zw7La4eONywulRo+IiEi68txiX8yso5mNT1g6JhFJT2BvoBEwB3g0LC9onI4XUl4ozd4SERGRYuPuvYBeW3jMvLzHZtYb+CBcnQnslrDrrsDssHzXAsoLpUyPiIhIusr14l+2QjhGJ8+/gLyZXe8DZ5lZGTPbk2DA8tfuPgdYbmZNwllbFwADi6pHmR4REZF0FcMVmc3sdaAlUN3MZgJdgZZm1oigi+p34AoAd59oZv2BSUA2cI2754SnuopgJlg5YHC4FEqNHhERESkx7n52AcUvFLJ/d6B7AeXjgQO2pG41ekRERNKV7r0lIiIisv1RpkdERCRd+dYNPN5WqdEjIiKSrtS9JSIiIrL9UaYnQa1davLg03dTvWY1PNfp3/dd+vZ+k8pVKtGjd3dq77Yzs2bMofNlt7Fs6XIA6jeoyz2P3MoOO1QgNzeX00+4iHVr18X8TIpf6RMuInPvg/BVy1nzctcNtmU1bk3plmew6pnrYfUKrFI1yl58H754LgA5s39l/dB+UKoMZc++Of8422FHsn/+ivWfv1mizyVOGRkZjB4ziNmz53Lavy/lttuv5+KLz2LhwuA2NHd3fYiPPx4eb5AxK1OmDEOH9qd06dJkZWXx7rsf0a3bYxx44H489dT9VKhQnj/+mMnFF/8fy5eviDvc2ASv01uUKfP363TffT3YccfK9Ov3LHXq7Moff8zk3HOvZsmSpXGHmzJ693qUdicex/wFC2l08LFxhxO/rbyuzrZKjZ4EOdk5PNT1CSb9OIXyFcozYGgfvhjxNf866yS+HDmO/z3Vh8uuvYDLr7uQR+97mszMTB569h5uvuZupkycSpUdK5O9PjvupxGJ7IljWP/dZ5Q58dINyq3ijmTWaUDuskUblPvSBazpc++GJ1m/doOysufdSc7UbyOLORVdc83FTJk8jYqVdsgve/qpF3jiid4xRpVa1q5dS5s2Z7Ny5SqysrL47LO3+eST4fTocQ+33NKd0aPHcsEFZ9C58xXce++jRZ9wOxW8TmclvE4D+Pjjz+nQoS2ffz6GRx55lhtvvJobb7yaO+54IO5wU0afPv159tmXeOmlJ+IORWKg7q0EC+YvYtKPUwBYtXIV03/5jZ12rkGrNi0Y+OaHAAx880OObXs0AM1aHsGUSdOYMnEqAEsWLyV3O+0fzZ05Fdas3KS81DFnsm7k21s8GM6q1ITyFYPzpoldateiTZtWvPzyG3GHkvJWrlwFQKlSWWRllcLdqVdvL0aPHgvAZ5+N4pRT2sYZYkpIfJ1KlcrC3Wnf/nj69XsbgH793ubkk1vHGWLKGTV6LH8tXhJ3GKkjgntvpbJIGz1m9n9mVskCL5jZt2a2TXwCd9ltZ/Y7cB++/2Yi1WpUZcH8IJOxYP4iqlYPbv66x967gzu933ySAUP7cGmn8+MMucRl7t0QX74EXzBzk21WuTplz7+LMmfeREbtepseu9/h5EwZVxJhpoyHHrqL2+94gNyN0slXXHkhY8cOpudzD1GlSqWYokstGRkZfPXVR/z557d89tkoxo2bwKRJv3DSSccDcOqp7dh1152LOMv2LyMjg7FjBzNjxncMGzaaceMmULNmdebOnQ/A3LnzqVGjesxRSkpLkdtQlJSoMz2XuPsyoDVQA7gYeDDiOv+x8hXK8eSLD/LgnT1YuWLT7EaezKxMDjm8ETdddSfntr+c405sSZOjDivBSGOUVZqsJu1YP2bTW534yqWsfr4La/rey7rh/Snd7nIoXXbDw/c9nOzJX5dUtLFr07YVCxYsYsJ3P21Q/r/e/Thg/xY0aXIic+fO54EH74gpwtSSm5tLkyYnUrduExo3bkSDBvW54oqbuOKKCxgz5gN22KEC69atjzvM2OXm5nLEEW3Ze+8jOOywhjRoUD/ukERSWtSNnrxbv58IvOTu31Pw7eCDnRNuR79k9fyIQytYVlYmT7z4XwYN+JhPPxwOwKIFf1GjZjUAatSsxl8LFwMwb/Z8xn35LUv+Wsqa1WsZOXQMDQ7aJ5a4S5pVqUFG5eqUvbArZS9/EKu4I2XPvxPKV4Kc7PyuMJ/3B750ARk77vT3sTV2BcvA5/0RV/gl7sgmjWnX7jgm/TyaV/o8xdFHN+WFFx5j/vyF5Obm4u689OIbND60YdyhppSlS5cxcuSXtG7dkl9+mU779ufTrNlJ9O//Pr/9lj7vn6IEr9NXtG7dkvnzF1KrVk0AatWqyYIFC2OOTlKZ5+YW+5LKom70fGNmnxA0ej42s4rAZl8Rd+/l7o3dvXGVcjUjDq1g3R6/k19/+Y1Xnnstv+yzj0fS4cx2AHQ4sx2fDRkJwOjPv2KfBnUpW64MmZmZHNb0EKZP+S2WuEuaL5zF6mdvYE3vW1jT+xZ8+WLW9L0PVi2DcjuABW1bq1wdq1KT3KV//+HN2vcIctIoywPQtetD1K93JA32a86FF1zLiBFfcOmlnalVq0b+PieffAITJ/0SY5SpoXr1qkLLmVQAACAASURBVFSuHHTzlS1bhlatmjNlyjRq1Aj+8TAzbrnlWnr3fjXOMGNX8Os0nQ8++JTzzjsNgPPOO41Bgz6NM0yRlBL17K1LgUbAr+6+ysyqEnRxpaRDjmhIhzNOZMqkqbzzWT8AHu/+LP97sg89et/PaeeezOyZ8+h82a0ALFu6nJefe423Pn4Fd2fksC8YMXRMnE8hMqXbXU7mbvtAuR0oe8VDrB/zPjk/jS5w38xd61OqWYfgoleey/pP+20wCDpzn8asfUczJwC6dbuVgw5qgLvzx58zue7a2+IOKXa1atWkd+8eZGZmkJGRwYABHzB48Gdcc83FXHHFBQAMHDiEPn36xxxpvGrVqsn//teDzMzMhNdpGGPHfsOrr/bkoovOZMaM2ZxzzpVxh5pS+vV9hqNbHEn16lX5/dfx3HPvI7yUzpMLUnwMTnEzj/AS1GbWDJjg7ivN7DzgEOAJdy8yL71fzcPT6zexlb7pclDcIWwzatw1NO4Qtgm5aXZZ+q0V5d/O7U12bk7cIWwzstfN2uwQkCis7H5Bsb+RK9zep0Sfw5aIunurJ7DKzBoCXYA/gD4R1ykiIiLJ0JT1YpXtwb9DHQgyPE8AFSOuU0RERJKRZlPWox7Ts9zMbgXOB44ys0ygVMR1ioiIiGwi6kzPmcBaguv1zAVqAw9HXKeIiIgkIze3+JcUFmmjJ2zoDADKhEULgXejrFNERESkIJF2b5nZ5UBHoCqwN0Gm5zlAt7YVERGJW4qPwSluUY/puQY4HBgL4O5TzSyeqw6KiIjIhlJ8tlVxi3pMz1p3X5e3YmZZQHo1K0VERCQlRJ3pGWFmtwHlzOx44GpgUMR1ioiISDLSrHsr6kzPLcAC4EfgCuAjQLeRFhERkRIXaabH3XOB3uEiIiIiKSTV74pe3KKevdUMuBuoE9ZlgLv7XlHWKyIiIklIs+6tqMf0vAB0Br4BdMc5ERERiU3UjZ6l7j444jpERERkayjTU6w+N7OHgXcIbkcBgLt/G3G9IiIiIhuIutFzRPizcUKZA60irldERESKkmYXJ4x69tYxUZ5fREREJFmRNHrM7Dx372dmNxS03d17RFGviIiIbAGN6SkWFcKfFSM6v4iIiPxDrkbPP+fuz4c/74ni/CIiIiJbKtLbUJjZQ2ZWycxKmdkwM1toZudFWaeIiIgkKdeLf0lhUd97q7W7LwNOAmYC9YGbIq5TREREZBNRT1kvFf48EXjd3f8ys4irFBERkaTo3lvFapCZTQZWA1ebWQ1gTcR1ioiISDJSvDuquEXaveXutwBHAo3dfT2wEugQZZ0iIiIiBYn6LusXJDxO3NQnynpFREQkCWmW6Ym6e+uwhMdlgWOBb1GjR0REREpY1LehuDZx3cwqA32jrFNERESS465MT5RWAfVKuE4REREpiLq3io+ZDSK4qzoEg6YbAP2jrFNERESkIFFneh5JeJwN/OHuMyOuU0RERJKhTE/xcfcReY/NrDqwKNljZ61cGElM25uadw2LO4RtxqPVmscdwjah07zP4w5hm5ChC60mTa+UpIpIrtNjZk3MbLiZvWNmB5vZT8BPwDwzaxNFnSIiIrJlPNeLfUllUWV6ngZuAyoDnwFt3f0rM9sXeB0YElG9IiIiIgWK6orMWe7+ibu/Bcx1968A3H1yRPWJiIjIlorhLutm9qKZzQ97gfLKHjazyWb2g5m9a2ZVwvI9zGy1mU0Il+cSjjnUzH40s2lm9qQlcXPPqBo9iXcwW73RttTOfYmIiKSL3AiWor0MbDzU5VPgAHc/CPgFuDVh23R3bxQuVyaU9wQ6ElwKp14B59xEVI2ehma2zMyWAweFj/PWD4yoThEREUlx7j4S+Gujsk/cPTtc/QrYtbBzmNnOQCV3/9KDKyz2AU4pqu5IxvS4e2YU5xUREZHik6IDjy8B3kxY39PMvgOWAXe4+yigNpB4CZyZYVmhSvqKzCIiIrIdM7OOBN1OeXq5e68kj72d4Lp+r4ZFc4Dd3X2RmR0KvGdm+1PwlRCKbMGp0SMiIpKuIsj0hA2cpBo5iczsQuAk4Niwywp3XwusDR9/Y2bTgfoEmZ3ELrBdgdlF1RHVmB4RERFJdfEMZN5EeA2/m4GT3X1VQnkNM8sMH+9FMGD5V3efAywPrwtowAXAwKLqUaZHRERESoyZvQ60BKqb2UygK8FsrTLAp+HM86/CmVotgHvNLBvIAa5097xB0FcRzAQrBwwOl0Kp0SMiIpKm4hjI7O5nF1D8wmb2HQAM2My28cABW1K3urdEREQkLSjTIyIikq62cgzOtkqNHhERkTSVotfpiYy6t0RERCQtKNMjIiKSrtKse0uZHhEREUkLyvSIiIikKU+zTI8aPSIiIukqzRo96t4SERGRtKBMj4iISJpKt+4tZXpEREQkLSjTIyIikq6U6RERERHZ/ijTIyIikqbSbUyPGj0iIiJpKt0aPereEhERkbSgTI+IiEiaUqZHREREZDukTI+IiEi6cos7ghKlRo+IiEiaSrfuLTV6ClG5ckWeeuZBGjSoj7tzzVU3077DCbRteyzr1q/nt1//4Ooru7B06fK4Q43VxJ9HsWL5CnJyc8nOzqZF8w4ceNB+PPFkd8qWLUN2djadr7+Lb8Z/H3eoJaryXjtz/LOd8tcr7V6TcY++TYVaValz3MHkrs9m2R/z+fw/vVi3bBU1G+1FiwcvDXY2GP/Yu/w+ZHxM0aeGMmXKMPyzAZQuU4asrEzeeedD7rn30bjDShm9nn+EE088jgULFnLwIccBcOcdN3DJJeewcOGiYP2u/zJkyGdxhplyru10KZdceg5mxosvvMaTT/0v7pCkhJi7xx1DgSpV2Cv2wJ7r9TBfjBlHn1f6U6pUKcqXL8uhjRsyYviX5OTkcM99NwPQ9c7/xhZjbgr8/ib+PIoWzU9m0aLF+WUD3+/D00+/wKefjKD1CS3p3PkK2rY5O8Yo4ZFqzWKr2zKM88c9xTsnd6XK3jsza8wkPCeXI249E4CxD7xJVtnS5KzPxnNyKV+zCqd/3J0+ja/Fc0r2X7FO8z4v0fqKUqFCeVauXEVWVhYjh79L5xu6Mvbrb+MOiwyLv1ugefMjWLFiJS+9+PgGjZ4VK1fy2GPPxxzd31Lpe2b//fehX79nadq0HevWrefDD16l07W3Mm3ab3GHBsD6dbNK9I01p/kxxf7L2Xn05/F/ODZDA5k3o2LFHWja7HD6vNIfgPXr17N06XI+GzaanJwcAMZ9/R21a9eKM8yU5e5UqrgDAJUrVWTOnHkxRxSv2s33Z9kf81kxaxEzR/6U35CZ9910dti5KgDZa9bll2eWKUUKfU/EauXKVQCUKpVFVqlSKfUFGrfRo8eyePGSuMPYpuy7bz2+Hvstq1evIScnh5GjvqJDhzZxhyUlJNJGj5lVMLOM8HF9MzvZzEpFWWdx2WPP3Vi08C96Pv8Qo74YxFPPPED58uU22Of8C07n00+GxxNgCnF3Bg7qw6gx73PxJUE25+Yu99Lt/luZ/MsYuj9wG13vejjmKONV9+QjmTrwy03K9z2jBX9+/kP+es1Ge3PG0Ac549MHGHnbSyWe5UlFGRkZjB/3CXNm/cCwYSP5etx3cYeU8q668iK+Gf8pvZ5/hCpVKscdTkqZOHEyzY9qQtWqO1KuXFnatmnFbrvuEndYsfHc4l9SWdSZnpFAWTOrDQwDLgZejrjOYpGVmUXDRvvzQu9XOappe1atWsUN/7kyf/uNN11NdnY2b74xMMYoU8Nxx55G86btOfWUi+nY8XyaNTucyy4/j1u6dGPf+s24pUs3nu35YNxhxiajVCZ1jj+EXz8cu0H5IdeejOfkMvXdMfll8ydMp/9xtzDgpLs45Jr2ZJbZJv5HiFRubi6ND2tNnT0bc1jjg9l//33iDimlPd+rD/vu14zGh7Vm7tz5PPTfO+MOKaVMnjyNRx5+hiGDX+fDD17lhx8mkZ2dE3dYsXG3Yl9SWdSNHnP3VcCpwFPu/i+gwWZ3NutoZuPNbPy67GURh1a4WbPnMGvWXMaHg2/fe3cIDRsdAMA5555Km7atuOySznGGmDLmzpkPwIIFixg06GMObdyQc849lYEDhwDwzjsfcmjjhnGGGKvdj2nIwp9+Z/XCv9/T9U87it2PPZhh1z5b4DFLps1m/aq1VN1n15IKM+UtXbqMESO/4ITWLeMOJaXNn7+Q3Nxc3J0XXnyNww5rFHdIKeell9/g8CPa0OrYf/PX4iUpM55Hohd5o8fMjgTOBT4MyzY7Y8zde7l7Y3dvXDqrUsShFW7+vIXMmjmHuvX2BKBly6ZMnjyV445vwfWdr+DMMzqyevWaWGNMBeXLl2OHHSrkP2517FFMmjSFuXPmc9RRRwDBazd9+u8xRhmvuh2OZFpC19ZuLQ+i0VUnMeSSHmSvWZdfXnG3Glhm8JHcoXY1quy9M8tnLCjxeFNJ9epVqVw5+FtQtmxZjm11FFOmTI85qtRWq1bN/McdOrRh4sQpMUaTmmrUqAbAbrvtwimntOWNN9+LOaL4pFv3VtRT1q8HbgXedfeJZrYXkFpTQwpx0413878XH6d06VL8/tufXH1lF4aPfI/SZUozcFAfAMZ9PYHO/3dHzJHGp2bN6rz+RjBLJCsrk/7932fopyPptOJWHnrkLrIys1izdi3Xdrot5kjjkVW2NLsedQAjb3kxv6z5fReSWTqLk167BYB5305j1G0vUeuw+hx8dXtys3PwXGfU7S+zZvGKuEJPCTvvvBMvvvA4mZkZZGRk8Pbbg/jwo6Fxh5Uy+vZ5mhYtjqR69ar8On0c9973KEe3OJKGDffH3fnjjxlcfc0tcYeZcvq/2Zuq1XYke3021113O0uWLI07JCkhmrK+jUuFKevbijinrG9LUm3KeqpKhSnr24pU/Z5JRSU9ZX3GYccW+y9nt3HDUvbDEfXsrU/NrErC+o5m9nGUdYqIiIgUJOrurRrunn8RCXdfbGY7RVyniIiIJCHdknBRN3pyzGx3d/8TwMzqACk+zElERCQ9eG7K9kRFIupGz+3AaDMbEa63ADpGXKeIiIjIJiJt9Lj7EDM7BGgCGNDZ3RdGWaeIiIgkJ90yPZEMZDazfcOfhwC7A7OBWcDuYZmIiIhIiYoq03MDQTfWowVsc6BVRPWKiIhIkjSQuRi4e964nbbuvsFli82sbBR1ioiIyJZR91bx+iLJMhEREZFIRZLpMbNaQG2gnJkdTDCIGaASUD6KOkVERGTLpPpd0YtbVGN6TgAuAnYFeiSULwfS8yZMIiIiEquoxvS8ArxiZv929wFR1CEiIiL/TKrfFb24RdW9dZ679wP2MLMbNt7u7j0KOExERERKUK66t4pFhfDnDhGdX0RERGSLRNW99Xz4854ozi8iIiL/nAYyFwMze7Kw7e5+XRT1ioiIiGxOVN1b30R0XhERESkm6XZxwihnb+Uzs4pBsa+Ioj4RERGRokR6l3UzOwDoC1QNVm0BcIG7T4yyXhERESma7r0VMrOnCG4OWqAkx+X0Am5w98/Dc7YEegNNtyxMERERKW5xdG+Z2YvAScB8dz8gLKsKvAnsAfwOnOHui8NttwKXAjnAde7+cVh+KPAyUA74CPg/98KbcYXde2s8wdiczS3JqJDX4AFw9+H8PZ1dRERE0s/LQJuNym4Bhrl7PWBYuI6ZNQDOAvYPj3nWzDLDY3oCHYF64bLxOTex2UzPxuNyttKvZnYnQRcXwHnAb8VwXhEREfmH4rg4obuPNLM9NiruALQMH78CDAduDsvfcPe1wG9mNg043Mx+Byq5+5cAZtYHOAUYXFjdRY7pMbMaYcUNgLIJQbcq6ljgEuAe4J1wfSRwcRLHiYiISPrYyd3nALj7HDOrGZbXBr5K2G9mWLY+fLxxeaGSGcj8KkE/WzvgSuBCYEFhB5hZ2XDfusCPwH/cfX0SdYmIiEgJieLihGbWkaDbKU8vd++1tacroMwLKS9UMo2eau7+gpn9n7uPAEaY2YgijnmFoBU2CmgL7Adcn0RdIiIiUkKimL0VNnC2tJEzz8x2DrM8OwPzw/KZwG4J++0KzA7Ldy2gvFCFDWTOk5ehmWNm7czs4I0qKkgDdz8vvB3FaUCLJOoRERGR9PQ+QU8S4c+BCeVnmVkZM9uTYMDy12FX2HIza2JmBlyQcMxmJZPp6WZmlYH/AE8BlYDORRyT35Xl7tlBPCIiIpJK4hjIbGavEwxarm5mM4GuwINAfzO7FPgTOB3A3SeaWX9gEpANXOPuOeGpruLvKeuDKWIQMyTR6HH3D8KHS4FjknxODc1sWfjYgHLhugWn9EpJnkdERES2I+5+9mY2HbuZ/bsD3QsoHw8csCV1JzN76yUKGBzk7pds7hh3z9zcNhEREUkNusv6pj5IeFwW+BdJDBYSERGR1KbbUGzE3Qckrod9cUMji0hEREQkAltzw9F6wO7FHYiIiIiUrDgGMscpmTE9y9lwTM9cgis0R6p8qTJRV7FdWJutaz4m69p5nxe9k9CmVqO4Q9gmfDLv+7hD2GakWQ+KpLBkurcqlkQgIiIiUrLSbSBzkRcnNLNhyZSJiIiIpLLNZnrC+2eVJ7h40I78fZ+LSsAuJRCbiIiIREhjev52BcH9snYBvuHvRs8y4JmI4xIREZGIpdt4q802etz9CeAJM7vW3Z8qwZhEREREil0yNxzNNbMqeStmtqOZXR1hTCIiIlICct2KfUllyTR6Lnf3JXkr7r4YuDy6kERERESKXzIXJ8wwM3MPLlZtZplA6WjDEhERkail25T1ZBo9HxPc7v05gjFPV5LE7dtFREQkteXGHUAJS6bRczPQEbiKYAbXd8DOUQYlIiIiUtySuSJzrpl9BewFnAlUBQYUfpSIiIikOkfdWwCYWX3gLOBsYBHwJoC7H1MyoYmIiIgUn8IyPZOBUUB7d58GYGadSyQqERERiVxuml2dsLBGz78JMj2fm9kQ4A1IszyYiIjIdiw3zb7WN3udHnd/193PBPYFhgOdgZ3MrKeZtS6h+ERERESKRZEXJ3T3le7+qrufBOwKTABuiTwyERERiZRjxb6ksmSuyJzP3f9y9+fdvVVUAYmIiIhEIZnr9IiIiMh2KN0uTrhFmR4RERGRbZUyPSIiImkq1cfgFDc1ekRERNKUurdEREREtkPK9IiIiKQpZXpEREREtkPK9IiIiKQpDWQWERGRtJCbXm0edW+JiIhIelCmR0REJE3pLusiIiIi2yFlekRERNKUxx1ACVOjR0REJE3pOj1p7LGnu/HT1NEM/+L9/LL2HU5gxJeDmP3XRBo22j+//OBDDmToqHcYOuodho1+l7YnHRdHyLGrW29PRn7xfv7yx+wJXHn1RQBcfuX5fP3tJ3wxbjD33Ncl3kBT0P9ddzkTJnzGd98No2/fZyhTpkzcIcWmVJlS9Hi/B08NeYpnhj7LOTecC0CXZ27mycFP8eTgp3hhzIs8OfgpADKzMuncozNPf/IMPYc9x+nXnB5n+LHp9fwjzJwxge++HbpB+dVXX8xPP45gwnfDeOD+22OKLnWd0LolE38ayeRJo+ly0zVxhyMlSJmeBG++9h4v9n6Np3o+mF82+eepXHL+tTz8+D0b7Dv556mc0PJ0cnJyqLlTDT4b/S6fDP6cnJyckg47VtOm/kaLpicDkJGRwaSpY/hw0Cc0b9GEE9sdR/MmJ7Fu3Tqq16gac6SpZZddanHNNZdwUMNjWLNmDa+99hxnntGBPn37xx1aLNavXc9tZ93GmlVryMzK5KEBD/PN5+N56Jr/5u9z6R2XsnL5KgCat2tOqdKl6NT6GsqULcOzw3oyYuAI5s+cH9dTiEWfvm/xbM+XeenFx/PLjj66Ke3bt+aQQ49n3bp11KhRLcYIU09GRgZPPtGdNieezcyZc/jqy48Y9MEn/Pzz1LhDi0WuaSBz2vrqi/EsWbxkg7Kpv/zK9Gm/b7Lv6tVr8hs4ZcuWxj3dekY3dXTLpvz+65/MmDGbSy47h8cffZ5169YBsHDBXzFHl3qysrIoV64smZmZlC9Xjtlz5sYdUqzWrFoDBK9LZlYmG3+kmp90FCMHjgDAHcqWL0tGZgaly5Yme302q8IGUToZPXosizf6m3VFx/N5+OFn8j97CxYsiiO0lHX4YQczffrv/Pbbn6xfv57+/QdycvsT4g5LSkjkjR4zK2dm+0RdTxwOPvQgRnw5iM/HDKTLDfekXZZnY6ee1o4Bb38AQN26e3Bks8P49PO3+WDIaxx8yIExR5daZs+ey2OPPcev079mxp/fsWzZMoYOHRl3WLHKyMjgycFP0e+7V5kwegK/TJiSv23/w/dnycIlzP59NgBjPhrNmlVr6Du+Hy999TLv9HqHFUtXxBV6SqlXby+aNzuC0aMGMfTTtzn00IZxh5RSdqldixkzZ+evz5w1h112qRVjRPHyCJZUFmmjx8zaAxOAIeF6IzN7v/Cjth3fffMDRx/ZnjatzuC6zpdTpkzpuEOKTalSpWjb7ljee/cjIPhvvUqVShx/zGncdfuDvNTnyZgjTC1VqlSmffsTqFe/CbvXOYTyFcpzzjmnxh1WrHJzc7mu7bVcdMSF1G9Ynzr16+RvO7rD0flZHoD6jeqTm5PLBYedz6XNLuFfl/+LnXZP3y+uRFlZmVTZsTLNj2rPLbd247XXesYdUkqxArpzlKlPH1Fneu4GDgeWALj7BGCPze1sZh3NbLyZjV+1bsnmdks5U3/5lVWrVrPvfvXiDiU2x7U+mu8nTGLB/CCVPmvWXAa9/wkA337zA7m5TrXqGteT59hjj+L33/9k4cK/yM7O5r33BnNkk8Zxh5USVi5byY9f/cAhLQ8FICMzgyPbNGXkoL8zYUd3aMk3I74hJzuHpYuW8vP4SdQ7qG5cIaeUmbPm8t57gwEYP34Cubm5VNdnL9+smXPYbddd8td3rb0zc+bMizGieOVGsKSyqBs92e6+NNmd3b2Xuzd298blS1eJMq5/bPc6tcnMzARg1912Ye+6ezLjz1kxRxWf004/iQFvDcpf/+iDT2lxdBMA9q67B6VLl2LRQo3ryTPjz1kcfsQhlCtXFoBWxzRn8uT0HEgJUKlqJSpUqgBA6TKladS8ETOnzwCgUfODmTl9Jovm/j02ZcHsBRzUNOi2KVOuDPscsi8zp80s+cBT0PvvD+GYls0AqFdvT0qXKs1CffbyjRs/gbp192SPPXajVKlSnHFGBwZ98EncYcUm14p/SWVRz976yczOATLNrB5wHfBFxHVutZ7/e4SmzQ+narUqfDvxcx5+8GmWLF5K9//eTrXqVenX/zl++nEyZ//7cg5vcijXXn8567PXk5vr3HLjvfz117aTnSpO5cqVpeUxzeh83R35Zf36vM3TPR/ki68/Yt269Vx1xU0xRph6vh73He+88yFff/0x2dnZfD9hIr3/92rcYcWmas2qdO5xAxmZGWRkGKM+GM24YeMAaHFyC0a+P2KD/T985QOuf7Qzzwx9FjNjaP9P+X3y7zFEHq++fZ6mRYsjqV69Kr9OH8e99z3Kyy+/Se9ej/Ldt0NZt249l152fdxhppScnBz+7/o7+OjD18jMyODlV95k0qRf4g5LSohF2ZdpZuWB24HWYdHHQDd3X1PUsbWq7KdO1iSszV4fdwjbjBXrVscdwjbhhFqN4g5hm/DJvO/jDmGbkasxM0nLXjerRHMlr+5yXrH/cs6d3S9l8z2RZnrcfRVBo0dXxxIREZFYRT1761Mzq5KwvqOZfRxlnSIiIpKcdJuyHvWYnurunj/Qxd0Xm1nNiOsUERGRJKT6wOPiFvXsrVwz2z1vxczqkPoNQREREYmIme1jZhMSlmVmdr2Z3W1msxLKT0w45lYzm2ZmU8xsqy+hHXWm53ZgtJnlTb1oAXSMuE4RERFJQhzX1XH3KUAjADPLBGYB7wIXA4+5+yOJ+5tZA+AsYH9gF2ComdV39y2+DULUA5mHmNkhQBPAgM7uvjDKOkVERGSbcSww3d3/KOhq2aEOwBvuvhb4zcymEVz4+MstrawkbjhaBvgLWAo0MLMWJVCniIiIFCGKgcyJd1cIl8J6eM4CXk9Y72RmP5jZi2a2Y1hWG5iRsM/MsGyLRZrpMbP/AmcCE/k7i+ZAet9ZUUREJAVEMZDZ3XsBvYraz8xKAycDt4ZFPYH7CNoJ9wGPApcQ9BRtUs3WxBb1mJ5TgH3ClJSIiIhInrbAt+4+DyDvJ4CZ9QY+CFdnArslHLcrMHtrKoy6e+tXoFTEdYiIiMhWiPmGo2eT0LVlZjsnbPsX8FP4+H3gLDMrY2Z7AvWAr7esqkDUmZ5VwAQzGwbkZ3vc/bqI6xUREZEUFd6m6njgioTih8ysEUHX1e9529x9opn1ByYB2cA1WzNzC6Jv9LwfLiIiIpJi4piyDvm3qaq2Udn5hezfHej+T+uNesr6K2ZWDtg9nJcvIiIiEouo773VHpgADAnXG5mZMj8iIiIpwK34l1QW9UDmuwkuILQEwN0nAHtGXKeIiIgkIeaBzCUu6kZPtrsv3ahM994SERGREhf1QOafzOwcINPM6gHXAV9EXKeIiIgkIdUzM8Ut6kzPtQQ3CFtLMBd/GXB9xHWKyP+3d+dxUlTn/sc/35lBQEAUQcF9w7gkbgEVd9EIalyiXjGbJibhmqg36jX+jLsmXI0aNS4/FXe9QdQg7goCiuIGCLihKCICAiJiQFDRYZ77R9fgMMzAzDjV1VDfN69+dXd1VZ2nDt3TT59zqo6ZmS0n7bO3vqAw0/q5aZZjZmZmjZe38SapJD2SromI0yQ9Sh11GhGHp1GumZmZNVwac2+VsrRaeu5J7q9Maf9mZmZmjZJK0hMRryYPxwJfRkQVgKRyoGUaZZqZmVnjeCBz8xoOrFnjeWtgWMplmpmZmS0ndvAIngAAIABJREFU7VPWW0XEwuonEbEwmWTMzMzMMuaWnua1SNIu1U8kdQO+TLlMMzMza4BI4VbK0m7pOQ14QNJMCnWxAdAn5TLNzMzMlpNKS4+k7pI6R8QYYBvgPqCSwsSjH6RRppmZmTVOlZr/VsrS6t66Gfg6edwDOAe4AfgM6J9SmWZmZmb1Sqt7qzwi5iWP+wD9I2IQMEjShJTKNDMzs0bwQObmUS6pOqE6ABhR47W0xxGZmZmZLSetBOReYKSkuRTO1noeQNJWwPyUyjQzM7NGKPWzrZpbWldk7idpONAFGBoR1fVaRmHm9ZX6/Guf2d4QVZG3t2zTuaYaZshs90A3xLvbbpd1CKuMrm9PzDoEq0dVzv4yptbVFBEv17Hs3bTKMzMzM1sRj68xMzPLKQ9kNjMzM1sNuaXHzMwsp/I1osdJj5mZWW65e8vMzMxsNeSWHjMzs5wq9bmymptbeszMzCwX3NJjZmaWU744oZmZmeVCvlIed2+ZmZlZTrilx8zMLKd8yrqZmZnZasgtPWZmZjnlgcxmZmaWC/lKedy9ZWZmZjnhlh4zM7Oc8kBmMzMzs9WQW3rMzMxyKm8Dmd3SY2ZmZrnglh4zM7Ocylc7j5MeMzOz3PJAZjMzM7PVkFt6zMzMcipy1sHllh4zMzPLBbf0mJmZ5VTexvQ46TEzM8spX6fHzMzMbDXkpMfMzCynIoVbQ0iaKukNSRMkjU2WdZD0tKT3kvt1aqz/Z0mTJU2S1Kupx+ukx8zMzLKwf0TsFBHdkudnA8MjoiswPHmOpO2A44Dtgd7A/5dU3pQCnfSYmZnlVBXR7Lfv4AjgruTxXcCRNZYPjIjFEfEBMBnYtSkFeCDzCpSVlTHqhUeZOXM2xxz9G+66+3q23noLANq3X4v58xfQY/dDMo4yWy1btmTYsPtZY401qKioYPDgJ/jrX69mhx2247rr+tGyZUsqK5dw2mnnMXbsa1mHWzK23npLBvzzxqXPt9h8Ey66+Equve7WDKMqTX/8r9/x6xN/SkTw5pvv8NvfnsHixYuzDqtoOl7836y5z24smfdvPjq6LwBla7VjvcvPpWKDzlTOnM2cP/2Vqs8X0uaQnrQ/4dil266x9ebMPO4PfPPhDNa74nwqNu4CVVV8MfJlPvvHbVkdUuZ6HbQfV111CeVlZdx+x71cfsUNWYeUmTTO3pLUF+hbY1H/iOhfa7UAhkoK4Obk9fUjYhZARMyStF6y7obAyzW2nZEsazQnPStw8sm/ZtI7k2m3VlsATjj+lKWvXXrpucxf8HlWoZWMxYsX07v3T1m06AsqKioYMeJfDB36LOeffwb9+v2DoUOfpVev/enX78/06nVc1uGWjHfffZ9u3Q8CCsn1tKmv8tDDT2YcVenZYIPOnHzyieyw4/589dVXDBhwE32OPYK777k/69CKZuHDQ1lw78N06nfW0mXtT+zDl6PHM//2+2h/Yh/a/+Y4PrvmVhY9MYJFT4wAoMVWm7H+Py7h60nvo1YtmX/3A3w15jWoqKDLLZfTes/ufPnCmKwOKzNlZWVc+49+9D7kp8yYMYuXX3qCRx8byttvv5d1aKuNJIGpneTUtmdEzEwSm6clvbOCdVVXMU2Jzd1b9dhgw8707t2TO+8cWOfrRx19KA/c/0iRoypNixZ9AUCLFhVUVLQgIogI1kqSxfbt2zFr1pwsQyxpB/TciylTPmTatI+yDqUkVVRU0Lp1K8rLy1mzdWtmzpqddUhF9dW4N6iq9QNrzf33YOEjTwOw8JGnWXP/PZbbru3BPVn05DMAxFeLCwkPQGUli9+eTMX6HdMNvETt2n1n3n9/Kh98MI1vvvmG++9/mMMPa/K42FVepPCvQeVGzEzu5wCDKXRXfSypC0ByX/3FMQPYuMbmGwEzm3K8RUt6JK0jaYdilfddXX75BZx73qVUVS3/H7jnnrsyZ85c3n9/avEDK0FlZWW8/PITTJs2jhEjnmfMmAn86U+X8D//cw7vvfcSl156Lhdc8LeswyxZxx57BAPveyjrMErSzJmzufrqm5jy/mimTxvPggULGDbsuazDylx5h3VYMnceAEvmzqO8w9rLrdOm174sfOqZ5ZaXtWvDmvvuzpevjE89zlK0wYadmT7j2+/LGR/NYoMNOmcYUf5IaiOpXfVj4CDgTeAR4IRktROAh5PHjwDHSWopaXOgKzC6KWWnmvRIelbSWpI6AK8Bd0i6Ks0ym0Pvg3vyySefMmH8m3W+/h/HHu5WnhqqqqrYffdD2Gqr3enWbSe2225r+vb9BWed9Re6du3BWWddwo03Xp51mCWpRYsWHPbjg/jXoMeyDqUkrb12ew47rBddt96dTTbdhTXbrMnPfnZU1mGVvJY/2Ib4ajHfTJ667AvlZXS67BwWDBhM5Uf5ajGrJi3fUxKRrwv01VSVwq0B1gdGSXqNQvLyeEQ8BVwG/EjSe8CPkudExFvA/cBE4Cng5IhY0pTjTbulp31ELACOAu6IiB8CB9a3sqS+ksZKGltZmd14mR67d+PQQw9k4tujuOvu69h33z247barASgvL+eIw3v5S6oO8+cv4LnnXuKgg/bj5z8/moceKoxRGTTocbp12zHj6EpT7977M378G8yZMzfrUErSAQfszdSp05g7dx6VlZU89NCT9Ni928o3XM0tmfcZ5R07AFDesQNL5v17mdfb9NqPhU8u38rT8YLT+WbaRyz45+CixFmKPpoxi4032mDp84027MKsWR9nGFH+RMSUiNgxuW0fEf2S5Z9GxAER0TW5n1djm34RsWVEfC8imjwAMu2kpyLplzsWWGmWEBH9I6JbRHSrqGiXcmj1u/DCy9m6aw+223YvTjj+VEaOfJHf/OZ0AHr23ItJ705hZk5/JdXWsWMH2rdfC4BWrVoW6mfSZGbNmsPee+8OwH777cnk2r84DYDj+hzprq0VmD7tI3bdbRdat24FQM/99+Kddzzg9ItnX6Lt4T8CoO3hP+KLZ1789kWJNgftw6JaXVvrnPwrytq2Yd7lN5JnY8ZOYKutNmezzTamRYsWHHvsETz62NCsw8pMVmN6spL22VsXA0OAURExRtIWwCr9F+uYYw7jgQfctVWtc+f1uOWWqygvL6OsrIxBgx7jySdHMH/+Aq644iIqKspZvHgxp5xydtahlpzWrVtx4AH78Ps//L+sQylZo8eM58EHH2f06CFUVlby2oS3uOXWf2YdVlF1uuwcWnXbgfK127Px0AF8duPdzL99IOtdcT7tjjyYytlzmHPmX5au3+qHP6Dy47nLdF+Vr9eRtfv+nK+nTGODgYWkZ8HAh1k4OH9nDC5ZsoQ/nnYeTzw+gPKyMu686z4mTnw367Ayk7cJR5VmX6akPSPihZUtq0ubNTcr7XSxRFTluC+6sb5ZUpl1CKuEus4NteW9u+12WYewyuj69sSsQ1hlVH79UVE/gidsdnSzf4ncNXVQyf4ZSbt767oGLjMzM7Miq4po9lspS6V7S1IPYA+gk6Qzary0FtCk+TLMzMzMvou0xvSsAbRN9l9zRPIC4JiUyjQzM7NGKO12meaXStITESMljQJ+EBEXp1GGmZmZfTffcYLQVU5qY3qSCwd1SGv/ZmZmZo2R9inr4yU9AjwALKpeGBEPplyumZmZrUSpX1enuaWd9HQAPgV61lgWgJMeMzMzK6pUk56I+HWa+zczM7Omy9vFCdOecHQjSYMlzZH0saRBkjZKs0wzMzNrmCqi2W+lLO2LE95BYUr4DYANgUeTZWZmZmZFlXbS0yki7oiIyuR2J9Ap5TLNzMysAfI24WjaSc9cSb+QVJ7cfkFhYLOZmZlZUaWd9JwIHAvMTm7HJMvMzMwsY1Up3EpZ2mdvTQMOT7MMMzMzs4ZI++ytLSQ9KumT5AyuhyVtkWaZZmZm1jAR0ey3UpZ299YA4H6gC4UzuB4A7k25TDMzM2sAn7LevBQR99Q4e+t/yd+krmZmZlYC0p6G4hlJZwMDKSQ7fYDHJXUAiIh5KZdvZmZm9Sj1gcfNLe2kp09y3ze5V3J/IoUkyON7zMzMrChSSXokdQemR8TmyfMTgKOBqcBFbuExMzPLXqlfTLC5pTWm52bgawBJ+wCXAncB84H+KZVpZmZmjZC3gcxpdW+V12jN6QP0j4hBwCBJE1Iq08zMzKxeqSU9kioiohI4gG/H9KRZppmZmTVCqV9Xp7mllYDcC4yUNBf4EngeQNJWFLq4zMzMzIoqlaQnIvpJGk7hooRD49tUsgw4NY0yzczMrHF8ynoziYiX61j2blrlmZmZWeP47C0zMzOz1ZAHFZuZmeVUqZ9i3tzc0mNmZma54JYeMzOznMrbKetu6TEzM7NccEuPmZlZTuVtTE/JJj2VVUuyDmGVkLemye+ioqw86xBWCf7sNUzXtydmHcIqY93W7bIOwerhU9bNzMzMVkMl29JjZmZm6arKWW+BW3rMzMwsF9zSY2ZmllP5audx0mNmZpZbeTt7y91bZmZmlgtu6TEzM8spt/SYmZmZrYbc0mNmZpZTebvArZMeMzOznHL3lpmZmdlqyEmPmZlZTkUK/1ZG0saSnpH0tqS3JP0xWX6RpI8kTUhuh9TY5s+SJkuaJKlXU4/X3VtmZmZWTJXAf0fEOEntgFclPZ28dnVEXFlzZUnbAccB2wMbAMMkbR0RjZ4d2UmPmZlZTmUxkDkiZgGzksefS3ob2HAFmxwBDIyIxcAHkiYDuwIvNbZsd2+ZmZlZJiRtBuwMvJIsOkXS65Jul7ROsmxDYHqNzWaw4iSpXk56zMzMcqqKaPabpL6Sxta49a2rbEltgUHAaRGxALgR2BLYiUJL0N+rV61j8yY1Ubl7y8zMLKfS6N6KiP5A/xWtI6kFhYTnnxHxYLLdxzVevwV4LHk6A9i4xuYbATObEptbeszMzKxoJAm4DXg7Iq6qsbxLjdV+AryZPH4EOE5SS0mbA12B0U0p2y09ZmZmOZXRxQn3BH4JvCFpQrLsHOCnknai0HU1FfhPgIh4S9L9wEQKZ36d3JQzt8BJj5mZmRVRRIyi7nE6T6xgm35Av+9atpMeMzOznGrIxQRXJ056zMzMcqoqZxOOeiCzmZmZ5YJbeszMzHIqb91bbukxMzOzXHBLj5mZWU7lbUyPkx4zM7OccveWmZmZ2WrILT1mZmY5lbfuLbf0mJmZWS64pcfMzCynPKbHANhooy4MGXIfr00Ywfhxwzjl5BMBOO+805ny/hhGv/IUo195it699s840uz1v/lKZkyfwPhxw5YuO/+8M/hgyljGjB7CmNFD6N27Z4YRloaWLVvy/POPMHr0U4wbN4zzzz8DgKOOOpRx44bxxRdT2WWXHTKOsrS0bNmSl154jFfHPs1rE0Zw4QX/nXVIJct1taxrru/HW5NfYORLjyxdtvY67bn/odt4adxT3P/QbbRfey0ANt5kQ6bOnsDw5wcz/PnBXH71RRlFbWlTlGh/XstWG2caWOfO69G583pMmPAmbdu24eWXnuCY//gtxxzzYxYt/IKrr7k5y/CWKoX/v7322o2FCxdxx+3XsPMuBwKFpGfhokVcfXVp1BNAmbLP8du0WZNFi76goqKCESMGceaZFzF//udUVVVxww2XcvbZ/Rg37vVMY6ysatLkxampWWfPPTuY08+4kFdGj8s6rJJUqnW1but2RS9z9z26sWjRF1x/02Xs2+NwAM6/5Ez+/dl8rrv6Fk49/Xe0X3st/nrh39l4kw353/tuXLpelj6e/05dE3GmZsuOuzT7l8j7c8cV9RgaI/tvgRI1e/YcJkx4E4CFCxfxzjuT2XDDzhlHVZpGjXqFzz77d9ZhrBIWLfoCgBYtKmjRooKIYNKkybz33pSMIytdNeusokWLkkj0S5Xr6lsvvziWf382f5llvQ85gPsGPATAfQMe4uBDD8witJISKfwrZU56GmDTTTdix522Z/To8QCc9PsTGDtmKDfffCVrr90+4+hK1+9P+hWvjn2a/q6npcrKynjllSeZPn08w4ePYsyYCVmHVPLKysoYO2Yosz56neHDn2P0mPFZh1SyXFcr1qnTusz5+BMA5nz8CR07dVj62iabbsSw5x9k8OP3sFuPH2YVoqWs6EmPpOzbWhuhTZs1GXjvzZx55kV8/vlC+ve/h2233Yvuu/Zi9uw5/O1v52cdYkm6uf/dbLPtnnTrfhCzZ8/hctcTAFVVVey228FsueVudO++I9ttt3XWIZW8qqoqunU/iE0370b3bjuz/fbfyzqkkuW6apqPZ89hl+17cuDeR3HhuZdx461X0rZdm6zDKoqIqma/lbKiJz0RsUt9r0nqK2mspLFLliwsZlh1qqio4L6B/Rk48CEefvgpAObMmUtVVRURwe23D6B7t50yjrI01ayn224fQPfurqea5s9fwHPPvcxBB+2XdSirjPnzFzDyuRfp5TpbKddV3T755FPWW78TAOut34m5n8wD4Ouvv1naRf/6hLeY+sF0ttxq88zitPQUJemRtK6kn0haYZthRPSPiG4R0a28vG0xQluhm2++gnfeeY9/XHvL0mWdO6+39PERh/fmrbcmZRFayVumno5wPQF07NiB9u0LZ4u0atWSnj33YtKk9zOOqrQtW2etOKDn3q6zeriuVm7IkyPo87MjAejzsyN56onhAKy77jqUlRW+DjfdbCO22HJTPpw6PbM4i6mKaPZbKUvlOj2SHgPOjog3JXUBxgFjgS0l9Y+Ia9IotzntsUd3fvHzY3jjjbcZ/UqhleeCC/7GsX2OYMcdtici+PDDGZx8ytkZR5q9e+6+nn326UHHjh2Y8v4YLvnL39l3nx7suGN1PU3nDye7njp3Xo9bb72K8vJyysrKGDToMZ58cjiHH96Lq666hE6dOjB48B28/vpEDjvsl1mHWxK6dFmf22+7hvLyMsrKyvjXvx7l8SeGrXzDHHJdLeum2/7OHnt1p8O66zB+4rNccel1XHfVLdxy19X87JdH89GMWfz2hNMA2H3P7px1zqksqVzCkqolnHX6RcsNgl5d5W2weyqnrEt6KyK2Tx6fA2wTEcdLage8EBErvRhJ1qesryry9ob9LkrhlPVVQamdsm6rvixOWV9VFfuU9U06/KDZv0SmzXujZE9ZT+uKzN/UeHwAcAtARHwuqbRHOZmZmeVEqXdHNbe0kp7pkk4FZgC7AE8BSGoNtEipTDMzM7N6pZX0/Aa4BDgQ6BMR1Veu2x24I6UyzczMrBHyNkQilaQnIuYAJ9Wx/BngmTTKNDMzs8apylnSk8XFCfsWu0wzMzOztLq3VqRkR3WbmZnlSanPldXc0rpOzxm1FgUwFxgVEaUz7baZmZnlRlrdW+1q3dYCugFPSjoupTLNzMysESKi2W+lLK2BzBfXtVxSB2AYMDCNcs3MzMzqU9QxPRExT5LH9JiZmZUAX5wwRZJ6Ap8Vs0wzMzOrW6l3RzW3tAYyvwHLpY8dgJnA8WmUaWZmZrYiabX0/LjW8wA+jYhFKZVnZmZmjZS3ixOmNZD5wzT2a2ZmZtZUWVyc0MzMzEqAx/SYmZlZLuTt7K2iz71lZmZmlgW39JiZmeVU3rq33NJjZmZmueCWHjMzs5zyKetmZmaWC+GBzGZmZmarH7f0mJmZ5VTeurfc0mNmZma54JYeMzOznPIp62ZmZmarIbf0mJmZ5VTezt5y0mNmZpZT7t4yMzMzS5Gk3pImSZos6exileuWHjMzs5zKoqVHUjlwA/AjYAYwRtIjETEx7bLd0mNmZmbFtCswOSKmRMTXwEDgiGIU7KTHzMwspyKFWwNsCEyv8XxGsix1Jdu9tfir6co6htok9Y2I/lnHsSpwXTWM66nhXFcN43pqGNdTQeXXHzX7d62kvkDfGov616rrusosSj+bW3oap+/KV7GE66phXE8N57pqGNdTw7ieUhIR/SOiW41b7eRyBrBxjecbATOLEZuTHjMzMyumMUBXSZtLWgM4DnikGAWXbPeWmZmZrX4iolLSKcAQoBy4PSLeKkbZTnoaJ/f9v43gumoY11PDua4axvXUMK6nDEXEE8ATxS5Xebsao5mZmeWTx/SYmZlZLuQm6ZG0RNKEGrfNmmGfl0g68LtHl60adfOapHGS9mjifk6SdHxzx1fKatTdm5IekLTmCtb9laTrk8dL6ypZvkGN9W6VtF360ReHpHMlvSXp9aSudqtnvW6Sri12fKWqofXWyH3u19TP96pE0maS3qy17CJJZ9Zatr6kx5K/fRMlFb27xYorT2N6voyInRq7kaSKiKis67WIuOC7h1USltaNpF7ApcC+jd1JRNzU3IGtAmrW3T+Bk4CrVrZRrbr6FfAmySmbEfHb5g8zG5J6AD8GdomIxZI6AmvUtW5EjAXGFjO+UtWYemvEPiuA/YCFwIvfOcjVwyXA0xHxDwBJO2Qcj6UsNy09dZG0k6SXk19SgyWtkyx/VtL/SBoJ/FHSDyWNlPSqpCGSuiTr3SnpmOTxIZLekTRK0rWSHkuWXyTp9mSfUyT9V2YH3DBrAZ/B0l+Fj1W/IOl6Sb9KHl+W/DJ6XdKVybKlv6SS4/2bpNGS3pW0d7K8XNIVksYk2/5nsryLpOdqtJrsnax7Z/L8DUmnF7cqGu15YCtJHSQ9lBzfy3X9Ia2uq+T90w34Z3LsrZO665as1ztpfXtN0vBk2b76tsVyvKR2RT3KxukCzI2IxQARMTciZkrqLunF5LhGS2pX8/0mqU3yuRmTHOMRyfJfSXpQ0lOS3pN0eXVB9dRVnftZBdRXb1NrfK5GS9oKQNKmkoYn77nhkjZJlt8p6SpJzwD3UUjKT0/eO3tL+o/k8/WapOeyOtgMdaFwzRgAIuL1DGOxIshTS09rSROSxx9ExE+Au4FTI2KkpEuAC4HTknXWjoh9JbUARgJHRMQnkvoA/YATq3csqRVwM7BPRHwg6d5aZW8D7A+0AyZJujEivknrQJugum5aUfgj0HNFK0vqAPwE2CYiQtLa9axaERG7SjqEQt0eCPwGmB8R3SW1BF6QNBQ4ChgSEf1UmIxuTWAnYMOI+H5Sbn3lZE6FX9EHA08BFwPjI+JIST0pvM/qbGWMiH+pcOrmmUlLB5Kq99kJuIVv31cdks3OBE6OiBcktQW+SvHQvquhwAWS3gWGUfjifSm57xMRYyStBXxZa7tzgRERcWLy/z5a0rDktZ2AnYHFFD5P11Gog7rqqs79RMSi9A65WSxXbxExMnltQfK5Oh64hkKL0PXA3RFxl6QTgWuBI5P1twYOjIglki4CFkZE9Q+VN4BeEfFRKX++UnQDcF/yGRwG3BERRblInmUjTy09X0bETsntJ5LaU0hsqv+Q3AXsU2P9+5L77wHfB55OEoPzKFw9sqZtgCkR8UHyvHbS83hELI6IucAcYP1mOqbmUl032wC9gbtV/c1btwUUvmRulXQU8EU96z2Y3L8KbJY8Pgg4PqnLV4B1ga4ULlb16+SP8g8i4nNgCrCFpOsk9U7KLTXVCeNYYBpwG7AXcA9ARIwA1k3eb421O/Bc9fsqIuYly18ArlKh1XDt+rpfS0FELAR+SOHqt59Q+Fz9JzArIsYk6yyo4xgOAs5O6vZZCgn5JslrwyNifkR8BUwENqX+ulrRfkpWXfWmpJWVb/++3Av0SB73AAYkj++h8B6s9kBELKmnqBeAOyX9jsL1UlYX9Z2WvMzyiBgCbEEhYd4GGJ/82LDVVJ5aehqr+peggLcioscK1l3Z3CWLazxeQgnXe0S8pML4gU5AJcsmxq2SdSol7QocQOFKmqdQd+tQ9XHXPGZRaF0bUntlSfsAhwL3SLoiIu6WtCPQCzgZOJYaLWwlYrmxYvUkjE25NoTq2i4iLpP0OHAI8LKkAyPinSbsvyiSL9xngWeTloWTWXl9CDg6IiYts7AwmLeuz1OddVXfflYFddTbCdUv1Vytvs1rPK63VSsiTkrq9FBggqSdIuLTpkddMj4F1qm1rAPwQe0VkwR5ADAg6V7dBxiUeoSWiTy19CwjIuYDnykZawL8kkI3Vm2TgE4qDCxEUgtJ29da5x0KLRKbJc/7NH/ExSFpGwq/+D4FPgS2k9Qyaak4IFmnLdA+ubjUadTTdVOPIcDvk25DJG2djLvYFJgTEbdQaC3ZJUm+yiJiEHA+sEvzHGXqngN+DoVxURTGZqyolepzCl2ftb0E7Ctp82RfHZL7LSPijYj4G4UWpm2aMfZmJel7krrWWLQT8DawgaTuyTrtku7BmoYAp1YnkJJ2XklRddZVE/ZTEuqptw+Tx31q3L+UPH6Rwg8QKLz3RtWz62Xea8l76ZXkpIy5LDsf0ioraSmbJan6b1YHCq3Yy9SLpJ5KzrhUYWzclhRabG01VbItDkVyAnBT8qafAvy69goR8bUKg02vTb74Kyj0o79VY50vJf0BeErSXGB0UaJvPjXHOwk4IfmVOV3S/cDrwHvA+GSddsDDyVgmAY0ZYHwrha6ucckX0ScUxh7sB/xJ0jcUzi45HtgQuENSdXL+56YdXtFdRCHu1yl0/Z2w4tW5k8L78Eu+7a4gGUPWF3gwqYM5wI+A0yTtT6GVYyLwZLMfQfNpC1yXjBepBCZT6LK5I1nemsJ4ntqXfvgLhc/Z68n7ZCqFsSt1WkFdNWo/JaS+evsx0FLSKxR+tP40Wf+/gNsl/YnCZ2q5v2WJR4F/qTCg+1QKg5q7UvgcDwdeS+l4snA8cIOkvyfPL46I9yWdBEvPoPwhcL2k6lbtW6u7XSVNaMoZv1bafEXmZiKpbUQsTP6w3gC8FxFXZx2Xma0+JE0FuiXjA82skXLbvZWC3yWtJW8B7SmczWVmZmYlwi09ZmZmlgtu6TEzM7NccNJjZmZmueCkx8zMzHLBSY/ZKkqNmOG9AfuqOY/cCmd5VxNn6lZh3qiOTY3RzOy7ctLUZV2DAAAB9ElEQVRjtuqqnj7k+8DXFCaTXEqFOcwaLSJ+GxETV7DKfkCjkx4zs6w56TFbPVTP8L6fpGckDQDeUP2z2kvS9ZImJlNarFe9I61glvfkquO1Z+ruJGlQUsYYSXsm264raagKs5vfzMqnazEzS1Xer8hstsrTsjO8A+wKfD+Zbbwvdc9qvzOFyXR/QGEC3InA7bX2u9ws7xExT9JNLDtT9wDg6ogYJWkTClM/bAtcCIyKiEskHUrhisJmZplx0mO26qo5fcjzFOYs2wMYXT3bOIVZxneoHq9D4cKZXSlMqnhvMt3ITEkj6th/fTOX13YghTnaqp+vlcxjtA9wVLLt45I+a+Jxmpk1Cyc9ZquuumZ4h2Vn1a5zVntJh9Cwmc4bcvXSMqBHRHxZRyy++qmZlQyP6TFbvdU5qz2FmeCPS8b8dAH2r2Pb+mYurz0r/FDglOonkqoTsZqzzR8MrNNsR2Vm1gROesxWb7dSGK8zTtKbFOaEqwAGA+8BbwA3AiNrbxgRn1AYh/OgpNeA+5KXHgV+Uj2QmcIM392SgdIT+fYssouBfSSNo9DNNi2lYzQzaxDPvWVmZma54JYeMzMzywUnPWZmZpYLTnrMzMwsF5z0mJmZWS446TEzM7NccNJjZmZmueCkx8zMzHLBSY+ZmZnlwv8B47Iruq2F6XYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "model, c_matrix, y_test, y_pred, indices_test = plot_confusion_matrix(LinearSVC(), df, features, labels, news_desk_to_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check misstakes.\n",
    "\n",
    "> Here we can have a look on the wrong preditions and indeed some of them are misleading... See that mentions of China nd G-20 inside Business desk, I would personally guess it as Foreign in this case. Anyways, the proper conduct here is to check once again with the business department if these mistakes are acceptable. If not we should go back into gathering more data, choose different models or do hyper parameter tunning for improving it's performance. Nevertheless we're still lacking some hard numbers to tell how well our model performs... Let's have a look at that!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "'Business' predicted as 'Foreign' : 206 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8275</th>\n",
       "      <td>Business</td>\n",
       "      <td>All eyes are on a pivotal meeting between President Trump and President Xi Jinping of China at the G-20.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12457</th>\n",
       "      <td>Business</td>\n",
       "      <td>One approach is a microgrid — creating a small power source that can keep the lights on in a small area when the main grid isn’t working.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "8275   Business   \n",
       "12457  Business   \n",
       "\n",
       "                                                                                                                                        abstract  \n",
       "8275                                    All eyes are on a pivotal meeting between President Trump and President Xi Jinping of China at the G-20.  \n",
       "12457  One approach is a microgrid — creating a small power source that can keep the lights on in a small area when the main grid isn’t working.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Politics' predicted as 'Foreign' : 62 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9968</th>\n",
       "      <td>Politics</td>\n",
       "      <td>The Senate majority leader did not back the legislation, but he said the chamber would consider it when lawmakers come back into session next month.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16259</th>\n",
       "      <td>Politics</td>\n",
       "      <td>The 35-day stalemate sapped the president’s poll numbers and raised concern about his ability to navigate a divided government.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "9968   Politics   \n",
       "16259  Politics   \n",
       "\n",
       "                                                                                                                                                   abstract  \n",
       "9968   The Senate majority leader did not back the legislation, but he said the chamber would consider it when lawmakers come back into session next month.  \n",
       "16259                       The 35-day stalemate sapped the president’s poll numbers and raised concern about his ability to navigate a divided government.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Science' predicted as 'Foreign' : 113 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16289</th>\n",
       "      <td>Science</td>\n",
       "      <td>The mysterious Denisovans may have occupied a cave in what is now Siberia for more than 250,000 years.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>Science</td>\n",
       "      <td>The country is home to most of the world’s wild tigers, and wildlife authorities announced steps to protect them.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "16289   Science   \n",
       "89      Science   \n",
       "\n",
       "                                                                                                                abstract  \n",
       "16289             The mysterious Denisovans may have occupied a cave in what is now Siberia for more than 250,000 years.  \n",
       "89     The country is home to most of the world’s wild tigers, and wildlife authorities announced steps to protect them.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Sports' predicted as 'Foreign' : 47 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21079</th>\n",
       "      <td>Sports</td>\n",
       "      <td>Climbers and the Nepalese government believe the recently exposed bodies are the result of global warming, which is rapidly melting the mountain’s glaciers.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8867</th>\n",
       "      <td>Sports</td>\n",
       "      <td>Police charged the former Yankee and Mets star after a traffic stop last month.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "21079    Sports   \n",
       "8867     Sports   \n",
       "\n",
       "                                                                                                                                                           abstract  \n",
       "21079  Climbers and the Nepalese government believe the recently exposed bodies are the result of global warming, which is rapidly melting the mountain’s glaciers.  \n",
       "8867                                                                                Police charged the former Yankee and Mets star after a traffic stop last month.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'U.S.' predicted as 'Foreign' : 25 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3873</th>\n",
       "      <td>U.S.</td>\n",
       "      <td>States that reopened early and relaxed social distancing restrictions may be contributing to the first pronounced increase in Covid-19 fatalities since April.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16546</th>\n",
       "      <td>U.S.</td>\n",
       "      <td>Every January, thousands of agricultural families come together for a 16-day celebration of Western heritage.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "3873       U.S.   \n",
       "16546      U.S.   \n",
       "\n",
       "                                                                                                                                                             abstract  \n",
       "3873   States that reopened early and relaxed social distancing restrictions may be contributing to the first pronounced increase in Covid-19 fatalities since April.  \n",
       "16546                                                   Every January, thousands of agricultural families come together for a 16-day celebration of Western heritage.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Foreign' predicted as 'Business' : 144 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10114</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>Whoever was responsible, experts say, the episode raised serious concerns about the state of Bulgaria’s cybersecurity.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16175</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>A year after a dam burst killed at least 259 people, prosecutors accused company executives of covering up safety concerns.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "10114   Foreign   \n",
       "16175   Foreign   \n",
       "\n",
       "                                                                                                                          abstract  \n",
       "10114       Whoever was responsible, experts say, the episode raised serious concerns about the state of Bulgaria’s cybersecurity.  \n",
       "16175  A year after a dam burst killed at least 259 people, prosecutors accused company executives of covering up safety concerns.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Politics' predicted as 'Business' : 58 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10576</th>\n",
       "      <td>Politics</td>\n",
       "      <td>The former investor is an unlikely populist. But he is spending his fortune to try and get money out of politics.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19962</th>\n",
       "      <td>Politics</td>\n",
       "      <td>Robert Mueller’s letter in March is another sign of anger toward Attorney General William Barr, whose initial statement let President Trump claim to have been vindicated.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "10576  Politics   \n",
       "19962  Politics   \n",
       "\n",
       "                                                                                                                                                                         abstract  \n",
       "10576                                                           The former investor is an unlikely populist. But he is spending his fortune to try and get money out of politics.  \n",
       "19962  Robert Mueller’s letter in March is another sign of anger toward Attorney General William Barr, whose initial statement let President Trump claim to have been vindicated.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Science' predicted as 'Business' : 76 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6725</th>\n",
       "      <td>Science</td>\n",
       "      <td>It’s likely you can, at least for some period of time. That is opening new opportunities for testing and treatment.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1514</th>\n",
       "      <td>Science</td>\n",
       "      <td>This is what happens when atmospheric chemists hang towels on drying racks around their chemistry building.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     news_desk  \\\n",
       "6725   Science   \n",
       "1514   Science   \n",
       "\n",
       "                                                                                                                 abstract  \n",
       "6725  It’s likely you can, at least for some period of time. That is opening new opportunities for testing and treatment.  \n",
       "1514          This is what happens when atmospheric chemists hang towels on drying racks around their chemistry building.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Sports' predicted as 'Business' : 38 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14599</th>\n",
       "      <td>Sports</td>\n",
       "      <td>After forcing his way out of Miami, Minkah Fitzpatrick has powered the Steelers’ surprising surge into contention.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5153</th>\n",
       "      <td>Sports</td>\n",
       "      <td>A forensic audit of Russia’s antidoping agency has suggested former officials funneled money to companies they controlled.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "14599    Sports   \n",
       "5153     Sports   \n",
       "\n",
       "                                                                                                                         abstract  \n",
       "14599          After forcing his way out of Miami, Minkah Fitzpatrick has powered the Steelers’ surprising surge into contention.  \n",
       "5153   A forensic audit of Russia’s antidoping agency has suggested former officials funneled money to companies they controlled.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'U.S.' predicted as 'Business' : 15 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1227</th>\n",
       "      <td>U.S.</td>\n",
       "      <td>More than 300 dams in the state are classified as having the potential to cause loss of life or economic and environmental damage in the case of a failure, according to the U.S. Army Corps of Engineers.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11628</th>\n",
       "      <td>U.S.</td>\n",
       "      <td>Mass shootings came one after another, sometimes only hours apart, leaving a reeling nation wondering why.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "1227       U.S.   \n",
       "11628      U.S.   \n",
       "\n",
       "                                                                                                                                                                                                         abstract  \n",
       "1227   More than 300 dams in the state are classified as having the potential to cause loss of life or economic and environmental damage in the case of a failure, according to the U.S. Army Corps of Engineers.  \n",
       "11628                                                                                                  Mass shootings came one after another, sometimes only hours apart, leaving a reeling nation wondering why.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Foreign' predicted as 'Politics' : 18 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>Chen Chien-jen has embraced a rare dual role, using his political authority as vice president to criticize China’s response to the virus even as he hunkers down to analyze trends in transmission.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8550</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>Mr. Sigley, whose family raised alarms last week after losing contact with him, declined to say on Thursday why he had been detained.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     news_desk  \\\n",
       "821    Foreign   \n",
       "8550   Foreign   \n",
       "\n",
       "                                                                                                                                                                                                 abstract  \n",
       "821   Chen Chien-jen has embraced a rare dual role, using his political authority as vice president to criticize China’s response to the virus even as he hunkers down to analyze trends in transmission.  \n",
       "8550                                                                Mr. Sigley, whose family raised alarms last week after losing contact with him, declined to say on Thursday why he had been detained.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Business' predicted as 'Politics' : 45 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12643</th>\n",
       "      <td>Business</td>\n",
       "      <td>Several candidates criticized Senator Elizabeth Warren’s health care proposal and backed proposals to impose big taxes on the wealthy.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14677</th>\n",
       "      <td>Business</td>\n",
       "      <td>A coal town in southwestern Virginia has been trying for years. Hope is running thin.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "12643  Business   \n",
       "14677  Business   \n",
       "\n",
       "                                                                                                                                     abstract  \n",
       "12643  Several candidates criticized Senator Elizabeth Warren’s health care proposal and backed proposals to impose big taxes on the wealthy.  \n",
       "14677                                                   A coal town in southwestern Virginia has been trying for years. Hope is running thin.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'U.S.' predicted as 'Politics' : 31 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15997</th>\n",
       "      <td>U.S.</td>\n",
       "      <td>Senators Elizabeth Warren and Bernie Sanders led the stage of six candidates.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9615</th>\n",
       "      <td>U.S.</td>\n",
       "      <td>See which candidates spoke the most and which candidates disagreed over policy.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "15997      U.S.   \n",
       "9615       U.S.   \n",
       "\n",
       "                                                                              abstract  \n",
       "15997    Senators Elizabeth Warren and Bernie Sanders led the stage of six candidates.  \n",
       "9615   See which candidates spoke the most and which candidates disagreed over policy.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Foreign' predicted as 'Science' : 58 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12140</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>Zarifa Ghafari expects to die in office. But in the meantime, she intends to do something about the litter problem.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2259</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>He caught a break this week, flustering his Labour opponent in Parliament and glorying in British researchers’ discovery of an effective treatment drug against Covid-19 symptoms.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "12140   Foreign   \n",
       "2259    Foreign   \n",
       "\n",
       "                                                                                                                                                                                 abstract  \n",
       "12140                                                                 Zarifa Ghafari expects to die in office. But in the meantime, she intends to do something about the litter problem.  \n",
       "2259   He caught a break this week, flustering his Labour opponent in Parliament and glorying in British researchers’ discovery of an effective treatment drug against Covid-19 symptoms.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Business' predicted as 'Science' : 39 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8305</th>\n",
       "      <td>Business</td>\n",
       "      <td>A survey finds that voters think the trade measures will be bad for the United States, and that even most Republicans say they will lead to higher prices.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4833</th>\n",
       "      <td>Business</td>\n",
       "      <td>The Dart Container Corporation, which makes foam products, is a manufacturing behemoth and produced a fortune for the family behind it. Environmentalists say its products are polluting the globe.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     news_desk  \\\n",
       "8305  Business   \n",
       "4833  Business   \n",
       "\n",
       "                                                                                                                                                                                                 abstract  \n",
       "8305                                           A survey finds that voters think the trade measures will be bad for the United States, and that even most Republicans say they will lead to higher prices.  \n",
       "4833  The Dart Container Corporation, which makes foam products, is a manufacturing behemoth and produced a fortune for the family behind it. Environmentalists say its products are polluting the globe.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Foreign' predicted as 'Sports' : 38 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14658</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>A headline alongside photos of two black players in the Corriere dello Sport brought new condemnations of racism in Italy. The newspaper said it had suffered a “lynching.”</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5071</th>\n",
       "      <td>Foreign</td>\n",
       "      <td>Fast on the heels of another big storm, Dennis has caused flooding and misery for the second straight weekend.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "14658   Foreign   \n",
       "5071    Foreign   \n",
       "\n",
       "                                                                                                                                                                          abstract  \n",
       "14658  A headline alongside photos of two black players in the Corriere dello Sport brought new condemnations of racism in Italy. The newspaper said it had suffered a “lynching.”  \n",
       "5071                                                                Fast on the heels of another big storm, Dennis has caused flooding and misery for the second straight weekend.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Business' predicted as 'Sports' : 30 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16189</th>\n",
       "      <td>Business</td>\n",
       "      <td>Ford Motor is transforming the long-vacant Michigan Central Station, part of the city’s overall resurgence.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3161</th>\n",
       "      <td>Business</td>\n",
       "      <td>Put through its paces on a motor club’s racetrack, the 2021 Porsche 911 Turbo S growls with a gaudy 640 horsepower and handling that lets even amateur drivers feel like superheroes.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      news_desk  \\\n",
       "16189  Business   \n",
       "3161   Business   \n",
       "\n",
       "                                                                                                                                                                                    abstract  \n",
       "16189                                                                            Ford Motor is transforming the long-vacant Michigan Central Station, part of the city’s overall resurgence.  \n",
       "3161   Put through its paces on a motor club’s racetrack, the 2021 Porsche 911 Turbo S growls with a gaudy 640 horsepower and handling that lets even amateur drivers feel like superheroes.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Politics' predicted as 'Sports' : 15 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5453</th>\n",
       "      <td>Politics</td>\n",
       "      <td>Getting into one of the Democratic debates is like getting a ticket to the Super Bowl or the World Series.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1118</th>\n",
       "      <td>Politics</td>\n",
       "      <td>He isn’t yet the pop culture star that Ruth Bader Ginsburg — “Notorious R.B.G.” to many — is. But his fans think he could be.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     news_desk  \\\n",
       "5453  Politics   \n",
       "1118  Politics   \n",
       "\n",
       "                                                                                                                           abstract  \n",
       "5453                     Getting into one of the Democratic debates is like getting a ticket to the Super Bowl or the World Series.  \n",
       "1118  He isn’t yet the pop culture star that Ruth Bader Ginsburg — “Notorious R.B.G.” to many — is. But his fans think he could be.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "'Science' predicted as 'Sports' : 16 examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_desk</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9184</th>\n",
       "      <td>Science</td>\n",
       "      <td>On Sunday night, The Times presents a play, built from the words of the men and women who made the Apollo missions happen.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9233</th>\n",
       "      <td>Science</td>\n",
       "      <td>Even when older plaintiffs win their suits, correcting institutional biases can take years.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     news_desk  \\\n",
       "9184   Science   \n",
       "9233   Science   \n",
       "\n",
       "                                                                                                                        abstract  \n",
       "9184  On Sunday night, The Times presents a play, built from the words of the men and women who made the Apollo missions happen.  \n",
       "9233                                 Even when older plaintiffs win their suits, correcting institutional biases can take years.  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "plot_prediction_mistakes(df, y_pred, y_test, id_to_news_desk, c_matrix, indices_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check classification report.\n",
    "\n",
    "> Last step is to calculate the model metrics, here it's possible to check precision (false positive rate), recall (false negative rate), f1-score (harmonic mean of precision and recall) and accuracy (true predicitions divided by all observations) for each class. It looks like the model performs fairly well with 83% accuracy in average, though there are some weak points at Science desk (with lowess precision and second lowest recall), and US as well (lowest recall). Maybe it's time to gather more data for these specific fields, or check if they are necessary at well (again with the business felows!)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Foreign       0.81      0.88      0.84      2175\n",
      "    Business       0.81      0.82      0.82      1778\n",
      "    Politics       0.87      0.83      0.85       870\n",
      "     Science       0.77      0.64      0.70       591\n",
      "      Sports       0.91      0.91      0.91      1172\n",
      "        U.S.       0.90      0.58      0.70       182\n",
      "\n",
      "    accuracy                           0.83      6768\n",
      "   macro avg       0.85      0.78      0.80      6768\n",
      "weighted avg       0.84      0.83      0.83      6768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(metrics.classification_report(y_test, y_pred, target_names=df['news_desk'].unique()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Export"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Export data.\n",
    "\n",
    "> Exporting models for later reproduction is a must! Specially here since we're gonna wrap it inside an API so it can be available on an endpoint. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['c:\\\\users\\\\muril\\\\documents\\\\github\\\\newspapers-text-mining\\\\models\\\\id_to_news_desk.sav']"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(model, LOCAL_MODELS_PATH / 'model.sav')\n",
    "joblib.dump(tfidf, LOCAL_MODELS_PATH / 'tfidf.sav')\n",
    "joblib.dump(id_to_news_desk, LOCAL_MODELS_PATH / 'id_to_news_desk.sav')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}